add preProcessHook

Author: Keyang

libs/core/Converter.js

@@ -10,9 +10,7 @@ var utils = require("./utils.js");
 var async = require("async");
 
 function Converter(params) {
-  Transform.call(this,{
-    highWaterMark:1024
-  }); //TODO what does this do? -->This calls the constructor of Transform and initialise anything the Transform needs.(like var initialisation)
+  Transform.call(this);
   var _param = {
     constructResult: true, //set to false to not construct result in memory. suitable for big csv data
     delimiter: ',', // change the delimiter of csv columns. It is able to use an array to specify potencial delimiters. e.g. [",","|",";"]
@@ -190,16 +188,25 @@ Converter.prototype.flushBuffer = function() {
   }
   this.checkAndFlush();
 }
-var size = 0;
+Converter.prototype.preProcessRaw=function(data,cb){
+  cb(data);
+}
 
 Converter.prototype._transformNoFork = function(data, encoding, cb) {
-  size += data.length;
   if (this.param.toArrayString && this.started === false) {
     this.started = true;
     this.push("[" + eol, "utf8");
   }
-  var lines = this.toCSVLines(this.toLines(data, encoding)); //lines of csv
-  this.processCSVLines(lines, cb);
+  data=data.toString("utf8");
+  var self=this;
+  this.preProcessRaw(data,function(d){
+    if (d && d.length>0){
+      var lines = self.toCSVLines(self.toLines(d)); //lines of csv
+      self.processCSVLines(lines, cb);
+    }else{
+      cb();
+    }
+  })
   // async.eachLimit(lines,1,function(line,scb){
   //   this.push(line.data);
   //   scb();
@@ -232,18 +239,17 @@ Converter.prototype.processCSVLines = function(csvLines, cb) {
     }
   }.bind(this), cb);
 }
-Converter.prototype.toLines = function(data, encoding) {
-  if (encoding === "buffer") {
-    encoding = "utf8";
-  }
-  data = this._lineBuffer + data.toString(encoding);
+Converter.prototype.toLines = function(data) {
+  data = this._lineBuffer + data;
   var eol = this.getEol(data);
   return data.split(eol);
 }
+var lineNumber=0;
 Converter.prototype.toCSVLines = function(fileLines, last) {
   var recordLine = "";
   var lines = [];
   while (fileLines.length > 1) {
+    lineNumber++;
     var line = fileLines.shift();
     lines = lines.concat(this._line(line));
   }

libs/core/Processor.js

@@ -32,7 +32,7 @@ function Processor(params) {
   this.runningWorker = 0;
   this.flushCb = null;
   if (this.param.workerNum > 1) {
-    for (var i = 0; i < this.param.workerNum; i++) {
+    for (var i = 0; i < this.param.workerNum-1; i++) {
       var worker = new Worker(this.param, false);
       // worker.on("error",onError);
       this.addWorker(worker);

package.json

@@ -18,7 +18,7 @@
       "email": "t3dodson@gmail.com"
     }
   ],
-  "version": "0.5.12",
+  "version": "0.5.13",
   "keywords": [
     "csv",
     "csvtojson",

readme.md

@@ -27,6 +27,7 @@ All you need nodejs csv to json converter.
   * [Synchronouse Transformer](#synchronouse-transformer)
   * [Asynchronouse Transformer](#asynchronouse-transformer)
   * [Convert to other data type](#convert-to-other-data-type)
+* [Hooks](#hooks)
 * [Events](#events)
 * [Flags](#flags)
 * [Big CSV File Streaming](#big-csv-file)
@@ -300,6 +301,43 @@ It will be converted to:
 }
 ```
 
+# Hooks
+## preProcessRaw
+This hook is called when parser received any data from upper stream and allow developers to change it. e.g.
+```js
+/*
+CSV data:
+a,b,c,d,e
+12,e3,fb,w2,dd
+*/
+
+var conv=new Converter();
+conv.preProcessRaw=function(data,cb){
+    //change all 12 to 23
+    cb(data.replace("12","23"));
+}
+conv.fromString(csv,function(err,json){
+  //json:{a:23 ....}
+})
+```
+By default, the preProcessRaw just returns the data from the source
+```js
+Converter.prototype.preProcessRaw=function(data,cb){
+  cb(data);
+}
+```
+It is also very good to sanitise/prepare the CSV data stream.
+```js
+var headWhiteSpaceRemoved=false;
+conv.preProcessRaw=function(data,cb){
+    if (!headWhiteSpaceRemoved){
+      data=data.replace(/^\s+/,"");
+      cb(data);
+    }else{
+      cb(data);
+    }
+}
+```
 
 # Events
 
@@ -727,7 +765,7 @@ The parameter of Parse function is a JSON object. It contains following fields:
 
 ## 0.5.12
 * Added support for scientific notation number support (#100)
-* Added "off" option to quote parameter 
+* Added "off" option to quote parameter
 
 ## 0.5.4
 * Added new feature: accept special delimiter "auto" and array

test/data/quoteTolerant

@@ -0,0 +1,2 @@
+"Style","Description","SKU","BusinessAreaCode","BusinessAreaDescription","DepartmentCode","DepartmentDescription","ClassCode","ClassDescription","ColorCode","ColorDescription","SizeCode","SizeDescription","PrimaryMaterialCode","PrimaryMaterialDescription","VendorCode","VendorDescription","CurrentRetail"
+"503951518","OVERSIZE ROUND GOLD PENDANT ON 32" THICK BLACK CORD","00505039515193","W12","Jewelry","1231","Jewelry","123111","D2 New Jewelry Necklace","70","GOLD","100","1 Size","065","Costume/Imitation","3522","KENNETH JAY LANE, INC.","79.0000"

test/testCSVConverter2.js

@@ -177,6 +177,20 @@ describe("CSV Converter", function() {
     });
     rs.pipe(conv);
   })
+  it ("should pre process data in the line",function(done){
+    var testData = __dirname + "/data/quoteTolerant";
+    var rs = fs.createReadStream(testData);
+    var conv=new Converter();
+    conv.preProcessRaw=function(d,cb){
+      d=d.replace('32"','32""');
+      cb(d);
+    }
+    conv.on("end_parsed",function(res){
+      assert(res[0].Description.indexOf('32"')>-1);
+      done();
+    });
+    rs.pipe(conv);
+  })
   // it ("should convert big csv",function(done){
   //   // var rs=fs.createReadStream(__dirname+"/data/large-csv-sample.csv");
   //   var rs=fs.createReadStream("/Users/kxiang/tmp/csvdata");