add preProcessLine hook

Author: Keyang

libs/core/Converter.js

@@ -42,6 +42,7 @@ function Converter(params) {
   this.pipe(this.resultObject); // it is important to have downstream for a transform otherwise it will stuck
   this.started = false;
   this.recordNum = 0;
+  this.lineNumber=0;
   this.runningProcess = 0;
   //this._pipe(this.lineParser).pipe(this.processor);
   if (this.param.fork) {
@@ -244,14 +245,18 @@ Converter.prototype.toLines = function(data) {
   var eol = this.getEol(data);
   return data.split(eol);
 }
-var lineNumber=0;
+Converter.prototype.preProcessLine=function(line,lineNumber){
+    return line;
+}
 Converter.prototype.toCSVLines = function(fileLines, last) {
   var recordLine = "";
   var lines = [];
   while (fileLines.length > 1) {
-    lineNumber++;
-    var line = fileLines.shift();
-    lines = lines.concat(this._line(line));
+    this.lineNumber++;
+    var line = this.preProcessLine(fileLines.shift(),this.lineNumber);
+    if (line && line.length>0){
+      lines = lines.concat(this._line(line));
+    }
   }
   this._lineBuffer = fileLines[0];
   if (last && this._csvLineBuffer.length > 0) {

package.json

@@ -18,7 +18,7 @@
       "email": "t3dodson@gmail.com"
     }
   ],
-  "version": "0.5.13",
+  "version": "0.5.14",
   "keywords": [
     "csv",
     "csvtojson",

readme.md

@@ -338,6 +338,29 @@ conv.preProcessRaw=function(data,cb){
     }
 }
 ```
+## preProcessLine
+this hook is called when a file line is emitted. It is called with two parameters `fileLineData,lineNumber`. The `lineNumber` is starting from 1.
+```js
+/*
+CSV data:
+a,b,c,d,e
+12,e3,fb,w2,dd
+*/
+
+var conv=new Converter();
+conv.preProcessLine=function(line,lineNumber){
+    //only change 12 to 23 for line 2
+    if (lineNumber === 2){
+      line=line.replace("12","23");
+    }
+    return line;
+}
+conv.fromString(csv,function(err,json){
+  //json:{a:23 ....}
+})
+```
+Notice that preProcessLine does not support async changes not like preProcessRaw hook.
+
 
 # Events
 

test/testCSVConverter2.js

@@ -177,7 +177,7 @@ describe("CSV Converter", function() {
     });
     rs.pipe(conv);
   })
-  it ("should pre process data in the line",function(done){
+  it ("should pre process raw data in the line",function(done){
     var testData = __dirname + "/data/quoteTolerant";
     var rs = fs.createReadStream(testData);
     var conv=new Converter();
@@ -191,6 +191,22 @@ describe("CSV Converter", function() {
     });
     rs.pipe(conv);
   })
+  it ("should pre process by line in the line",function(done){
+    var testData = __dirname + "/data/quoteTolerant";
+    var rs = fs.createReadStream(testData);
+    var conv=new Converter();
+    conv.preProcessLine=function(line,lineNumber){
+      if (lineNumber === 2){
+        line=line.replace('32"','32""');
+      }
+      return line;
+    }
+    conv.on("end_parsed",function(res){
+      assert(res[0].Description.indexOf('32"')>-1);
+      done();
+    });
+    rs.pipe(conv);
+  })
   // it ("should convert big csv",function(done){
   //   // var rs=fs.createReadStream(__dirname+"/data/large-csv-sample.csv");
   //   var rs=fs.createReadStream("/Users/kxiang/tmp/csvdata");