rewrite readme.md

Author: Keyang

.npmignore

@@ -0,0 +1,19 @@
+.DS_Store
+Thumbs.db
+node_modules/
+*.swp
+*.swo
+
+# intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+pids
+logs
+results
+build
+graph.png
+.project
+.settings
+.vscode
+docs
+devops

bin/options.json

@@ -1,7 +1,6 @@
 {
     "commands": {
         "parse": "(Default)Parse a csv file to json",
-        "startserver": "(Deprecated)Start a web server",
         "version": "Show version of current csvtojson"
     },
     "options": {

docs/performance.md

@@ -0,0 +1,26 @@
+# Performance Optimisation
+
+Version 1.1.0 is much faster than versions before. 
+
+Below test is parsing a 300K records csv
+
+ Time | workNum = 1 | workNum = 2 (fork)* | workNum = 3 |  workNum = 4
+--- | --- | --- | --- | ---
+**1.0.3** | 11.806s | 15.945s | 8.611s | 8.314s
+**1.1.0** | 9.707s | 10.065s | 5.955s | 4.563s
+
+*when workNum=2, it only creates 1 extra worker to unblock main process. It is reasonalbe it is slightly slower than workerNum=1 (just use main process).
+
+The result shows V1.1.0 has about 30% - 50% performance boost.
+
+# CPU usage leverage
+
+Below is CPU core usages for v1.1.0 when running the test:
+
+ Core | workNum = 1 | workNum = 2| workNum = 3 |  workNum = 4
+--- | --- | --- | --- | ---
+Core 1 (Main) | 100% | 25% | 60% | 90%
+Core 2 | N/A | 80% | 70% | 70%
+Core 3 | N/A | N/A | 70% | 70%
+Core 4 | N/A | N/A | N/A | 70%
+

libs/core/Converter.js

@@ -39,28 +39,41 @@ function Converter(params,options) {
   this._needJson=null;
   this._needEmitResult=null;
   this._needEmitFinalResult=null;
+  this._needEmitJson=null;
   this._needPush=null;
+  this._needEmitCsv=null;
+  this._csvTransf=null;
   this.finalResult=[];
   // this.on("data", function() {});
   this.on("error", function() {});
   this.initWorker();
+  process.nextTick(function(){
+    if (this._needEmitFinalResult === null){
+      this._needEmitFinalResult=this.listeners("end_parsed").length > 0
+    }
+    if (this._needEmitResult===null){
+      this._needEmitResult=this.listeners("record_parsed").length>0
+    }
+    if (this._needEmitJson === null){
+      this._needEmitJson=this.listeners("json").length>0
+    }
+    if (this._needEmitCsv === null){
+      this._needEmitCsv=this.listeners("csv").length>0
+    }
+    if (this._needJson === null){
+      this._needJson=this._needEmitJson || this._needEmitFinalResult || this._needEmitResult || this.transform || this._options.objectMode;
+    }
+    if (this._needPush === null){
+      this._needPush = this.listeners("data").length > 0 || this.listeners("readable").length>0
+      // this._needPush=false;
+    }
+    this.param._needParseJson=this._needJson || this._needPush; 
+
+  }.bind(this))
   return this;
 }
 util.inherits(Converter, Transform);
 Converter.prototype._transform = function(data, encoding, cb) {
-  if (this._needEmitFinalResult === null){
-    this._needEmitFinalResult=this.listeners("end_parsed").length > 0
-  }
-  if (this._needEmitResult===null){
-    this._needEmitResult=this.listeners("record_parsed").length>0
-  }
-  if (this._needJson === null){
-    this._needJson=this._needEmitFinalResult || this._needEmitResult || this.transform || this._options.objectMode;
-  }
-  if (this._needPush === null){
-    this._needPush = this.listeners("data").length > 0 || this.listeners("readable").length>0
-    // this._needPush=false;
-  }
   if (this.param.toArrayString && this.started === false) {
     this.started = true;
     if (this._needPush){
@@ -206,11 +219,21 @@ Converter.prototype.emitResult=function(r){
       row=JSON.parse(row)
     }
   }
-  if (this.param.constructResult && this._needEmitFinalResult){
-    this.finalResult.push(resultJson)
-  }
   if (this.transform && typeof this.transform==="function"){
     this.transform(resultJson,row,index);
+    resultStr=null;
+  }
+  if (this._needEmitJson){
+    this.emit("json",resultJson,index)
+  }
+  if (this._needEmitCsv){
+    if (typeof row ==="string"){
+      row=JSON.parse(row)
+    }
+    this.emit("csv",row,index)
+  }
+  if (this.param.constructResult && this._needEmitFinalResult){
+    this.finalResult.push(resultJson)
   }
   if (this._needEmitResult){
     this.emit("record_parsed", resultJson, row, index);
@@ -305,51 +328,65 @@ Converter.prototype.getEol = function(data) {
 };
 Converter.prototype.fromFile = function(filePath, cb) {
   var fs = require('fs');
+  var rs=null;
+  this.wrapCallback(cb, function() {
+    if (rs && rs.destroy){
+      rs.destroy();
+    }
+  });
   fs.exists(filePath, function(exist) {
     if (exist) {
-      var rs = fs.createReadStream(filePath);
+      rs = fs.createReadStream(filePath);
       rs.pipe(this);
-      this.wrapCallback(cb, function() {
-        rs.destroy();
-      });
     } else {
-      cb(new Error(filePath + " cannot be found."));
+      this.emit('error',new Error("File not exist"))
     }
   }.bind(this));
   return this;
 }
+Converter.prototype.fromStream=function(readStream,cb){
+  if (cb && typeof cb ==="function"){
+    this.wrapCallback(cb);
+  }
+  process.nextTick(function(){
+    readStream.pipe(this);
+  }.bind(this))
+  return this;
+}
+Converter.prototype.transf=function(func){
+  this.transform=func;
+  return this;
+}
 Converter.prototype.fromString = function(csvString, cb) {
-  var rs = new Readable();
-  var offset = 0;
   if (typeof csvString != "string") {
     return cb(new Error("Passed CSV Data is not a string."));
   }
-  rs._read = function(len) {
-    // console.log(offset,len,csvString.length);
-    var sub = csvString.substr(offset, len);
-    this.push(sub);
-    offset += len;
-    if (offset >= csvString.length) {
-      this.push(null);
-    }
-  };
-  rs.pipe(this);
   if (cb && typeof cb === "function") {
     this.wrapCallback(cb, function() {
-      rs.pause();
     });
   }
+  process.nextTick(function(){
+    this.end(csvString)
+  }.bind(this))
   return this;
 };
 Converter.prototype.wrapCallback = function(cb, clean) {
-  this.once("end_parsed", function(res) {
-    if (!this.hasError) {
-      cb(null, res);
-    }
-  }.bind(this));
+
+  if (clean === undefined){
+    clean=function(){}
+  }
+  if (cb && typeof cb ==="function"){
+    this.once("end_parsed", function(res) {
+      if (!this.hasError) {
+        cb(null, res);
+      }
+    }.bind(this));
+  }
   this.once("error", function(err) {
     this.hasError=true;
-    cb(err);
+    if (cb && typeof cb  ==="function"){
+      cb(err);
+    }
     clean();
   }.bind(this));
 }

libs/core/defParam.js

@@ -16,6 +16,9 @@ module.exports=function(params){
     checkColumn: false, //whether check column number of a row is the same as headers. If column number mismatched headers number, an error of "mismatched_column" will be emitted.. default: false
     escape:'"' //escape char for quoted column
   };
+  if (!params){
+    params={};
+  }
   for (var key in params) {
     if (params.hasOwnProperty(key)) {
       _param[key] = params[key];

libs/core/linesToJson.js

@@ -9,14 +9,31 @@ var CSVError=require("./CSVError");
  * @return {[type]}   [{err:null,json:obj,index:line,row:[csv row]}]
  */
 module.exports=function(lines,params,idx){
-  if (!params._headers){
-    params._headers=[];
+  if (params._needParseJson){
+    if (!params._headers){
+      params._headers=[];
+    }
+    if (!params.parseRules){
+      var row=params._headers;
+      params.parseRules=parserMgr.initParsers(row,params);
+    }
+    return processRows(lines,params,idx);
+  }else{
+    return justReturnRows(lines,params,idx);
   }
-  if (!params.parseRules){
-    var row=params._headers;
-    params.parseRules=parserMgr.initParsers(row,params);
+}
+
+function justReturnRows(lines,params,idx){
+  var rtn=[];
+  for (var i=0;i<lines.length;i++){
+    rtn.push({
+      err:null,
+      json:{},
+      index:idx++,
+      row:lines[i]
+    })
   }
-  return processRows(lines,params,idx);
+  return rtn;
 }
 function processRows(csvRows, params,startIndex) {
   var count = csvRows.length;

package.json

@@ -18,7 +18,7 @@
       "email": "t3dodson@gmail.com"
     }
   ],
-  "version": "1.0.3",
+  "version": "1.1.0",
   "keywords": [
     "csv",
     "csvtojson",