merge from master

Author: Keyang

bin/options.json

@@ -61,6 +61,10 @@
         "--quiet": {
             "desc": "If any error happens, quit the process quietly rather than log out the error. Default is false.",
             "type": "boolean"
+        },
+        "--escape":{
+            "desc":"escape character used in quoted column. Default is double quote (\") according to RFC4108. Change to back slash (\\) or other chars for your own case.",
+            "type":"string"
         }
     },
     "examples": [

dist/csvtojson.js

@@ -18,8 +18,8 @@ var Worker = require("./Worker.js");
 var utils = require("./utils.js");
 var async = require("async");
 
-function Converter(params) {
-  Transform.call(this); //TODO what does this do? -->This calls the constructor of Transform and initialise anything the Transform needs.(like var initialisation)
+function Converter(params,options) {
+  Transform.call(this,options);
   var _param = {
     constructResult: true, //set to false to not construct result in memory. suitable for big csv data
     delimiter: ',', // change the delimiter of csv columns. It is able to use an array to specify potencial delimiters. e.g. [",","|",";"]
@@ -46,11 +46,14 @@ function Converter(params) {
     console.warn("Parameter should be a JSON object like {'constructResult':false}");
     _param.constructResult = params;
   }
+  this._options=options || {};
   this.param = _param;
+  this.param._options=this._options;
   this.resultObject = new Result(this);
   this.pipe(this.resultObject); // it is important to have downstream for a transform otherwise it will stuck
   this.started = false;
   this.recordNum = 0;
+  this.lineNumber=0;
   this.runningProcess = 0;
   //this._pipe(this.lineParser).pipe(this.processor);
   if (this.param.fork) {
@@ -192,21 +195,34 @@ Converter.prototype.flushBuffer = function() {
     if (this.param.toArrayString && this.recordNum > 0) {
       this.push("," + eol);
     }
-    this.push(resultJSONStr, "utf8");
+    if (this._options && this._options.objectMode){
+      this.push(resultRow);
+    }else{
+      this.push(resultJSONStr, "utf8");
+    }
     this.recordNum++;
   }
   this.checkAndFlush();
 }
-var size = 0;
+Converter.prototype.preProcessRaw=function(data,cb){
+  cb(data);
+}
 
 Converter.prototype._transformNoFork = function(data, encoding, cb) {
-  size += data.length;
   if (this.param.toArrayString && this.started === false) {
     this.started = true;
     this.push("[" + eol, "utf8");
   }
-  var lines = this.toCSVLines(this.toLines(data, encoding)); //lines of csv
-  this.processCSVLines(lines, cb);
+  data=data.toString("utf8");
+  var self=this;
+  this.preProcessRaw(data,function(d){
+    if (d && d.length>0){
+      var lines = self.toCSVLines(self.toLines(d)); //lines of csv
+      self.processCSVLines(lines, cb);
+    }else{
+      cb();
+    }
+  })
   // async.eachLimit(lines,1,function(line,scb){
   //   this.push(line.data);
   //   scb();
@@ -239,20 +255,23 @@ Converter.prototype.processCSVLines = function(csvLines, cb) {
     }
   }.bind(this), cb);
 }
-Converter.prototype.toLines = function(data, encoding) {
-  if (encoding === "buffer") {
-    encoding = "utf8";
-  }
-  data = this._lineBuffer + data.toString(encoding);
+Converter.prototype.toLines = function(data) {
+  data = this._lineBuffer + data;
   var eol = this.getEol(data);
   return data.split(eol);
 }
+Converter.prototype.preProcessLine=function(line,lineNumber){
+    return line;
+}
 Converter.prototype.toCSVLines = function(fileLines, last) {
   var recordLine = "";
   var lines = [];
   while (fileLines.length > 1) {
-    var line = fileLines.shift();
-    lines = lines.concat(this._line(line));
+    this.lineNumber++;
+    var line = this.preProcessLine(fileLines.shift(),this.lineNumber);
+    if (line && line.length>0){
+      lines = lines.concat(this._line(line));
+    }
   }
   this._lineBuffer = fileLines[0];
   if (last && this._csvLineBuffer.length > 0) {
@@ -417,7 +436,7 @@ function Processor(params) {
   this.runningWorker = 0;
   this.flushCb = null;
   if (this.param.workerNum > 1) {
-    for (var i = 0; i < this.param.workerNum; i++) {
+    for (var i = 0; i < this.param.workerNum-1; i++) {
       var worker = new Worker(this.param, false);
       // worker.on("error",onError);
       this.addWorker(worker);
@@ -558,20 +577,40 @@ var Writable = require("stream").Writable;
 var util = require("util");
 var eol=require("os").EOL;
 function Result(csvParser) {
-  Writable.call(this);
+  Writable.call(this,csvParser._options);
+  this._option=csvParser._options || {};
   this.parser = csvParser;
   this.param = csvParser.param;
-  this.buffer = this.param.toArrayString?"":"["+eol;
+  this.buffer =this._option.objectMode?[]:"["+eol;
   this.started = false;
   var self = this;
   this.parser.on("end", function() {
-    if (!self.param.toArrayString){
+    if (typeof self.buffer === "string"){
       self.buffer += eol+ "]";
     }
   });
+  this._write=this._option.objectMode?_writeObject:_writeBuffer;
 }
 util.inherits(Result, Writable);
-Result.prototype._write = function(data, encoding, cb) {
+
+Result.prototype.getBuffer = function() {
+  return typeof this.buffer ==="string"?JSON.parse(this.buffer):this.buffer;
+};
+
+Result.prototype.disableConstruct = function() {
+  this._write = function(d, e, cb) {
+    // console.log(typeof d,d);
+    cb(); //do nothing just dropit
+  };
+};
+
+
+function _writeObject (data, encoding, cb) {
+  this.buffer.push(data);
+  cb();
+};
+
+function _writeBuffer(data, encoding, cb) {
   if (encoding === "buffer") {
     encoding = "utf8";
   }
@@ -588,17 +627,6 @@ Result.prototype._write = function(data, encoding, cb) {
   cb();
 };
 
-Result.prototype.getBuffer = function() {
-  // console.log(this.buffer);
-  return JSON.parse(this.buffer);
-};
-
-Result.prototype.disableConstruct = function() {
-  this._write = function(d, e, cb) {
-    cb(); //do nothing just dropit
-  };
-};
-
 module.exports = Result;
 
 },{"os":35,"stream":54,"util":67}],6:[function(require,module,exports){
@@ -772,21 +800,33 @@ module.exports = {
     var headArr = (params.config && params.config.flatKeys) ? [fieldStr] : fieldStr.split('.');
     var match, index, key, pointer;
     //now the pointer is pointing the position to add a key/value pair.
-    pointer = processHead(params.resultRow, headArr, arrReg, params.config && params.config.flatKeys);
+    var pointer = processHead(params.resultRow, headArr, arrReg, params.config && params.config.flatKeys);
     key = headArr.shift();
     match = (params.config && params.config.flatKeys) ? false : key.match(arrReg);
     if (match) { // the last element is an array, we need check and treat it as an array.
-      key = key.replace(match[0], '');
-      if (!pointer[key] || !(pointer[key] instanceof Array)) {
-        pointer[key] = [];
-      }
-      index = match[1];
-      if (index === '') {
-        index = pointer[key].length;
+      try {
+        key = key.replace(match[0], '');
+        if (!pointer[key] || !(pointer[key] instanceof Array)) {
+          pointer[key] = [];
+        }
+        if (pointer[key]) {
+          index = match[1];
+          if (index === '') {
+            index = pointer[key].length;
+          }
+          pointer[key][index] = params.item;
+        } else {
+          params.resultRow[fieldStr] = params.item;
+        }
+      } catch (e) {
+        params.resultRow[fieldStr] = params.item;
       }
-      pointer[key][index] = params.item;
     } else {
-      pointer[key] =  params.item;
+      if (typeof pointer=== "string"){
+        params.resultRow[fieldStr] = params.item;
+      }else{
+        pointer[key] = params.item;
+      }
     }
   }
 };
@@ -847,7 +887,7 @@ function Parser(name, regExp, parser, processSafe) {
     this.parse = parser;
   }
 }
-var numReg = /^[-+]?[0-9]*\.?[0-9]+$/;
+var numReg = /^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?$/;
 Parser.prototype.convertType = function(item) {
   var type=this.type;
   if (type === 'number') {
@@ -1049,6 +1089,9 @@ function isQuoteClose(str,param){
   return count%2!==0;
 }
 function rowSplit(rowStr, param) {
+  if (rowStr ===""){
+    return [];
+  }
   var quote=param.quote;
   var trim=param.trim;
   if (param.needCheckDelimiter===true){
@@ -1057,6 +1100,9 @@ function rowSplit(rowStr, param) {
   }
   var delimiter=param.delimiter;
   var rowArr = rowStr.split(delimiter);
+  if (quote ==="off"){
+    return rowArr;
+  }
   var row = [];
   var inquote = false;
   var quoteBuff = '';
@@ -10147,7 +10193,7 @@ module.exports={
       "email": "t3dodson@gmail.com"
     }
   ],
-  "version": "0.5.10",
+  "version": "1.0.1",
   "keywords": [
     "csv",
     "csvtojson",
@@ -10180,10 +10226,11 @@ module.exports={
     "mocha": "^2.4.5"
   },
   "dependencies": {
-    "async": "^1.2.1"
+    "async": "^1.2.1",
+    "minimist": "^1.2.0"
   },
-  "scripts":{
-    "test":"mocha ./test -R spec"
+  "scripts": {
+    "test": "mocha ./test -R spec"
   }
 }
 

dist/csvtojson.min.js

@@ -13,7 +13,8 @@ module.exports=function(params){
     headers: null, //an array of header strings. If noheader is false and headers is array, csv header will be ignored.
     flatKeys: false, // Don't interpret dots and square brackets in header fields as nested object or array identifiers at all.
     maxRowLength: 0, //the max character a csv row could have. 0 means infinite. If max number exceeded, parser will emit "error" of "row_exceed". if a possibly corrupted csv data provided, give it a number like 65535 so the parser wont consume memory. default: 0
-    checkColumn: false //whether check column number of a row is the same as headers. If column number mismatched headers number, an error of "mismatched_column" will be emitted.. default: false
+    checkColumn: false, //whether check column number of a row is the same as headers. If column number mismatched headers number, an error of "mismatched_column" will be emitted.. default: false
+    escape:'"' //escape char for quoted column
   };
   for (var key in params) {
     if (params.hasOwnProperty(key)) {

libs/core/defParam.js

@@ -2,10 +2,9 @@
 module.exports = {
   getDelimiter: getDelimiter, // Handle auto delimiter: return explicitely specified delimiter or try auto detect
   rowSplit: rowSplit, //Split a csv row to an array based on delimiter and quote
-  isToogleQuote: isToogleQuote, //returns if a segmenthas even number of quotes
-  twoDoubleQuote: twoDoubleQuote //converts two double quotes to one
+  isToogleQuote: isToogleQuote //returns if a segmenthas even number of quotes
 }
-var cachedRegExp = {};
+
 var defaulDelimiters=[",","|","\t",";",":"];
 function getDelimiter(rowStr,param) {
   var checker;
@@ -29,21 +28,29 @@ function getDelimiter(rowStr,param) {
 }
 function isQuoteOpen(str,param){
   var quote=param.quote;
-  return str[0] === quote && (str[1]!==quote || str[1]===quote && (str[2] === quote || str.length ===2));
+  var escape=param.escape;
+  return str[0] === quote && (
+    str[1]!==quote || 
+    str[1]===escape && (str[2] === quote || str.length ===2));
 }
 function isQuoteClose(str,param){
   var quote=param.quote;
   var count=0;
   var idx=str.length-1;
-  while (str[idx] === quote){
+  var escape=param.escape;
+  while (str[idx] === quote || str[idx]===escape){
     idx--;
     count++;
   }
   return count%2!==0;
 }
 function rowSplit(rowStr, param) {
+  if (rowStr ===""){
+    return [];
+  }
   var quote=param.quote;
   var trim=param.trim;
+  var escape=param.escape;
   if (param.needCheckDelimiter===true){
       param.delimiter=getDelimiter(rowStr,param);
       param.needCheckDelimiter=false;
@@ -67,7 +74,7 @@ function rowSplit(rowStr, param) {
           e=e.substr(1);
           if (isQuoteClose(e,param)){ //quote close
               e=e.substring(0,e.length-1);
-              e=twoDoubleQuote(e,quote);
+              e=_escapeQuote(e,quote,escape);
               row.push(e);
               continue;
           }else{
@@ -84,7 +91,7 @@ function rowSplit(rowStr, param) {
         inquote=false;
         e=e.substr(0,len-1);
         quoteBuff+=delimiter+e;
-        quoteBuff=twoDoubleQuote(quoteBuff,quote);
+        quoteBuff=_escapeQuote(quoteBuff,quote,escape);
         if (trim){
           quoteBuff=quoteBuff.trimRight();
         }
@@ -122,25 +129,28 @@ function rowSplit(rowStr, param) {
   return row;
 }
 
-function _getRegExpObj(quote) {
-  if (cachedRegExp[quote]) {
-    return cachedRegExp[quote];
-  } else {
-    cachedRegExp[quote] = {
-      single: new RegExp(quote, 'g'),
-      double: new RegExp(quote + quote, 'g')
-    }
-    return _getRegExpObj(quote);
-  }
-}
+
+var cachedRegExp = {};
 
 function isToogleQuote(segment, quote) {
-  var reg = _getRegExpObj(quote).single;
+  var key="s|"+quote;
+  if ( cachedRegExp[key]=== undefined){
+    cachedRegExp[key]=new RegExp(quote, 'g');
+  }
+  var reg =  cachedRegExp[key];
   var match = segment.match(reg);
   return match && match.length % 2 !== 0;
 }
 
-function twoDoubleQuote(segment, quote) {
-  var regExp = _getRegExpObj(quote).double;
+function _escapeQuote(segment, quote,escape) {
+  
+  var key="es|"+quote+"|"+escape;
+  if (cachedRegExp[key] === undefined){
+    if (escape ==="\\"){
+      escape="\\\\";
+    }
+    cachedRegExp[key]=new RegExp(escape+quote,'g');
+  }
+  var regExp = cachedRegExp[key];
   return segment.replace(regExp, quote);
 }