add escape char support

Author: Keyang

bin/options.json

@@ -61,6 +61,10 @@
         "--quiet": {
             "desc": "If any error happens, quit the process quietly rather than log out the error. Default is false.",
             "type": "boolean"
+        },
+        "--escape":{
+            "desc":"escape character used in quoted column. Default is double quote (\") according to RFC4108. Change to back slash (\\) or other chars for your own case.",
+            "type":"string"
         }
     },
     "examples": [

libs/core/Converter.js

@@ -15,6 +15,7 @@ function Converter(params,options) {
     constructResult: true, //set to false to not construct result in memory. suitable for big csv data
     delimiter: ',', // change the delimiter of csv columns. It is able to use an array to specify potencial delimiters. e.g. [",","|",";"]
     quote: '"', //quote for a column containing delimiter.
+    escape:'"', //escape char for quoted column
     trim: true, //trim column's space charcters
     checkType: true, //whether check column type
     toArrayString: false, //stream down stringified json array instead of string of json. (useful if downstream is file writer etc)

libs/core/utils.js

@@ -2,10 +2,9 @@
 module.exports = {
   getDelimiter: getDelimiter, // Handle auto delimiter: return explicitely specified delimiter or try auto detect
   rowSplit: rowSplit, //Split a csv row to an array based on delimiter and quote
-  isToogleQuote: isToogleQuote, //returns if a segmenthas even number of quotes
-  twoDoubleQuote: twoDoubleQuote //converts two double quotes to one
+  isToogleQuote: isToogleQuote //returns if a segmenthas even number of quotes
 }
-var cachedRegExp = {};
+
 var defaulDelimiters=[",","|","\t",";",":"];
 function getDelimiter(rowStr,param) {
   var checker;
@@ -29,13 +28,17 @@ function getDelimiter(rowStr,param) {
 }
 function isQuoteOpen(str,param){
   var quote=param.quote;
-  return str[0] === quote && (str[1]!==quote || str[1]===quote && (str[2] === quote || str.length ===2));
+  var escape=param.escape;
+  return str[0] === quote && (
+    str[1]!==quote || 
+    str[1]===escape && (str[2] === quote || str.length ===2));
 }
 function isQuoteClose(str,param){
   var quote=param.quote;
   var count=0;
   var idx=str.length-1;
-  while (str[idx] === quote){
+  var escape=param.escape;
+  while (str[idx] === quote || str[idx]===escape){
     idx--;
     count++;
   }
@@ -47,6 +50,7 @@ function rowSplit(rowStr, param) {
   }
   var quote=param.quote;
   var trim=param.trim;
+  var escape=param.escape;
   if (param.needCheckDelimiter===true){
       param.delimiter=getDelimiter(rowStr,param);
       param.needCheckDelimiter=false;
@@ -70,7 +74,7 @@ function rowSplit(rowStr, param) {
           e=e.substr(1);
           if (isQuoteClose(e,param)){ //quote close
               e=e.substring(0,e.length-1);
-              e=twoDoubleQuote(e,quote);
+              e=_escapeQuote(e,quote,escape);
               row.push(e);
               continue;
           }else{
@@ -87,7 +91,7 @@ function rowSplit(rowStr, param) {
         inquote=false;
         e=e.substr(0,len-1);
         quoteBuff+=delimiter+e;
-        quoteBuff=twoDoubleQuote(quoteBuff,quote);
+        quoteBuff=_escapeQuote(quoteBuff,quote,escape);
         if (trim){
           quoteBuff=quoteBuff.trimRight();
         }
@@ -125,25 +129,28 @@ function rowSplit(rowStr, param) {
   return row;
 }
 
-function _getRegExpObj(quote) {
-  if (cachedRegExp[quote]) {
-    return cachedRegExp[quote];
-  } else {
-    cachedRegExp[quote] = {
-      single: new RegExp(quote, 'g'),
-      double: new RegExp(quote + quote, 'g')
-    }
-    return _getRegExpObj(quote);
-  }
-}
+
+var cachedRegExp = {};
 
 function isToogleQuote(segment, quote) {
-  var reg = _getRegExpObj(quote).single;
+  var key="s|"+quote;
+  if ( cachedRegExp[key]=== undefined){
+    cachedRegExp[key]=new RegExp(quote, 'g');
+  }
+  var reg =  cachedRegExp[key];
   var match = segment.match(reg);
   return match && match.length % 2 !== 0;
 }
 
-function twoDoubleQuote(segment, quote) {
-  var regExp = _getRegExpObj(quote).double;
+function _escapeQuote(segment, quote,escape) {
+  
+  var key="es|"+quote+"|"+escape;
+  if (cachedRegExp[key] === undefined){
+    if (escape ==="\\"){
+      escape="\\\\";
+    }
+    cachedRegExp[key]=new RegExp(escape+quote,'g');
+  }
+  var regExp = cachedRegExp[key];
   return segment.replace(regExp, quote);
 }

package.json

@@ -18,7 +18,7 @@
       "email": "t3dodson@gmail.com"
     }
   ],
-  "version": "1.0.2",
+  "version": "1.0.3",
   "keywords": [
     "csv",
     "csvtojson",

readme.md

@@ -160,6 +160,7 @@ Following parameters are supported:
 * **maxRowLength**: the max character a csv row could have. 0 means infinite. If max number exceeded, parser will emit "error" of "row_exceed". if a possibly corrupted csv data provided, give it a number like 65535 so the parser wont consume memory. default: 0
 * **checkColumn**: whether check column number of a row is the same as headers. If column number mismatched headers number, an error of "mismatched_column" will be emitted.. default: false
 * **eol**: End of line character. If omitted, parser will attempt retrieve it from first chunk of CSV data. If no valid eol found, then operation system eol will be used.
+* **escape**: escape character used in quoted column. Default is double quote (") according to RFC4108. Change to back slash (\) or other chars for your own case.
 
 All parameters can be used in Command Line tool. see
 

test/data/dataWithSlashEscape

@@ -0,0 +1,2 @@
+id,raw
+0,"{\"hello\":\"world\",\"test\":true}"

test/testCSVConverter2.js

@@ -240,6 +240,20 @@ describe("CSV Converter", function() {
       done();
     });
   });
+  it ("should process escape chars",function(done){
+    var test_converter = new Converter({
+      escape:"\\"
+    });
+
+    var testData = __dirname + "/data/dataWithSlashEscape";
+    var rs = fs.createReadStream(testData);
+    test_converter.on("end_parsed",function(res){
+      assert.equal(res[0].raw.hello,"world");
+      assert.equal(res[0].raw.test,true);
+      done();
+    });
+    rs.pipe(test_converter);
+  });
   // it ("should convert big csv",function(done){
   //   // var rs=fs.createReadStream(__dirname+"/data/large-csv-sample.csv");
   //   var rs=fs.createReadStream("/Users/kxiang/tmp/csvdata");