fix: #139

Author: Keyang

libs/core/defParam.js

@@ -4,7 +4,7 @@ module.exports=function(params){
     delimiter: ',', // change the delimiter of csv columns. It is able to use an array to specify potencial delimiters. e.g. [",","|",";"]
     quote: '"', //quote for a column containing delimiter.
     trim: true, //trim column's space charcters
-    checkType: true, //whether check column type
+    checkType: false, //whether check column type
     toArrayString: false, //stream down stringified json array instead of string of json. (useful if downstream is file writer etc)
     ignoreEmpty: false, //Ignore empty value while parsing. if a value of the column is empty, it will be skipped parsing.
     workerNum: getEnv("CSV_WORKER",1), //number of parallel workers. If multi-core CPU available, increase the number will get better performance for large csv data.

libs/core/linesToJson.js

@@ -172,7 +172,7 @@ function checkType(item, head, headIdx, param) {
     } else if (head.indexOf('string#!') > -1) {
       return param._headerType[headIdx] = stringType
     } else if (param.checkType) {
-      return param._headerType[headIdx] = dynamicType(item)
+      return param._headerType[headIdx] = dynamicType
     } else {
       return param._headerType[headIdx] = stringType
     }
@@ -194,17 +194,17 @@ function stringType(item) {
 function dynamicType(item) {
   var trimed = item.trim();
   if (trimed === "") {
-    return stringType;
+    return stringType(item);
   }
   if (numReg.test(trimed)) {
-    return numberType
+    return numberType(item)
   } else if (trimed.length === 5 && trimed.toLowerCase() === "false" || trimed.length === 4 && trimed.toLowerCase() === "true") {
-    return booleanType;
+    return booleanType(item);
   } else if (trimed[0] === "{" && trimed[trimed.length - 1] === "}" || trimed[0] === "[" && trimed[trimed.length - 1] === "]") {
-    return jsonType;
+    return jsonType(item);
 
   } else {
-    return stringType;
+    return stringType(item);
   }
 }
 

libs/core/rowSplit.js

@@ -62,7 +62,6 @@ module.exports=function rowSplit(rowStr, param) {
       }
     }
   }
-
   return {cols:row,closed:!inquote};
   // if (param.workerNum<=1){
   // }else{

readme.md

@@ -238,7 +238,7 @@ Following parameters are supported:
 * **delimiter**: delimiter used for seperating columns. Use "auto" if delimiter is unknown in advance, in this case, delimiter will be auto-detected (by best attempt). Use an array to give a list of potential delimiters e.g. [",","|","$"]. default: ","
 * **quote**: If a column contains delimiter, it is able to use quote character to surround the column content. e.g. "hello, world" wont be split into two columns while parsing. Set to "off" will ignore all quotes. default: " (double quote)
 * **trim**: Indicate if parser trim off spaces surrounding column content. e.g. "  content  " will be trimmed to "content". Default: true
-* **checkType**: This parameter turns on and off whether check field type. default is true. 
+* **checkType**: This parameter turns on and off whether check field type. default is false.
 * **toArrayString**: Stringify the stream output to JSON array. This is useful when pipe output to a file which expects stringified JSON array. default is false and only stringified JSON (without []) will be pushed to downstream.
 * **ignoreEmpty**: Ignore the empty value in CSV columns. If a column value is not giving, set this to true to skip them. Defalut: false.
 * **workerNum**: Number of worker processes. The worker process will use multi-cores to help process CSV data. Set to number of Core to improve the performance of processing large csv file. Keep 1 for small csv files. Default 1.
@@ -283,7 +283,7 @@ csv()
 })
 ```
 
-`csvRow` is always an array of strings no matter `checkType` value.
+`csvRow` is always an array of strings without types.
 
 `csv` event is the fastest parse event while `json` and `data` event is about 2 times slower. Thus if `csv` is enough, for best performance, just use it without `json` and `data` event.
 
@@ -549,6 +549,12 @@ There are some limitations when using multi-core feature:
 
 #Change Log
 
+
+## 1.1.4
+
+* default value of `checkType` is now false as it causes problems on some csv docs.
+* Added ignoreColumns and includeColumns features. #138
+
 ## 1.1.1
 
 * Fix bugs: preProcessLine is not emitted

test/data/data#139

@@ -0,0 +1,4 @@
+field1, field2, field3, field4,
+2005088801,A1,9001009395,
+2005088806,A6,9001009395 9001009990,
+2005088807,A7,9001009989,

test/data/dataIgnoreEmpty

@@ -7,3 +7,7 @@ d1,,d3,,world
 d2,d,d,d,d
 ,,,,
 d4,d2,d3
+
+
+
+