add csv generator

Author: Keyang

bin/genCsv.js

@@ -0,0 +1,33 @@
+#!/usr/bin/env node
+var minimist=require("minimist");
+var argv=process.argv;
+argv.shift();
+argv.shift();
+var args=minimist(argv);
+var headers=["name","header1.filed1","header1.file2","description","header2.field1[]","header2.field1[]","header2.filed2"];
+
+if (args.headers){
+  headers=JSON.parse(args.headers);
+}
+var rowNum=args.row?args.row:10000;
+var chars=args.chars?args.chars:"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
+var maxLength=parseInt(args.max?args.max:"15");
+console.log(headers.join(","));
+for (var i=0;i<rowNum;i++){
+  var row=[];
+  for (var j=0;j<headers.length;j++){
+    row.push(genWord());
+  }
+  console.log(row.join(","));
+}
+
+
+function genWord(){
+  var len=Math.round(Math.random()*maxLength);
+  var rtn="";
+  for (var i=0;i<len;i++){
+    var pos=Math.round(Math.random()*chars.length);
+    rtn+=chars[pos];
+  }
+  return rtn;
+}

libs/core/Converter.js

@@ -10,7 +10,9 @@ var utils = require("./utils.js");
 var async = require("async");
 
 function Converter(params) {
-  Transform.call(this); //TODO what does this do? -->This calls the constructor of Transform and initialise anything the Transform needs.(like var initialisation)
+  Transform.call(this,{
+    highWaterMark:1024
+  }); //TODO what does this do? -->This calls the constructor of Transform and initialise anything the Transform needs.(like var initialisation)
   var _param = {
     constructResult: true, //set to false to not construct result in memory. suitable for big csv data
     delimiter: ',', // change the delimiter of csv columns. It is able to use an array to specify potencial delimiters. e.g. [",","|",";"]

libs/core/defaultParsers/parser_json.js

@@ -38,21 +38,33 @@ module.exports = {
     var headArr = (params.config && params.config.flatKeys) ? [fieldStr] : fieldStr.split('.');
     var match, index, key, pointer;
     //now the pointer is pointing the position to add a key/value pair.
-    pointer = processHead(params.resultRow, headArr, arrReg, params.config && params.config.flatKeys);
+    var pointer = processHead(params.resultRow, headArr, arrReg, params.config && params.config.flatKeys);
     key = headArr.shift();
     match = (params.config && params.config.flatKeys) ? false : key.match(arrReg);
     if (match) { // the last element is an array, we need check and treat it as an array.
-      key = key.replace(match[0], '');
-      if (!pointer[key] || !(pointer[key] instanceof Array)) {
-        pointer[key] = [];
+      try {
+        key = key.replace(match[0], '');
+        if (!pointer[key] || !(pointer[key] instanceof Array)) {
+          pointer[key] = [];
+        }
+        if (pointer[key]) {
+          index = match[1];
+          if (index === '') {
+            index = pointer[key].length;
+          }
+          pointer[key][index] = params.item;
+        } else {
+          params.resultRow[fieldStr] = params.item;
+        }
+      } catch (e) {
+        params.resultRow[fieldStr] = params.item;
       }
-      index = match[1];
-      if (index === '') {
-        index = pointer[key].length;
-      }
-      pointer[key][index] = params.item;
     } else {
-      pointer[key] =  params.item;
+      if (typeof pointer=== "string"){
+        params.resultRow[fieldStr] = params.item;
+      }else{
+        pointer[key] = params.item;
+      }
     }
   }
 };

package.json

@@ -51,9 +51,10 @@
     "mocha": "^2.4.5"
   },
   "dependencies": {
-    "async": "^1.2.1"
+    "async": "^1.2.1",
+    "minimist": "^1.2.0"
   },
-  "scripts":{
-    "test":"mocha ./test -R spec"
+  "scripts": {
+    "test": "mocha ./test -R spec"
   }
 }

test/data/invalidHeader

@@ -0,0 +1,6 @@
+header1,header1.filed1,header1.file2,header2,header2.field1[],header2.field1[],header2.filed2
+5OlFPc,q7,ejpJdw,DIgNVqB7h9jI,f8ayrzv,undefinedzvTY3Qd3pSkKOk,S7cVvW7m50t9U
+0TaUGQVPqPkOr,lT,GA,UPUuORnuaDjXdl,V6G4QFddmPH8b,65NxWPl,Lclhl0fy
+,ex4,1gjT4YPJ,QtJ8S5TQ,M4zO4OppCAR4,Pg7VipESqZmHwY5,mPzz
+FEEa,,B1B,U9,kjxHGy,McW,6FWKBP0JoCiWn
+0,,Fj0o9IPB,jwFuChbeqZdVVR,hGqs3Ps7dMEo6,Zundefined00Xqvw,hT7lN6iVUHDx

test/testCSVConverter2.js

@@ -166,6 +166,17 @@ describe("CSV Converter", function() {
     rs.pipe(conv);
 
   });
+  it ("should auto flat header if header is not valid nested json keys",function(done){
+    var testData = __dirname + "/data/invalidHeader";
+    var rs = fs.createReadStream(testData);
+    var conv=new Converter();
+    conv.on("end_parsed",function(res){
+      assert.equal(res[0]["header1.filed1"],"q7");
+      assert.equal(res[0]["header2.field1[]"],"undefinedzvTY3Qd3pSkKOk");
+      done();
+    });
+    rs.pipe(conv);
+  })
   // it ("should convert big csv",function(done){
   //   // var rs=fs.createReadStream(__dirname+"/data/large-csv-sample.csv");
   //   var rs=fs.createReadStream("/Users/kxiang/tmp/csvdata");