Skip to content

Commit a027cea

Browse files
KeyangKeyang
authored
Merge pull request #150 from kakts/refactor-libs
Refactored libs/core and tests
2 parents f8bf42b + 64b7291 commit a027cea

8 files changed

Lines changed: 126 additions & 169 deletions

File tree

‎libs/core/Converter.js‎

Lines changed: 35 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -176,26 +176,26 @@ Converter.prototype.workerProcess = function (fileLine, cb) {
176176
var eol = this.getEol();
177177
this.setPartialData(line.partial);
178178
this.workerMgr.sendWorker(line.lines.join(eol) + eol, this.lastIndex, cb, function (results, lastIndex) {
179+
var buf;
179180
var cur = self.sequenceBuffer[0];
180181
if (cur.idx === lastIndex) {
181182
cur.result = results;
182183
var records = [];
183184
while (self.sequenceBuffer[0] && self.sequenceBuffer[0].result) {
184-
var buf = self.sequenceBuffer.shift();
185+
buf = self.sequenceBuffer.shift();
185186
records = records.concat(buf.result);
186187
}
187188
self.processResult(records);
188189
self.recordNum += records.length;
189190
} else {
190-
for (var i = 0; i < self.sequenceBuffer.length; i++) {
191-
var buf = self.sequenceBuffer[i];
191+
for (var i = 0, len = self.sequenceBuffer.length; i < len; i++) {
192+
buf = self.sequenceBuffer[i];
192193
if (buf.idx === lastIndex) {
193194
buf.result = results;
194195
break;
195196
}
196197
}
197198
}
198-
// self.processResult(JSON.parse(results),function(){},true);
199199
});
200200
this.sequenceBuffer.push({
201201
idx: this.lastIndex,
@@ -206,38 +206,38 @@ Converter.prototype.workerProcess = function (fileLine, cb) {
206206

207207
Converter.prototype.processHead = function (fileLine, cb) {
208208
var params = this.param;
209-
if (!params._headers) { //header is not inited. init header
210-
var lines = fileLineToCSVLine(fileLine, params);
211-
this.setPartialData(lines.partial);
212-
if (params.noheader) {
213-
if (params.headers) {
214-
params._headers = params.headers;
215-
} else {
216-
params._headers = [];
217-
}
209+
if (params._headers) {
210+
return cb();
211+
}
212+
213+
// if header is not inited. init header
214+
var lines = fileLineToCSVLine(fileLine, params);
215+
this.setPartialData(lines.partial);
216+
if (params.noheader) {
217+
if (params.headers) {
218+
params._headers = params.headers;
218219
} else {
219-
var headerRow = lines.lines.shift();
220-
if (params.headers) {
221-
params._headers = params.headers;
222-
} else {
223-
params._headers = headerRow;
224-
}
225-
}
226-
if (this.param.workerNum > 1) {
227-
this.workerMgr.setParams(params);
220+
params._headers = [];
228221
}
229-
var res = linesToJson(lines.lines, params, 0);
230-
this.processResult(res);
231-
this.lastIndex += res.length;
232-
this.recordNum += res.length;
233-
cb();
234222
} else {
235-
cb();
223+
var headerRow = lines.lines.shift();
224+
if (params.headers) {
225+
params._headers = params.headers;
226+
} else {
227+
params._headers = headerRow;
228+
}
236229
}
230+
if (this.param.workerNum > 1) {
231+
this.workerMgr.setParams(params);
232+
}
233+
var res = linesToJson(lines.lines, params, 0);
234+
this.processResult(res);
235+
this.lastIndex += res.length;
236+
this.recordNum += res.length;
237+
cb();
237238
};
238239

239240
Converter.prototype.processResult = function (result) {
240-
241241
for (var i = 0, len = result.length; i < len; i++) {
242242
var r = result[i];
243243
if (r.err) {
@@ -246,8 +246,6 @@ Converter.prototype.processResult = function (result) {
246246
this.emitResult(r);
247247
}
248248
}
249-
// this.lastIndex+=result.length;
250-
// cb();
251249
};
252250

253251
Converter.prototype.emitResult = function (r) {
@@ -305,6 +303,7 @@ Converter.prototype.preProcessRaw = function (data, cb) {
305303
cb(data);
306304
};
307305

306+
// FIXME: lineNumber is not used.
308307
Converter.prototype.preProcessLine = function (line, lineNumber) {
309308
return line;
310309
};
@@ -322,8 +321,9 @@ Converter.prototype._flush = function (cb) {
322321
}
323322
};
324323
if (this._csvLineBuffer.length > 0) {
325-
if (this._csvLineBuffer[this._csvLineBuffer.length - 1] != this.getEol()) {
326-
this._csvLineBuffer += this.getEol();
324+
var eol = this.getEol();
325+
if (this._csvLineBuffer[this._csvLineBuffer.length - 1] !== eol) {
326+
this._csvLineBuffer += eol;
327327
}
328328
this.processData(this._csvLineBuffer, function () {
329329
this.checkAndFlush();
@@ -333,13 +333,7 @@ Converter.prototype._flush = function (cb) {
333333
}
334334
return;
335335
};
336-
// Converter.prototype._transformFork = function(data, encoding, cb) {
337-
// this.child.stdin.write(data, encoding, cb);
338-
// }
339-
// Converter.prototype._flushFork = function(cb) {
340-
// this.child.stdin.end();
341-
// this.child.on("exit", cb);
342-
// }
336+
343337
Converter.prototype.checkAndFlush = function () {
344338
if (this._csvLineBuffer.length !== 0) {
345339
this.emit("error", CSVError.unclosed_quote(this.recordNum, this._csvLineBuffer), this._csvLineBuffer);
@@ -410,7 +404,7 @@ Converter.prototype.transf = function (func) {
410404
};
411405

412406
Converter.prototype.fromString = function (csvString, cb) {
413-
if (typeof csvString != "string") {
407+
if (typeof csvString !== "string") {
414408
return cb(new Error("Passed CSV Data is not a string."));
415409
}
416410
if (cb && typeof cb === "function") {

‎libs/core/defaultParsers/parser_flat.js‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
module.exports = {
22
"name": "flat",
3-
"processSafe":true,
3+
"processSafe": true,
44
"regExp": /^\*flat\*/,
55
"parserFunc": function parser_flat (params) {
6-
var key=this.getHeadStr();
7-
var val=params.item;
8-
params.resultRow[key]=val;
6+
var key = this.getHeadStr();
7+
var val = params.item;
8+
params.resultRow[key] = val;
99
}
1010
};

‎libs/core/defaultParsers/parser_json.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ module.exports = {
3636
"parserFunc": function parser_json(params) {
3737
var fieldStr = this.getHeadStr();
3838
var headArr = (params.config && params.config.flatKeys) ? [fieldStr] : fieldStr.split('.');
39-
var match, index, key, pointer;
39+
var match, index, key;
4040
//now the pointer is pointing the position to add a key/value pair.
4141
var pointer = processHead(params.resultRow, headArr, arrReg, params.config && params.config.flatKeys);
4242
key = headArr.shift();
@@ -60,7 +60,7 @@ module.exports = {
6060
params.resultRow[fieldStr] = params.item;
6161
}
6262
} else {
63-
if (typeof pointer=== "string"){
63+
if (typeof pointer === "string"){
6464
params.resultRow[fieldStr] = params.item;
6565
}else{
6666
pointer[key] = params.item;

‎libs/core/fileline.js‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,5 @@ module.exports = function(data, param) {
99
var eol = getEol(data,param);
1010
var lines = data.split(eol);
1111
var partial = lines.pop();
12-
// if (param.ignoreEmpty){
13-
// var trimmedLines=[];
14-
// for (var i=0;i<lines.length;i++){
15-
// trimmedLines.push(lines[i].trim())
16-
// }
17-
// return {lines:trimmedLines,partial:partial};
18-
// }else{
19-
return {lines: lines, partial: partial};
20-
// }
12+
return {lines: lines, partial: partial};
2113
};

‎libs/core/linesToJson.js‎

Lines changed: 18 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
var parserMgr = require("./parserMgr.js");
2-
var Parser = require("./parser");
32
var CSVError = require("./CSVError");
43
var numReg = /^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?$/;
54
/**
@@ -48,18 +47,9 @@ function processRows(csvRows, params, startIndex) {
4847
return res;
4948
}
5049

51-
function getConstParser(number, param) {
52-
var inst = new Parser("field" + number, /.*/, function (params) {
53-
var name = this.getName();
54-
params.resultRow[name] = params.item;
55-
}, true);
56-
inst.setParam(param);
57-
return inst;
58-
}
59-
6050
function processRow(row, param, index) {
6151
var parseRules = param.parseRules;
62-
if (param.checkColumn && row.length != parseRules.length) {
52+
if (param.checkColumn && row.length !== parseRules.length) {
6353
return {
6454
err: CSVError.column_mismatched(index)
6555
};
@@ -91,14 +81,9 @@ function convertRowToJson(row, headRow, param) {
9181
}
9282
hasValue = true;
9383

94-
// parser = parseRules[i];
95-
// if (!parser) {
96-
// parser = parseRules[i] = getConstParser(i + 1, param);
97-
// }
9884
head = headRow[i];
9985
if (!head || head === "") {
10086
head = headRow[i] = "field" + (i + 1);
101-
// parser.initHead(head);
10287
}
10388
var flag = getFlag(head, i, param);
10489
if (flag === 'omit') {
@@ -114,16 +99,6 @@ function convertRowToJson(row, headRow, param) {
11499
} else {
115100
setPath(resultRow, title, item);
116101
}
117-
// _.set(resultRow,head,item)
118-
// parser.parse({
119-
// head: head,
120-
// item: item,
121-
// itemIndex: i,
122-
// rawRow: row,
123-
// resultRow: resultRow,
124-
// rowIndex: index,
125-
// config: param || {}
126-
// });
127102
}
128103
if (hasValue) {
129104
return resultRow;
@@ -145,41 +120,37 @@ function setPath(json, path, value) {
145120
function getFlag(head, i, param) {
146121
if (typeof param._headerFlag[i] === "string") {
147122
return param._headerFlag[i];
123+
} else if (head.indexOf('*omit*') > -1) {
124+
return param._headerFlag[i] = 'omit';
125+
} else if (head.indexOf('*flat*') > -1) {
126+
return param._headerFlag[i] = 'flat';
148127
} else {
149-
if (head.indexOf('*omit*') > -1) {
150-
return param._headerFlag[i] = 'omit';
151-
} else if (head.indexOf('*flat*') > -1) {
152-
return param._headerFlag[i] = 'flat';
153-
} else {
154-
return param._headerFlag[i] = '';
155-
}
128+
return param._headerFlag[i] = '';
156129
}
157130
}
158131

159132
function getTitle(head, i, param) {
160133
if (param._headerTitle[i]) {
161134
return param._headerTitle[i];
162-
} else {
163-
var flag = getFlag(head, i, param);
164-
var str = head.replace(flag, '');
165-
str = str.replace('string#!', '').replace('number#!', '');
166-
return param._headerTitle[i] = str;
167135
}
136+
137+
var flag = getFlag(head, i, param);
138+
var str = head.replace(flag, '');
139+
str = str.replace('string#!', '').replace('number#!', '');
140+
return param._headerTitle[i] = str;
168141
}
169142

170143
function checkType(item, head, headIdx, param) {
171144
if (param._headerType[headIdx]) {
172145
return param._headerType[headIdx];
146+
} else if (head.indexOf('number#!') > -1) {
147+
return param._headerType[headIdx] = numberType;
148+
} else if (head.indexOf('string#!') > -1) {
149+
return param._headerType[headIdx] = stringType;
150+
} else if (param.checkType) {
151+
return param._headerType[headIdx] = dynamicType;
173152
} else {
174-
if (head.indexOf('number#!') > -1) {
175-
return param._headerType[headIdx] = numberType;
176-
} else if (head.indexOf('string#!') > -1) {
177-
return param._headerType[headIdx] = stringType;
178-
} else if (param.checkType) {
179-
return param._headerType[headIdx] = dynamicType;
180-
} else {
181-
return param._headerType[headIdx] = stringType;
182-
}
153+
return param._headerType[headIdx] = stringType;
183154
}
184155
}
185156

@@ -206,7 +177,6 @@ function dynamicType(item) {
206177
return booleanType(item);
207178
} else if (trimed[0] === "{" && trimed[trimed.length - 1] === "}" || trimed[0] === "[" && trimed[trimed.length - 1] === "]") {
208179
return jsonType(item);
209-
210180
} else {
211181
return stringType(item);
212182
}
@@ -228,25 +198,3 @@ function jsonType(item) {
228198
return item;
229199
}
230200
}
231-
// function dynamicType(item) {
232-
// var trimed = item.trim();
233-
// if (trimed === "") {
234-
// return trimed;
235-
// }
236-
// if (!isNaN(trimed)) {
237-
// return parseFloat(trimed);
238-
// } else if (trimed.length === 5 && trimed.toLowerCase() === "false") {
239-
// return false;
240-
// } else if (trimed.length === 4 && trimed.toLowerCase() === "true") {
241-
// return true;
242-
// } else if (trimed[0] === "{" && trimed[trimed.length - 1] === "}" || trimed[0] === "[" && trimed[trimed.length - 1] === "]") {
243-
// try {
244-
// return JSON.parse(trimed);
245-
// } catch (e) {
246-
// return item;
247-
// }
248-
// } else {
249-
// return item;
250-
251-
// }
252-
// }

‎libs/core/parser.js‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,6 @@ Parser.prototype.clone = function() {
107107
newParser[key] = obj[key];
108108
}
109109
return newParser;
110-
//return new Parser(this.name, this.regExp, this.parse, this.processSafe);
111110
};
112111

113112
Parser.prototype.getName = function() {

0 commit comments

Comments
 (0)