Skip to content

Commit 6bf62a9

Browse files
KeyangKeyang
committed
update quote algorithm
1 parent a6fea75 commit 6bf62a9

5 files changed

Lines changed: 82 additions & 22 deletions

File tree

‎libs/core/utils.js‎

Lines changed: 64 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -42,28 +42,73 @@ function rowSplit(rowStr, param) {
4242
var quoteBuff = '';
4343
for (var i=0;i<rowArr.length;i++){
4444
var e=rowArr[i];
45-
if (isToogleQuote(e, quote)) { //if current col has odd quotes, switch quote status
46-
if (inquote) { //if currently in open quote status, close it and output data
47-
quoteBuff += delimiter;
48-
quoteBuff += twoDoubleQuote(e.substr(0, e.length - 1), quote);
49-
row.push(trim ? quoteBuff.trim() : quoteBuff);
50-
quoteBuff = '';
51-
} else { // currently not in open quote status, open it
52-
quoteBuff += twoDoubleQuote(e.substring(1), quote);
53-
}
54-
inquote = !inquote;
55-
} else if (inquote) { // if current col has even quotes, do not switch quote status
56-
//if current status is in quote, add to buffer wait to close
57-
quoteBuff += delimiter + twoDoubleQuote(e, quote);
58-
} else { // if current status is not in quote, out put data
59-
if (e.indexOf(quote) === 0 && e[e.length - 1] === quote) { //if current col contain full quote segment,remove quote first
60-
e = e.substring(1, e.length - 1);
45+
if (!inquote && trim){
46+
e=e.trim();
47+
}
48+
var len=e.length;
49+
if (!inquote){
50+
if (e[0] === quote){ //possible open
51+
e=e.substr(1);
52+
len-=1;
53+
if (e[0]!==quote){ // open quote
54+
if (e[len-1] === quote){ // possible close
55+
if (e[len-2]!==quote){ // close quote
56+
e=e.substring(0,len-1);
57+
e=twoDoubleQuote(e,quote);
58+
row.push(e);
59+
continue;
60+
}
61+
}
62+
inquote=true;
63+
quoteBuff+=e;
64+
continue;
65+
}else{
66+
row.push(e);
67+
continue;
68+
}
69+
}else{
70+
row.push(e);
71+
continue;
6172
}
62-
if (trim) {
63-
e = e.trim();
73+
}else{ //previous quote not closed
74+
if (e[len-1] === quote && e[len-2] !==quote){ //close double quote
75+
inquote=false;
76+
e=e.substr(0,len-1);
77+
quoteBuff+=delimiter+e;
78+
quoteBuf=twoDoubleQuote(quoteBuff,quote);
79+
if (trim){
80+
quoteBuff=quoteBuff.trimRight();
81+
}
82+
row.push(quoteBuff);
83+
quoteBuff="";
84+
}else{
85+
quoteBuff+=delimiter+e;
6486
}
65-
row.push(twoDoubleQuote(e, quote));
6687
}
88+
89+
90+
// if (isToogleQuote(e, quote)) { //if current col has odd quotes, switch quote status
91+
// if (inquote) { //if currently in open quote status, close it and output data
92+
// quoteBuff += delimiter;
93+
// quoteBuff += twoDoubleQuote(e.substr(0, e.length - 1), quote);
94+
// row.push(trim ? quoteBuff.trim() : quoteBuff);
95+
// quoteBuff = '';
96+
// } else { // currently not in open quote status, open it
97+
// quoteBuff += twoDoubleQuote(e.substring(1), quote);
98+
// }
99+
// inquote = !inquote;
100+
// } else if (inquote) { // if current col has even quotes, do not switch quote status
101+
// //if current status is in quote, add to buffer wait to close
102+
// quoteBuff += delimiter + twoDoubleQuote(e, quote);
103+
// } else { // if current status is not in quote, out put data
104+
// if (trim) {
105+
// e = e.trim();
106+
// }
107+
// if (e.indexOf(quote) === 0 && e[e.length - 1] === quote) { //if current col contain full quote segment,remove quote first
108+
// e = e.substring(1, e.length - 1);
109+
// }
110+
// row.push(twoDoubleQuote(e, quote));
111+
// }
67112
}
68113
return row;
69114
}

‎test/data/dataWithWhiteSpace‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
"Column 1" , "Column 2", "Column 3" ,"Email"
2+
Column1Row1,Column2Row1,Column3Row1,Row1@example.com

‎test/data/twodoublequotes‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
title,actors,data,uuid,fieldA,fieldB
2-
hello world,"[""Neil"", ""Bill"""", ""Carl"",""Richard"",""Linus""]",xyabcde,"fejal""eifa",bnej""""falkfe,eisjfes
2+
hello world,"[""Neil"", ""Bill"""", ""Carl"",""Richard"",""Linus""]",xyabcde,"fejal""eifa","bnej""""falkfe",eisjfes

‎test/testCSVConverter.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ describe("CSV Converter", function () {
6666
});
6767
obj.on("end_parsed", function (result) {
6868
assert(result[0].col1 === "\"Mini. Sectt");
69-
assert(result[3].col2 === "125001,fenvkdsf");
69+
assert.equal(result[3].col2 , "125001,fenvkdsf");
7070
// console.log(result);
7171
done();
7272
});
@@ -136,8 +136,8 @@ describe("CSV Converter", function () {
136136
var csvConverter = new Converter();
137137
csvConverter.on("end_parsed", function () {});
138138
csvConverter.fromString(data, function (err, jsonObj) {
139-
//console.log(jsonObj);
140139
assert(jsonObj[0].TIMESTAMP === '13954264"22', JSON.stringify(jsonObj[0].TIMESTAMP));
140+
141141
assert(jsonObj[1].TIMESTAMP === 'abc, def, ccc', JSON.stringify(jsonObj[1].TIMESTAMP));
142142
done();
143143
});

‎test/testCSVConverter2.js‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,19 @@ describe("CSV Converter", function() {
139139
});
140140
rs.pipe(conv);
141141

142+
});
143+
it ("should stripe out whitespaces if trim is true",function(done){
144+
var testData = __dirname + "/data/dataWithWhiteSpace";
145+
var rs = fs.createReadStream(testData);
146+
var conv=new Converter({trim:true});
147+
conv.on("end_parsed",function(res){
148+
// console.log(res);
149+
assert.equal(res[0]["Column 1"],"Column1Row1");
150+
assert.equal(res[0]["Column 2"],"Column2Row1");
151+
done();
152+
});
153+
rs.pipe(conv);
154+
142155
})
143156
// it ("should convert big csv",function(done){
144157
// // var rs=fs.createReadStream(__dirname+"/data/large-csv-sample.csv");

0 commit comments

Comments
 (0)