Skip to content

Commit e0fd517

Browse files
gms1gms1
committed
fixed benchmark
1 parent 2bce512 commit e0fd517

5 files changed

Lines changed: 134 additions & 63 deletions

File tree

‎memory-bank/development.md‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,43 @@ node tools/benchmark/run.js
208208

209209
See [Project Overview](project-overview.md) for benchmark details.
210210

211+
### Critical: Statement.finalize() Requires Callback
212+
213+
**IMPORTANT**: When using `stmt.finalize()` with queued operations (e.g., multiple `stmt.run()` calls inside `db.serialize()`), you **must** provide a callback to wait for operations to complete.
214+
215+
**Without callback (WRONG):**
216+
```javascript
217+
db.serialize(() => {
218+
const stmt = db.prepare('INSERT INTO foo VALUES (?, ?)');
219+
for (let i = 0; i < 10000; i++) {
220+
stmt.run(i, 'Row ' + i);
221+
}
222+
stmt.finalize(); // Returns immediately, cancels queued operations!
223+
});
224+
// Result: Only 1 row inserted, operations cancelled
225+
```
226+
227+
**With callback (CORRECT):**
228+
```javascript
229+
db.serialize(() => {
230+
const stmt = db.prepare('INSERT INTO foo VALUES (?, ?)');
231+
for (let i = 0; i < 10000; i++) {
232+
stmt.run(i, 'Row ' + i);
233+
}
234+
stmt.finalize((err) => {
235+
if (err) throw err;
236+
// All 10,000 rows now inserted
237+
});
238+
});
239+
// Result: All 10,000 rows inserted
240+
```
241+
242+
**Why this matters:**
243+
- `finalize()` without a callback returns immediately
244+
- Queued operations (`stmt.run()` calls) are cancelled when the statement is finalized
245+
- This caused benchmark results to appear ~10x faster than reality because operations never completed
246+
- Always use callbacks with `finalize()` when operations are queued
247+
211248
## Making Changes
212249

213250
**IMPORTANT**: After making any code changes, always run:

‎tools/benchmark/README.md‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -60,16 +60,17 @@ node tools/benchmark/run.js
6060
```
6161
Task Name ops/sec Average Time (ns) Margin Samples
6262
------------------------------------------------------------------------------------------------------
63-
insert: literal file 42.34 23620025909.09 3.71% 22
64-
insert: transaction with two statements 44.45 22495146260.87 3.01% 23
65-
insert: with transaction 43.08 23212004409.09 6.05% 22
66-
insert: without transaction 44.64 22399535956.52 5.67% 23
63+
insert: literal file 101.79 9867669.32 1.36% 102
64+
insert: transaction with two statements 15.72 64101675.83 1.81% 100
65+
insert: with transaction 9.53 105168845.93 0.98% 100
66+
insert: without transaction 8.76 114706903.08 1.33% 100
6767
```
6868

6969
**Key findings:**
70-
- Using transactions (`BEGIN`/`COMMIT`) significantly improves performance
71-
- Parallelized statements can further improve performance for concurrent operations
72-
- Without transactions, each statement is committed individually, which is much slower
70+
- `literal file` is fastest because it uses a single `db.exec()` call with all SQL in one batch
71+
- `transaction with two statements` uses parallelized statements within a transaction for better throughput
72+
- `with transaction` wraps all inserts in a single transaction
73+
- `without transaction` is slowest because each INSERT is committed individually
7374

7475
### select.js
7576

@@ -89,14 +90,15 @@ node tools/benchmark/run.js
8990
```
9091
Task Name ops/sec Average Time (ns) Margin Samples
9192
------------------------------------------------------------------------------------------------------
92-
select: db.each 0.49 2057124731700.00 3.65% 10
93-
select: db.all 0.41 2457100030700.00 4.77% 10
94-
select: db.all with statement reset 0.21 4847062019200.00 1.33% 10
93+
select: db.each 0.82 1232450196.30 5.37% 10
94+
select: db.all 0.86 1166833768.40 2.27% 10
95+
select: db.all with statement reset 0.42 2357234044.80 0.77% 10
9596
```
9697

9798
**Key findings:**
98-
- `db.all()` is typically faster for small to medium result sets
99+
- `db.all()` and `db.each()` have similar performance for large result sets (1 million rows)
99100
- `db.each()` is more memory-efficient for large result sets as it processes rows one at a time
101+
- `db.all with statement reset` is slower because it executes the query twice
100102

101103
## Benchmark Data Files
102104

@@ -185,4 +187,4 @@ module.exports = {
185187

186188
## Dependencies
187189

188-
- `tinybench`: ^2.9.0 - Modern benchmark library with proper setup/teardown support
190+
- `tinybench`: ^6.0.0 - Modern benchmark library with proper setup/teardown support

‎tools/benchmark/insert.js‎

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -87,12 +87,17 @@ module.exports = {
8787
i++;
8888
stmt2.run(i, 'Row ' + i);
8989
}
90-
stmt1.finalize();
91-
stmt2.finalize();
90+
let finalized = 0;
91+
const checkDone = (err) => {
92+
if (err) reject(err);
93+
finalized++;
94+
if (finalized === 2) {
95+
promisifyRun(db, 'COMMIT').then(resolve).catch(reject);
96+
}
97+
};
98+
stmt1.finalize(checkDone);
99+
stmt2.finalize(checkDone);
92100
});
93-
94-
await promisifyRun(db, 'COMMIT');
95-
resolve();
96101
});
97102
});
98103
},
@@ -123,9 +128,14 @@ module.exports = {
123128
for (let i = 0; i < iterations; i++) {
124129
stmt.run(i, 'Row ' + i);
125130
}
126-
stmt.finalize();
127-
await promisifyRun(db, 'COMMIT');
128-
resolve();
131+
stmt.finalize(async (err) => {
132+
if (err) {
133+
reject(err);
134+
return;
135+
}
136+
await promisifyRun(db, 'COMMIT');
137+
resolve();
138+
});
129139
});
130140
});
131141
},
@@ -155,8 +165,10 @@ module.exports = {
155165
for (let i = 0; i < iterations; i++) {
156166
stmt.run(i, 'Row ' + i);
157167
}
158-
stmt.finalize();
159-
resolve();
168+
stmt.finalize((err) => {
169+
if (err) reject(err);
170+
else resolve();
171+
});
160172
});
161173
});
162174
},

‎tools/benchmark/run.js‎

Lines changed: 58 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -11,32 +11,61 @@ const selectBenchmarks = require('./select');
1111
async function runBenchmarks() {
1212
console.log('Running sqlite3 benchmarks...\n');
1313

14-
const suite = new Bench({
15-
time: 500, // Run each benchmark for 500ms
16-
warmupTime: 50, // Warmup time in ms
17-
warmupIterations: 5, // Minimum warmup iterations
18-
});
14+
const results = [];
1915

20-
// Add insert benchmarks
16+
// Run insert benchmarks
2117
console.log('=== INSERT BENCHMARKS ===\n');
2218
for (const [name, benchmark] of Object.entries(insertBenchmarks.benchmarks)) {
19+
console.log(`Running: insert: ${name}`);
20+
const suite = new Bench({
21+
iterations: 100,
22+
warmupIterations: 5,
23+
});
24+
2325
suite.add(`insert: ${name}`, benchmark.fn, {
2426
beforeEach: benchmark.beforeEach,
2527
afterEach: benchmark.afterEach,
2628
});
29+
30+
await suite.run();
31+
const task = suite.tasks[0];
32+
if (task.result && task.result.latency) {
33+
results.push({
34+
name: `insert: ${name}`,
35+
throughput: task.result.throughput?.mean,
36+
latency: task.result.latency.mean,
37+
rme: task.result.latency.rme,
38+
samples: task.result.latency.samplesCount,
39+
});
40+
}
2741
}
2842

29-
// Add select benchmarks
30-
console.log('=== SELECT BENCHMARKS ===\n');
43+
// Run select benchmarks (fewer iterations since they're slower)
44+
console.log('\n=== SELECT BENCHMARKS ===\n');
3145
for (const [name, benchmark] of Object.entries(selectBenchmarks.benchmarks)) {
46+
console.log(`Running: select: ${name}`);
47+
const suite = new Bench({
48+
iterations: 10,
49+
warmupIterations: 1,
50+
});
51+
3252
suite.add(`select: ${name}`, benchmark.fn, {
3353
beforeAll: benchmark.beforeAll,
3454
afterAll: benchmark.afterAll,
3555
});
36-
}
3756

38-
// Run all benchmarks
39-
await suite.run();
57+
await suite.run();
58+
const task = suite.tasks[0];
59+
if (task.result && task.result.latency) {
60+
results.push({
61+
name: `select: ${name}`,
62+
throughput: task.result.throughput?.mean,
63+
latency: task.result.latency.mean,
64+
rme: task.result.latency.rme,
65+
samples: task.result.latency.samplesCount,
66+
});
67+
}
68+
}
4069

4170
// Output results with full precision
4271
console.log('\n=== RESULTS ===\n');
@@ -53,31 +82,25 @@ async function runBenchmarks() {
5382
console.log('-'.repeat(102));
5483

5584
// Results
56-
for (const task of suite.tasks) {
57-
if (task.result && task.result.samples && task.result.samples.length > 0) {
58-
// tinybench v6: result has hz (ops/sec), mean (ms), rme (relative margin of error)
59-
const opsSec = task.result.hz.toFixed(2);
60-
const avgTime = (task.result.mean * 1e6).toFixed(2); // Convert ms to nanoseconds
61-
const margin = task.result.rme.toFixed(2);
62-
const samples = task.result.samples.length;
85+
for (const result of results) {
86+
const opsSec = result.throughput?.toFixed(2) || 'N/A';
87+
const avgTime = (result.latency * 1e6).toFixed(2); // Convert ms to nanoseconds
88+
const margin = result.rme?.toFixed(2) || 'N/A';
89+
const samples = result.samples || 0;
6390

64-
const row = [
65-
task.name.padEnd(45),
66-
opsSec.padStart(15),
67-
avgTime.padStart(20),
68-
(margin + '%').padStart(12),
69-
samples.toString().padStart(10)
70-
].join('');
71-
console.log(row);
72-
} else if (task.result?.error) {
73-
console.log(`${task.name.padEnd(45)} ERROR: ${task.result.error.message}`);
74-
} else {
75-
console.log(`${task.name.padEnd(45)} No results`);
76-
}
91+
const row = [
92+
result.name.padEnd(45),
93+
opsSec.padStart(15),
94+
avgTime.padStart(20),
95+
(margin + '%').padStart(12),
96+
samples.toString().padStart(10)
97+
].join('');
98+
console.log(row);
7799
}
78100
}
79101

80-
runBenchmarks().catch((err) => {
81-
console.error('Benchmark failed:', err);
82-
process.exit(1);
83-
});
102+
runBenchmarks()
103+
.catch((err) => {
104+
console.error('Benchmark failed:', err);
105+
process.exit(1);
106+
});

‎tools/benchmark/select.js‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -105,19 +105,16 @@ module.exports = {
105105
const stmt = db.prepare('SELECT * FROM foo');
106106
stmt.all((err, rows) => {
107107
if (err) {
108-
stmt.finalize();
109-
reject(err);
108+
stmt.finalize(() => reject(err));
110109
return;
111110
}
112111
stmt.reset();
113112
stmt.all((err2, rows2) => {
114113
if (err2) {
115-
stmt.finalize();
116-
reject(err2);
114+
stmt.finalize(() => reject(err2));
117115
return;
118116
}
119-
stmt.finalize();
120-
resolve();
117+
stmt.finalize(() => resolve());
121118
});
122119
});
123120
});

0 commit comments

Comments
 (0)