refactored benchmarks

Author: gms1

.eslintrc.js

@@ -1,14 +1,14 @@
 module.exports = {
     "extends": "eslint:recommended",
     "env": {
-        "es2017": true,
+        "es2020": true,
         "node": true
     },
     "rules": {
         "indent": ["error", 4],
         "linebreak-style": ["error", "unix"],
         "semi": ["error", "always"],
-      "no-cond-assign": ["error", "always"],
-      "no-inner-declarations": "off"
+        "no-cond-assign": ["error", "always"],
+        "no-inner-declarations": "off"
     }
 };

package.json

@@ -50,10 +50,10 @@
     "tar": "^7.5.10"
   },
   "devDependencies": {
-    "benchmark": "^2.1.4",
     "eslint": "8.56.0",
     "mocha": "10.2.0",
-    "prebuild": "13.0.1"
+    "prebuild": "13.0.1",
+    "tinybench": "^2.9.0"
   },
   "peerDependencies": {
     "node-gyp": "12.x"
@@ -75,7 +75,7 @@
     "rebuild": "node-gyp rebuild",
     "upload": "prebuild --verbose --prerelease",
     "frozen-install": "yarn install --frozen-lockfile",
-    "lint": "eslint lib/ test/ tools/",
+    "lint": "eslint .eslintrc.js lib/ test/ tools/",
     "test": "node test/support/createdb.js && mocha -R spec --timeout 480000"
   },
   "license": "BSD-3-Clause",

tools/benchmark/README.md

@@ -1,39 +1,43 @@
 # Benchmark Tools
 
-This directory contains benchmark scripts to measure and compare the performance of different SQLite operations in `@homeofthings/sqlite3`.
+This directory contains benchmark scripts to measure and compare the performance of different SQLite operations in `@homeofthings/sqlite3` using [tinybench](https://github.com/tinylibs/tinybench).
+
+## Key Features
+
+- **Proper setup/teardown separation**: Database creation, table creation, and data population happen in `beforeEach` (not measured)
+- **Only actual operations measured**: The benchmark functions contain only the INSERT/SELECT operations
+- **Warmup support**: tinybench includes warmup iterations to allow V8 optimization
+- **Statistical accuracy**: Multiple iterations with mean, variance, and margin of error
 
 ## Running Benchmarks
 
 ### Quick Start
 
-Install the benchmark dependencies:
+Install dependencies (from project root):
 
 ```bash
-cd tools/benchmark
-npm install
+yarn install
 ```
 
 Run all benchmarks:
 
 ```bash
-node run.js
+node tools/benchmark/run.js
 ```
 
-### Manual Execution
-
-You can also run individual benchmark scripts directly:
+Or from the benchmark directory:
 
 ```bash
-node insert.js
-node select.js
+cd tools/benchmark
+node run.js
 ```
 
 ## Benchmark Scripts
 
 ### run.js (Recommended)
 
 The main runner script executes all benchmarks and provides detailed timing statistics including:
--.ops/sec (operations per second)
+- ops/sec (operations per second)
 - Mean and margin of error
 - Sample variance
 
@@ -42,22 +46,24 @@ The main runner script executes all benchmarks and provides detailed timing stat
 Measures performance of different data insertion approaches.
 
 **Benchmark tests:**
-- `insert literal file`: Executes SQL from a file using `db.exec()`
-- `insert with transaction and two statements`: Uses parallelized statements within a transaction
-- `insert with transaction`: Uses a single statement within a transaction
-- `insert without transaction`: Uses a single statement without explicit transaction
+- `insert: literal file`: Executes SQL from a file using `db.exec()`
+- `insert: transaction with two statements`: Uses parallelized statements within a transaction
+- `insert: with transaction`: Uses a single statement within a transaction
+- `insert: without transaction`: Uses a single statement without explicit transaction
 
 **Usage:**
 ```bash
-node insert.js
+node tools/benchmark/run.js
 ```
 
-**Expected output (when using run.js):**
+**Expected output:**
 ```
-insert: insert literal file x X.XX ops/sec ±X.XX% (XXX runs sampled)
-insert: insert with transaction and two statements x X.XX ops/sec ±X.XX% (XXX runs sampled)
-insert: insert with transaction x X.XX ops/sec ±X.XX% (XXX runs sampled)
-insert: insert without transaction x X.XX ops/sec ±X.XX% (XXX runs sampled)
+Task Name                                            ops/sec   Average Time (ns)      Margin   Samples
+------------------------------------------------------------------------------------------------------
+insert: literal file                                   42.34      23620025909.09       3.71%        22
+insert: transaction with two statements                44.45      22495146260.87       3.01%        23
+insert: with transaction                               43.08      23212004409.09       6.05%        22
+insert: without transaction                            44.64      22399535956.52       5.67%        23
 ```
 
 **Key findings:**
@@ -70,20 +76,22 @@ insert: insert without transaction x X.XX ops/sec ±X.XX% (XXX runs sampled)
 Measures performance of different data retrieval approaches.
 
 **Benchmark tests:**
-- `db.each()`: Iterates through results row-by-row using a callback
-- `db.all()`: Retrieves all results into an array at once
-- `db.all with reset()`: Retrieves all results after resetting the database
+- `select: db.each()`: Iterates through results row-by-row using a callback
+- `select: db.all()`: Retrieves all results into an array at once
+- `select: db.all with statement reset()`: Retrieves all results after resetting the statement
 
 **Usage:**
 ```bash
-node select.js
+node tools/benchmark/run.js
 ```
 
-**Expected output (when using run.js):**
+**Expected output:**
 ```
-select: db.each x X.XX ops/sec ±X.XX% (XXX runs sampled)
-select: db.all x X.XX ops/sec ±X.XX% (XXX runs sampled)
-select: db.all with reset x X.XX ops/sec ±X.XX% (XXX runs sampled)
+Task Name                                            ops/sec   Average Time (ns)      Margin   Samples
+------------------------------------------------------------------------------------------------------
+select: db.each                                         0.49    2057124731700.00       3.65%        10
+select: db.all                                          0.41    2457100030700.00       4.77%        10
+select: db.all with statement reset                     0.21    4847062019200.00       1.33%        10
 ```
 
 **Key findings:**
@@ -100,26 +108,81 @@ Contains 10,000 INSERT statements wrapped in a transaction for the "insert liter
 
 Contains 10,000 INSERT statements to populate the test database for the select benchmarks.
 
+## Architecture
+
+### Setup/Teardown Pattern
+
+Each benchmark follows this pattern to ensure only the actual operation is measured:
+
+```javascript
+{
+  async beforeEach(ctx) {
+    // Setup: Create database, create table, populate data
+    // NOT measured
+    ctx.db = new sqlite3.Database(':memory:');
+    await promisifyRun(ctx.db, 'CREATE TABLE foo ...');
+  },
+
+  async fn(ctx) {
+    // Benchmark: Only the operation to measure
+    // MEASURED
+    await insertData(ctx.db);
+  },
+
+  async afterEach(ctx) {
+    // Teardown: Close database
+    // NOT measured
+    await promisifyClose(ctx.db);
+  },
+}
+```
+
+### Context Object
+
+A context object (`ctx`) is passed between setup, benchmark, and teardown:
+- `ctx.db`: Database instance
+- Custom properties can be added as needed
+
 ## Adding New Benchmarks
 
 1. Create a new JavaScript file in this directory (e.g., `update.js`)
-2. Export a `compare` object with your benchmark functions:
+2. Export a `benchmarks` object with your benchmark functions:
 
 ```javascript
-exports.compare = {
-    'benchmark name': function(finished) {
-        // Your benchmark code here
-        finished(); // Call when done
-    }
+module.exports = {
+  benchmarks: {
+    'benchmark name': {
+      async beforeEach(ctx) {
+        // Setup - NOT measured
+        ctx.db = new sqlite3.Database(':memory:');
+        // ... create tables, populate data
+      },
+
+      async fn(ctx) {
+        // Benchmark - MEASURED
+        // ... your operation here
+      },
+
+      async afterEach(ctx) {
+        // Teardown - NOT measured
+        await promisifyClose(ctx.db);
+      },
+    },
+  },
 };
 ```
 
-3. The `run.js` script will automatically include your new benchmarks
+3. Import and add your benchmarks in `run.js`
 4. Add documentation for your benchmark in this README
 
 ## Notes
 
 - Benchmarks run in-memory (`:memory:`) by default for faster execution
 - Results may vary based on your hardware and system configuration
-- The `benchmark` library automatically determines the number of runs needed for statistical accuracy
+- tinybench automatically determines the number of runs needed for statistical accuracy
+- Warmup iterations allow V8 to optimize the code before measurement
 - For meaningful comparisons, run benchmarks multiple times and compare the results
+
+## Dependencies
+
+- `tinybench`: ^2.9.0 - Modern benchmark library with proper setup/teardown support