Merge pull request #24 from shuiyisong/add-greptimedb-formal

Add GreptimeDB

Author: rschu1ze

README.md

@@ -148,6 +148,7 @@ While the main benchmark uses a specific machine configuration for reproducibili
 - [ ] FerretDB
 - [ ] Apache Drill
 - [ ] GlareDB
+- [x] GreptimeDB
 
 ## Similar projects
 

greptimedb/count.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+curl -s --fail http://localhost:4000/v1/sql \
+    -H 'Content-Type: application/x-www-form-urlencoded' \
+    -d "sql=select count(*) as cnt from bluesky"  \
+    -d "format=json" \
+    | grep -o "cnt\":[0-9]*" | sed 's/cnt\"://g'

greptimedb/data_size.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+curl -s --fail http://localhost:4000/v1/sql \
+    -H 'Content-Type: application/x-www-form-urlencoded' \
+    -d "sql=SELECT sum(r.sst_size) as data_size FROM information_schema.REGION_STATISTICS r LEFT JOIN information_schema.TABLES t on r.table_id = t.table_id WHERE t.table_name = 'bluesky'"  \
+    -d "format=json"  \
+    | grep -o "data_size\":[0-9]*" | sed 's/data_size\"://g'

greptimedb/drop_tables.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo "Stopping GreptimeDB"
+pidof greptime && kill `pidof greptime`
+
+echo "Dropping all data"
+rm -rf ./greptimedb_data

greptimedb/index_size.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+curl -s --fail http://localhost:4000/v1/sql \
+    -H 'Content-Type: application/x-www-form-urlencoded' \
+    -d "sql=SELECT sum(r.index_size) as index_size FROM information_schema.REGION_STATISTICS r LEFT JOIN information_schema.TABLES t on r.table_id = t.table_id WHERE t.table_name = 'bluesky'"  \
+    -d "format=json"  \
+    | grep -o "index_size\":[0-9]*" | sed 's/index_size\"://g'

greptimedb/install.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+RELEASE_VERSION=v0.13.0-nightly-20250315
+
+# download greptimedb
+wget -N "https://github.com/GreptimeTeam/greptimedb/releases/download/${RELEASE_VERSION}/greptime-linux-amd64-${RELEASE_VERSION}.tar.gz"
+tar xzf greptime-linux-amd64-${RELEASE_VERSION}.tar.gz
+mv greptime-linux-amd64-${RELEASE_VERSION}/greptime ./
+rm -rf greptime-linux-amd64-${RELEASE_VERSION}

greptimedb/load_data.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Check if the required arguments are provided
+if [[ $# -lt 4 ]]; then
+    echo "Usage: $0 <DATA_DIRECTORY> <MAX_FILES> <SUCCESS_LOG> <ERROR_LOG>"
+    exit 1
+fi
+
+# Arguments
+DATA_DIRECTORY="$1"
+MAX_FILES="$2"
+SUCCESS_LOG="$3"
+ERROR_LOG="$4"
+
+# Validate arguments
+[[ ! -d "$DATA_DIRECTORY" ]] && { echo "Error: Data directory '$DATA_DIRECTORY' does not exist."; exit 1; }
+[[ ! "$MAX_FILES" =~ ^[0-9]+$ ]] && { echo "Error: MAX_FILES must be a positive integer."; exit 1; }
+
+pushd $DATA_DIRECTORY
+counter=0
+for file in $(ls *.json.gz | head -n $MAX_FILES); do
+    echo "Processing file: $file"
+
+    curl "http://localhost:4000/v1/events/logs?table=bluesky&pipeline_name=jsonbench&ignore_errors=true" \
+         -H "Content-Type: application/x-ndjson" \
+         -H "Content-Encoding: gzip" \
+         --data-binary @$file
+    echo ""
+
+    first_attempt=$?
+    if [[ $first_attempt -eq 0 ]]; then
+        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Successfully imported $file." >> "$SUCCESS_LOG"
+    else
+        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Failed for $file. Giving up." >> "$ERROR_LOG"
+    fi
+
+    counter=$((counter + 1))
+    if [[ $counter -ge $MAX_FILES ]]; then
+        break
+    fi
+done
+
+curl -XPOST -H 'Content-Type: application/x-www-form-urlencoded' \
+          http://localhost:4000/v1/sql \
+          -d "sql=admin flush_table('bluesky')" \
+          -d "format=json"
+
+echo -e "\nLoaded $MAX_FILES data files from $DATA_DIRECTORY to GreptimeDB."

greptimedb/main.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# Default data directory
+DEFAULT_DATA_DIRECTORY=~/data/bluesky
+
+# Allow the user to optionally provide the data directory as an argument
+DATA_DIRECTORY="${1:-$DEFAULT_DATA_DIRECTORY}"
+
+# Define success and error log files
+SUCCESS_LOG="${2:-success.log}"
+ERROR_LOG="${3:-error.log}"
+
+# Define prefix for output files
+OUTPUT_PREFIX="${4:-_m6i.8xlarge}"
+
+# Check if the directory exists
+if [[ ! -d "$DATA_DIRECTORY" ]]; then
+    echo "Error: Data directory '$DATA_DIRECTORY' does not exist."
+    exit 1
+fi
+
+echo "Select the dataset size to benchmark:"
+echo "1) 1m (default)"
+echo "2) 10m"
+echo "3) 100m"
+echo "4) 1000m"
+echo "5) all"
+read -p "Enter the number corresponding to your choice: " choice
+
+./install.sh
+
+benchmark() {
+    local size=$1
+    # Check DATA_DIRECTORY contains the required number of files to run the benchmark
+    file_count=$(find "$DATA_DIRECTORY" -type f | wc -l)
+    if (( file_count < size )); then
+        echo "Error: Not enough files in '$DATA_DIRECTORY'. Required: $size, Found: $file_count."
+        exit 1
+    fi
+
+    ./start.sh
+    ./load_data.sh "$DATA_DIRECTORY" "$size" "$SUCCESS_LOG" "$ERROR_LOG"
+    ./total_size.sh | tee "${OUTPUT_PREFIX}_bluesky_${size}m.total_size"
+    ./data_size.sh | tee "${OUTPUT_PREFIX}_bluesky_${size}m.data_size"
+    ./index_size.sh | tee "${OUTPUT_PREFIX}_bluesky_${size}m.index_size"
+    ./count.sh | tee "${OUTPUT_PREFIX}_bluesky_${size}m.count"
+    ./run_queries.sh | tee "${OUTPUT_PREFIX}_bluesky_${size}m.results_runtime"
+    ./drop_tables.sh 
+}
+
+case $choice in
+    2)
+        benchmark 10
+        ;;
+    3)
+        benchmark 100
+        ;;
+    4)
+        benchmark 1000
+        ;;
+    5)
+        benchmark 1
+        benchmark 10
+        benchmark 100
+        benchmark 1000
+        ;;
+    *)
+        benchmark 1
+        ;;
+esac

greptimedb/pipeline.yaml

@@ -0,0 +1,35 @@
+processors:
+  - epoch:
+      fields:
+        - time_us
+      resolution: microsecond
+  - simple_extract:
+      fields:
+        - commit, commit_collection
+      key: "collection"
+      ignore_missing: true
+  - simple_extract:
+      fields:
+        - commit, commit_operation
+      key: "operation"
+      ignore_missing: true
+
+transform:
+  - fields:
+      - did
+    type: string
+  - fields:
+      - kind
+      - commit_collection
+      - commit_operation
+    type: string
+    index: inverted
+    tag: true
+  - fields:
+      - commit
+    type: json
+    on_failure: ignore
+  - fields:
+      - time_us
+    type: epoch, us
+    index: timestamp

greptimedb/queries.sql

@@ -0,0 +1,5 @@
+SELECT commit_collection AS event, count(1) AS cnt FROM bluesky GROUP BY event ORDER BY cnt DESC;
+SELECT commit_collection AS event, count(1) AS cnt, count(DISTINCT did) AS users FROM bluesky WHERE kind = 'commit' AND commit_operation = 'create' GROUP BY event ORDER BY cnt DESC;
+SELECT commit_collection AS event, date_part('hour', time_us) AS hour_of_day, count(1) AS cnt FROM bluesky WHERE kind = 'commit' AND commit_operation = 'create' AND commit_collection IN('app.bsky.feed.post', 'app.bsky.feed.repost', 'app.bsky.feed.like') GROUP BY event, hour_of_day ORDER BY hour_of_day, event;
+SELECT did AS user_id, min(time_us) AS first_post_ts FROM bluesky WHERE kind = 'commit' AND commit_operation = 'create' AND commit_collection = 'app.bsky.feed.post' GROUP BY user_id ORDER BY first_post_ts ASC LIMIT 3;
+SELECT did AS user_id, date_part('millisecond',(max(time_us) - min(time_us))) AS activity_span FROM bluesky WHERE kind = 'commit' AND commit_operation = 'create' AND commit_collection = 'app.bsky.feed.post' GROUP BY user_id ORDER BY activity_span DESC LIMIT 3;