Skip to content

Commit e5ce3bd

Browse files
steve-chavezsteve-chavez
committed
init: add csv_agg function
- Adds csv_agg aggregate function, RFC RFC 4180 - Compatible with pg >= 12 - Almost x2 faster than postgREST default csv query + loadtests with pgbench included
0 parents  commit e5ce3bd

21 files changed

Lines changed: 845 additions & 0 deletions

‎.clang-format‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
AlignAfterOpenBracket: Align
2+
AlignConsecutiveAssignments: true
3+
AlignConsecutiveDeclarations: true
4+
AlignOperands: true
5+
AllowAllParametersOfDeclarationOnNextLine: false
6+
AllowShortFunctionsOnASingleLine: Empty
7+
AllowShortLoopsOnASingleLine: false
8+
AlwaysBreakAfterDefinitionReturnType: None
9+
AlwaysBreakAfterReturnType: None
10+
AlwaysBreakBeforeMultilineStrings: false
11+
BinPackArguments: true
12+
BinPackParameters: true
13+
ColumnLimit: 1000
14+
IndentPPDirectives: AfterHash
15+
MaxEmptyLinesToKeep: 1
16+
PointerAlignment: Right
17+
SpaceBeforeAssignmentOperators: true
18+
SpaceBeforeParens: ControlStatements
19+
SpaceInEmptyParentheses: false
20+
SpacesBeforeTrailingComments: 1
21+
BracedInitializerIndentWidth: 2
22+
23+
# includes
24+
IncludeBlocks: Preserve
25+
SortIncludes: true
26+
27+
# indentation
28+
IndentWidth: 2
29+
TabWidth: 2
30+
UseTab: Never
31+
32+
# if
33+
AllowShortIfStatementsOnASingleLine: true
34+
35+
# case
36+
IndentCaseLabels: false
37+
AllowShortCaseLabelsOnASingleLine: true
38+
AlignConsecutiveShortCaseStatements:
39+
Enabled: true
40+
AcrossEmptyLines: true
41+
AcrossComments: true
42+
AlignCaseColons: true

‎.github/workflows/ci.yaml‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
name: CI
2+
3+
on: [push, pull_request]
4+
5+
jobs:
6+
test:
7+
runs-on: ubuntu-latest
8+
strategy:
9+
matrix:
10+
pg-version: ['12', '13', '14', '15', '16', '17']
11+
12+
steps:
13+
- uses: actions/checkout@v4
14+
15+
- name: Install Nix
16+
uses: cachix/install-nix-action@v30
17+
with:
18+
nix_path: nixpkgs=channel:nixos-unstable
19+
20+
- name: Use Cachix Cache
21+
uses: cachix/cachix-action@v10
22+
with:
23+
name: nxpg
24+
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
25+
26+
- name: Build
27+
run: nix-shell --run "xpg -v ${{ matrix.pg-version }} build"
28+
29+
- name: Run tests
30+
run: nix-shell --run "xpg -v ${{ matrix.pg-version }} test"
31+
32+
- if: ${{ failure() }}
33+
run: |
34+
cat regression.out
35+
cat regression.diffs
36+
37+
loadtest:
38+
strategy:
39+
matrix:
40+
kind: ['csv_agg', 'postgrest']
41+
name: Loadtest
42+
runs-on: ubuntu-24.04
43+
steps:
44+
- uses: actions/checkout@v4
45+
46+
- name: Install Nix
47+
uses: cachix/install-nix-action@v30
48+
with:
49+
nix_path: nixpkgs=channel:nixos-unstable
50+
51+
- name: Use Cachix Cache
52+
uses: cachix/cachix-action@v10
53+
with:
54+
name: nxpg
55+
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
56+
57+
- name: Run loadtest
58+
run: nix-shell --run "./bench/loadtest.sh ${{ matrix.kind }}"
59+
60+
coverage:
61+
62+
runs-on: ubuntu-latest
63+
64+
strategy:
65+
matrix:
66+
pg-version: ['17']
67+
68+
steps:
69+
- uses: actions/checkout@v4
70+
71+
- name: Install Nix
72+
uses: cachix/install-nix-action@v30
73+
with:
74+
nix_path: nixpkgs=channel:nixos-unstable
75+
76+
- name: Use Cachix Cache
77+
uses: cachix/cachix-action@v10
78+
with:
79+
name: nxpg
80+
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
81+
82+
- name: Run coverage
83+
run: nix-shell --run "xpg -v ${{ matrix.pg-version }} coverage"
84+
85+
- name: Send coverage to Coveralls
86+
uses: coverallsapp/github-action@v2.3.6
87+
with:
88+
github-token: ${{ secrets.GITHUB_TOKEN }}
89+
files: ./build-${{ matrix.pg-version }}/coverage.info
90+
91+
style:
92+
93+
runs-on: ubuntu-latest
94+
95+
steps:
96+
- uses: actions/checkout@v4
97+
98+
- name: Install Nix
99+
uses: cachix/install-nix-action@v30
100+
with:
101+
nix_path: nixpkgs=channel:nixos-unstable
102+
103+
- name: Use Cachix Cache
104+
uses: cachix/cachix-action@v10
105+
with:
106+
name: nxpg
107+
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
108+
109+
- name: Run style check
110+
run: nix-shell --run "pg_csv-style-check"

‎.gitignore‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
*.control
2+
regression.*
3+
results/
4+
*.so
5+
*.o
6+
*.bc
7+
*.diffs
8+
pgbench_log.*
9+
.history

‎LICENSE‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
Copyright (c) 2025 Steve Chavez
2+
3+
Permission is hereby granted, free of charge, to any person obtaining
4+
a copy of this software and associated documentation files (the
5+
"Software"), to deal in the Software without restriction, including
6+
without limitation the rights to use, copy, modify, merge, publish,
7+
distribute, sublicense, and/or sell copies of the Software, and to
8+
permit persons to whom the Software is furnished to do so, subject to
9+
the following conditions:
10+
11+
The above copyright notice and this permission notice shall be included
12+
in all copies or substantial portions of the Software.
13+
14+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
17+
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
18+
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
19+
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
20+
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

‎Makefile‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
SRC_DIR = src
2+
3+
# the `-Wno`s quiet C90 warnings
4+
PG_CFLAGS = -std=c11 -Wextra -Wall -Werror \
5+
-Wno-declaration-after-statement \
6+
-Wno-vla \
7+
-Wno-long-long
8+
ifeq ($(COVERAGE), 1)
9+
PG_CFLAGS += --coverage
10+
endif
11+
12+
ifeq ($(CC),gcc)
13+
GCC_MAJ := $(firstword $(subst ., ,$(shell $(CC) -dumpfullversion -dumpversion)))
14+
GCC_GE14 = $(shell test $(GCC_MAJ) -ge 14; echo $$?)
15+
ifeq ($(GCC_GE14),0)
16+
PG_CFLAGS += -Wmissing-variable-declarations
17+
endif
18+
endif
19+
20+
UNAME_S := $(shell uname -s)
21+
22+
ifeq ($(UNAME_S),Darwin)
23+
SHARED_EXT := dylib
24+
else
25+
SHARED_EXT := so
26+
endif
27+
28+
EXTENSION = pg_csv
29+
EXTVERSION = 0.1
30+
31+
DATA = $(wildcard sql/*--*.sql)
32+
33+
EXTRA_CLEAN = sql/$(EXTENSION)--$(EXTVERSION).sql $(EXTENSION).control
34+
35+
TESTS = $(wildcard test/sql/*.sql)
36+
REGRESS = $(patsubst test/sql/%.sql,%,$(TESTS))
37+
REGRESS_OPTS = --use-existing --inputdir=test
38+
39+
MODULE_big = $(EXTENSION)
40+
SRC = $(wildcard $(SRC_DIR)/*.c)
41+
42+
ifdef BUILD_DIR
43+
OBJS = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC))
44+
else
45+
OBJS = $(patsubst $(SRC_DIR)/%.c, src/%.o, $(SRC)) # if no BUILD_DIR, just build on src so standard PGXS `make` works
46+
endif
47+
48+
PG_CONFIG = pg_config
49+
50+
PG_CPPFLAGS := $(CPPFLAGS) -DEXTVERSION=\"$(EXTVERSION)\"
51+
52+
all: sql/$(EXTENSION)--$(EXTVERSION).sql $(EXTENSION).control
53+
54+
build: $(BUILD_DIR)/$(EXTENSION).$(SHARED_EXT) sql/$(EXTENSION)--$(EXTVERSION).sql $(EXTENSION).control
55+
56+
$(BUILD_DIR)/.gitignore: sql/$(EXTENSION)--$(EXTVERSION).sql $(EXTENSION).control
57+
mkdir -p $(BUILD_DIR)/extension
58+
cp $(EXTENSION).control $(BUILD_DIR)/extension
59+
cp sql/$(EXTENSION)--$(EXTVERSION).sql $(BUILD_DIR)/extension
60+
echo "*" > $(BUILD_DIR)/.gitignore
61+
62+
$(BUILD_DIR)/%.o: $(SRC_DIR)/%.c $(BUILD_DIR)/.gitignore
63+
$(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@
64+
65+
$(BUILD_DIR)/$(EXTENSION).$(SHARED_EXT): $(EXTENSION).$(SHARED_EXT)
66+
mv $? $@
67+
68+
sql/$(EXTENSION)--$(EXTVERSION).sql: sql/$(EXTENSION).sql
69+
cp $< $@
70+
71+
$(EXTENSION).control:
72+
sed "s/@EXTVERSION@/$(EXTVERSION)/g" $(EXTENSION).control.in > $@
73+
74+
PGXS := $(shell $(PG_CONFIG) --pgxs)
75+
include $(PGXS)
76+
77+
.PHONY: test
78+
test:
79+
make installcheck

‎README.md‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# pg_csv
2+
3+
![PostgreSQL version](https://img.shields.io/badge/postgresql-12+-blue.svg)
4+
[![Coverage Status](https://coveralls.io/repos/github/PostgREST/pg_csv/badge.svg)](https://coveralls.io/github/PostgREST/pg_csv)
5+
[![Tests](https://github.com/PostgREST/pg_csv/actions/workflows/ci.yml/badge.svg)](https://github.com/PostgREST/pg_csv/actions)
6+
7+
## csv_agg
8+
9+
Aggregate that builds a CSV as per [RFC 4180](https://www.ietf.org/rfc/rfc4180.txt), quoting as required.
10+
11+
```
12+
select csv_agg(x) from projects x;
13+
csv_agg
14+
-------------------
15+
id,name,client_id+
16+
1,Windows 7,1 +
17+
2,Windows 10,1 +
18+
3,IOS,2 +
19+
4,OSX,2 +
20+
5,Orphan,
21+
(1 row)
22+
```

‎bench/csv_agg.sql‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
\set lim random(1000, 2000)
2+
3+
select csv_agg(t) from (
4+
select * from student_emotion_assessments limit :lim
5+
) as t;

0 commit comments

Comments
 (0)