Skip to content

Commit 0293c43

Browse files
codedmonkeycodedmonkey
committed
Merge branch 'database-workflows'
2 parents 6c0a8e3 + da3e736 commit 0293c43

3 files changed

Lines changed: 358 additions & 53 deletions

File tree

‎.github/scripts/normalize-psql-dump.py‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
import sys
2+
import re
3+
4+
5+
def filter_lines(text):
6+
"""Remove irrelevant lines from the dump"""
7+
excluded_prefixes = ('SET ', 'SELECT ', 'GRANT ', 'REVOKE ')
8+
lines = [
9+
line for line in text.splitlines(keepends=True)
10+
if line.strip()
11+
and not line.startswith(excluded_prefixes)
12+
and not (line.startswith('\\') and not line.startswith('\\.'))
13+
]
14+
return ''.join(lines)
15+
16+
17+
def sort_copy_columns(text):
18+
"""Sort columns in COPY blocks and reorder the data rows accordingly"""
19+
result = []
20+
lines = text.split('\n')
21+
i = 0
22+
while i < len(lines):
23+
copy_match = re.match(r'(COPY \S+ \()([^)]+)(\) FROM stdin;)', lines[i])
24+
if copy_match:
25+
cols = [c.strip() for c in copy_match.group(2).split(',')]
26+
sorted_indices = sorted(range(len(cols)), key=lambda j: cols[j])
27+
sorted_cols = [cols[j] for j in sorted_indices]
28+
result.append(copy_match.group(1) + ', '.join(sorted_cols) + copy_match.group(3))
29+
i += 1
30+
while i < len(lines) and lines[i] != '\\.':
31+
values = lines[i].split('\t')
32+
result.append('\t'.join(values[j] for j in sorted_indices))
33+
i += 1
34+
if i < len(lines):
35+
result.append(lines[i]) # \.
36+
else:
37+
result.append(lines[i])
38+
i += 1
39+
return '\n'.join(result)
40+
41+
42+
def sort_create_table_columns(match):
43+
"""Sort column lines in a CREATE TABLE block alphabetically"""
44+
lines = [
45+
line.strip().rstrip(',')
46+
for line in match.group(1).split('\n')
47+
if line.strip()
48+
]
49+
return '(\n ' + ',\n '.join(sorted(lines)) + '\n)'
50+
51+
52+
text = filter_lines(sys.stdin.read())
53+
text = re.sub(r'\(((?:\n [^\n]+)+)\n\)', sort_create_table_columns, text)
54+
text = sort_copy_columns(text)
55+
sys.stdout.write(text)

‎.github/workflows/database.yaml‎

Lines changed: 303 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,303 @@
1+
name: Validate database components
2+
3+
# Runs tests to validate database components. It's only triggered for new
4+
# commits. If any of the jobs fail for new commits they're not allowed to be
5+
# merged into the default branch.
6+
7+
on: [pull_request, push]
8+
9+
env:
10+
GITHUB_TOKEN: ${{ github.token }}
11+
12+
jobs:
13+
schema:
14+
name: Schema
15+
16+
runs-on: ubuntu-latest
17+
services:
18+
database:
19+
image: postgres:16-alpine
20+
ports:
21+
- "5432:5432"
22+
env:
23+
POSTGRES_DB: dirigent
24+
POSTGRES_PASSWORD: "!ChangeMe!"
25+
POSTGRES_USER: dirigent
26+
27+
steps:
28+
- name: Checkout code
29+
uses: actions/checkout@v4
30+
31+
- name: Install PHP with extensions
32+
uses: shivammathur/setup-php@v2
33+
with:
34+
php-version: 8.3
35+
tools: composer:v2
36+
37+
- name: Set Composer cache directory
38+
id: composer-cache
39+
run: echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT
40+
41+
- name: Cache Composer output
42+
uses: actions/cache@v4
43+
with:
44+
path: ${{ steps.composer-cache.outputs.dir }}
45+
key: ${{ runner.os }}-composer-${{ hashFiles('composer.lock') }}
46+
restore-keys: ${{ runner.os }}-composer-
47+
48+
- name: Install Composer dependencies
49+
run: composer install --ansi --no-interaction --no-progress
50+
51+
- name: Generate encryption keys
52+
run: bin/console encryption:generate-keys
53+
54+
- name: Validate mapping
55+
run: bin/console doctrine:schema:validate --skip-sync -vvv --ansi --no-interaction
56+
57+
- name: Execute migrations
58+
run: bin/console doctrine:migrations:migrate -vvv --ansi --no-interaction
59+
60+
- name: Validate schema
61+
run: bin/console doctrine:schema:validate --skip-mapping --skip-property-types -vvv --ansi --no-interaction
62+
63+
- name: Load fixtures
64+
run: bin/console doctrine:fixtures:load -vvv --ansi --no-interaction
65+
66+
migration-integrity:
67+
name: Migration integrity
68+
69+
# Verifies the integrity of migration files:
70+
# 1. Existing migrations from the base branch have not been modified or deleted
71+
# 2. New migrations have a higher version number than existing migrations
72+
73+
runs-on: ubuntu-latest
74+
75+
steps:
76+
- name: Checkout code
77+
uses: actions/checkout@v4
78+
with:
79+
fetch-depth: 0
80+
81+
- name: Determine base branch
82+
id: base
83+
run: |
84+
if [ -n "${{ github.base_ref }}" ]; then
85+
BASE_REF="${{ github.base_ref }}"
86+
else
87+
git fetch --all --quiet
88+
BEST_BRANCH="main"
89+
BEST_DISTANCE=$(git rev-list --count origin/main..HEAD 2>/dev/null || echo 999999)
90+
for branch in $(git branch -r | grep -oE 'origin/[0-9]+\.x' | sed 's|origin/||'); do
91+
DISTANCE=$(git rev-list --count origin/$branch..HEAD 2>/dev/null || echo 999999)
92+
if [ "$DISTANCE" -lt "$BEST_DISTANCE" ]; then
93+
BEST_DISTANCE=$DISTANCE
94+
BEST_BRANCH=$branch
95+
fi
96+
done
97+
BASE_REF=$BEST_BRANCH
98+
fi
99+
echo "base_ref=$BASE_REF" >> $GITHUB_OUTPUT
100+
git fetch origin $BASE_REF
101+
102+
- name: Verify existing migrations are not modified or deleted
103+
run: |
104+
BASE_REF="${{ steps.base.outputs.base_ref }}"
105+
CHANGED=$(git diff --name-only --diff-filter=MD origin/$BASE_REF...HEAD -- migrations/)
106+
if [ -n "$CHANGED" ]; then
107+
echo "Error: The following existing migrations were modified or deleted:"
108+
echo "$CHANGED"
109+
exit 1
110+
fi
111+
echo "No existing migrations were modified or deleted."
112+
113+
- name: Verify new migrations have higher version numbers than existing migrations
114+
run: |
115+
BASE_REF="${{ steps.base.outputs.base_ref }}"
116+
BASE_MAX=$(git ls-tree -r --name-only origin/$BASE_REF -- migrations/ | grep -oE '[0-9]{14}' | sort | tail -1)
117+
if [ -z "$BASE_MAX" ]; then
118+
echo "No existing migrations found in base branch, skipping version check."
119+
exit 0
120+
fi
121+
echo "Highest existing migration version: $BASE_MAX"
122+
123+
NEW_MIGRATIONS=$(git diff --name-only --diff-filter=A origin/$BASE_REF...HEAD -- migrations/)
124+
if [ -z "$NEW_MIGRATIONS" ]; then
125+
echo "No new migrations added."
126+
exit 0
127+
fi
128+
129+
FAILED=false
130+
for file in $NEW_MIGRATIONS; do
131+
VERSION=$(echo "$file" | grep -oE '[0-9]{14}')
132+
if [ -z "$VERSION" ]; then
133+
echo "Warning: Could not extract version number from $file"
134+
continue
135+
fi
136+
if [ "$VERSION" -le "$BASE_MAX" ]; then
137+
echo "Error: $file has version $VERSION which is not higher than the existing maximum $BASE_MAX"
138+
FAILED=true
139+
else
140+
echo "OK: $file (version $VERSION > $BASE_MAX)"
141+
fi
142+
done
143+
144+
if [ "$FAILED" = true ]; then
145+
exit 1
146+
fi
147+
148+
migrations:
149+
name: Migrations
150+
151+
# Verifies that new database migrations are compatible with existing data.
152+
# When migration files changed, this workflow:
153+
# 1. Checks out the base branch and applies all existing migrations
154+
# 2. Loads fixtures to simulate a database with real-world data
155+
# 3. Switches to the current branch and applies the new migrations on top
156+
# 4. Reverts the current migrations to verify the down() methods work
157+
# 5. Compares the schema and data against the baseline to ensure the revert is clean
158+
159+
runs-on: ubuntu-latest
160+
services:
161+
database:
162+
image: postgres:16-alpine
163+
ports:
164+
- "5432:5432"
165+
env:
166+
POSTGRES_DB: dirigent
167+
POSTGRES_PASSWORD: "!ChangeMe!"
168+
POSTGRES_USER: dirigent
169+
170+
steps:
171+
- name: Checkout code
172+
uses: actions/checkout@v4
173+
with:
174+
fetch-depth: 0
175+
176+
- name: Check for migration changes
177+
id: check
178+
run: |
179+
if [ -n "${{ github.base_ref }}" ]; then
180+
BASE_REF="${{ github.base_ref }}"
181+
else
182+
git fetch --all --quiet
183+
BEST_BRANCH="main"
184+
BEST_DISTANCE=$(git rev-list --count origin/main..HEAD 2>/dev/null || echo 999999)
185+
for branch in $(git branch -r | grep -oE 'origin/[0-9]+\.x' | sed 's|origin/||'); do
186+
DISTANCE=$(git rev-list --count origin/$branch..HEAD 2>/dev/null || echo 999999)
187+
if [ "$DISTANCE" -lt "$BEST_DISTANCE" ]; then
188+
BEST_DISTANCE=$DISTANCE
189+
BEST_BRANCH=$branch
190+
fi
191+
done
192+
BASE_REF=$BEST_BRANCH
193+
fi
194+
echo "base_ref=$BASE_REF" >> $GITHUB_OUTPUT
195+
git fetch origin $BASE_REF
196+
CHANGED=$(git diff --name-only origin/$BASE_REF...HEAD -- migrations/)
197+
if [ -n "$CHANGED" ]; then
198+
echo "has_changes=true" >> $GITHUB_OUTPUT
199+
echo "New migration files:"
200+
echo "$CHANGED"
201+
else
202+
echo "has_changes=false" >> $GITHUB_OUTPUT
203+
echo "No migration changes detected, skipping."
204+
fi
205+
206+
- name: Install PHP with extensions
207+
if: steps.check.outputs.has_changes == 'true'
208+
uses: shivammathur/setup-php@v2
209+
with:
210+
php-version: 8.3
211+
tools: composer:v2
212+
213+
- name: Set Composer cache directory
214+
if: steps.check.outputs.has_changes == 'true'
215+
id: composer-cache
216+
run: echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT
217+
218+
- name: Cache Composer output
219+
if: steps.check.outputs.has_changes == 'true'
220+
uses: actions/cache@v4
221+
with:
222+
path: ${{ steps.composer-cache.outputs.dir }}
223+
key: ${{ runner.os }}-composer-${{ hashFiles('composer.lock') }}
224+
restore-keys: ${{ runner.os }}-composer-
225+
226+
- name: Checkout base branch
227+
if: steps.check.outputs.has_changes == 'true'
228+
run: git checkout origin/${{ steps.check.outputs.base_ref }}
229+
230+
- name: Install Composer dependencies for base branch
231+
if: steps.check.outputs.has_changes == 'true'
232+
run: composer install --ansi --no-interaction --no-progress
233+
234+
- name: Generate encryption keys
235+
if: steps.check.outputs.has_changes == 'true'
236+
run: bin/console encryption:generate-keys
237+
238+
- name: Execute migrations from base branch
239+
if: steps.check.outputs.has_changes == 'true'
240+
run: bin/console doctrine:migrations:migrate -vvv --ansi --no-interaction
241+
242+
- name: Capture base branch migration version
243+
id: base-version
244+
if: steps.check.outputs.has_changes == 'true'
245+
run: |
246+
VERSION=$(bin/console doctrine:migrations:latest --no-ansi | awk '{print $1}')
247+
echo "version=$VERSION" >> $GITHUB_OUTPUT
248+
249+
- name: Load fixtures
250+
if: steps.check.outputs.has_changes == 'true'
251+
run: bin/console doctrine:fixtures:load -vvv --ansi --no-interaction
252+
253+
- name: Checkout current branch
254+
if: steps.check.outputs.has_changes == 'true'
255+
run: git checkout ${{ github.sha }}
256+
257+
- name: Snapshot baseline schema and data
258+
if: steps.check.outputs.has_changes == 'true'
259+
env:
260+
PGPASSWORD: "!ChangeMe!"
261+
run: |
262+
pg_dump -U dirigent -h localhost dirigent \
263+
--schema-only --no-comments --no-privileges --no-owner \
264+
| python3 .github/scripts/normalize-psql-dump.py \
265+
> /tmp/schema_baseline.sql
266+
pg_dump -U dirigent -h localhost dirigent \
267+
--data-only --no-comments --no-privileges --no-owner \
268+
| python3 .github/scripts/normalize-psql-dump.py \
269+
> /tmp/data_baseline.sql
270+
271+
- name: Install Composer dependencies for current branch
272+
if: steps.check.outputs.has_changes == 'true'
273+
run: composer install --ansi --no-interaction --no-progress
274+
275+
- name: Execute new migrations
276+
if: steps.check.outputs.has_changes == 'true'
277+
run: bin/console doctrine:migrations:migrate -vvv --ansi --no-interaction
278+
279+
- name: Revert new migrations
280+
if: steps.check.outputs.has_changes == 'true'
281+
run: bin/console doctrine:migrations:migrate "${{ steps.base-version.outputs.version }}" -vvv --ansi --no-interaction
282+
283+
- name: Snapshot reverted schema and data
284+
if: steps.check.outputs.has_changes == 'true'
285+
env:
286+
PGPASSWORD: "!ChangeMe!"
287+
run: |
288+
pg_dump -U dirigent -h localhost dirigent \
289+
--schema-only --no-comments --no-privileges --no-owner \
290+
| python3 .github/scripts/normalize-psql-dump.py \
291+
> /tmp/schema_reverted.sql
292+
pg_dump -U dirigent -h localhost dirigent \
293+
--data-only --no-comments --no-privileges --no-owner \
294+
| python3 .github/scripts/normalize-psql-dump.py \
295+
> /tmp/data_reverted.sql
296+
297+
- name: Verify schema after revert matches baseline
298+
if: steps.check.outputs.has_changes == 'true'
299+
run: diff -u /tmp/schema_baseline.sql /tmp/schema_reverted.sql
300+
301+
- name: Verify data after revert matches baseline
302+
if: steps.check.outputs.has_changes == 'true'
303+
run: diff -u /tmp/data_baseline.sql /tmp/data_reverted.sql

0 commit comments

Comments
 (0)