Skip to content

Commit 02f5e4b

Browse files
thejcfactorthejcfactor
authored
Update Github Workflows (#7)
Changes ======= * Update tests workflow to use linux arm runners * Split alpine tests into separate job (and cache docker image) * PYCO-56 & PYCO-63: Add publish workflow to handle releasing artifacts and API docs * PYCO-64: Add verify_release workflow to download published versions and run tests for validation
1 parent 09c9a0f commit 02f5e4b

File tree

3 files changed

+1190
-61
lines changed

3 files changed

+1190
-61
lines changed

‎.github/workflows/publish.yml‎

Lines changed: 378 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,378 @@
1+
name: publish
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
sha:
7+
description: "The git SHA to use for release. Only set if needing to publish"
8+
required: true
9+
default: ""
10+
type: string
11+
version:
12+
description: "The Release version. Allowed format: x.y.z[-alphaN | -betaN | -rcN | -devN | -postN]"
13+
required: true
14+
default: ""
15+
type: string
16+
tests_run_id:
17+
description: "The workflow run ID of a tests workflow run. Set if wanting to use artifacts from an already completed run."
18+
required: false
19+
default: ""
20+
type: string
21+
config:
22+
description: "JSON formatted object representing various build system input parameters."
23+
required: false
24+
default: ""
25+
type: string
26+
is_dry_run:
27+
description: "Set to true in order to do a publish dry-run."
28+
required: true
29+
default: false
30+
type: boolean
31+
skip_validation:
32+
description: "Set to true in order to skip validating the published packages."
33+
required: true
34+
default: false
35+
type: boolean
36+
37+
env:
38+
CBCI_PROJECT_TYPE: "ANALYTICS"
39+
CBCI_DEFAULT_PYTHON: "3.9"
40+
CBCI_SUPPORTED_PYTHON_VERSIONS: "3.9 3.10 3.11 3.12 3.13"
41+
CBCI_SUPPORTED_X86_64_PLATFORMS: "linux alpine macos windows"
42+
CBCI_SUPPORTED_ARM64_PLATFORMS: "linux macos"
43+
CBCI_DEFAULT_LINUX_X86_64_PLATFORM: "ubuntu-22.04"
44+
CBCI_DEFAULT_LINUX_ARM64_PLATFORM: "ubuntu-22.04-arm"
45+
CBCI_DEFAULT_MACOS_X86_64_PLATFORM: "macos-13"
46+
CBCI_DEFAULT_MACOS_ARM64_PLATFORM: "macos-14"
47+
CBCI_DEFAULT_WINDOWS_PLATFORM: "windows-2022"
48+
CBCI_DEFAULT_LINUX_CONTAINER: "slim-bookworm"
49+
CBCI_DEFAULT_ALPINE_CONTAINER: "alpine"
50+
CBCI_CBDINO_VERSION: "v0.0.80"
51+
CI_SCRIPTS_URL: "https://raw.githubusercontent.com/couchbaselabs/sdkbuild-jenkinsfiles/master/python/ci_scripts_v1"
52+
53+
jobs:
54+
default-linux-runner:
55+
runs-on: ubuntu-latest
56+
outputs:
57+
runner: ${{ steps.set_runner.outputs.runner }}
58+
steps:
59+
- id: set_runner
60+
run: echo "runner=${{ env.CBCI_DEFAULT_LINUX_X86_64_PLATFORM }}" >> $GITHUB_OUTPUT
61+
62+
ci-scripts:
63+
needs: default-linux-runner
64+
runs-on: ${{ needs.default-linux-runner.outputs.runner }}
65+
steps:
66+
- name: Download CI Scripts
67+
run: |
68+
mkdir ci_scripts
69+
cd ci_scripts
70+
curl -o gha.sh ${CI_SCRIPTS_URL}/gha.sh
71+
curl -o pygha.py ${CI_SCRIPTS_URL}/pygha.py
72+
ls -alh
73+
- name: Upload CI scripts
74+
uses: actions/upload-artifact@v4
75+
with:
76+
retention-days: 1
77+
name: ci_scripts_publish
78+
path: |
79+
ci_scripts/
80+
81+
validate-input:
82+
needs: [ci-scripts, default-linux-runner]
83+
runs-on: ${{ needs.default-linux-runner.outputs.runner }}
84+
env:
85+
CBCI_IS_RELEASE: true
86+
CBCI_SHA: ${{ inputs.sha }}
87+
CBCI_VERSION: ${{ inputs.version }}
88+
CBCI_CONFIG: ${{ inputs.config }}
89+
steps:
90+
- name: Download CI scripts
91+
uses: actions/download-artifact@v4
92+
with:
93+
name: ci_scripts_publish
94+
path: ci_scripts
95+
- name: Verify Scripts
96+
run: |
97+
ls -alh ci_scripts
98+
chmod +x ci_scripts/gha.sh
99+
ls -alh ci_scripts
100+
- name: Display workflow info
101+
run: |
102+
./ci_scripts/gha.sh display_info
103+
- name: Validate workflow info
104+
run: |
105+
./ci_scripts/gha.sh validate_input ${{ github.workflow }}
106+
107+
publish-config:
108+
needs: [validate-input, default-linux-runner]
109+
runs-on: ${{ needs.default-linux-runner.outputs.runner }}
110+
env:
111+
CBCI_IS_RELEASE: true
112+
CBCI_SHA: ${{ inputs.sha }}
113+
CBCI_VERSION: ${{ inputs.version }}
114+
CBCI_CONFIG: ${{ inputs.config }}
115+
CBCI_PUBLISH_DRY_RUN: ${{ inputs.is_dry_run }}
116+
CBCI_TESTS_RUN_ID: ${{ inputs.tests_run_id }}
117+
outputs:
118+
publish_config: ${{ steps.set_config.outputs.publish_config }}
119+
steps:
120+
- name: Download CI scripts
121+
uses: actions/download-artifact@v4
122+
with:
123+
name: ci_scripts_publish
124+
path: ci_scripts
125+
- name: Enable CI Scripts
126+
run: |
127+
chmod +x ci_scripts/gha.sh
128+
- name: Setup Python ${{ env.CBCI_DEFAULT_PYTHON }}
129+
uses: actions/setup-python@v5
130+
with:
131+
python-version: ${{ env.CBCI_DEFAULT_PYTHON }}
132+
- name: Confirm Python version
133+
run: python -c "import sys; print(sys.version)"
134+
- name: Set public config
135+
id: set_config
136+
run: |
137+
exit_code=0
138+
PUBLISH_CONFIG=$(./ci_scripts/gha.sh build_publish_config) || exit_code=$?
139+
if [ $exit_code -ne 0 ]; then
140+
echo "Failed to obtain publish config."
141+
exit 1
142+
fi
143+
publish_config_json=$(jq -cn --argjson pubconfig "$PUBLISH_CONFIG" '$pubconfig')
144+
echo "PUBLISH_CONFIG_JSON=$publish_config_json"
145+
echo "publish_config=$publish_config_json" >> "$GITHUB_OUTPUT"
146+
147+
confirm-publish-config:
148+
needs: [publish-config, default-linux-runner]
149+
runs-on: ${{ needs.default-linux-runner.outputs.runner }}
150+
steps:
151+
- name: Publish Config
152+
run: |
153+
echo "${{ toJson(fromJson(needs.publish-config.outputs.publish_config)) }}"
154+
155+
run-tests:
156+
needs: publish-config
157+
if: ${{ fromJson(needs.publish-config.outputs.publish_config).publish_config.tests_run_id == '' }}
158+
uses: ./.github/workflows/tests.yml
159+
with:
160+
is_release: true
161+
sha: ${{ inputs.sha && inputs.sha || '' }}
162+
version: ${{ inputs.version && inputs.version || '' }}
163+
config: ${{ inputs.config && inputs.config || '' }}
164+
165+
set-tests-run-id:
166+
needs: [run-tests, default-linux-runner]
167+
if: >-
168+
${{ !cancelled()
169+
&& (needs.run-tests.result == 'success' || needs.run-tests.result == 'skipped') }}
170+
runs-on: ${{ needs.default-linux-runner.outputs.runner }}
171+
outputs:
172+
tests_run_id: ${{ inputs.tests_run_id == '' && steps.save_tests_run_id.outputs.tests_run_id || inputs.tests_run_id }}
173+
steps:
174+
- name: Save tests_run_id
175+
if: ${{ inputs.tests_run_id == '' }}
176+
id: save_tests_run_id
177+
run: |
178+
echo "tests_run_id=${{ needs.run-tests.outputs.workflow_run_id }}" >> $GITHUB_OUTPUT
179+
180+
upload-api-docs:
181+
needs: [publish-config, run-tests, default-linux-runner]
182+
if: >-
183+
${{ !cancelled()
184+
&& fromJson(needs.publish-config.outputs.publish_config).publish_config.publish_api_docs
185+
&& (needs.run-tests.result == 'success' || needs.run-tests.result == 'skipped') }}
186+
runs-on: ${{ needs.default-linux-runner.outputs.runner }}
187+
steps:
188+
- name: Checkout (with SHA)
189+
if: inputs.sha != ''
190+
uses: actions/checkout@v4
191+
with:
192+
ref: ${{ inputs.sha }}
193+
fetch-depth: 0
194+
- name: Checkout (no SHA)
195+
if: inputs.sha == ''
196+
uses: actions/checkout@v4
197+
with:
198+
fetch-depth: 0
199+
- name: Setup Python ${{ env.CBCI_DEFAULT_PYTHON }}
200+
uses: actions/setup-python@v5
201+
with:
202+
python-version: ${{ env.CBCI_DEFAULT_PYTHON }}
203+
- name: Confirm Python version
204+
run: python -c "import sys; print(sys.version)"
205+
- name: Set SDK git tag
206+
if: ${{ fromJson(needs.publish-config.outputs.publish_config).publish_config.set_git_tag != '' }}
207+
run: |
208+
git config user.name "Couchbase SDK Team"
209+
git config user.email "sdk_dev@couchbase.com"
210+
git tag -a $PYCBAC_VERSION -m "Release of client version $PYCBAC_VERSION"
211+
env:
212+
PYCBAC_VERSION: ${{ fromJson(needs.publish-config.outputs.publish_config).publish_config.set_git_tag }}
213+
- name: Set SDK version
214+
run: |
215+
python couchbase_analytics_version.py --mode make
216+
- name: Build API docs
217+
run: |
218+
python -m pip install --upgrade pip setuptools wheel
219+
python -m pip install -r requirements.txt
220+
python -m pip install -r requirements-sphinx.txt
221+
mkdir sphinx
222+
sphinx-build -M html ./docs ./sphinx
223+
- name: Upload Python API docs as artifact
224+
uses: actions/upload-artifact@v4
225+
with:
226+
retention-days: 1
227+
name: pycbac-api-docs
228+
path: sphinx/
229+
230+
publish-test-pypi:
231+
needs: [publish-config, run-tests, set-tests-run-id, default-linux-runner]
232+
if: >-
233+
${{ !cancelled()
234+
&& fromJson(needs.publish-config.outputs.publish_config).publish_config.publish_test_pypi
235+
&& (needs.run-tests.result == 'success' || needs.run-tests.result == 'skipped') }}
236+
runs-on: ${{ needs.default-linux-runner.outputs.runner }}
237+
environment:
238+
name: publish
239+
url: https://pypi.org/p/couchbase-analytics
240+
permissions:
241+
# This permission is required for pypi's "trusted publisher" feature
242+
id-token: write
243+
steps:
244+
- uses: actions/download-artifact@v4
245+
with:
246+
pattern: pycbac-artifact-*
247+
path: dist
248+
merge-multiple: true
249+
run-id: ${{ needs.set-tests-run-id.outputs.tests_run_id }}
250+
github-token: ${{ github.token }}
251+
- name: Setup Python ${{ env.CBCI_DEFAULT_PYTHON }}
252+
uses: actions/setup-python@v5
253+
with:
254+
python-version: ${{ env.CBCI_DEFAULT_PYTHON }}
255+
- name: Confirm Python version
256+
run: python -c "import sys; print(sys.version)"
257+
- name: Install dependencies
258+
run: |
259+
python -m pip install --upgrade pip setuptools wheel twine
260+
- name: Twine validate
261+
run: |
262+
ls -alh dist
263+
twine check dist/*
264+
- name: Publish package distributions to TestPyPI (dry run)
265+
if: ${{ inputs.is_dry_run }}
266+
run: |
267+
echo "Dry run set, not releasing to TestPyPI"
268+
- name: Publish package distributions to TestPyPI
269+
if: ${{ inputs.is_dry_run == false }}
270+
uses: pypa/gh-action-pypi-publish@release/v1
271+
with:
272+
repository-url: https://test.pypi.org/legacy/
273+
274+
validate-test-pypi:
275+
needs: publish-test-pypi
276+
if: >-
277+
${{ !cancelled()
278+
&& (needs.publish-test-pypi.result == 'success' && needs.publish-test-pypi.result != 'skipped')
279+
&& inputs.skip_validation == false }}
280+
uses: ./.github/workflows/verify_release.yml
281+
with:
282+
version: ${{ inputs.version }}
283+
sha: ${{ inputs.sha }}
284+
packaging_index: "TEST_PYPI"
285+
config: ${{ inputs.config && inputs.config || '' }}
286+
287+
publish-pypi:
288+
needs: [publish-config, run-tests, set-tests-run-id, validate-test-pypi, default-linux-runner]
289+
if: >-
290+
${{ !cancelled()
291+
&& fromJson(needs.publish-config.outputs.publish_config).publish_config.publish_pypi
292+
&& (needs.run-tests.result == 'success' || needs.run-tests.result == 'skipped')
293+
&& (needs.validate-test-pypi.result == 'success' || needs.validate-test-pypi.result == 'skipped') }}
294+
runs-on: ${{ needs.default-linux-runner.outputs.runner }}
295+
environment:
296+
name: publish
297+
url: https://pypi.org/p/couchbase-analytics
298+
permissions:
299+
# This permission is required for pypi's "trusted publisher" feature
300+
id-token: write
301+
steps:
302+
- uses: actions/download-artifact@v4
303+
with:
304+
pattern: pycbac-artifact-*
305+
path: dist
306+
merge-multiple: true
307+
run-id: ${{ needs.set-tests-run-id.outputs.tests_run_id }}
308+
github-token: ${{ github.token }}
309+
- name: Setup Python ${{ env.CBCI_DEFAULT_PYTHON }}
310+
uses: actions/setup-python@v5
311+
with:
312+
python-version: ${{ env.CBCI_DEFAULT_PYTHON }}
313+
- name: Confirm Python version
314+
run: python -c "import sys; print(sys.version)"
315+
- name: Install dependencies
316+
run: |
317+
python -m pip install --upgrade pip setuptools wheel twine
318+
- name: Twine validate
319+
run: |
320+
ls -alh dist
321+
twine check dist/*
322+
- name: Publish package distributions to PyPI (dry run)
323+
if: ${{ inputs.is_dry_run }}
324+
run: |
325+
echo "Dry run set, not releasing to PyPI"
326+
- name: Publish package distributions to PyPI
327+
if: ${{ inputs.is_dry_run == false }}
328+
uses: pypa/gh-action-pypi-publish@release/v1
329+
330+
validate-pypi:
331+
needs: publish-pypi
332+
if: >-
333+
${{ !cancelled()
334+
&& (needs.publish-pypi.result == 'success' && needs.publish-pypi.result != 'skipped')
335+
&& inputs.skip_validation == false }}
336+
uses: ./.github/workflows/verify_release.yml
337+
with:
338+
version: ${{ inputs.version }}
339+
sha: ${{ inputs.sha }}
340+
packaging_index: "PYPI"
341+
config: ${{ inputs.config && inputs.config || '' }}
342+
343+
publish-api-docs:
344+
needs: [upload-api-docs, validate-pypi, default-linux-runner]
345+
if: >-
346+
${{ !cancelled()
347+
&& fromJson(needs.publish-config.outputs.publish_config).publish_config.publish_api_docs
348+
&& (needs.validate-pypi.result == 'success' || needs.validate-pypi.result == 'skipped') }}
349+
environment: publish
350+
permissions:
351+
id-token: write
352+
contents: read
353+
runs-on: ${{ needs.default-linux-runner.outputs.runner }}
354+
steps:
355+
- name: Download API docs
356+
uses: actions/download-artifact@v4
357+
with:
358+
name: pycbac-api-docs
359+
- name: Setup AWS Credentials
360+
if: ${{ inputs.is_dry_run == false }}
361+
uses: aws-actions/configure-aws-credentials@v3
362+
with:
363+
role-to-assume: arn:aws:iam::786014483886:role/SDK_GHA
364+
aws-region: us-west-1
365+
- name: Upload API docs to S3
366+
if: ${{ inputs.is_dry_run == false }}
367+
run: |
368+
ls -alh ./html
369+
aws s3 cp ./html s3://docs.couchbase.com/sdk-api/analytics-python-client-$SDK_VERSION/ --recursive --acl public-read
370+
env:
371+
SDK_VERSION: ${{ inputs.version }}
372+
- name: Upload API docs to S3 (dry run)
373+
if: ${{ inputs.is_dry_run }}
374+
run: |
375+
ls -alh ./html
376+
echo "aws s3 cp ./html s3://docs.couchbase.com/sdk-api/analytics-python-client-$SDK_VERSION/ --recursive --acl public-read"
377+
env:
378+
SDK_VERSION: ${{ inputs.version }}

0 commit comments

Comments
 (0)