Add experimental Content-Addressable Storage (CAS) backups #2054
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build | |
| on: | |
| pull_request: | |
| branches: | |
| - "*" | |
| push: | |
| branches: | |
| - master | |
| jobs: | |
| build: | |
| name: Build | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| matrix: | |
| golang-version: | |
| - "1.26" | |
| steps: | |
| - name: Checkout project | |
| uses: actions/checkout@v6 | |
| - name: Setup golang | |
| id: setup-go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version: '^${{ matrix.golang-version }}' | |
| # we don't need musl after switch go 1.26 | |
| # - name: Setup musl | |
| # id: setup-musl | |
| # run: | | |
| # sudo apt-get install -y musl-tools musl-dev | |
| # - name: Cache golang | |
| # id: cache-golang | |
| # uses: actions/cache@v5 | |
| # with: | |
| # path: | | |
| # ~/go/pkg/mod | |
| # ~/.cache/go-build | |
| # key: clickhouse-backup-golang-${{ matrix.golang-version }}-${{ hashFiles('go.mod', '.github/workflows/*.yaml') }} | |
| - name: Install golang dependencies | |
| run: go mod download -x | |
| # if: | | |
| # steps.cache-golang.outputs.cache-hit != 'true' | |
| - name: Build clickhouse-backup binary | |
| id: make-race | |
| env: | |
| GOROOT: ${{ env.GOROOT_1_26_X64 }} | |
| run: | | |
| make build/linux/amd64/clickhouse-backup build/linux/arm64/clickhouse-backup | |
| make build/linux/amd64/clickhouse-backup-fips build/linux/arm64/clickhouse-backup-fips | |
| make build-race build-race-fips config test | |
| - name: Report unittest coverage | |
| uses: coverallsapp/github-action@v2 | |
| with: | |
| fail-on-error: false | |
| file: _coverage_/coverage.out | |
| parallel: true | |
| format: golang | |
| flag-name: unittest-${{ matrix.clickhouse }} | |
| - name: Extract GCS credentials | |
| id: secrets | |
| uses: ./.github/actions/extract-gcs-creds | |
| with: | |
| VAULT_PASSWORD: ${{ secrets.VAULT_PASSWORD }} | |
| - uses: actions/upload-artifact@v6 | |
| with: | |
| name: build-artifacts | |
| path: | | |
| build/linux/amd64/clickhouse-backup | |
| build/linux/arm64/clickhouse-backup | |
| build/linux/amd64/clickhouse-backup-fips | |
| build/linux/arm64/clickhouse-backup-fips | |
| if-no-files-found: error | |
| retention-days: 1 | |
| - uses: actions/upload-artifact@v6 | |
| with: | |
| name: build-test-artifacts | |
| path: | | |
| clickhouse-backup/clickhouse-backup-race | |
| clickhouse-backup/clickhouse-backup-race-fips | |
| if-no-files-found: error | |
| retention-days: 1 | |
| outputs: | |
| GCS_TESTS: ${{ steps.secrets.outputs.GCS_TESTS }} | |
| testflows: | |
| needs: build | |
| name: Testflows | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| golang-version: | |
| - "1.26" | |
| clickhouse: | |
| - '22.3' | |
| - '22.8' | |
| - '23.3' | |
| - '23.8' | |
| - '24.3' | |
| - '24.8' | |
| - '25.3' | |
| # todo wait when resolve https://github.com/ClickHouse/ClickHouse/issues/103232 | |
| - '25.8.21' | |
| - '26.3' | |
| steps: | |
| - name: Checkout project | |
| uses: actions/checkout@v6 | |
| - name: Setup golang | |
| id: setup-go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version: '^${{ matrix.golang-version }}' | |
| - uses: actions/download-artifact@v7 | |
| with: | |
| name: build-test-artifacts | |
| path: ./clickhouse-backup/ | |
| - name: Install python venv | |
| run: | | |
| set -x | |
| (dpkg -l | grep venv) || (apt-get update && apt-get install -y python3-venv) | |
| python3 -m venv ~/venv/qa | |
| - name: Cache python | |
| uses: actions/cache@v5 | |
| id: cache-python | |
| with: | |
| path: ~/venv/qa | |
| key: clickhouse-backup-python-${{ hashFiles('test/testflows/requirements.txt','.github/workflows/*.yaml') }} | |
| - name: Install python dependencies | |
| run: | | |
| set -x | |
| ~/venv/qa/bin/pip3 install -U -r ./test/testflows/requirements.txt | |
| if: | | |
| steps.cache-python.outputs.cache-hit != 'true' | |
| - name: Running TestFlows tests | |
| env: | |
| CLICKHOUSE_VERSION: ${{ matrix.clickhouse }} | |
| QA_AWS_ACCESS_KEY: ${{ secrets.QA_AWS_ACCESS_KEY }} | |
| QA_AWS_ENDPOINT: ${{ secrets.QA_AWS_ENDPOINT }} | |
| QA_AWS_SECRET_KEY: ${{ secrets.QA_AWS_SECRET_KEY }} | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_BUCKET: ${{ secrets.QA_AWS_BUCKET }} | |
| QA_GCS_CRED_JSON: ${{ secrets.QA_GCS_CRED_JSON }} | |
| RUN_PARALLEL: 3 | |
| DEBUG: 1 | |
| NO_COLORS: 1 | |
| GOROOT: ${{ env.GOROOT_1_26_X64 }} | |
| # don't change it to avoid not working CI/CD | |
| RUN_TESTS: "*" | |
| run: | | |
| set -xe | |
| export CLICKHOUSE_TESTS_DIR=$(pwd)/test/testflows/clickhouse_backup | |
| chmod +x $(pwd)/clickhouse-backup/clickhouse-backup* | |
| source ~/venv/qa/bin/activate | |
| ./test/testflows/run.sh | |
| - name: Report testflows coverage | |
| uses: coverallsapp/github-action@v2 | |
| with: | |
| fail-on-error: false | |
| base-path: ./ | |
| file: test/testflows/_coverage_/coverage.out | |
| parallel: true | |
| format: golang | |
| flag-name: testflows-${{ matrix.clickhouse }} | |
| # todo possible failures https://github.com/actions/upload-artifact/issues/270 | |
| - name: Upload testflows logs | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: testflows-logs-and-reports-${{ matrix.clickhouse }}-${{ github.run_id }} | |
| path: | | |
| test/testflows/*.log | |
| test/testflows/*.log.txt | |
| test/testflows/_logs_/**/*.log | |
| test/testflows/clickhouse_backup/_instances/**/*.log | |
| test/testflows/*.html | |
| retention-days: 7 | |
| test: | |
| needs: build | |
| name: Test | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| golang-version: | |
| - "1.26" | |
| clickhouse: | |
| - '1.1.54394' | |
| - '19.17' | |
| - '20.3' | |
| - '20.8' | |
| - '21.3' | |
| - '21.8' | |
| - '22.3' | |
| - '22.8' | |
| - '23.3' | |
| - '23.8' | |
| - '24.3' | |
| - '24.8' | |
| - '25.3' | |
| # todo wait when resolve https://github.com/ClickHouse/ClickHouse/issues/103232 | |
| - '25.8.21' | |
| - '26.3' | |
| steps: | |
| - name: Checkout project | |
| uses: actions/checkout@v6 | |
| - name: Setup golang | |
| id: setup-go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version: '^${{ matrix.golang-version }}' | |
| # - name: Cache golang | |
| # id: cache-golang | |
| # uses: actions/cache@v5 | |
| # with: | |
| # path: | | |
| # ~/go/pkg/mod | |
| # ~/.cache/go-build | |
| # key: clickhouse-backup-golang-${{ matrix.golang-version }}-${{ hashFiles('go.mod', '.github/workflows/*.yaml') }} | |
| - uses: actions/download-artifact@v7 | |
| with: | |
| name: build-test-artifacts | |
| path: ./clickhouse-backup/ | |
| - name: Extract GCS credentials | |
| id: secrets | |
| uses: ./.github/actions/extract-gcs-creds | |
| with: | |
| VAULT_PASSWORD: ${{ secrets.VAULT_PASSWORD }} | |
| - name: Running integration tests | |
| env: | |
| RUN_PARALLEL: 4 | |
| GOROOT: ${{ env.GOROOT_1_26_X64 }} | |
| CLICKHOUSE_VERSION: ${{ matrix.clickhouse }} | |
| # LOG_LEVEL: "info" | |
| # TEST_LOG_LEVEL: "debug" | |
| # options for advanced debug CI/CD | |
| # RUN_TESTS: "^TestSkipDisk$" | |
| # CLICKHOUSE_DEBUG: "true" | |
| # GCS_DEBUG: "true" | |
| # SFTP_DEBUG: "true" | |
| # AZBLOB_DEBUG: "true" | |
| # FTP_DEBUG: "true" | |
| # S3_DEBUG: "true" | |
| CGO_ENABLED: 1 | |
| GCS_TESTS: ${{ needs.build.outputs.GCS_TESTS }} | |
| KEEPER_TLS_ENABLED: 1 | |
| RUN_ADVANCED_TESTS: 1 | |
| AZURE_TESTS: 1 | |
| # need for FIPS | |
| QA_AWS_ACCESS_KEY: ${{ secrets.QA_AWS_ACCESS_KEY }} | |
| QA_AWS_SECRET_KEY: ${{ secrets.QA_AWS_SECRET_KEY }} | |
| QA_AWS_BUCKET: ${{ secrets.QA_AWS_BUCKET }} | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| # need for GCP over S3 | |
| QA_GCS_OVER_S3_ACCESS_KEY: ${{ secrets.QA_GCS_OVER_S3_ACCESS_KEY }} | |
| QA_GCS_OVER_S3_SECRET_KEY: ${{ secrets.QA_GCS_OVER_S3_SECRET_KEY }} | |
| QA_GCS_OVER_S3_BUCKET: ${{ secrets.QA_GCS_OVER_S3_BUCKET }} | |
| run: | | |
| set -xe | |
| echo "CLICKHOUSE_VERSION=${CLICKHOUSE_VERSION}" | |
| echo "GCS_TESTS=${GCS_TESTS}" | |
| GCS_ENCRYPTION_KEY=$(openssl rand -base64 32) | |
| export GCS_ENCRYPTION_KEY | |
| chmod +x $(pwd)/clickhouse-backup/clickhouse-backup* | |
| if [[ "${CLICKHOUSE_VERSION}" =~ ^2[1-9]+ || "${CLICKHOUSE_VERSION}" == "head" ]]; then | |
| export CLICKHOUSE_IMAGE=clickhouse/clickhouse-server | |
| else | |
| export CLICKHOUSE_IMAGE=yandex/clickhouse-server | |
| fi | |
| export CUR_DIR="$(pwd)/test/integration" | |
| export CLICKHOUSE_BACKUP_BIN="$(pwd)/clickhouse-backup/clickhouse-backup-race" | |
| set +e | |
| go test -count=1 -parallel "${RUN_PARALLEL}" -race -timeout "${TEST_TIMEOUT:-120m}" -failfast -tags=integration -shuffle=$(date +%Y%m%d) -run "${RUN_TESTS:-.+}" -v "./test/integration/..." | |
| TEST_FAILED=$? | |
| set -e | |
| if [[ "0" != "${TEST_FAILED}" ]]; then | |
| exit 1 | |
| fi | |
| - name: Format integration coverage | |
| env: | |
| GOROOT: ${{ env.GOROOT_1_26_X64 }} | |
| run: | | |
| sudo chmod -Rv a+rw test/integration/_coverage_/ | |
| ls -la test/integration/_coverage_ | |
| go tool covdata textfmt -i test/integration/_coverage_/ -o test/integration/_coverage_/coverage.out | |
| - name: Report integration coverage | |
| uses: coverallsapp/github-action@v2 | |
| with: | |
| fail-on-error: false | |
| base-path: ./ | |
| file: test/integration/_coverage_/coverage.out | |
| parallel: true | |
| format: golang | |
| flag-name: integration-${{ matrix.clickhouse }} | |
| coverage: | |
| needs: | |
| - test | |
| - testflows | |
| name: coverage | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Coveralls Finished | |
| uses: coverallsapp/github-action@v2 | |
| with: | |
| fail-on-error: false | |
| base-path: ./ | |
| parallel-finished: true | |
| docker: | |
| needs: | |
| - test | |
| - testflows | |
| name: Docker | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Checkout project | |
| uses: actions/checkout@v6 | |
| - uses: actions/download-artifact@v7 | |
| with: | |
| name: build-artifacts | |
| path: ./build/linux/ | |
| - name: Extract DOCKER_TAG version | |
| id: docker_tag | |
| run: | | |
| DOCKER_TAG=${GITHUB_REF##*/} | |
| export DOCKER_TAG=${DOCKER_TAG##*\\} | |
| echo "docker_tag=${DOCKER_TAG:-dev}" >> $GITHUB_OUTPUT | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Building docker image | |
| env: | |
| DOCKER_REPO: ${{ secrets.DOCKER_REPO }} | |
| DOCKER_IMAGE: ${{ secrets.DOCKER_IMAGE }} | |
| DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} | |
| DOCKER_USER: ${{ secrets.DOCKER_USER }} | |
| DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }} | |
| DOCKER_TAG: ${{ steps.docker_tag.outputs.docker_tag }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| if [[ "${DOCKER_TOKEN}" != "" ]]; then | |
| export DOCKER_REGISTRY=${DOCKER_REGISTRY:-docker.io} | |
| echo ${DOCKER_TOKEN} | docker login -u ${DOCKER_USER} --password-stdin ${DOCKER_REGISTRY} | |
| export VERSION=$(git describe --always --tags --abbrev=0 2>/dev/null | sed 's/^v//;s/\([^-]*-g\)/c\1/;s/-/./g') | |
| docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}-full:${DOCKER_TAG} --build-arg VERSION=$VERSION --target=image_full --pull --push . | |
| docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}:${DOCKER_TAG} --build-arg VERSION=$VERSION --target=image_short --pull --push . | |
| docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}:${DOCKER_TAG}-fips --build-arg VERSION=$VERSION --target=image_fips --pull --push . | |
| # Publish to GitHub Container Registry | |
| echo ${GITHUB_TOKEN} | docker login -u $GITHUB_ACTOR --password-stdin ghcr.io | |
| docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=ghcr.io/${DOCKER_REPO}/${DOCKER_IMAGE}:${DOCKER_TAG} --build-arg VERSION=$VERSION --target=image_short --pull --push . | |
| docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=ghcr.io/${DOCKER_REPO}/${DOCKER_IMAGE}-full:${DOCKER_TAG} --build-arg VERSION=$VERSION --target=image_full --pull --push . | |
| docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=ghcr.io/${DOCKER_REPO}/${DOCKER_IMAGE}:${DOCKER_TAG}-fips --build-arg VERSION=$VERSION --target=image_fips --pull --push . | |
| fi | |