From 513660bc87ec3c872e5a43e8692afe82258246d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Garillot?= <4142+huitseeker@users.noreply.github.com> Date: Thu, 30 Apr 2026 19:56:59 -0400 Subject: [PATCH 1/3] ci: add shear, workspace inheritance, and zizmor checks, and move heavy GitHub Actions jobs to WarpBuild (#2013) --- .github/actionlint.yaml | 4 + .github/actions/debian/action.yml | 133 +++++++++++--------- .github/workflows/book.yml | 26 ++-- .github/workflows/build-docker.yml | 4 +- .github/workflows/build-docs.yml | 6 +- .github/workflows/changelog.yml | 5 +- .github/workflows/ci.yml | 107 +++++++++++----- .github/workflows/cleanup-workflows.yml | 39 ++++-- .github/workflows/nightly.yml | 41 ++++-- .github/workflows/publish-crates.yml | 12 +- .github/workflows/publish-debian-all.yml | 18 ++- .github/workflows/publish-debian.yml | 8 +- .github/workflows/publish-docker.yml | 11 +- .github/workflows/trigger-deploy-docs.yml | 1 + .github/workflows/zizmor.yml | 39 ++++++ Cargo.lock | 17 --- Cargo.toml | 14 ++- Makefile | 12 +- bin/genesis/Cargo.toml | 1 - bin/network-monitor/Cargo.toml | 4 +- bin/node/Dockerfile | 4 +- bin/remote-prover/Cargo.toml | 29 ++--- bin/remote-prover/src/server/mod.rs | 5 +- bin/stress-test/Cargo.toml | 4 +- crates/block-producer/Cargo.toml | 28 ++--- crates/grpc-error-macro/Cargo.toml | 6 +- crates/large-smt-backend-rocksdb/Cargo.toml | 2 +- crates/ntx-builder/Cargo.toml | 19 ++- crates/proto/Cargo.toml | 7 +- crates/remote-prover-client/Cargo.toml | 12 +- crates/rocksdb-cxx-linkage-fix/Cargo.toml | 4 +- crates/rpc/Cargo.toml | 7 +- crates/rpc/src/tests.rs | 23 +++- crates/store/Cargo.toml | 17 +-- crates/store/src/db/query_plan/mod.rs | 73 ----------- crates/store/src/db/query_plan/renderer.rs | 49 -------- crates/test-macro/Cargo.toml | 6 +- crates/utils/Cargo.toml | 5 +- crates/validator/Cargo.toml | 3 + proto/Cargo.toml | 6 +- zizmor.yml | 4 + 41 files changed, 435 insertions(+), 380 deletions(-) create mode 100644 .github/actionlint.yaml create mode 100644 .github/workflows/zizmor.yml delete mode 100644 crates/store/src/db/query_plan/mod.rs delete mode 100644 crates/store/src/db/query_plan/renderer.rs create mode 100644 zizmor.yml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000000..e69e33a7ac --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,4 @@ +self-hosted-runner: + labels: + - Linux-ARM64-Runner + - warp-ubuntu-latest-x64-8x diff --git a/.github/actions/debian/action.yml b/.github/actions/debian/action.yml index 99d65d069c..2b52257f9e 100644 --- a/.github/actions/debian/action.yml +++ b/.github/actions/debian/action.yml @@ -10,46 +10,24 @@ inputs: arch: required: true description: Machine architecture to build packages for. - type: choice - options: - - amd64 - - arm64 crate: required: true description: Name of binary crate being packaged. - type: choice - options: - - miden-node - - miden-remote-prover crate_dir: required: true description: Name of crate being packaged. - type: choice - options: - - miden-node - - miden-remote-prover package: required: true description: The Debian package name. - type: choice - options: - - miden-node - - miden-prover - - miden-prover-proxy packaging_dir: required: true description: Name of packaging directory. - type: choice - options: - - node - - prover - - prover-proxy runs: using: "composite" steps: - name: Rust cache - uses: Swatinem/rust-cache@v2 + uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: # Only update the cache on push onto the next branch. This strikes a nice balance between # cache hits and cache evictions (github has a 10GB cache limit). @@ -63,47 +41,61 @@ runs: - name: Identify target git SHA id: git-sha shell: bash + env: + INPUT_GITREF: ${{ inputs.gitref }} run: | - if git show-ref -q --verify "refs/remotes/origin/${{ inputs.gitref }}" 2>/dev/null; then - echo "sha=$(git show-ref --hash --verify 'refs/remotes/origin/${{ inputs.gitref }}')" >> $GITHUB_OUTPUT - elif git show-ref -q --verify "refs/tags/${{ inputs.gitref }}" 2>/dev/null; then - echo "sha=$(git show-ref --hash --verify 'refs/tags/${{ inputs.gitref }}')" >> $GITHUB_OUTPUT - elif git rev-parse --verify "${{ inputs.gitref }}^{commit}" >/dev/null 2>&1; then - echo "sha=$(git rev-parse --verify '${{ inputs.gitref }}^{commit}')" >> $GITHUB_OUTPUT + if git show-ref -q --verify "refs/remotes/origin/${INPUT_GITREF}" 2>/dev/null; then + ref="refs/remotes/origin/${INPUT_GITREF}" + elif git show-ref -q --verify "refs/tags/${INPUT_GITREF}" 2>/dev/null; then + ref="refs/tags/${INPUT_GITREF}" + elif git rev-parse --verify "${INPUT_GITREF}^{commit}" >/dev/null 2>&1; then + ref="${INPUT_GITREF}" else echo "::error::Unknown git reference type" exit 1 fi + sha=$(git rev-parse --verify "${ref}^{commit}") + echo "sha=${sha}" >> "$GITHUB_OUTPUT" + - name: Create package directories shell: bash + env: + INPUT_PACKAGE: ${{ inputs.package }} run: | - pkg=${{ inputs.package }} + pkg="${INPUT_PACKAGE}" mkdir -p \ - packaging/deb/$pkg/DEBIAN \ - packaging/deb/$pkg/usr/bin \ - packaging/deb/$pkg/lib/systemd/system \ - packaging/deb/$pkg/opt/$pkg \ - done + "packaging/deb/${pkg}/DEBIAN" \ + "packaging/deb/${pkg}/usr/bin" \ + "packaging/deb/${pkg}/lib/systemd/system" \ + "packaging/deb/${pkg}/opt/${pkg}" - name: Copy package install scripts shell: bash + env: + INPUT_CRATE_DIR: ${{ inputs.crate_dir }} + INPUT_PACKAGE: ${{ inputs.package }} + INPUT_PACKAGING_DIR: ${{ inputs.packaging_dir }} + TARGET_SHA: ${{ steps.git-sha.outputs.sha }} run: | - pkg=${{ inputs.package }} - pkg_dir=${{ inputs.packaging_dir }} - crate=${{ inputs.crate_dir }} - git show ${{ steps.git-sha.outputs.sha }}:packaging/$pkg_dir/postinst > packaging/deb/$pkg/DEBIAN/postinst - git show ${{ steps.git-sha.outputs.sha }}:packaging/$pkg_dir/postrm > packaging/deb/$pkg/DEBIAN/postrm - for service_file in $(ls packaging/$pkg_dir/*.service | sed "s/.*miden/miden/g"); do - svc=$(echo $service_file | sed "s/.service//g") - git show ${{ steps.git-sha.outputs.sha }}:packaging/$pkg_dir/$service_file > packaging/deb/$pkg/lib/systemd/system/$service_file - git show ${{ steps.git-sha.outputs.sha }}:bin/$crate/.env > packaging/deb/$pkg/lib/systemd/system/$svc.env - done - chmod 0775 packaging/deb/$pkg/DEBIAN/postinst - chmod 0775 packaging/deb/$pkg/DEBIAN/postrm + pkg="${INPUT_PACKAGE}" + pkg_dir="${INPUT_PACKAGING_DIR}" + crate="${INPUT_CRATE_DIR}" + git show "${TARGET_SHA}:packaging/${pkg_dir}/postinst" > "packaging/deb/${pkg}/DEBIAN/postinst" + git show "${TARGET_SHA}:packaging/${pkg_dir}/postrm" > "packaging/deb/${pkg}/DEBIAN/postrm" + while IFS= read -r service_file; do + service_file="${service_file##*/}" + svc="${service_file%.service}" + git show "${TARGET_SHA}:packaging/${pkg_dir}/${service_file}" > "packaging/deb/${pkg}/lib/systemd/system/${service_file}" + git show "${TARGET_SHA}:bin/${crate}/.env" > "packaging/deb/${pkg}/lib/systemd/system/${svc}.env" + done < <(find "packaging/${pkg_dir}" -maxdepth 1 -name '*.service' -print) + chmod 0775 "packaging/deb/${pkg}/DEBIAN/postinst" + chmod 0775 "packaging/deb/${pkg}/DEBIAN/postrm" - name: Create control files shell: bash + env: + INPUT_PACKAGE: ${{ inputs.package }} run: | # Map the architecture to the format required by Debian. # i.e. arm64 and amd64 instead of aarch64 and x86_64. @@ -111,8 +103,8 @@ runs: # Control file's version field must be x.y.z format so strip the rest. version=$(git describe --tags --abbrev=0 | sed 's/[^0-9.]//g' ) - pkg=${{ inputs.package }} - cat > packaging/deb/$pkg/DEBIAN/control << EOF + pkg="${INPUT_PACKAGE}" + cat > "packaging/deb/${pkg}/DEBIAN/control" << EOF Package: $pkg Version: $version Section: base @@ -128,47 +120,66 @@ runs: - name: Build binaries shell: bash env: - repo-url: ${{ github.server_url }}/${{ github.repository }} + INPUT_CRATE: ${{ inputs.crate }} + REPO_URL: ${{ github.server_url }}/${{ github.repository }} + TARGET_SHA: ${{ steps.git-sha.outputs.sha }} run: | - cargo install ${{ inputs.crate }} --root . --locked --git ${{ env.repo-url }} --rev ${{ steps.git-sha.outputs.sha }} + cargo install "${INPUT_CRATE}" --root . --locked --git "${REPO_URL}" --rev "${TARGET_SHA}" - name: Copy binary files shell: bash + env: + INPUT_CRATE: ${{ inputs.crate }} + INPUT_PACKAGE: ${{ inputs.package }} run: | - pkg=${{ inputs.package }} - bin=${{ inputs.crate }} - cp -p ./bin/$bin packaging/deb/$pkg/usr/bin/ + pkg="${INPUT_PACKAGE}" + bin="${INPUT_CRATE}" + cp -p "./bin/${bin}" "packaging/deb/${pkg}/usr/bin/" - name: Build packages shell: bash + env: + INPUT_PACKAGE: ${{ inputs.package }} run: | - dpkg-deb --build --root-owner-group packaging/deb/${{ inputs.package }} + dpkg-deb --build --root-owner-group "packaging/deb/${INPUT_PACKAGE}" # Save the .deb files, delete the rest. mv packaging/deb/*.deb . rm -rf packaging - name: Package names + id: package-names shell: bash + env: + INPUT_ARCH: ${{ inputs.arch }} + INPUT_GITREF: ${{ inputs.gitref }} + INPUT_PACKAGE: ${{ inputs.package }} run: | - echo "package=${{ inputs.package }}-${{ inputs.gitref }}-${{ inputs.arch }}.deb" >> $GITHUB_ENV + echo "package=${INPUT_PACKAGE}-${INPUT_GITREF}-${INPUT_ARCH}.deb" >> "$GITHUB_OUTPUT" - name: Rename package files shell: bash + env: + INPUT_PACKAGE: ${{ inputs.package }} + PACKAGE_FILE: ${{ steps.package-names.outputs.package }} run: | - mv ${{ inputs.package}}.deb ${{ env.package }} + mv "${INPUT_PACKAGE}.deb" "${PACKAGE_FILE}" - name: shasum packages shell: bash + env: + PACKAGE_FILE: ${{ steps.package-names.outputs.package }} run: | - sha256sum ${{ env.package }} > ${{ env.package }}.checksum + sha256sum "${PACKAGE_FILE}" > "${PACKAGE_FILE}.checksum" - name: Publish packages shell: bash env: GH_TOKEN: ${{ inputs.github_token }} + INPUT_GITREF: ${{ inputs.gitref }} + PACKAGE_FILE: ${{ steps.package-names.outputs.package }} run: | - gh release upload ${{ inputs.gitref }} \ - ${{ env.package }} \ - ${{ env.package }}.checksum \ + gh release upload "${INPUT_GITREF}" \ + "${PACKAGE_FILE}" \ + "${PACKAGE_FILE}.checksum" \ --clobber diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index 10e48c5f1d..de48a3fd49 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -11,16 +11,13 @@ name: book on: workflow_dispatch: pull_request: - path: ["docs/internal/**"] + paths: ["docs/internal/**"] push: branches: [next] - path: ["docs/internal/**"] + paths: ["docs/internal/**"] -# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages permissions: contents: read - pages: write - id-token: write # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. @@ -34,13 +31,18 @@ jobs: # The documentation is uploaded as a github artifact IFF it is required for deployment i.e. on push into next. build: name: Build documentation + permissions: + contents: read + pages: write runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@main + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false # Installation from source takes a fair while, so we install the binaries directly instead. - name: Install mdbook and plugins - uses: taiki-e/install-action@v2 + uses: taiki-e/install-action@055f5df8c3f65ea01cd41e9dc855becd88953486 # v2.75.18 with: tool: mdbook@0.4, mdbook-linkcheck@0.7, mdbook-alerts@0.8, mdbook-katex@0.9 @@ -51,11 +53,11 @@ jobs: - name: Setup Pages if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/next' }} id: pages - uses: actions/configure-pages@v5 + uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5 - name: Upload book artifact if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/next' }} - uses: actions/upload-pages-artifact@v3 + uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3 with: # We specify multiple [output] sections in our book.toml which causes mdbook to create separate folders for each. This moves the generated `html` into its own `html` subdirectory. path: ./docs/internal/book/html @@ -63,6 +65,10 @@ jobs: # Deployment job only runs on push to next. deploy: name: Deploy documentation + permissions: + contents: read + id-token: write + pages: write environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} @@ -72,4 +78,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v4 + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4 diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index b259c23fd9..e2e2e15e84 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -15,10 +15,10 @@ jobs: runs-on: Linux-ARM64-Runner steps: - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - name: Build and push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: push: false file: ./bin/node/Dockerfile diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 56cc7795f4..105f1bc0e9 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -28,10 +28,12 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false - name: Setup Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: "20" cache: "npm" diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index be2667efd0..9a254f6a78 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -15,12 +15,13 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout code - uses: actions/checkout@main + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: fetch-depth: 0 + persist-credentials: false - name: Check for changes in changelog env: BASE_REF: ${{ github.event.pull_request.base.ref }} NO_CHANGELOG_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'no changelog') }} - run: ./scripts/check-changelog.sh "${{ inputs.changelog }}" + run: ./scripts/check-changelog.sh shell: bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 674d4ddd49..9cc7d2364b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,16 +66,16 @@ jobs: # Normal cargo build that saves either the persistent trunk cache or a # single per-run cache for downstream jobs to restore immediately. build: - runs-on: ubuntu-24.04 + runs-on: warp-ubuntu-latest-x64-8x steps: - - uses: actions/checkout@v6 - - name: Cleanup large tools for build space - uses: ./.github/actions/cleanup-runner + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - uses: ./.github/actions/install-rocksdb - uses: ./.github/actions/install-protobuf-compiler - name: Rustup run: rustup update --no-self-update - - uses: Swatinem/rust-cache@v2 + - uses: WarpBuilds/rust-cache@9d0cc3090d9c87de74ea67617b246e978735b1a1 # v2.9.1 with: shared-key: ${{ env.RUST_CACHE_SUFFIX }} prefix-key: ${{ env.RUST_CACHE_PREFIX }} @@ -119,17 +119,19 @@ jobs: exit 3 fi done - echo "Static linkage check passed for all of ${bin_targets[@]}" + printf "Static linkage check passed for all of %s\n" "${bin_targets[*]}" clippy: name: lint - clippy - runs-on: ubuntu-24.04 + runs-on: warp-ubuntu-latest-x64-8x needs: [build] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - name: Rustup run: rustup update --no-self-update - - uses: Swatinem/rust-cache@v2 + - uses: WarpBuilds/rust-cache@9d0cc3090d9c87de74ea67617b246e978735b1a1 # v2.9.1 with: shared-key: ${{ env.RUST_CACHE_SUFFIX }} prefix-key: ${{ env.RUST_CACHE_PREFIX }} @@ -139,17 +141,19 @@ jobs: run: cargo clippy --locked --all-targets --all-features --workspace -- -D warnings tests: - runs-on: ubuntu-24.04 + runs-on: warp-ubuntu-latest-x64-8x needs: [build] timeout-minutes: 30 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - name: Rustup run: rustup update --no-self-update - - uses: taiki-e/install-action@v2 + - uses: taiki-e/install-action@055f5df8c3f65ea01cd41e9dc855becd88953486 # v2.75.18 with: tool: nextest@0.9.122 - - uses: Swatinem/rust-cache@v2 + - uses: WarpBuilds/rust-cache@9d0cc3090d9c87de74ea67617b246e978735b1a1 # v2.9.1 with: shared-key: ${{ env.RUST_CACHE_SUFFIX }} prefix-key: ${{ env.RUST_CACHE_PREFIX }} @@ -164,14 +168,14 @@ jobs: doc: needs: [build] - runs-on: ubuntu-24.04 + runs-on: warp-ubuntu-latest-x64-8x steps: - - uses: actions/checkout@v6 - - name: Cleanup large tools for build space - uses: ./.github/actions/cleanup-runner + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - name: Rustup run: rustup update --no-self-update - - uses: Swatinem/rust-cache@v2 + - uses: WarpBuilds/rust-cache@9d0cc3090d9c87de74ea67617b246e978735b1a1 # v2.9.1 with: shared-key: ${{ env.RUST_CACHE_SUFFIX }} prefix-key: ${{ env.RUST_CACHE_PREFIX }} @@ -184,21 +188,23 @@ jobs: stress-test: name: stress test needs: [build] - runs-on: ubuntu-24.04 + runs-on: warp-ubuntu-latest-x64-8x timeout-minutes: 20 env: DATA_DIR: /tmp/store steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - name: Rustup run: rustup update --no-self-update - - uses: Swatinem/rust-cache@v2 + - uses: WarpBuilds/rust-cache@9d0cc3090d9c87de74ea67617b246e978735b1a1 # v2.9.1 with: shared-key: ${{ env.RUST_CACHE_SUFFIX }} prefix-key: ${{ env.RUST_CACHE_PREFIX }} cache-workspace-crates: true save-if: false - - uses: taiki-e/install-action@v2 + - uses: taiki-e/install-action@055f5df8c3f65ea01cd41e9dc855becd88953486 # v2.75.18 with: tool: nextest@0.9.122 - name: Build @@ -271,9 +277,11 @@ jobs: # quick so we don't need a separate cache here. client-wasm: name: wasm targets - runs-on: ubuntu-24.04 + runs-on: warp-ubuntu-latest-x64-8x steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - name: Rustup run: rustup update --no-self-update - name: cargo build @@ -296,8 +304,10 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - - uses: actions/checkout@v6 - - uses: taiki-e/install-action@v2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + - uses: taiki-e/install-action@055f5df8c3f65ea01cd41e9dc855becd88953486 # v2.75.18 with: tool: typos@1.42.0 - run: make typos-check @@ -306,7 +316,9 @@ jobs: name: lint - rustfmt runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - name: Rustup +nightly run: | rustup update --no-self-update nightly @@ -319,8 +331,10 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - - uses: actions/checkout@v6 - - uses: taiki-e/install-action@v2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + - uses: taiki-e/install-action@055f5df8c3f65ea01cd41e9dc855becd88953486 # v2.75.18 with: tool: taplo-cli@0.10.0 - run: make toml-check @@ -330,16 +344,41 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - - uses: actions/checkout@v6 - - uses: taiki-e/install-action@v2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + - uses: taiki-e/install-action@055f5df8c3f65ea01cd41e9dc855becd88953486 # v2.75.18 with: tool: cargo-workspace-lints@0.1.4 - run: make workspace-check + workspace-inheritance: + name: lint - workspace inheritance + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + - name: Install cargo-binstall + uses: taiki-e/install-action@055f5df8c3f65ea01cd41e9dc855becd88953486 # v2.75.18 + with: + tool: cargo-binstall + - name: Install workspace inheritance checker + run: cargo binstall --no-confirm cargo-workspace-inheritance-check@1.2.0 + - name: Check workspace inheritance + run: cargo workspace-inheritance-check --promotion-threshold 2 --promotion-failure + unused_deps: name: lint - unused deps runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 - - name: machete - uses: bnjbvr/cargo-machete@main + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + - name: Install cargo-shear + uses: taiki-e/install-action@055f5df8c3f65ea01cd41e9dc855becd88953486 # v2.75.18 + with: + tool: cargo-shear@1.11.2 + - name: Check for unused dependencies + run: make shear diff --git a/.github/workflows/cleanup-workflows.yml b/.github/workflows/cleanup-workflows.yml index a7a6d2b428..eff2aa99e2 100644 --- a/.github/workflows/cleanup-workflows.yml +++ b/.github/workflows/cleanup-workflows.yml @@ -27,16 +27,17 @@ jobs: steps: - name: Checkout repo - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: fetch-depth: 0 + persist-credentials: false - name: Workflows on main id: main run: | git fetch origin main WORKFLOWS=$(git ls-tree -r origin/main --name-only | grep '^.github/workflows/') - printf "%s\n" $WORKFLOWS + printf "%s\n" "$WORKFLOWS" { echo "workflows<> "$GITHUB_OUTPUT" - name: Show package info + env: + MANIFEST_PATH: ${{ steps.pkg.outputs.manifest_path }} + PACKAGE: ${{ matrix.package }} run: | - echo "Package: ${{ matrix.package }}" - echo "Manifest path: ${{ steps.pkg.outputs.manifest_path }}" - cargo msrv show --manifest-path "${{ steps.pkg.outputs.manifest_path }}" + echo "Package: ${PACKAGE}" + echo "Manifest path: ${MANIFEST_PATH}" + cargo msrv show --manifest-path "${MANIFEST_PATH}" - name: Check MSRV + env: + MANIFEST_PATH: ${{ steps.pkg.outputs.manifest_path }} run: | - cargo msrv verify --manifest-path "${{ steps.pkg.outputs.manifest_path }}" + cargo msrv verify --manifest-path "${MANIFEST_PATH}" diff --git a/.github/workflows/publish-crates.yml b/.github/workflows/publish-crates.yml index be4f62ded6..813564e8b8 100644 --- a/.github/workflows/publish-crates.yml +++ b/.github/workflows/publish-crates.yml @@ -12,17 +12,21 @@ jobs: name: Cargo publish release runs-on: Linux-ARM64-Runner if: ${{ github.repository_owner == '0xMiden' }} + environment: publish-crates steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: fetch-depth: 0 + persist-credentials: false ref: ${{ github.event.release.tag_name }} - uses: ./.github/actions/install-rocksdb - uses: ./.github/actions/install-protobuf-compiler - name: Log release info + env: + RELEASE_TAG: ${{ github.event.release.tag_name }} run: | - echo "Publishing release ${{ github.event.release.tag_name }}" + echo "Publishing release ${RELEASE_TAG}" echo "Commit: $(git rev-parse HEAD)" - name: Cleanup large tools for build space uses: ./.github/actions/cleanup-runner @@ -30,8 +34,8 @@ jobs: run: sudo apt-get update && sudo apt-get install -y jq - name: Update Rust toolchain run: rustup update --no-self-update - - uses: Swatinem/rust-cache@v2 - - uses: taiki-e/install-action@v2 + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + - uses: taiki-e/install-action@055f5df8c3f65ea01cd41e9dc855becd88953486 # v2.75.18 with: tool: cargo-binstall@1.16.6 # - name: Install cargo-msrv diff --git a/.github/workflows/publish-debian-all.yml b/.github/workflows/publish-debian-all.yml index 1b5ccdb742..201412e39b 100644 --- a/.github/workflows/publish-debian-all.yml +++ b/.github/workflows/publish-debian-all.yml @@ -15,12 +15,13 @@ env: version: ${{ inputs.version || github.ref_name }} permissions: - id-token: write - contents: write + contents: read jobs: publish-node: name: Publish Node ${{ matrix.arch }} Debian + permissions: + contents: write strategy: matrix: arch: [amd64, arm64] @@ -28,9 +29,10 @@ jobs: labels: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} steps: - name: Checkout repo - uses: actions/checkout@main + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: fetch-depth: 0 + persist-credentials: false - uses: ./.github/actions/install-rocksdb - uses: ./.github/actions/install-protobuf-compiler - name: Build and Publish Node @@ -46,6 +48,8 @@ jobs: publish-prover: name: Publish Prover ${{ matrix.arch }} Debian + permissions: + contents: write strategy: matrix: arch: [amd64, arm64] @@ -53,9 +57,10 @@ jobs: labels: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} steps: - name: Checkout repo - uses: actions/checkout@main + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: fetch-depth: 0 + persist-credentials: false - name: Build and Publish Prover uses: ./.github/actions/debian with: @@ -69,6 +74,8 @@ jobs: publish-network-monitor: name: Publish Network Monitor ${{ matrix.arch }} Debian + permissions: + contents: write strategy: matrix: arch: [amd64, arm64] @@ -76,9 +83,10 @@ jobs: labels: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} steps: - name: Checkout repo - uses: actions/checkout@main + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: fetch-depth: 0 + persist-credentials: false - name: Build and Publish Network Monitor uses: ./.github/actions/debian with: diff --git a/.github/workflows/publish-debian.yml b/.github/workflows/publish-debian.yml index be01b9d1e7..a9550ff33b 100644 --- a/.github/workflows/publish-debian.yml +++ b/.github/workflows/publish-debian.yml @@ -43,12 +43,13 @@ on: type: string permissions: - id-token: write - contents: write + contents: read jobs: publish: name: Publish ${{ inputs.package }} ${{ matrix.arch }} Debian + permissions: + contents: write strategy: matrix: arch: [amd64, arm64] @@ -56,9 +57,10 @@ jobs: labels: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} steps: - name: Checkout repo - uses: actions/checkout@main + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: fetch-depth: 0 + persist-credentials: false - uses: ./.github/actions/install-rocksdb - uses: ./.github/actions/install-protobuf-compiler diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index 990ef1d94c..c23d5a5a1d 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -25,16 +25,18 @@ jobs: publish: runs-on: labels: "ubuntu-24.04" + environment: publish-docker strategy: matrix: component: [node] name: Publish ${{ matrix.component }} ${{ inputs.version }} steps: - name: Checkout repo - uses: actions/checkout@main + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: ref: ${{ env.version }} fetch-depth: 0 + persist-credentials: false - name: Log in to the Container registry uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 @@ -44,23 +46,22 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4 with: aws-region: ${{ secrets.AWS_REGION }} role-to-assume: ${{ secrets.AWS_ROLE }} role-session-name: GithubActionsSession - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 with: cache-binary: true - name: Build and push Docker image id: push - uses: docker/build-push-action@v5 + uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5 with: push: true - labels: ${{ steps.meta.outputs.labels }} file: ./bin/${{ matrix.component }}/Dockerfile tags: ${{ env.registry }}/0xmiden/miden-${{ matrix.component }}:${{ env.version }} cache-from: type=s3,region=${{ secrets.AWS_REGION }},bucket=${{ secrets.AWS_CACHE_BUCKET }},name=miden-${{ matrix.component }} diff --git a/.github/workflows/trigger-deploy-docs.yml b/.github/workflows/trigger-deploy-docs.yml index ca54a442d2..568e6541fb 100644 --- a/.github/workflows/trigger-deploy-docs.yml +++ b/.github/workflows/trigger-deploy-docs.yml @@ -9,6 +9,7 @@ on: jobs: notify: runs-on: ubuntu-24.04 + environment: deploy-docs permissions: contents: read diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml new file mode 100644 index 0000000000..9d6ecbc162 --- /dev/null +++ b/.github/workflows/zizmor.yml @@ -0,0 +1,39 @@ +name: GitHub Actions Security Analysis with zizmor + +on: + push: + branches: [main, next] + paths: + - ".github/workflows/**" + - ".github/actions/**" + - "zizmor.yml" + pull_request: + types: [opened, reopened, synchronize] + paths: + - ".github/workflows/**" + - ".github/actions/**" + - "zizmor.yml" + merge_group: + workflow_dispatch: + +permissions: {} + +jobs: + zizmor: + name: Run zizmor + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + + - name: Run zizmor + uses: zizmorcore/zizmor-action@71321a20a9ded102f6e9ce5718a2fcec2c4f70d8 # v0.5.2 + with: + advanced-security: false + config: zizmor.yml + version: 1.23.1 diff --git a/Cargo.lock b/Cargo.lock index f16e6e7040..e794916ead 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3066,7 +3066,6 @@ dependencies = [ "hex", "miden-agglayer", "miden-node-store", - "miden-node-utils", "miden-protocol", "miden-standards", "rand 0.9.2", @@ -3195,18 +3194,15 @@ dependencies = [ "miden-node-proto", "miden-node-proto-build", "miden-node-store", - "miden-node-test-macro", "miden-node-utils", "miden-node-validator", "miden-protocol", "miden-remote-prover-client", "miden-standards", - "miden-tx", "miden-tx-batch-prover", "pretty_assertions", "rand 0.9.2", "rand_chacha", - "rstest", "serial_test", "tempfile", "thiserror 2.0.18", @@ -3253,7 +3249,6 @@ dependencies = [ "miden-node-db", "miden-node-proto", "miden-node-proto-build", - "miden-node-test-macro", "miden-node-utils", "miden-protocol", "miden-remote-prover-client", @@ -3278,7 +3273,6 @@ name = "miden-node-proto" version = "0.14.10" dependencies = [ "anyhow", - "assert_matches", "build-rs", "fs-err", "hex", @@ -3376,10 +3370,8 @@ dependencies = [ "pretty_assertions", "rand 0.9.2", "rand_chacha", - "regex", "serde", "tempfile", - "termtree", "thiserror 2.0.18", "tokio", "tokio-stream", @@ -3598,14 +3590,12 @@ dependencies = [ "build-rs", "clap", "fs-err", - "http 1.4.0", "humantime", "miden-block-prover", "miden-node-proto", "miden-node-proto-build", "miden-node-utils", "miden-protocol", - "miden-standards", "miden-testing", "miden-tx", "miden-tx-batch-prover", @@ -3642,7 +3632,6 @@ dependencies = [ "tonic", "tonic-prost", "tonic-prost-build", - "tonic-web", "tonic-web-wasm-client", ] @@ -6068,12 +6057,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "termtree" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d1330fe7f7f872cd05165130b10602d667b205fd85be09be2814b115d4ced9" - [[package]] name = "textwrap" version = "0.16.2" diff --git a/Cargo.toml b/Cargo.toml index 8720181051..e2416f9e82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -85,20 +85,26 @@ indexmap = { version = "2.12" } itertools = { version = "0.14" } libsqlite3-sys = { features = ["bundled"], version = "0.35" } lru = { default-features = false, version = "0.16" } +miette = { version = "7.6" } +opentelemetry = { version = "0.31" } pretty_assertions = { version = "1.4" } # prost and protox are from different authors and are _not_ released in # lockstep, nor are they adhering to semver semantics. We keep this # to avoid future breakage. prost = { default-features = false, version = "=0.14.3" } protox = { version = "=0.9.1" } +quote = { version = "1.0" } rand = { version = "0.9" } rand_chacha = { default-features = false, version = "0.9" } +rayon = { version = "1.10" } reqwest = { version = "0.13" } rstest = { version = "0.26" } serde = { features = ["derive"], version = "1" } +serial_test = { version = "3.2" } +syn = { version = "2.0" } tempfile = { version = "3.12" } thiserror = { default-features = false, version = "2.0" } -tokio = { features = ["rt-multi-thread"], version = "1.46" } +tokio = { default-features = false, version = "1.46" } tokio-stream = { version = "0.1" } tokio-util = { version = "0.7" } toml = { version = "1.0" } @@ -107,12 +113,18 @@ tonic-health = { version = "0.14" } tonic-prost = { version = "0.14" } tonic-prost-build = { version = "0.14" } tonic-reflection = { version = "0.14" } +tonic-web = { version = "0.14" } tower = { version = "0.5" } tower-http = { features = ["cors", "trace"], version = "0.6" } tracing = { version = "0.1" } tracing-subscriber = { features = ["env-filter", "fmt", "json"], version = "0.3" } url = { features = ["serde"], version = "2.5" } +[workspace.metadata.cargo-shear] +# libsqlite3-sys is kept to control the bundled SQLite linkage. +# tonic-prost is used by generated gRPC code rather than handwritten Rust. +ignored = ["libsqlite3-sys", "tonic-prost"] + # Lints are set to warn for development, which are promoted to errors in CI. [workspace.lints.clippy] # Pedantic lints are set to a lower priority which allows lints in the group to be selectively enabled. diff --git a/Makefile b/Makefile index 33ab72a885..531833a3d8 100644 --- a/Makefile +++ b/Makefile @@ -35,9 +35,9 @@ format-check: ## Runs Format using nightly toolchain but only in check mode cargo +nightly fmt --all --check -.PHONY: machete -machete: ## Runs machete to find unused dependencies - cargo machete +.PHONY: shear +shear: ## Runs cargo-shear to find unused or misplaced dependencies + cargo shear .PHONY: toml @@ -59,7 +59,7 @@ workspace-check: ## Runs a check that all packages have `lints.workspace = true` .PHONY: lint -lint: typos-check format fix clippy toml machete ## Runs all linting tasks at once (Clippy, fixing, formatting, machete) +lint: typos-check format fix clippy toml shear ## Runs all linting tasks at once (Clippy, fixing, formatting, cargo-shear) # --- docs ---------------------------------------------------------------------------------------- @@ -155,7 +155,7 @@ check-tools: ## Checks if development tools are installed @command -v typos >/dev/null 2>&1 && echo "[OK] typos is installed" || echo "[MISSING] typos (make install-tools)" @command -v cargo nextest >/dev/null 2>&1 && echo "[OK] cargo-nextest is installed" || echo "[MISSING] cargo-nextest(make install-tools)" @command -v taplo >/dev/null 2>&1 && echo "[OK] taplo is installed" || echo "[MISSING] taplo (make install-tools)" - @command -v cargo-machete >/dev/null 2>&1 && echo "[OK] cargo-machete is installed" || echo "[MISSING] cargo-machete (make install-tools)" + @command -v cargo-shear >/dev/null 2>&1 && echo "[OK] cargo-shear is installed" || echo "[MISSING] cargo-shear is not installed (run: make install-tools)" @command -v npm >/dev/null 2>&1 && echo "[OK] npm is installed" || echo "[MISSING] npm is not installed (run: make install-tools)" .PHONY: install-tools @@ -166,7 +166,7 @@ install-tools: ## Installs tools required by the Makefile cargo install typos-cli --locked cargo install cargo-nextest --locked cargo install taplo-cli --locked - cargo install cargo-machete --locked + cargo install cargo-shear --version 1.11.2 --locked @if ! command -v node >/dev/null 2>&1; then \ echo "Node.js not found. Please install Node.js from https://nodejs.org/ or using your package manager"; \ echo "On macOS: brew install node"; \ diff --git a/bin/genesis/Cargo.toml b/bin/genesis/Cargo.toml index c37ab026fd..2be6b35595 100644 --- a/bin/genesis/Cargo.toml +++ b/bin/genesis/Cargo.toml @@ -29,6 +29,5 @@ rand_chacha = { workspace = true } [dev-dependencies] miden-node-store = { workspace = true } -miden-node-utils = { workspace = true } tempfile = { workspace = true } tokio = { features = ["macros", "rt-multi-thread"], workspace = true } diff --git a/bin/network-monitor/Cargo.toml b/bin/network-monitor/Cargo.toml index 2f3a7704b1..fec7946372 100644 --- a/bin/network-monitor/Cargo.toml +++ b/bin/network-monitor/Cargo.toml @@ -18,7 +18,7 @@ workspace = true anyhow = { workspace = true } axum = { version = "0.8" } clap = { features = ["env"], workspace = true } -hex = { version = "0.4" } +hex = { workspace = true } humantime = { workspace = true } miden-node-proto = { workspace = true } miden-node-utils = { workspace = true } @@ -29,7 +29,7 @@ miden-tx = { features = ["concurrent", "std"], workspace = true } rand = { workspace = true } rand_chacha = { workspace = true } reqwest = { features = ["json", "query"], workspace = true } -serde = { features = ["derive"], version = "1.0" } +serde = { workspace = true } serde_json = { version = "1.0" } sha2 = { version = "0.10" } tokio = { features = ["full"], workspace = true } diff --git a/bin/node/Dockerfile b/bin/node/Dockerfile index 5986451a28..0c8ff95602 100644 --- a/bin/node/Dockerfile +++ b/bin/node/Dockerfile @@ -51,5 +51,5 @@ LABEL org.opencontainers.image.created=$CREATED \ # Expose RPC port EXPOSE 57291 -# Miden node does not spawn sub-processes, so it can be used as the PID1 -CMD miden-node +# Miden node does not spawn sub-processes, so it can be used as the PID1. +CMD ["miden-node"] diff --git a/bin/remote-prover/Cargo.toml b/bin/remote-prover/Cargo.toml index 7a05151cc3..e14a083736 100644 --- a/bin/remote-prover/Cargo.toml +++ b/bin/remote-prover/Cargo.toml @@ -16,9 +16,8 @@ workspace = true [dependencies] anyhow = { workspace = true } -async-trait = { version = "0.1" } +async-trait = { workspace = true } clap = { features = ["env"], workspace = true } -http = { workspace = true } humantime = { workspace = true } miden-block-prover = { workspace = true } miden-node-proto = { workspace = true } @@ -27,36 +26,34 @@ miden-node-utils = { workspace = true } miden-protocol = { features = ["std"], workspace = true } miden-tx = { features = ["concurrent", "std"], workspace = true } miden-tx-batch-prover = { features = ["std"], workspace = true } -opentelemetry = { version = "0.31" } +opentelemetry = { workspace = true } prost = { default-features = false, features = ["derive"], workspace = true } tokio = { features = ["full"], workspace = true } -tokio-stream = { features = ["net"], version = "0.1" } -tonic = { default-features = false, features = ["codegen", "router", "transport"], version = "0.14" } -tonic-health = { version = "0.14" } +tokio-stream = { features = ["net"], workspace = true } +tonic = { default-features = false, features = ["codegen", "router", "transport"], workspace = true } +tonic-health = { workspace = true } tonic-prost = { workspace = true } tonic-reflection = { workspace = true } -tonic-web = { version = "0.14" } +tonic-web = { workspace = true } tower-http = { features = ["trace"], workspace = true } tracing = { workspace = true } [dev-dependencies] -assert_matches = { workspace = true } -miden-protocol = { features = ["testing"], workspace = true } -miden-standards = { features = ["testing"], workspace = true } -miden-testing = { workspace = true } -miden-tx = { features = ["concurrent", "testing"], workspace = true } -serial_test = { version = "3" } +assert_matches = { workspace = true } +miden-protocol = { features = ["testing"], workspace = true } +miden-testing = { workspace = true } +miden-tx = { features = ["concurrent", "testing"], workspace = true } +serial_test = { workspace = true } [build-dependencies] build-rs = { workspace = true } fs-err = { workspace = true } miden-node-proto-build = { features = ["internal"], workspace = true } -miette = { features = ["fancy"], version = "7.5" } +miette = { features = ["fancy"], workspace = true } tonic-prost-build = { workspace = true } -[package.metadata.cargo-machete] +[package.metadata.cargo-shear] ignored = [ - "http", "prost", "tonic-prost", # used in generated OUT_DIR code ] diff --git a/bin/remote-prover/src/server/mod.rs b/bin/remote-prover/src/server/mod.rs index 3b084c42fc..6d3038e6c8 100644 --- a/bin/remote-prover/src/server/mod.rs +++ b/bin/remote-prover/src/server/mod.rs @@ -1,7 +1,6 @@ use std::num::NonZeroUsize; use anyhow::Context; -use miden_node_utils::clap::GrpcOptionsInternal; use miden_node_utils::cors::cors_for_grpc_web_layer; use miden_node_utils::panic::catch_panic_layer_fn; use miden_node_utils::tracing::grpc::grpc_trace_fn; @@ -83,11 +82,9 @@ impl Server { // Mark the service as serving health_reporter.set_serving::>().await; - let grpc_options = GrpcOptionsInternal::default(); - let server = tonic::transport::Server::builder() .accept_http1(true) - .timeout(grpc_options.request_timeout) + .timeout(self.timeout) .layer(CatchPanicLayer::custom(catch_panic_layer_fn)) .layer(TraceLayer::new_for_grpc().make_span_with(grpc_trace_fn)) .layer(cors_for_grpc_web_layer()) diff --git a/bin/stress-test/Cargo.toml b/bin/stress-test/Cargo.toml index 452da58606..0743e2b751 100644 --- a/bin/stress-test/Cargo.toml +++ b/bin/stress-test/Cargo.toml @@ -17,7 +17,7 @@ version.workspace = true workspace = true [dependencies] -clap = { features = ["derive"], version = "4.5" } +clap = { features = ["derive"], workspace = true } fs-err = { workspace = true } futures = { workspace = true } miden-node-block-producer = { workspace = true } @@ -27,7 +27,7 @@ miden-node-utils = { workspace = true } miden-protocol = { workspace = true } miden-standards = { workspace = true } rand = { workspace = true } -rayon = { version = "1.10" } +rayon = { workspace = true } tokio = { workspace = true } tonic = { default-features = true, workspace = true } url = { workspace = true } diff --git a/crates/block-producer/Cargo.toml b/crates/block-producer/Cargo.toml index 65f99e01a1..d3349bf7cc 100644 --- a/crates/block-producer/Cargo.toml +++ b/crates/block-producer/Cargo.toml @@ -14,6 +14,9 @@ version.workspace = true [lints] workspace = true +[lib] +doctest = false + [features] testing = [] tracing-forest = ["miden-node-utils/tracing-forest"] @@ -40,17 +43,14 @@ tracing = { workspace = true } url = { workspace = true } [dev-dependencies] -assert_matches = { workspace = true } -miden-node-store = { workspace = true } -miden-node-test-macro = { workspace = true } -miden-node-utils = { features = ["testing"], workspace = true } -miden-node-validator = { workspace = true } -miden-protocol = { default-features = true, features = ["testing"], workspace = true } -miden-standards = { features = ["testing"], workspace = true } -miden-tx = { features = ["concurrent", "testing"], workspace = true } -pretty_assertions = "1.4" -rand_chacha = { default-features = false, workspace = true } -rstest = { workspace = true } -serial_test = "3.2" -tempfile = { workspace = true } -tokio = { features = ["test-util"], workspace = true } +assert_matches = { workspace = true } +miden-node-store = { workspace = true } +miden-node-utils = { features = ["testing"], workspace = true } +miden-node-validator = { workspace = true } +miden-protocol = { default-features = true, features = ["testing"], workspace = true } +miden-standards = { features = ["testing"], workspace = true } +pretty_assertions = { workspace = true } +rand_chacha = { default-features = false, workspace = true } +serial_test = { workspace = true } +tempfile = { workspace = true } +tokio = { features = ["test-util"], workspace = true } diff --git a/crates/grpc-error-macro/Cargo.toml b/crates/grpc-error-macro/Cargo.toml index 4fd40b0b2d..dd040a1f10 100644 --- a/crates/grpc-error-macro/Cargo.toml +++ b/crates/grpc-error-macro/Cargo.toml @@ -15,8 +15,10 @@ version.workspace = true workspace = true [lib] +doctest = false proc-macro = true +test = false [dependencies] -quote = "1.0" -syn = { features = ["full"], version = "2.0" } +quote = { workspace = true } +syn = { features = ["full"], workspace = true } diff --git a/crates/large-smt-backend-rocksdb/Cargo.toml b/crates/large-smt-backend-rocksdb/Cargo.toml index 5109ecc9c3..3b790c39cb 100644 --- a/crates/large-smt-backend-rocksdb/Cargo.toml +++ b/crates/large-smt-backend-rocksdb/Cargo.toml @@ -17,7 +17,7 @@ workspace = true [dependencies] miden-crypto = { features = ["concurrent", "std"], workspace = true } miden-protocol = { features = ["std"], workspace = true } -rayon = { version = "1.10" } +rayon = { workspace = true } rocksdb = { default-features = false, features = ["bindgen-runtime", "lz4"], version = "0.24" } [build-dependencies] diff --git a/crates/ntx-builder/Cargo.toml b/crates/ntx-builder/Cargo.toml index c4e2a991cc..629776da1e 100644 --- a/crates/ntx-builder/Cargo.toml +++ b/crates/ntx-builder/Cargo.toml @@ -13,6 +13,9 @@ version.workspace = true [lints] workspace = true +[lib] +doctest = false + [dependencies] anyhow = { workspace = true } diesel = { features = ["numeric", "sqlite"], workspace = true } @@ -41,13 +44,9 @@ url = { workspace = true } build-rs = { workspace = true } [dev-dependencies] -miden-node-test-macro = { path = "../test-macro" } -miden-node-utils = { features = ["testing"], workspace = true } -miden-protocol = { default-features = true, features = ["testing"], workspace = true } -miden-standards = { features = ["testing"], workspace = true } -rand_chacha = { workspace = true } -rstest = { workspace = true } -tempfile = { version = "3.20" } - -[package.metadata.cargo-machete] -ignored = ["libsqlite3-sys"] +miden-node-utils = { features = ["testing"], workspace = true } +miden-protocol = { default-features = true, features = ["testing"], workspace = true } +miden-standards = { features = ["testing"], workspace = true } +rand_chacha = { workspace = true } +rstest = { workspace = true } +tempfile = { workspace = true } diff --git a/crates/proto/Cargo.toml b/crates/proto/Cargo.toml index fa48024ce5..8165f1ea29 100644 --- a/crates/proto/Cargo.toml +++ b/crates/proto/Cargo.toml @@ -16,7 +16,7 @@ workspace = true [dependencies] anyhow = { workspace = true } -hex = { version = "0.4" } +hex = { workspace = true } http = { workspace = true } miden-node-grpc-error-macro = { workspace = true } miden-node-utils = { workspace = true } @@ -29,14 +29,13 @@ tonic-prost = { workspace = true } url = { workspace = true } [dev-dependencies] -assert_matches = { workspace = true } -proptest = { version = "1.7" } +proptest = { version = "1.7" } [build-dependencies] build-rs = { workspace = true } fs-err = { workspace = true } miden-node-proto-build = { features = ["internal"], workspace = true } -miette = { version = "7.6" } +miette = { workspace = true } tonic-prost-build = { workspace = true } [package.metadata.cargo-machete] diff --git a/crates/remote-prover-client/Cargo.toml b/crates/remote-prover-client/Cargo.toml index 3edb6ea546..f9157d2390 100644 --- a/crates/remote-prover-client/Cargo.toml +++ b/crates/remote-prover-client/Cargo.toml @@ -12,6 +12,8 @@ version.workspace = true [lib] crate-type = ["lib"] +doctest = false +test = false [features] batch-prover = ["dep:miden-protocol", "dep:tokio"] @@ -26,8 +28,7 @@ tonic = { features = ["codegen"], workspace = true } tonic-web-wasm-client = { default-features = false, version = "0.9" } [target.'cfg(not(all(target_arch = "wasm32", target_os = "unknown")))'.dependencies] -tonic = { features = ["codegen", "tls-native-roots", "tls-ring", "transport"], workspace = true } -tonic-web = { optional = true, version = "0.14" } +tonic = { features = ["codegen", "tls-native-roots", "tls-ring", "transport"], workspace = true } [lints] workspace = true @@ -37,18 +38,19 @@ miden-protocol = { optional = true, workspace = true } miden-tx = { optional = true, workspace = true } prost = { default-features = false, features = ["derive"], workspace = true } thiserror = { workspace = true } -tokio = { default-features = false, features = ["sync"], optional = true, version = "1.44" } +tokio = { default-features = false, features = ["sync"], optional = true, workspace = true } tonic-prost = { workspace = true } [build-dependencies] build-rs = { workspace = true } fs-err = { workspace = true } miden-node-proto-build = { workspace = true } -miette = { features = ["fancy"], version = "7.5" } +miette = { features = ["fancy"], workspace = true } tonic-prost-build = { workspace = true } -[package.metadata.cargo-machete] +[package.metadata.cargo-shear] ignored = [ + "getrandom", "prost", "tonic-prost", # used in generated OUT_DIR code ] diff --git a/crates/rocksdb-cxx-linkage-fix/Cargo.toml b/crates/rocksdb-cxx-linkage-fix/Cargo.toml index 9e0eb23f7a..3e80a35bfd 100644 --- a/crates/rocksdb-cxx-linkage-fix/Cargo.toml +++ b/crates/rocksdb-cxx-linkage-fix/Cargo.toml @@ -11,7 +11,9 @@ rust-version.workspace = true version.workspace = true [lib] -path = "src/lib.rs" +doctest = false +path = "src/lib.rs" +test = false [lints] workspace = true diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index 8c483ac319..ed3d8ff903 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -14,9 +14,12 @@ version.workspace = true [lints] workspace = true +[lib] +doctest = false + [dependencies] anyhow = { workspace = true } -futures = { version = "0.3" } +futures = { workspace = true } http = { workspace = true } mediatype = { version = "0.21" } miden-node-proto = { workspace = true } @@ -31,7 +34,7 @@ tokio = { features = ["macros", "net", "rt-multi-thread"], work tokio-stream = { features = ["net"], workspace = true } tonic = { default-features = true, features = ["tls-native-roots", "tls-ring"], workspace = true } tonic-reflection = { workspace = true } -tonic-web = { version = "0.14" } +tonic-web = { workspace = true } tower = { workspace = true } tower-http = { features = ["trace"], workspace = true } tracing = { workspace = true } diff --git a/crates/rpc/src/tests.rs b/crates/rpc/src/tests.rs index 29613be0d6..a1e85e7737 100644 --- a/crates/rpc/src/tests.rs +++ b/crates/rpc/src/tests.rs @@ -38,6 +38,7 @@ use tempfile::TempDir; use tokio::net::TcpListener; use tokio::runtime::{self, Runtime}; use tokio::task; +use tokio::time::sleep; use url::Url; use crate::Rpc; @@ -228,7 +229,7 @@ async fn rpc_startup_is_robust_to_network_failures() { let (store_runtime, data_directory, _genesis, store_addr) = start_store(store_listener).await; // Test: send request against RPC api and should succeed - let response = send_request(&mut rpc_client).await; + let response = send_request_until_success(&mut rpc_client).await; assert!(response.unwrap().into_inner().block_header.is_some()); // Test: shutdown the store and should fail @@ -238,7 +239,7 @@ async fn rpc_startup_is_robust_to_network_failures() { // Test: restart the store and request should succeed let store_runtime = restart_store(store_addr, data_directory.path()).await; - let response = send_request(&mut rpc_client).await; + let response = send_request_until_success(&mut rpc_client).await; assert_eq!(response.unwrap().into_inner().block_header.unwrap().block_num, 0); // Shutdown the store before data_directory is dropped to allow RocksDB to flush properly @@ -440,6 +441,24 @@ async fn send_request( rpc_client.get_block_header_by_number(request).await } +async fn send_request_until_success( + rpc_client: &mut RpcClient, +) -> std::result::Result, tonic::Status> { + let mut attempts = 0; + loop { + attempts += 1; + + match send_request(rpc_client).await { + Ok(response) => return Ok(response), + Err(err) if attempts < 30 => { + sleep(Duration::from_millis(200)).await; + tracing::warn!(%attempts, %err, "RPC request failed, retrying"); + }, + Err(err) => return Err(err), + } + } +} + async fn connect_rpc(url: Url, local_address: Option) -> RpcClient { let mut endpoint = tonic::transport::Endpoint::from_shared(url.to_string()) .expect("Url type always results in valid endpoint") diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index aa8cfc3a27..08ca35f3d5 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -14,14 +14,17 @@ version.workspace = true [lints] workspace = true +[lib] +doctest = false + [dependencies] anyhow = { workspace = true } -deadpool = { default-features = false, features = ["managed", "rt_tokio_1"], version = "0.12" } -deadpool-diesel = { features = ["sqlite"], version = "0.6" } -diesel = { features = ["numeric", "sqlite"], version = "2.3" } -diesel_migrations = { features = ["sqlite"], version = "2.3" } +deadpool = { features = ["managed", "rt_tokio_1"], workspace = true } +deadpool-diesel = { features = ["sqlite"], workspace = true } +diesel = { features = ["numeric", "sqlite"], workspace = true } +diesel_migrations = { features = ["sqlite"], workspace = true } fs-err = { workspace = true } -hex = { version = "0.4" } +hex = { workspace = true } indexmap = { workspace = true } libsqlite3-sys = { workspace = true } miden-block-prover = { workspace = true } @@ -38,7 +41,7 @@ miden-protocol = { features = ["std", "testing"], workspace = true } pretty_assertions = { workspace = true } rand = { workspace = true } rand_chacha = { workspace = true } -serde = { features = ["derive"], version = "1" } +serde = { workspace = true } thiserror = { workspace = true } tokio = { features = ["fs", "rt-multi-thread"], workspace = true } tokio-stream = { features = ["net"], workspace = true } @@ -66,9 +69,7 @@ miden-node-utils = { features = ["testing", "tracing-forest"], workspace = miden-protocol = { default-features = true, features = ["testing"], workspace = true } miden-standards = { features = ["testing"], workspace = true } rand = { workspace = true } -regex = { version = "1.11" } tempfile = { workspace = true } -termtree = "1.0" [features] default = ["rocksdb"] diff --git a/crates/store/src/db/query_plan/mod.rs b/crates/store/src/db/query_plan/mod.rs deleted file mode 100644 index ebeb805630..0000000000 --- a/crates/store/src/db/query_plan/mod.rs +++ /dev/null @@ -1,73 +0,0 @@ -use std::{ - fmt::{Display, Formatter}, - ops::Not, -}; - -pub mod renderer; - -impl Transaction<'_> { - /// Panics if the query plan contains a slow table scan. - pub fn check_query_plan(&self, sql: &str) { - let query_plan = self.query_plan(sql).expect("Must be a valid SQL"); - - assert!( - query_plan.contains_unnecessary_scans().not(), - "Query plan contains unnecessary table scan(s):\n{query_plan}" - ); - } - - /// Renders query plan as tree using ASCII graphics. - fn query_plan(&self, sql: &str) -> rusqlite::Result { - let explain_sql = format!("EXPLAIN QUERY PLAN {sql}"); - let explain_stmt = self.prepare(&explain_sql)?; - let query_plan = QueryPlanRenderer::new().render_tree(explain_stmt)?; - - Ok(QueryPlan { explain_sql, query_plan }) - } -} - -/// Represents rendered query plan with `EXPLAIN QUERY PLAN ...` SQL query. -struct QueryPlan { - explain_sql: String, - query_plan: String, -} - -impl Display for QueryPlan { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - writeln!(f, "{}\n\n{}", self.explain_sql, self.query_plan) - } -} - -impl QueryPlan { - /// Checks if the given query plan contains unnecessary table scans not utilizing indices. - fn contains_unnecessary_scans(&self) -> bool { - use std::sync::LazyLock; - - static RE_IDX_SCAN: LazyLock = LazyLock::new(|| { - regex::RegexBuilder::new( - r"SCAN ([A-Za-z0-9_]+) (USING( COVERING)?|VIRTUAL TABLE) INDEX ([A-Za-z0-9_]+)", - ) - .case_insensitive(true) - .build() - .expect("Must be a valid regex pattern") - }); - - // Some index scans might cause false positiveness if we just look for `SCAN` keyword: - // ``` - // SCAN rarray VIRTUAL TABLE INDEX 1 - // SCAN table1 USING INDEX sqlite_autoindex_table1_1 - // SCAN table1 USING COVERING INDEX idx_table1_column - // ``` - // Preprocess query plan in order to replace such cases to `SEARCH` expressions instead of - // `SCAN`. Another solution would be to find only scan expressions which don't accompanied - // by known index suffixes, but current `regex` implementation doesn't support look-around. - let query_plan = RE_IDX_SCAN.replace_all(&self.query_plan, r"SEARCH $1 $2 INDEX $4"); - - // Don't flag `SELECT` queries without `WHERE` clause, since they don't usually use indexes - if self.explain_sql.contains("SELECT") && !self.explain_sql.contains("WHERE") { - return false; - } - - query_plan.contains(" SCAN ") - } -} diff --git a/crates/store/src/db/query_plan/renderer.rs b/crates/store/src/db/query_plan/renderer.rs deleted file mode 100644 index 8a08785918..0000000000 --- a/crates/store/src/db/query_plan/renderer.rs +++ /dev/null @@ -1,49 +0,0 @@ -use rusqlite::Statement; -use termtree::Tree; - -/// SQL query plan renderer which represents result as a tree using ASCII graphics. -pub struct QueryPlanRenderer { - path: Vec<(u64, Tree)>, -} - -impl QueryPlanRenderer { - /// Constructs and initializes query plan renderer. - pub fn new() -> Self { - Self { - path: vec![(0_u64, Tree::new("QUERY PLAN".to_string()))], - } - } - - /// Runs `EXPLAIN QUERY PLAN` statement and renders result as tree using ASCII graphics. - /// - /// # Note - /// Current algorithm relies on the row ordering (all child rows go after corresponding parent - /// row) of the current implementation of SQLite's `EXPLAIN QUERY PLAN` command. This is not - /// bad, because this makes algorithm simple and effective, and it is intended to be used only - /// for debugging and testing. - pub fn render_tree(mut self, mut explain_stmt: Statement) -> rusqlite::Result { - let mut rows = explain_stmt.raw_query(); - while let Some(row) = rows.next()? { - let id: u64 = row.get(0)?; - let parent_id: u64 = row.get(1)?; - let label: String = row.get(3)?; - - self.fold_up_to(parent_id); - self.path.push((id, label.into())); - } - self.fold_up_to(0); - - let (_, root) = self.path.pop().expect("Always present"); - - Ok(root.to_string()) - } - - /// Folds all elements from the top of the path stack up to the element with the given ID. - /// All elements with higher indexes become children of the elements with lower indexes. - fn fold_up_to(&mut self, id: u64) { - while self.path.last().expect("Always present").0 > id { - let (_, top) = self.path.pop().expect("Always present"); - self.path.last_mut().map(|(_, last)| last.push(top)).expect("Always present"); - } - } -} diff --git a/crates/test-macro/Cargo.toml b/crates/test-macro/Cargo.toml index dc1b9fec17..7064479d0f 100644 --- a/crates/test-macro/Cargo.toml +++ b/crates/test-macro/Cargo.toml @@ -15,8 +15,10 @@ version = "0.1.0" workspace = true [dependencies] -quote = { version = "1.0" } -syn = { features = ["extra-traits", "full"], version = "2.0" } +quote = { workspace = true } +syn = { features = ["extra-traits", "full"], workspace = true } [lib] +doctest = false proc-macro = true +test = false diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 01f536ffee..f3f6770185 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -14,6 +14,9 @@ version.workspace = true [lints] workspace = true +[lib] +doctest = false + [features] # Enables utility functions for testing traces created by some other crate's stack. rocksdb = ["dep:miden-large-smt-backend-rocksdb"] @@ -31,7 +34,7 @@ humantime = { workspace = true } itertools = { workspace = true } lru = { workspace = true } miden-protocol = { workspace = true } -opentelemetry = { version = "0.31" } +opentelemetry = { workspace = true } opentelemetry-otlp = { default-features = false, features = ["grpc-tonic", "tls-roots", "trace"], version = "0.31" } opentelemetry_sdk = { features = ["rt-tokio", "testing"], version = "0.31" } rand = { workspace = true } diff --git a/crates/validator/Cargo.toml b/crates/validator/Cargo.toml index 2cf6fb650e..bbe4d4df0a 100644 --- a/crates/validator/Cargo.toml +++ b/crates/validator/Cargo.toml @@ -14,6 +14,9 @@ version.workspace = true [lints] workspace = true +[lib] +doctest = false + [features] [dependencies] diff --git a/proto/Cargo.toml b/proto/Cargo.toml index ee79d7adc1..2011f090f7 100644 --- a/proto/Cargo.toml +++ b/proto/Cargo.toml @@ -26,5 +26,9 @@ tonic-prost-build = { workspace = true } [build-dependencies] build-rs = { workspace = true } fs-err = { workspace = true } -miette = { version = "7.6" } +miette = { workspace = true } protox = { workspace = true } + +[package.metadata.cargo-shear] +# cargo-shear misses this because it is required by files generated by build.rs. +ignored = ["tonic-prost-build"] diff --git a/zizmor.yml b/zizmor.yml new file mode 100644 index 0000000000..f8a62efe78 --- /dev/null +++ b/zizmor.yml @@ -0,0 +1,4 @@ +rules: + use-trusted-publishing: + ignore: + - publish-crates.yml From b491987ba93857c024506c80afc673c32aeaa0f2 Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Mon, 4 May 2026 14:44:34 +0200 Subject: [PATCH 2/3] ci: install toolchain properly (#2026) --- .github/workflows/ci.yml | 16 +++++++--------- .github/workflows/nightly.yml | 2 +- .github/workflows/publish-crates.yml | 2 +- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9cc7d2364b..91a4ba468b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,7 +74,7 @@ jobs: - uses: ./.github/actions/install-rocksdb - uses: ./.github/actions/install-protobuf-compiler - name: Rustup - run: rustup update --no-self-update + run: rustup toolchain install --no-self-update - uses: WarpBuilds/rust-cache@9d0cc3090d9c87de74ea67617b246e978735b1a1 # v2.9.1 with: shared-key: ${{ env.RUST_CACHE_SUFFIX }} @@ -130,7 +130,7 @@ jobs: with: persist-credentials: false - name: Rustup - run: rustup update --no-self-update + run: rustup toolchain install --no-self-update - uses: WarpBuilds/rust-cache@9d0cc3090d9c87de74ea67617b246e978735b1a1 # v2.9.1 with: shared-key: ${{ env.RUST_CACHE_SUFFIX }} @@ -149,7 +149,7 @@ jobs: with: persist-credentials: false - name: Rustup - run: rustup update --no-self-update + run: rustup toolchain install --no-self-update - uses: taiki-e/install-action@055f5df8c3f65ea01cd41e9dc855becd88953486 # v2.75.18 with: tool: nextest@0.9.122 @@ -174,7 +174,7 @@ jobs: with: persist-credentials: false - name: Rustup - run: rustup update --no-self-update + run: rustup toolchain install --no-self-update - uses: WarpBuilds/rust-cache@9d0cc3090d9c87de74ea67617b246e978735b1a1 # v2.9.1 with: shared-key: ${{ env.RUST_CACHE_SUFFIX }} @@ -197,7 +197,7 @@ jobs: with: persist-credentials: false - name: Rustup - run: rustup update --no-self-update + run: rustup toolchain install --no-self-update - uses: WarpBuilds/rust-cache@9d0cc3090d9c87de74ea67617b246e978735b1a1 # v2.9.1 with: shared-key: ${{ env.RUST_CACHE_SUFFIX }} @@ -283,7 +283,7 @@ jobs: with: persist-credentials: false - name: Rustup - run: rustup update --no-self-update + run: rustup toolchain install --no-self-update - name: cargo build run: | cargo build --locked -p miden-remote-prover-client \ @@ -320,9 +320,7 @@ jobs: with: persist-credentials: false - name: Rustup +nightly - run: | - rustup update --no-self-update nightly - rustup +nightly component add rustfmt + run: rustup toolchain install --no-self-update nightly --component rustfmt - name: Fmt run: make format-check diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 398321d5f2..bbde7c792e 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -50,7 +50,7 @@ jobs: - uses: ./.github/actions/install-rocksdb - uses: ./.github/actions/install-protobuf-compiler - name: Install rust - run: rustup update --no-self-update + run: rustup toolchain install --no-self-update - name: Install cargo-hack uses: taiki-e/install-action@055f5df8c3f65ea01cd41e9dc855becd88953486 # v2.75.18 with: diff --git a/.github/workflows/publish-crates.yml b/.github/workflows/publish-crates.yml index 813564e8b8..a0b20fca46 100644 --- a/.github/workflows/publish-crates.yml +++ b/.github/workflows/publish-crates.yml @@ -33,7 +33,7 @@ jobs: - name: Install dependencies run: sudo apt-get update && sudo apt-get install -y jq - name: Update Rust toolchain - run: rustup update --no-self-update + run: rustup toolchain install --no-self-update - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 - uses: taiki-e/install-action@055f5df8c3f65ea01cd41e9dc855becd88953486 # v2.75.18 with: From a5c3f3e890eaf48b5a11486ce61e046414f90a2f Mon Sep 17 00:00:00 2001 From: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> Date: Mon, 4 May 2026 14:58:43 +0200 Subject: [PATCH 3/3] ci: cache each job (#2025) --- .github/workflows/ci.yml | 129 +++++++++++---------------------------- 1 file changed, 34 insertions(+), 95 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91a4ba468b..ff4d4c0d8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,6 @@ # Continuous integration jobs. # -# These get run on every pull-request. +# These get run on every pull-request, with Warp caches updated on push into `main` or `next`. name: CI permissions: @@ -23,33 +23,12 @@ on: - "docs/**" env: - # Shared prefix key for the rust cache. + # Shared prefix key for the Rust caches. # - # This provides a convenient way to evict old or corrupted cache. - RUST_CACHE_PREFIX: rust-cache - # Shared branch-aware cache namespace for rust-cache. - # - # Pushes save to the persistent trunk cache, while pull requests append the - # PR number so they save into an ephemeral PR-specific cache. - # - # The format is
with an optional `-pr-` for pull requests. - # We default to next if no suitable is found. - RUST_CACHE_SUFFIX: >- - ${{ - format( - '{0}{1}', - github.base_ref == 'main' && 'main' - || github.base_ref == 'next' && 'next' - || github.ref == 'refs/heads/main' && 'main' - || github.ref == 'refs/heads/next' && 'next' - || 'next', - github.event_name == 'pull_request' - && format('-pr-{0}', github.event.pull_request.number) - || '' - ) - }} - # Match rust-cache's compilation mode so restored outputs stay reusable downstream. - CARGO_INCREMENTAL: 0 + # Cache is trunk specific (next|main). + CACHE_PREFIX: rust-cache-${{ github.base_ref || github.ref_name }} + # Only trusted branch pushes should update caches; PRs restore from the target branch cache. + SAVE_CACHE: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/next') }} # Reduce cache usage by removing debug information. CARGO_PROFILE_DEV_DEBUG: 0 @@ -60,11 +39,10 @@ concurrency: jobs: # =============================================================================================== - # Conventional builds, lints and tests that re-use a single cache for efficiency + # Conventional builds, lints and tests # =============================================================================================== - # Normal cargo build that saves either the persistent trunk cache or a - # single per-run cache for downstream jobs to restore immediately. + # Normal cargo build that verifies the workspace compiles. build: runs-on: warp-ubuntu-latest-x64-8x steps: @@ -77,10 +55,9 @@ jobs: run: rustup toolchain install --no-self-update - uses: WarpBuilds/rust-cache@9d0cc3090d9c87de74ea67617b246e978735b1a1 # v2.9.1 with: - shared-key: ${{ env.RUST_CACHE_SUFFIX }} - prefix-key: ${{ env.RUST_CACHE_PREFIX }} - cache-workspace-crates: true - save-if: true + shared-key: ${{ github.job }} + prefix-key: ${{ env.CACHE_PREFIX }} + save-if: ${{ env.SAVE_CACHE }} - name: cargo build run: cargo build --workspace --all-targets --locked - name: Check static linkage @@ -124,7 +101,6 @@ jobs: clippy: name: lint - clippy runs-on: warp-ubuntu-latest-x64-8x - needs: [build] steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: @@ -133,16 +109,14 @@ jobs: run: rustup toolchain install --no-self-update - uses: WarpBuilds/rust-cache@9d0cc3090d9c87de74ea67617b246e978735b1a1 # v2.9.1 with: - shared-key: ${{ env.RUST_CACHE_SUFFIX }} - prefix-key: ${{ env.RUST_CACHE_PREFIX }} - cache-workspace-crates: true - save-if: false + shared-key: ${{ github.job }} + prefix-key: ${{ env.CACHE_PREFIX }} + save-if: ${{ env.SAVE_CACHE }} - name: clippy run: cargo clippy --locked --all-targets --all-features --workspace -- -D warnings tests: runs-on: warp-ubuntu-latest-x64-8x - needs: [build] timeout-minutes: 30 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -155,10 +129,9 @@ jobs: tool: nextest@0.9.122 - uses: WarpBuilds/rust-cache@9d0cc3090d9c87de74ea67617b246e978735b1a1 # v2.9.1 with: - shared-key: ${{ env.RUST_CACHE_SUFFIX }} - prefix-key: ${{ env.RUST_CACHE_PREFIX }} - cache-workspace-crates: true - save-if: false + shared-key: ${{ github.job }} + prefix-key: ${{ env.CACHE_PREFIX }} + save-if: ${{ env.SAVE_CACHE }} - name: Build tests run: cargo nextest run --all-features --workspace --no-run - name: Run tests @@ -167,7 +140,6 @@ jobs: run: cargo test --doc --workspace --all-features doc: - needs: [build] runs-on: warp-ubuntu-latest-x64-8x steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -177,17 +149,15 @@ jobs: run: rustup toolchain install --no-self-update - uses: WarpBuilds/rust-cache@9d0cc3090d9c87de74ea67617b246e978735b1a1 # v2.9.1 with: - shared-key: ${{ env.RUST_CACHE_SUFFIX }} - prefix-key: ${{ env.RUST_CACHE_PREFIX }} - cache-workspace-crates: true - save-if: false + shared-key: ${{ github.job }} + prefix-key: ${{ env.CACHE_PREFIX }} + save-if: ${{ env.SAVE_CACHE }} - name: Build docs run: cargo doc --no-deps --workspace --all-features --locked # Ensure the stress-test still functions by running some cheap benchmarks. stress-test: name: stress test - needs: [build] runs-on: warp-ubuntu-latest-x64-8x timeout-minutes: 20 env: @@ -200,10 +170,9 @@ jobs: run: rustup toolchain install --no-self-update - uses: WarpBuilds/rust-cache@9d0cc3090d9c87de74ea67617b246e978735b1a1 # v2.9.1 with: - shared-key: ${{ env.RUST_CACHE_SUFFIX }} - prefix-key: ${{ env.RUST_CACHE_PREFIX }} - cache-workspace-crates: true - save-if: false + shared-key: ${{ github.job }} + prefix-key: ${{ env.CACHE_PREFIX }} + save-if: ${{ env.SAVE_CACHE }} - uses: taiki-e/install-action@055f5df8c3f65ea01cd41e9dc855becd88953486 # v2.75.18 with: tool: nextest@0.9.122 @@ -216,12 +185,12 @@ jobs: cargo run --bin miden-node-stress-test seed-store \ --data-directory ${{ env.DATA_DIR }} \ --num-accounts 500 --public-accounts-percentage 50 - # TODO re-introduce + # TODO re-introduce # - name: Benchmark state sync - # run: | - # cargo run --bin miden-node-stress-test benchmark-store \ - # --data-directory ${{ env.DATA_DIR }} \ - # --iterations 10 --concurrency 1 sync-state + # run: | + # cargo run --bin miden-node-stress-test benchmark-store \ + # --data-directory ${{ env.DATA_DIR }} \ + # --iterations 10 --concurrency 1 sync-state - name: Benchmark notes sync run: | cargo run --bin miden-node-stress-test benchmark-store \ @@ -233,48 +202,13 @@ jobs: --data-directory ${{ env.DATA_DIR }} \ --iterations 10 --concurrency 1 sync-nullifiers --prefixes 10 - cleanup-run-cache: - name: cleanup run cache - runs-on: ubuntu-24.04 - if: ${{ always() && github.event_name == 'pull_request' }} - needs: - - build - - clippy - - tests - - doc - - stress-test - permissions: - actions: write - contents: read - steps: - - name: Delete PR rust cache - env: - GH_TOKEN: ${{ github.token }} - run: | - mapfile -t cache_entries < <( - gh api \ - -H "Accept: application/vnd.github+json" \ - "/repos/${{ github.repository }}/actions/caches?key=${{ env.RUST_CACHE_PREFIX }}&ref=${{ github.ref }}" \ - --jq '.actions_caches[] | @json' - ) - for cache_entry in "${cache_entries[@]}"; do - cache_id="$(jq -r '.id' <<< "${cache_entry}")" - cache_key="$(jq -r '.key' <<< "${cache_entry}")" - echo "Deleting rust cache key=${cache_key}" - gh api \ - --method DELETE \ - -H "Accept: application/vnd.github+json" \ - "/repos/${{ github.repository }}/actions/caches/${cache_id}" - done - # =============================================================================================== # WASM related jobs # =============================================================================================== # Tests the miden-remote-prover-client WASM support. # - # The WASM build is incompatible with the build job's cache, thankfully this compilation is fairly - # quick so we don't need a separate cache here. + # The WASM build is incompatible with native build caches, so it uses a dedicated cache. client-wasm: name: wasm targets runs-on: warp-ubuntu-latest-x64-8x @@ -284,6 +218,11 @@ jobs: persist-credentials: false - name: Rustup run: rustup toolchain install --no-self-update + - uses: WarpBuilds/rust-cache@9d0cc3090d9c87de74ea67617b246e978735b1a1 # v2.9.1 + with: + shared-key: ${{ github.job }} + prefix-key: ${{ env.CACHE_PREFIX }} + save-if: ${{ env.SAVE_CACHE }} - name: cargo build run: | cargo build --locked -p miden-remote-prover-client \