Include graph runtime benchmarks in CI perf regression runs #3745
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Profiling Changes | |
| on: | |
| pull_request: | |
| env: | |
| CARGO_TERM_COLOR: always | |
| jobs: | |
| profile: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Install Valgrind | |
| run: | | |
| sudo apt update | |
| sudo apt install -y valgrind | |
| - name: Cache Rust dependencies | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| # Cache on Cargo.lock file | |
| cache-on-failure: true | |
| - name: Cache iai-callgrind binary | |
| id: cache-iai | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cargo/bin/iai-callgrind-runner | |
| key: ${{ runner.os }}-iai-callgrind-runner-0.12.3 | |
| - name: Install iai-callgrind | |
| if: steps.cache-iai.outputs.cache-hit != 'true' | |
| run: | | |
| cargo install iai-callgrind-runner@0.12.3 | |
| - name: Checkout master branch | |
| run: | | |
| git fetch origin master:master | |
| git checkout master | |
| - name: Get master commit SHA | |
| id: master-sha | |
| run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT | |
| - name: Cache benchmark baselines | |
| id: cache-benchmark-baselines | |
| uses: actions/cache@v4 | |
| with: | |
| path: target/iai | |
| key: ${{ runner.os }}-benchmark-baselines-master-${{ steps.master-sha.outputs.sha }} | |
| restore-keys: | | |
| ${{ runner.os }}-benchmark-baselines-master- | |
| - name: Run baseline benchmarks | |
| if: steps.cache-benchmark-baselines.outputs.cache-hit != 'true' | |
| run: | | |
| # Compile benchmarks | |
| cargo bench --bench compile_demo_art_iai -- --save-baseline=master | |
| # Runtime benchmarks | |
| cargo bench --bench update_executor_iai -- --save-baseline=master | |
| cargo bench --bench run_once_iai -- --save-baseline=master | |
| cargo bench --bench run_cached_iai -- --save-baseline=master | |
| - name: Checkout PR branch | |
| run: | | |
| git checkout ${{ github.event.pull_request.head.sha }} | |
| - name: Run PR benchmarks | |
| id: benchmark | |
| run: | | |
| # Compile benchmarks | |
| COMPILE_OUTPUT=$(cargo bench --bench compile_demo_art_iai -- --baseline=master --output-format=json | jq -sc | sed 's/\\"//g') | |
| # Runtime benchmarks | |
| UPDATE_OUTPUT=$(cargo bench --bench update_executor_iai -- --baseline=master --output-format=json | jq -sc | sed 's/\\"//g') | |
| RUN_ONCE_OUTPUT=$(cargo bench --bench run_once_iai -- --baseline=master --output-format=json | jq -sc | sed 's/\\"//g') | |
| RUN_CACHED_OUTPUT=$(cargo bench --bench run_cached_iai -- --baseline=master --output-format=json | jq -sc | sed 's/\\"//g') | |
| # Store outputs | |
| echo "COMPILE_OUTPUT<<EOF" >> $GITHUB_OUTPUT | |
| echo "$COMPILE_OUTPUT" >> $GITHUB_OUTPUT | |
| echo "EOF" >> $GITHUB_OUTPUT | |
| echo "UPDATE_OUTPUT<<EOF" >> $GITHUB_OUTPUT | |
| echo "$UPDATE_OUTPUT" >> $GITHUB_OUTPUT | |
| echo "EOF" >> $GITHUB_OUTPUT | |
| echo "RUN_ONCE_OUTPUT<<EOF" >> $GITHUB_OUTPUT | |
| echo "$RUN_ONCE_OUTPUT" >> $GITHUB_OUTPUT | |
| echo "EOF" >> $GITHUB_OUTPUT | |
| echo "RUN_CACHED_OUTPUT<<EOF" >> $GITHUB_OUTPUT | |
| echo "$RUN_CACHED_OUTPUT" >> $GITHUB_OUTPUT | |
| echo "EOF" >> $GITHUB_OUTPUT | |
| - name: Make old comments collapsed by default | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{secrets.GITHUB_TOKEN}} | |
| script: | | |
| const { data: comments } = await github.rest.issues.listComments({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| }); | |
| const botComments = comments.filter((comment) => | |
| comment.user.type === 'Bot' && comment.body.includes('Performance Benchmark Results') && comment.body.includes('<details open>') | |
| ); | |
| for (const comment of botComments) { | |
| // Edit the comment to remove the "open" attribute from the <details> tag | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: comment.id, | |
| body: comment.body.replace('<details open>', '<details>') | |
| }); | |
| } | |
| - name: Comment PR | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{secrets.GITHUB_TOKEN}} | |
| script: | | |
| const compileOutput = JSON.parse(`${{ steps.benchmark.outputs.COMPILE_OUTPUT }}`); | |
| const updateOutput = JSON.parse(`${{ steps.benchmark.outputs.UPDATE_OUTPUT }}`); | |
| const runOnceOutput = JSON.parse(`${{ steps.benchmark.outputs.RUN_ONCE_OUTPUT }}`); | |
| const runCachedOutput = JSON.parse(`${{ steps.benchmark.outputs.RUN_CACHED_OUTPUT }}`); | |
| let significantChanges = false; | |
| let commentBody = ""; | |
| function formatNumber(num) { | |
| return num.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ","); | |
| } | |
| function formatPercentage(pct) { | |
| const sign = pct >= 0 ? '+' : ''; | |
| return `${sign}${pct.toFixed(2)}%`; | |
| } | |
| function padRight(str, len) { | |
| return str.padEnd(len); | |
| } | |
| function padLeft(str, len) { | |
| return str.padStart(len); | |
| } | |
| function processBenchmarkOutput(benchmarkOutput, sectionTitle) { | |
| let sectionBody = ""; | |
| let hasResults = false; | |
| let hasSignificantChanges = false; | |
| for (const benchmark of benchmarkOutput) { | |
| if (benchmark.callgrind_summary && benchmark.callgrind_summary.summaries) { | |
| const summary = benchmark.callgrind_summary.summaries[0]; | |
| const irDiff = summary.events.Ir; | |
| if (irDiff.diff_pct !== null) { | |
| hasResults = true; | |
| const changePercentage = formatPercentage(irDiff.diff_pct); | |
| const color = irDiff.diff_pct > 0 ? "red" : "lime"; | |
| sectionBody += `**${benchmark.module_path} ${benchmark.id}:${benchmark.details}**\n`; | |
| sectionBody += `Instructions: \`${formatNumber(irDiff.old)}\` (master) → \`${formatNumber(irDiff.new)}\` (HEAD) : `; | |
| sectionBody += `$$\\color{${color}}${changePercentage.replace("%", "\\\\%")}$$\n\n`; | |
| sectionBody += "<details>\n<summary>Detailed metrics</summary>\n\n```\n"; | |
| sectionBody += `Baselines: master| HEAD\n`; | |
| for (const [eventKind, costsDiff] of Object.entries(summary.events)) { | |
| if (costsDiff.diff_pct !== null) { | |
| const changePercentage = formatPercentage(costsDiff.diff_pct); | |
| const line = `${padRight(eventKind, 20)} ${padLeft(formatNumber(costsDiff.old), 11)}|${padLeft(formatNumber(costsDiff.new), 11)} ${padLeft(changePercentage, 15)}`; | |
| sectionBody += `${line}\n`; | |
| } | |
| } | |
| sectionBody += "```\n</details>\n\n"; | |
| if (Math.abs(irDiff.diff_pct) > 5) { | |
| significantChanges = true; | |
| hasSignificantChanges = true; | |
| } | |
| } | |
| } | |
| } | |
| if (hasResults) { | |
| // Wrap section in collapsible details, open only if there are significant changes | |
| const openAttribute = hasSignificantChanges ? " open" : ""; | |
| return `<details${openAttribute}>\n<summary><h2>${sectionTitle}</h2></summary>\n\n${sectionBody}\n---\n</details>`; | |
| } | |
| return ""; | |
| } | |
| // Process each benchmark category | |
| const compileSection = processBenchmarkOutput(compileOutput, "🔧 Graph Compile Time"); | |
| const updateSection = processBenchmarkOutput(updateOutput, "🔄 Executor Update"); | |
| const runOnceSection = processBenchmarkOutput(runOnceOutput, "🚀 Render: Cold Execution"); | |
| const runCachedSection = processBenchmarkOutput(runCachedOutput, "⚡ Render: Cached Execution"); | |
| // Combine all sections | |
| commentBody = [compileSection, updateSection, runOnceSection, runCachedSection] | |
| .filter(section => section.length > 0) | |
| .join("\n\n"); | |
| if (commentBody.length > 0) { | |
| const output = ` | |
| <details open> | |
| <summary>Performance Benchmark Results</summary> | |
| ${commentBody} | |
| </details> | |
| `; | |
| if (significantChanges) { | |
| github.rest.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: output | |
| }); | |
| } else { | |
| console.log("No significant performance changes detected. Skipping comment."); | |
| console.log(output); | |
| } | |
| } else { | |
| console.log("No benchmark results to display."); | |
| } |