vortex/.github/workflows/ci-python.yml at cd2fbee7eba70be5281e5e5f27d33f4f844fcf62 · vortex-data/vortex · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
name: CI / Python

# Concurrency control:
# - PRs: new commits on a feature branch will cancel in-progress (outdated) runs.
# - Push to develop: runs queue sequentially, never cancelled.
# - `workflow_dispatch`: groups by branch and queues if run on develop.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }}
on:
  push:
    branches: [develop]
  pull_request: { }
  workflow_dispatch: { }

permissions:
  actions: read
  contents: read

env:
  CARGO_TERM_COLOR: auto
  RUST_BACKTRACE: 1

jobs:
  python-lint:
    name: "Python (lint)"
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=python-lint', github.run_id)
          || 'ubuntu-latest' }}
    timeout-minutes: 40
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@v6
      - uses: ./.github/actions/setup-prebuild
      # Use uvx for ruff to avoid building the Rust extension (saves ~4.5 min)
      - name: Python Lint - Format
        run: uvx ruff format --check .
      - name: Python Lint - Ruff
        run: uvx ruff check .
      # PyRight needs the project for type information, so use uv run
      - name: Python Lint - PyRight
        env:
          MATURIN_PEP517_ARGS: "--profile dev"
        run: uv run basedpyright vortex-python

  python-test:
    name: "Python (test)"
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/tag=python-test', github.run_id)
          || 'ubuntu-latest' }}
    timeout-minutes: 40
    env:
      RUST_LOG: "info,uv=debug"
      MATURIN_PEP517_ARGS: "--profile dev"
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@v6
      - uses: ./.github/actions/setup-prebuild

      - name: Pytest - Vortex
        run: |
          uv run --all-packages pytest --benchmark-disable -n auto test/
        working-directory: vortex-python/

      - name: Setup benchmark environment
        run: sudo bash scripts/setup-benchmark.sh

      - name: Pytest Benchmarks - Vortex
        run: |
          bash ../scripts/bench-taskset.sh uv run --all-packages pytest --benchmark-only benchmark/
        working-directory: vortex-python/

      - name: Doctest - PyVortex
        run: |
          uv run --all-packages make doctest
        working-directory: docs/

      - name: Ensure docs build - PyVortex
        run: |
          uv run --all-packages make html
        working-directory: docs/

  python-wheel-build:
    name: "Python (wheel build)"
    runs-on: ubuntu-latest
    timeout-minutes: 40
    steps:
      - uses: actions/checkout@v6
      - name: Rust Dependency Cache
        uses: Swatinem/rust-cache@v2
        with:
          save-if: ${{ github.ref_name == 'develop' }}
      - uses: ./.github/actions/setup-rust
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
          enable-sccache: "false"
      - uses: mlugg/setup-zig@v2.2.1
      - name: Install uv
        uses: spiraldb/actions/.github/actions/setup-uv@0.18.5
        with:
          sync: false
          prune-cache: false
      - name: Ensure wheel and sdist can be built on Linux - PyVortex
        shell: bash
        run: |
          echo "Clearing wheel target directory"
          rm -rf ../target/wheels/

          uv venv
          uv tool run maturin@1.10 build --interpreter python3.11 --zig
          uv tool run maturin@1.10 build --interpreter python3.11 --zig --sdist

          file_count=$(ls -1 ../target/wheels/ | wc -l)

          if [[ $file_count -ne 2 ]]; then
            echo "Unexpected number of files detected ${file_count}:"
            ls ../target/wheels/
            exit 1
          else
            echo "Generated two files"
          fi
        working-directory: vortex-python/