WIP: populate cache. #2
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Build C++ and Python Clients on Ubuntu 24.04" | |
| on: | |
| pull_request: | |
| branches: [ 'main', 'rc/v*' ] | |
| push: | |
| branches: [ 'main', 'check/**', 'release/v*' ] | |
| permissions: | |
| packages: write | |
| #=============================================================================== | |
| # Background: in this workflow there are three different kinds of caching going | |
| # on: | |
| # | |
| # 1. The outputs are Github Build artifacts, attached to every action run. | |
| # These artifacts can be downloaded from e.g. | |
| # https://github.com/deephaven/deephaven-core/actions/runs/17416308139 | |
| # but the specific URL depends on the build. | |
| # We use this to upload: | |
| # - The C++ install directory | |
| # - The .whl file for the static Python client | |
| # - The .whl file for the ticking Python client | |
| # | |
| # 2. The C++ install directory is cached via actions/cache, and keyed | |
| # by the hash of the cpp-client and proto directories. This allows us | |
| # to skip recompilation if these files don't change. This is an optional | |
| # optimization that might not be very significant. | |
| # | |
| # 3. Github Packages, which is a NuGet-compatible repo accessible from | |
| # https://github.com/orgs/deephaven/packages?repo_name=deephaven-core | |
| # and controlled via the NuGet API. We use this package repository to hold | |
| # the binaries for the C++ dependencies e build (grpc, arrow, etc.). It is | |
| # managed automatically for us by cmake/vcpkg/nuget. This optimization is | |
| # important because building the C++ dependencies takes significant time | |
| # (45 minutes maybe). | |
| #=============================================================================== | |
| env: | |
| GH_PACKAGES_USERNAME: ${{ github.repository_owner }} | |
| GH_PACKAGES_FEED: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json | |
| DHINSTALL: ${{ github.workspace }}/dhinstall | |
| jobs: | |
| #============================================================================= | |
| # Identify version of Deephaven Core | |
| #============================================================================= | |
| version_job: | |
| uses: ./.github/workflows/dhc-version.yml | |
| #============================================================================= | |
| # Build the C++ Client | |
| #============================================================================= | |
| cpp_job: | |
| name: Build C++ client on Ubuntu or fetch from cache | |
| runs-on: ubuntu-24.04 | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Check out this repo | |
| uses: actions/checkout@v6 | |
| # Try to fetch the whole installation directory (the final product of the | |
| # build) from the Github Cache. The key to this cache is derived | |
| # from the hash of the contents of the files in the cpp-client and proto | |
| # directories. If this fetch is successful, we skip over all the build | |
| # work and just copy this cached install directory to the output artifact | |
| # for this run. | |
| - name: Restore the cached installation directory, or prepare a new one | |
| id: cache-cpp-install | |
| uses: actions/cache@v5 | |
| with: | |
| path: ${{ env.DHINSTALL }} | |
| key: ${{ runner.os }}-cpp-client-${{ hashFiles('cpp-client/**', 'proto/**') }} | |
| # [Note: skip this step if cache-cpp-install was successful] | |
| # Check out vcpkg | |
| - name: Check out vcpkg | |
| id: checkout-vcpkg | |
| if: steps.cache-cpp-install.outputs.cache-hit != 'true' | |
| uses: actions/checkout@v6 | |
| with: | |
| repository: microsoft/vcpkg | |
| path: vcpkg | |
| # [Note: skip this step if cache-cpp-install was successful] | |
| # Bootstrap vcpkg | |
| - name: Bootstrap vcpkg | |
| if: steps.cache-cpp-install.outputs.cache-hit != 'true' | |
| run: ${{ github.workspace }}/vcpkg/bootstrap-vcpkg.sh | |
| # [Note: skip this step if cache-cpp-install was successful] | |
| # Get mono so that we can run nuget. This is because the nuget.exe that | |
| # ships with vcpkg is an old .NET Framework binary (Windows format), and | |
| # only mono knows how to run such a thing on Linux. Some day the maintainers | |
| # will fix this so it runs with the cross-platform "dotnet". | |
| - name: Get mono so that we can run nuget | |
| if: steps.cache-cpp-install.outputs.cache-hit != 'true' | |
| run: sudo apt install mono-complete | |
| # [Note: skip this step if cache-cpp-install was successful] | |
| # Adapted from | |
| # https://learn.microsoft.com/en-us/vcpkg/consume/binary-caching-github-packages?pivots=linux-runner | |
| # [Note: skip this step if cache-cpp-install was successful] | |
| - name: "Configure nuget to know how to download (or upload) each cached package that we need (or create)" | |
| if: steps.cache-cpp-install.outputs.cache-hit != 'true' | |
| run: | | |
| vcpkg_exe=${{ github.workspace }}/vcpkg/vcpkg | |
| nuget_dotnet=$($vcpkg_exe fetch nuget | tail -n 1) | |
| mono $nuget_dotnet \ | |
| sources add \ | |
| -Source "${{ env.GH_PACKAGES_FEED }}" \ | |
| -StorePasswordInClearText \ | |
| -Name GitHubPackages \ | |
| -UserName "${{ env.GH_PACKAGES_USERNAME }}" \ | |
| -Password "${{ secrets.GITHUB_TOKEN }}" | |
| mono $nuget_dotnet \ | |
| setapikey "${{ secrets.GITHUB_TOKEN }}" \ | |
| -Source "${{ env.GH_PACKAGES_FEED }}" | |
| # [Note: skip this step if cache-cpp-install was successful] | |
| # 1. Invoke vcpkg to: | |
| # a) Try to fetch each of the dependent libraries from the nuget cache | |
| # b) If not found, build them and insert them in the nuget cache | |
| # 2. Configure cmake to prepare for the build/install step, which comes next | |
| - name: build packages and configure cmake. Cached packages will be found, or stored in github's nuget repo | |
| if: steps.cache-cpp-install.outputs.cache-hit != 'true' | |
| env: | |
| VCPKG_BINARY_SOURCES: "clear;nuget,${{ env.GH_PACKAGES_FEED }},readwrite" | |
| run: cmake -S cpp-client/deephaven -B cpp-client/deephaven/build --toolchain ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_INSTALL_PREFIX=${{ env.DHINSTALL }} -DVCPKG_TARGET_TRIPLET=x64-linux-dynamic-release -DVCPKG_OVERLAY_TRIPLETS=cpp-client/deephaven/custom-triplets | |
| # [Note: skip this step if cache-cpp-install was successful] | |
| # Build our executables and install them in the configured install directory {{ env.DHINSTALL }} | |
| - name: Run cmake build and install | |
| if: steps.cache-cpp-install.outputs.cache-hit != 'true' | |
| run: cmake --build cpp-client/deephaven/build --config RelWithDebInfo --target install --parallel | |
| # Upload results as build artifact. Note: if cache-cpp-install was | |
| # then none of the intervening steps were executed. In that case, | |
| # this effectively just copies the cache to the build artifact. | |
| - name: Upload install directory as build artifact | |
| id: upload-install-directory | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: cpp-client-install | |
| path: ${{ env.DHINSTALL }} | |
| #============================================================================= | |
| # Build the Python Static Client | |
| #============================================================================= | |
| python_static_job: | |
| name: Build Python static client on Ubuntu | |
| runs-on: ubuntu-24.04 | |
| needs: [version_job] | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Check out this repo | |
| uses: actions/checkout@v6 | |
| - name: Install Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.10' | |
| - name: Install the 'wheel' package | |
| run: pip3 install wheel | |
| - name: Install requirements-dev.txt | |
| run: | | |
| cd ./py/client | |
| pip3 install -r requirements-dev.txt | |
| - name: Run setup.py | |
| env: | |
| DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} | |
| run: | | |
| cd ./py/client | |
| python setup.py bdist_wheel | |
| - name: Upload static .whl file as build artifact, to be used directly or by dependent clients like Python ticking | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: py-static-wheel | |
| path: ./py/client/dist/*.whl | |
| #============================================================================= | |
| # Build the Python Ticking Client | |
| #============================================================================= | |
| python_ticking_job: | |
| name: Build Python ticking client on Ubuntu | |
| runs-on: ubuntu-24.04 | |
| needs: [cpp_job, python_static_job, version_job] | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Check out this repo | |
| uses: actions/checkout@v6 | |
| - name: Get the C++ installation as a build artifact | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: cpp-client-install | |
| path: ${{ env.DHINSTALL }} | |
| - name: Get the Python static installation as a build artifact | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: py-static-wheel | |
| path: ./py/client/dist/ | |
| - name: Install Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.10' | |
| - name: Install the 'wheel' and 'cython' packages | |
| run: pip3 install wheel cython | |
| - name: Install requirements-dev.txt | |
| run: | | |
| cd ./py/client | |
| pip3 install -r requirements-dev.txt | |
| - name: Install the static client | |
| run: pip3 install --force --no-deps ./py/client/dist/*.whl | |
| - name: Build cython code | |
| env: | |
| DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} | |
| CPPFLAGS: -I${{ env.DHINSTALL }}/include | |
| LDFLAGS: -L${{ env.DHINSTALL }}/lib | |
| run: | | |
| cd ./py/client-ticking | |
| python setup.py build_ext -i | |
| - name: Run setup.py | |
| env: | |
| DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} | |
| run: | | |
| cd ./py/client-ticking | |
| python setup.py bdist_wheel | |
| - name: Upload ticking .whl file as build artifact | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: py-ticking-wheel | |
| path: ./py/client-ticking/dist/*.whl |