clients-cpp-python-ubuntu2404.yml

name: "Build C++ and Python Clients on Ubuntu 24.04"
on:
  pull_request:
    branches: [ 'main', 'rc/v*' ]
  push:
    branches: [ 'main', 'check/**', 'release/v*' ]
permissions:
  packages: write

#===============================================================================
# Background: in this workflow there are three different kinds of caching going
# on:
#
# 1. The outputs are Github Build artifacts, attached to every action run.
#    These artifacts can be downloaded from e.g.
#    https://github.com/deephaven/deephaven-core/actions/runs/17416308139
#    but the specific URL depends on the build.
#    We use this to upload:
#    - The C++ install directory
#    - The .whl file for the static Python client
#    - The .whl file for the ticking Python client
#
# 2. The C++ install directory is cached via actions/cache, and keyed
#    by the hash of the cpp-client and proto directories. This allows us
#    to skip recompilation if these files don't change. This is an optional
#    optimization that might not be very significant.
#
# 3. Github Packages, which is a NuGet-compatible repo accessible from
#    https://github.com/orgs/deephaven/packages?repo_name=deephaven-core
#    and controlled via the NuGet API. We use this package repository to hold
#    the binaries for the C++ dependencies e build (grpc, arrow, etc.). It is
#    managed automatically for us by cmake/vcpkg/nuget. This optimization is
#    important because building the C++ dependencies takes significant time
#    (45 minutes maybe).
#===============================================================================

env:
  GH_PACKAGES_USERNAME: ${{ github.repository_owner }}
  GH_PACKAGES_FEED: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json
  DHINSTALL: ${{ github.workspace }}/dhinstall

jobs:
  #=============================================================================
  # Identify version of Deephaven Core
  #=============================================================================
  version_job:
    uses: ./.github/workflows/dhc-version.yml

  #=============================================================================
  # Build the C++ Client
  #=============================================================================
  cpp_job:
    name: Build C++ client on Ubuntu or fetch from cache
    runs-on: ubuntu-24.04
    defaults:
      run:
        shell: bash

    steps:
      - name: Check out this repo
        uses: actions/checkout@v6

      # Try to fetch the whole installation directory (the final product of the
      # build) from the Github Cache. The key to this cache is derived
      # from the hash of the contents of the files in the cpp-client and proto
      # directories. If this fetch is successful, we skip over all the build
      # work and just copy this cached install directory to the output artifact
      # for this run.

      - name: Restore the cached installation directory, or prepare a new one
        id: cache-cpp-install
        uses: actions/cache@v5
        with:
          path: ${{ env.DHINSTALL }}
          key: ${{ runner.os }}-cpp-client-${{ hashFiles('cpp-client/**', 'proto/**') }}

      # [Note: skip this step if cache-cpp-install was successful]
      # Check out vcpkg

      - name: Check out vcpkg
        id: checkout-vcpkg
        if: steps.cache-cpp-install.outputs.cache-hit != 'true'
        uses: actions/checkout@v6
        with:
          repository: microsoft/vcpkg
          path: vcpkg

      # [Note: skip this step if cache-cpp-install was successful]
      # Bootstrap vcpkg

      - name: Bootstrap vcpkg
        if: steps.cache-cpp-install.outputs.cache-hit != 'true'
        run: ${{ github.workspace }}/vcpkg/bootstrap-vcpkg.sh

      # [Note: skip this step if cache-cpp-install was successful]
      # Get mono so that we can run nuget. This is because the nuget.exe that
      # ships with vcpkg is an old .NET Framework binary (Windows format), and
      # only mono knows how to run such a thing on Linux. Some day the maintainers
      # will fix this so it runs with the cross-platform "dotnet".

      - name: Get mono so that we can run nuget
        if: steps.cache-cpp-install.outputs.cache-hit != 'true'
        run: sudo apt install mono-complete

      # [Note: skip this step if cache-cpp-install was successful]
      # Adapted from
      # https://learn.microsoft.com/en-us/vcpkg/consume/binary-caching-github-packages?pivots=linux-runner

      # [Note: skip this step if cache-cpp-install was successful]
      - name: "Configure nuget to know how to download (or upload) each cached package that we need (or create)"
        if: steps.cache-cpp-install.outputs.cache-hit != 'true'
        run: |
          vcpkg_exe=${{ github.workspace }}/vcpkg/vcpkg
          nuget_dotnet=$($vcpkg_exe fetch nuget | tail -n 1)
          mono $nuget_dotnet \
            sources add \
            -Source "${{ env.GH_PACKAGES_FEED }}" \
            -StorePasswordInClearText \
            -Name GitHubPackages \
            -UserName "${{ env.GH_PACKAGES_USERNAME }}" \
            -Password "${{ secrets.GITHUB_TOKEN }}"
          mono $nuget_dotnet \
            setapikey "${{ secrets.GITHUB_TOKEN }}" \
            -Source "${{ env.GH_PACKAGES_FEED }}"

      # [Note: skip this step if cache-cpp-install was successful]
      # 1. Invoke vcpkg to:
      #    a) Try to fetch each of the dependent libraries from the nuget cache
      #    b) If not found, build them and insert them in the nuget cache
      # 2. Configure cmake to prepare for the build/install step, which comes next

      - name: build packages and configure cmake. Cached packages will be found, or stored in github's nuget repo
        if: steps.cache-cpp-install.outputs.cache-hit != 'true'
        env:
          VCPKG_BINARY_SOURCES: "clear;nuget,${{ env.GH_PACKAGES_FEED }},readwrite"
        run: cmake -S cpp-client/deephaven -B cpp-client/deephaven/build --toolchain ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_INSTALL_PREFIX=${{ env.DHINSTALL }} -DVCPKG_TARGET_TRIPLET=x64-linux-dynamic-release -DVCPKG_OVERLAY_TRIPLETS=cpp-client/deephaven/custom-triplets

      # [Note: skip this step if cache-cpp-install was successful]
      # Build our executables and install them in the configured install directory {{ env.DHINSTALL }}

      - name: Run cmake build and install
        if: steps.cache-cpp-install.outputs.cache-hit != 'true'
        run: cmake --build cpp-client/deephaven/build --config RelWithDebInfo --target install --parallel

      # Upload results as build artifact. Note: if cache-cpp-install was
      # then none of the intervening steps were executed. In that case,
      # this effectively just copies the cache to the build artifact.

      - name: Upload install directory as build artifact
        id: upload-install-directory
        uses: actions/upload-artifact@v7
        with:
          name: cpp-client-install
          path: ${{ env.DHINSTALL }}

  #=============================================================================
  # Build the Python Static Client
  #=============================================================================

  python_static_job:
    name: Build Python static client on Ubuntu
    runs-on: ubuntu-24.04
    needs: [version_job]
    defaults:
      run:
        shell: bash

    steps:
      - name: Check out this repo
        uses: actions/checkout@v6

      - name: Install Python
        uses: actions/setup-python@v6
        with:
          python-version: '3.10'

      - name: Install the 'wheel' package
        run: pip3 install wheel

      - name: Install requirements-dev.txt
        run: |
          cd ./py/client
          pip3 install -r requirements-dev.txt

      - name: Run setup.py
        env:
          DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }}
        run: |
          cd ./py/client
          python setup.py bdist_wheel

      - name: Upload static .whl file as build artifact, to be used directly or by dependent clients like Python ticking
        uses: actions/upload-artifact@v7
        with:
          name: py-static-wheel
          path: ./py/client/dist/*.whl

  #=============================================================================
  # Build the Python Ticking Client
  #=============================================================================

  python_ticking_job:
    name: Build Python ticking client on Ubuntu
    runs-on: ubuntu-24.04
    needs: [cpp_job, python_static_job, version_job]
    defaults:
      run:
        shell: bash

    steps:
      - name: Check out this repo
        uses: actions/checkout@v6

      - name: Get the C++ installation as a build artifact
        uses: actions/download-artifact@v8
        with:
          name: cpp-client-install
          path: ${{ env.DHINSTALL }}

      - name: Get the Python static installation as a build artifact
        uses: actions/download-artifact@v8
        with:
          name: py-static-wheel
          path: ./py/client/dist/

      - name: Install Python
        uses: actions/setup-python@v6
        with:
          python-version: '3.10'

      - name: Install the 'wheel' and 'cython' packages
        run: pip3 install wheel cython

      - name: Install requirements-dev.txt
        run: |
          cd ./py/client
          pip3 install -r requirements-dev.txt

      - name: Install the static client
        run: pip3 install --force --no-deps ./py/client/dist/*.whl

      - name: Build cython code
        env:
          DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }}
          CPPFLAGS: -I${{ env.DHINSTALL }}/include
          LDFLAGS: -L${{ env.DHINSTALL }}/lib
        run: |
          cd ./py/client-ticking
          python setup.py build_ext -i

      - name: Run setup.py
        env:
          DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }}
        run: |
          cd ./py/client-ticking
          python setup.py bdist_wheel

      - name: Upload ticking .whl file as build artifact
        uses: actions/upload-artifact@v7
        with:
          name: py-ticking-wheel
          path: ./py/client-ticking/dist/*.whl