feat(cpp/client, py/client, csharp/client): DH-22472, DH-22473, DH-22475: Github Actions to build (cpp-client, python, csharp) on (Ubuntu 24.04, Windows)

.github/workflows/clients-cpp-python.yml

@@ -0,0 +1,321 @@
+name: "C++/Python"
+on:
+  pull_request:
+    branches: [ 'main', 'rc/v*' ]
+  push:
+    branches: [ 'main', 'check/**', 'release/v*' ]
+permissions:
+  contents: read
+
+#===============================================================================
+# Background: in this workflow there are three different kinds of caching going
+# on:
+#
+# 1. The outputs are Github Build artifacts, attached to every action run.
+#    These artifacts can be downloaded from e.g.
+#    https://github.com/deephaven/deephaven-core/actions/runs/17416308139
+#    but the specific URL depends on the build.
+#    We use this to upload:
+#    - The C++ install directory
+#    - The .whl file for the static Python client
+#    - The .whl file for the ticking Python client
+#
+# 2. The C++ install directory is cached via actions/cache, and keyed
+#    by the hash of the cpp-client and proto directories. This allows us
+#    to skip recompilation if these files don't change. This is an optional
+#    optimization that might not be very significant.
+#
+# 3. Github Packages, which is a NuGet-compatible repo accessible from
+#    https://github.com/orgs/deephaven/packages?repo_name=deephaven-core
+#    and controlled via the NuGet API. We use this package repository to hold
+#    the binaries for the C++ dependencies e build (grpc, arrow, etc.). It is
+#    managed automatically for us by cmake/vcpkg/nuget. This optimization is
+#    important because building the C++ dependencies takes significant time
+#    (45 minutes maybe).
+#===============================================================================
+
+env:
+  GH_PACKAGES_USERNAME: ${{ github.repository_owner }}
+  GH_PACKAGES_FEED: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json
+  DHINSTALL: ${{ github.workspace }}/dhinstall
+
+jobs:
+  #=============================================================================
+  # Identify version of Deephaven Core
+  #=============================================================================
+  version_job:
+    uses: ./.github/workflows/dhc-version.yml
+
+  #=============================================================================
+  # Build the C++ Client
+  #=============================================================================
+  cpp_job:
+    name: "Build C++ (or use cache)"
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-24.04
+            target_triplet: x64-linux-dynamic-release
+          - os: windows-2025-vs2026
+            target_triplet: x64-windows-release
+      fail-fast: false
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        # This gets you Git Bash on Windows
+        shell: bash
+
+    steps:
+      - name: Check out this repo
+        uses: actions/checkout@v6
+
+      # Try to fetch the whole installation directory (the final product of the
+      # build) from the Github Cache. The key to this cache is derived
+      # from the hash of the contents of the files in the cpp-client and proto
+      # directories. If this fetch is successful, we skip over all the build
+      # work and just copy this cached install directory to the output artifact
+      # for this run.
+
+      - name: Try to restore cached installdir
+        id: cache-cpp-install
+        uses: actions/cache@v5
+        with:
+          path: ${{ env.DHINSTALL }}
+          key: ${{ runner.os }}-cpp-client-${{ hashFiles('cpp-client/**', 'proto/**') }}
+
+      # Make sure we check out the same version of the vcpkg tool that we
+      # have configured as our registry baseline. This (we suppose) will help
+      # improve the problem we have with our package hash drift.
+
+      - name: Extract vcpkg baseline
+        id: vcpkg-baseline
+        run: |
+          baseline=$(jq -r '.["default-registry"].baseline' ./cpp-client/deephaven/vcpkg-configuration.json)
+          echo "baseline=$baseline"
+          echo "baseline=$baseline" >> "$GITHUB_OUTPUT"
+
+      # [Note: skip this step if cache-cpp-install was successful]
+      # Check out vcpkg
+
+      - name: Check out vcpkg
+        id: checkout-vcpkg
+        if: steps.cache-cpp-install.outputs.cache-hit != 'true'
+        uses: actions/checkout@v6
+        with:
+          repository: microsoft/vcpkg
+          ref: ${{ steps.vcpkg-baseline.outputs.baseline }}
+          path: vcpkg
+
+      # [Note: skip this step if cache-cpp-install was successful]
+      # Bootstrap vcpkg
+
+      - name: Bootstrap vcpkg
+        if: steps.cache-cpp-install.outputs.cache-hit != 'true'
+        run: "$GITHUB_WORKSPACE/vcpkg/bootstrap-vcpkg.sh"
+
+      # [Note: Linux only]
+      # [Note: skip this step if cache-cpp-install was successful]
+      # Get mono so that we can run nuget. This is because the nuget.exe that
+      # ships with vcpkg is an old .NET Framework binary (Windows format), and
+      # only mono knows how to run such a thing on Linux. Some day the
+      # maintainers will fix this so it runs with the cross-platform "dotnet".
+
+      - name: Install mono (Linux)
+        if: runner.os == 'Linux' && steps.cache-cpp-install.outputs.cache-hit != 'true'
+        run: sudo apt install -y mono-complete
+
+      # [Note: skip this step if cache-cpp-install was successful]
+      # Adapted from
+      # https://learn.microsoft.com/en-us/vcpkg/consume/binary-caching-github-packages?pivots=linux-runner
+      # Configure nuget to know how to download (or upload) each cached
+      # package that we need (or create)
+
+      - name: "Configure nuget (Linux)"
+        if: runner.os == 'Linux' && steps.cache-cpp-install.outputs.cache-hit != 'true'
+        run: |
+          vcpkg_exe=${{ github.workspace }}/vcpkg/vcpkg
+          nuget_dotnet=$($vcpkg_exe fetch nuget | tail -n 1)
+          mono $nuget_dotnet \
+            sources add \
+            -Source "${{ env.GH_PACKAGES_FEED }}" \
+            -StorePasswordInClearText \
+            -Name GitHubPackages \
+            -UserName "${{ env.GH_PACKAGES_USERNAME }}" \
+            -Password "${{ secrets.GITHUB_TOKEN }}"
+          mono $nuget_dotnet \
+            setapikey "${{ secrets.GITHUB_TOKEN }}" \
+            -Source "${{ env.GH_PACKAGES_FEED }}"
+
+      - name: "Configure nuget (Windows)"
+        if: runner.os == 'Windows' && steps.cache-cpp-install.outputs.cache-hit != 'true'
+        shell: pwsh
+        run: |
+          $nuget_exe = & ./vcpkg/vcpkg.exe fetch nuget
+          & $nuget_exe sources add `
+          -Source "${{ env.GH_PACKAGES_FEED }}" `
+          -StorePasswordInClearText `
+          -Name GitHubPackages `
+          -UserName "${{ env.GH_PACKAGES_USERNAME }}" `
+          -Password "${{ secrets.GITHUB_TOKEN }}"
+          & $nuget_exe setapikey "${{ secrets.GITHUB_TOKEN }}" -Source "${{ env.GH_PACKAGES_FEED }}"
+
+      # [Note: skip this step if cache-cpp-install was successful]
+      # Build packages and configure cmake. Check nupkg cache for existing
+      # packages, otherwise build and upload.
+      # 1. Invoke vcpkg to:
+      #    a) Try to fetch each of the dependent libraries from the nuget cache
+      #    b) If not found, build them and insert them in the nuget cache
+      # 2. Configure cmake to prepare for the build/install step, which comes next
+
+      - name: "Build/cache dependencies"
+        if: steps.cache-cpp-install.outputs.cache-hit != 'true'
+        env:
+          VCPKG_BINARY_SOURCES: "clear;nuget,${{ env.GH_PACKAGES_FEED }},readwrite"
+        run: cmake -S cpp-client/deephaven -B cpp-client/deephaven/build --toolchain "$GITHUB_WORKSPACE/vcpkg/scripts/buildsystems/vcpkg.cmake" -DCMAKE_INSTALL_PREFIX="$DHINSTALL" -DVCPKG_TARGET_TRIPLET="${{ matrix.target_triplet }}" -DVCPKG_OVERLAY_TRIPLETS=cpp-client/deephaven/custom-triplets
+
+      # [Note: skip this step if cache-cpp-install was successful]
+      # Build our executables and install them in the configured install
+      # directory {{ env.DHINSTALL }}
+
+      - name: Build and install
+        if: steps.cache-cpp-install.outputs.cache-hit != 'true'
+        run: cmake --build cpp-client/deephaven/build --config RelWithDebInfo --target install --parallel
+
+      # Upload results as build artifact. Note: if cache-cpp-install was
+      # successful, then none of the intervening steps were executed. In that
+      # case, this effectively just copies the cache to the build artifact.
+
+      - name: Upload install dir
+        id: upload-install-directory
+        uses: actions/upload-artifact@v7
+        with:
+          name: cpp-client-install-${{ matrix.os }}
+          path: ${{ env.DHINSTALL }}
+
+  #=============================================================================
+  # Build the Python Static Client (not OS dependent)
+  #=============================================================================
+
+  python_static_job:
+    name: Python static
+    runs-on: ubuntu-24.04
+    needs: [version_job]
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out this repo
+        uses: actions/checkout@v6
+
+      - name: Install Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.10'
+
+      - name: Install the 'wheel' package
+        run: pip3 install wheel
+
+      - name: Install requirements-dev.txt
+        run: |
+          cd ./py/client
+          pip3 install -r requirements-dev.txt
+
+      - name: Run setup.py
+        env:
+          DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }}
+        run: |
+          cd ./py/client
+          python setup.py bdist_wheel
+
+      # Upload static .whl file as build artifact, to be used directly or by          # dependent clients like Python ticking
+      - name: Upload .whl file
+        uses: actions/upload-artifact@v7
+        with:
+          name: py-static-wheel
+          path: ./py/client/dist/*.whl
+
+  #=============================================================================
+  # Build the Python Ticking Client (OS dependent)
+  #=============================================================================
+
+  python_ticking_job:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-24.04, windows-2025-vs2026]
+    name: Python ticking
+    runs-on: ${{ matrix.os }}
+    needs: [cpp_job, python_static_job, version_job]
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out this repo
+        uses: actions/checkout@v6
+
+      # Get the C++ installation as a build artifact
+      - name: Fetch C++ installation
+        uses: actions/download-artifact@v8
+        with:
+          name: cpp-client-install-${{ matrix.os }}
+          path: ${{ env.DHINSTALL }}
+
+      # Get the Python static installation as a build artifact
+      - name: Fetch Python static installation
+        uses: actions/download-artifact@v8
+        with:
+          name: py-static-wheel
+          path: ./py/client/dist/
+
+      - name: Install Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.10'
+
+      - name: Install the 'wheel' and 'cython' packages
+        run: pip3 install wheel cython
+
+      - name: Install requirements-dev.txt
+        run: |
+          cd ./py/client
+          pip3 install -r requirements-dev.txt
+
+      - name: Install static client
+        run: pip3 install --force --no-deps ./py/client/dist/*.whl
+
+      # Pinned to a specific SHA to avoid future surprises
+      - name: set up PATH for cl.exe
+        if: runner.os == 'Windows'
+        uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756
+
+      - name: Build our Cython code (Linux)
+        if: runner.os == 'Linux'
+        env:
+          DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }}
+          CPPFLAGS: -I${{ env.DHINSTALL }}/include
+          LDFLAGS: -L${{ env.DHINSTALL }}/lib
+        run: |
+          cd ./py/client-ticking
+          python setup.py build_ext -i
+          python setup.py bdist_wheel
+
+      - name: Build our Cython code (Windows)
+        if: runner.os == 'Windows'
+        shell: cmd
+        env:
+          DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }}
+        run: |
+          cd .\py\client-ticking
+          python setup.py build_ext -i
+          python setup.py bdist_wheel
+
+      - name: Upload ticking .whl file as build artifact
+        uses: actions/upload-artifact@v7
+        with:
+          name: py-ticking-wheel-${{ matrix.os }}
+          path: ./py/client-ticking/dist/*.whl

.github/workflows/clients-csharp.yml

@@ -0,0 +1,58 @@
+name: "C# Client"
+on:
+  pull_request:
+    branches: [ 'main', 'rc/v*' ]
+  push:
+    branches: [ 'main', 'check/**', 'release/v*' ]
+permissions:
+  packages: write
+
+env:
+  GH_PACKAGES_USERNAME: ${{ github.repository_owner }}
+  GH_PACKAGES_FEED: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json
+  DHINSTALL: ${{ github.workspace }}/dhinstall
+
+jobs:
+  #=============================================================================
+  # Identify version of Deephaven Core
+  #=============================================================================
+  version_job:
+    uses: ./.github/workflows/dhc-version.yml
+
+  #=============================================================================
+  # Build the .NET Client (cross-platform, but we use the Ubuntu runner)
+  #=============================================================================
+  dotnet_job:
+    name: Build .NET client
+    runs-on: ubuntu-24.04
+    needs: version_job
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out this repo
+        uses: actions/checkout@v6
+
+      - name: Setup dotnet
+        uses: actions/setup-dotnet@v5
+        with:
+          dotnet-version: '10.0.x'
+
+      - name: Build Deephaven Core C# Client
+        run: |
+          cd csharp/client/Dh_NetClient
+          dotnet build -c Release
+
+      - name: Create unsigned NuGet package
+        env:
+          DHC_VERSION: ${{ needs.version_job.outputs.dhc_version }}
+        run: |
+          cd csharp/client/Dh_NetClient
+          dotnet pack --no-build -c Release /p:Platform="Any CPU" /p:PackageVersion=$DHC_VERSION
+
+      - name: Upload unsigned NuGet package as build artifact
+        uses: actions/upload-artifact@v7
+        with:
+          name: dotnet-client
+          path: ./csharp/client/Dh_NetClient/bin/Release/*.nupkg