From ac939fcfad9e70df7cd0bf70b459850c5608f116 Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Mon, 27 Apr 2026 13:15:04 -0400 Subject: [PATCH 01/39] feat(cpp/client, py/client, csharp/client): DH-22472, DH-22473, DH-22475: Github Actions to build (cpp-client, python, csharp) on (Ubuntu 24.04, Windows) --- .../clients-cpp-python-ubuntu2404.yml | 262 ++++++++++++++++++ .../workflows/clients-cpp-python-windows.yml | 255 +++++++++++++++++ .github/workflows/clients-csharp.yml | 58 ++++ .github/workflows/dhc-version.yml | 39 +++ cpp-client/deephaven/CMakeLists.txt | 11 + .../x64-linux-dynamic-release.cmake | 14 + .../custom-triplets/x64-windows-release | 10 + 7 files changed, 649 insertions(+) create mode 100644 .github/workflows/clients-cpp-python-ubuntu2404.yml create mode 100644 .github/workflows/clients-cpp-python-windows.yml create mode 100644 .github/workflows/clients-csharp.yml create mode 100644 .github/workflows/dhc-version.yml create mode 100644 cpp-client/deephaven/custom-triplets/x64-linux-dynamic-release.cmake create mode 100644 cpp-client/deephaven/custom-triplets/x64-windows-release diff --git a/.github/workflows/clients-cpp-python-ubuntu2404.yml b/.github/workflows/clients-cpp-python-ubuntu2404.yml new file mode 100644 index 00000000000..97e454d3267 --- /dev/null +++ b/.github/workflows/clients-cpp-python-ubuntu2404.yml @@ -0,0 +1,262 @@ +name: "Build C++ and Python Clients on Ubuntu 24.04" +on: + pull_request: + branches: [ 'main', 'rc/v*' ] + push: + branches: [ 'main', 'check/**', 'release/v*' ] +permissions: + packages: write + +#=============================================================================== +# Background: in this workflow there are three different kinds of caching going +# on: +# +# 1. The outputs are Github Build artifacts, attached to every action run. +# These artifacts can be downloaded from e.g. +# https://github.com/deephaven/deephaven-core/actions/runs/17416308139 +# but the specific URL depends on the build. +# We use this to upload: +# - The C++ install directory +# - The .whl file for the static Python client +# - The .whl file for the ticking Python client +# +# 2. The C++ install directory is cached via actions/cache, and keyed +# by the hash of the cpp-client and proto directories. This allows us +# to skip recompilation if these files don't change. This is an optional +# optimization that might not be very significant. +# +# 3. Github Packages, which is a NuGet-compatible repo accessible from +# https://github.com/orgs/deephaven/packages?repo_name=deephaven-core +# and controlled via the NuGet API. We use this package repository to hold +# the binaries for the C++ dependencies e build (grpc, arrow, etc.). It is +# managed automatically for us by cmake/vcpkg/nuget. This optimization is +# important because building the C++ dependencies takes significant time +# (45 minutes maybe). +#=============================================================================== + +env: + GH_PACKAGES_USERNAME: ${{ github.repository_owner }} + GH_PACKAGES_FEED: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json + DHINSTALL: ${{ github.workspace }}/dhinstall + +jobs: + #============================================================================= + # Identify version of Deephaven Core + #============================================================================= + version_job: + uses: ./.github/workflows/dhc-version.yml + + #============================================================================= + # Build the C++ Client + #============================================================================= + cpp_job: + name: Build C++ client on Ubuntu or fetch from cache + runs-on: ubuntu-24.04 + defaults: + run: + shell: bash + + steps: + - name: Check out this repo + uses: actions/checkout@v6 + + # Try to fetch the whole installation directory (the final product of the + # build) from the Github Cache. The key to this cache is derived + # from the hash of the contents of the files in the cpp-client and proto + # directories. If this fetch is successful, we skip over all the build + # work and just copy this cached install directory to the output artifact + # for this run. + + - name: Restore the cached installation directory, or prepare a new one + id: cache-cpp-install + uses: actions/cache@v5 + with: + path: ${{ env.DHINSTALL }} + key: ${{ runner.os }}-cpp-client-${{ hashFiles('cpp-client/**', 'proto/**') }} + + # [Note: skip this step if cache-cpp-install was successful] + # Check out vcpkg + + - name: Check out vcpkg + id: checkout-vcpkg + if: steps.cache-cpp-install.outputs.cache-hit != 'true' + uses: actions/checkout@v6 + with: + repository: microsoft/vcpkg + path: vcpkg + + # [Note: skip this step if cache-cpp-install was successful] + # Bootstrap vcpkg + + - name: Bootstrap vcpkg + if: steps.cache-cpp-install.outputs.cache-hit != 'true' + run: ${{ github.workspace }}/vcpkg/bootstrap-vcpkg.sh + + # [Note: skip this step if cache-cpp-install was successful] + # Get mono so that we can run nuget. This is because the nuget.exe that + # ships with vcpkg is an old .NET Framework binary (Windows format), and + # only mono knows how to run such a thing on Linux. Some day the maintainers + # will fix this so it runs with the cross-platform "dotnet". + + - name: Get mono so that we can run nuget + if: steps.cache-cpp-install.outputs.cache-hit != 'true' + run: sudo apt install mono-complete + + # [Note: skip this step if cache-cpp-install was successful] + # Adapted from + # https://learn.microsoft.com/en-us/vcpkg/consume/binary-caching-github-packages?pivots=linux-runner + + # [Note: skip this step if cache-cpp-install was successful] + - name: "Configure nuget to know how to download (or upload) each cached package that we need (or create)" + if: steps.cache-cpp-install.outputs.cache-hit != 'true' + run: | + vcpkg_exe=${{ github.workspace }}/vcpkg/vcpkg + nuget_dotnet=$($vcpkg_exe fetch nuget | tail -n 1) + mono $nuget_dotnet \ + sources add \ + -Source "${{ env.GH_PACKAGES_FEED }}" \ + -StorePasswordInClearText \ + -Name GitHubPackages \ + -UserName "${{ env.GH_PACKAGES_USERNAME }}" \ + -Password "${{ secrets.GITHUB_TOKEN }}" + mono $nuget_dotnet \ + setapikey "${{ secrets.GITHUB_TOKEN }}" \ + -Source "${{ env.GH_PACKAGES_FEED }}" + + # [Note: skip this step if cache-cpp-install was successful] + # 1. Invoke vcpkg to: + # a) Try to fetch each of the dependent libraries from the nuget cache + # b) If not found, build them and insert them in the nuget cache + # 2. Configure cmake to prepare for the build/install step, which comes next + + - name: build packages and configure cmake. Cached packages will be found, or stored in github's nuget repo + if: steps.cache-cpp-install.outputs.cache-hit != 'true' + env: + VCPKG_BINARY_SOURCES: "clear;nuget,${{ env.GH_PACKAGES_FEED }},readwrite" + run: cmake -S cpp-client/deephaven -B cpp-client/deephaven/build --toolchain ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_INSTALL_PREFIX=${{ env.DHINSTALL }} -DVCPKG_TARGET_TRIPLET=x64-linux-dynamic-release -DVCPKG_OVERLAY_TRIPLETS=cpp-client/deephaven/custom-triplets + + # [Note: skip this step if cache-cpp-install was successful] + # Build our executables and install them in the configured install directory {{ env.DHINSTALL }} + + - name: Run cmake build and install + if: steps.cache-cpp-install.outputs.cache-hit != 'true' + run: cmake --build cpp-client/deephaven/build --config RelWithDebInfo --target install --parallel + + # Upload results as build artifact. Note: if cache-cpp-install was + # then none of the intervening steps were executed. In that case, + # this effectively just copies the cache to the build artifact. + + - name: Upload install directory as build artifact + id: upload-install-directory + uses: actions/upload-artifact@v7 + with: + name: cpp-client-install + path: ${{ env.DHINSTALL }} + + #============================================================================= + # Build the Python Static Client + #============================================================================= + + python_static_job: + name: Build Python static client on Ubuntu + runs-on: ubuntu-24.04 + needs: [version_job] + defaults: + run: + shell: bash + + steps: + - name: Check out this repo + uses: actions/checkout@v6 + + - name: Install Python + uses: actions/setup-python@v6 + with: + python-version: '3.10' + + - name: Install the 'wheel' package + run: pip3 install wheel + + - name: Install requirements-dev.txt + run: | + cd ./py/client + pip3 install -r requirements-dev.txt + + - name: Run setup.py + env: + DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} + run: | + cd ./py/client + python setup.py bdist_wheel + + - name: Upload static .whl file as build artifact, to be used directly or by dependent clients like Python ticking + uses: actions/upload-artifact@v7 + with: + name: py-static-wheel + path: ./py/client/dist/*.whl + + #============================================================================= + # Build the Python Ticking Client + #============================================================================= + + python_ticking_job: + name: Build Python ticking client on Ubuntu + runs-on: ubuntu-24.04 + needs: [cpp_job, python_static_job, version_job] + defaults: + run: + shell: bash + + steps: + - name: Check out this repo + uses: actions/checkout@v6 + + - name: Get the C++ installation as a build artifact + uses: actions/download-artifact@v8 + with: + name: cpp-client-install + path: ${{ env.DHINSTALL }} + + - name: Get the Python static installation as a build artifact + uses: actions/download-artifact@v8 + with: + name: py-static-wheel + path: ./py/client/dist/ + + - name: Install Python + uses: actions/setup-python@v6 + with: + python-version: '3.10' + + - name: Install the 'wheel' and 'cython' packages + run: pip3 install wheel cython + + - name: Install requirements-dev.txt + run: | + cd ./py/client + pip3 install -r requirements-dev.txt + + - name: Install the static client + run: pip3 install --force --no-deps ./py/client/dist/*.whl + + - name: Build cython code + env: + DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} + CPPFLAGS: -I${{ env.DHINSTALL }}/include + LDFLAGS: -L${{ env.DHINSTALL }}/lib + run: | + cd ./py/client-ticking + python setup.py build_ext -i + + - name: Run setup.py + env: + DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} + run: | + cd ./py/client-ticking + python setup.py bdist_wheel + + - name: Upload ticking .whl file as build artifact + uses: actions/upload-artifact@v7 + with: + name: py-ticking-wheel + path: ./py/client-ticking/dist/*.whl diff --git a/.github/workflows/clients-cpp-python-windows.yml b/.github/workflows/clients-cpp-python-windows.yml new file mode 100644 index 00000000000..d43bb3d9fbe --- /dev/null +++ b/.github/workflows/clients-cpp-python-windows.yml @@ -0,0 +1,255 @@ +name: "Build C++ and Python Clients on Windows" +on: + pull_request: + branches: [ 'main', 'rc/v*' ] + push: + branches: [ 'main', 'check/**', 'release/v*' ] +permissions: + packages: write + +#=============================================================================== +# Background: in this workflow there are three different kinds of caching going +# on: +# +# 1. The outputs are Github Build artifacts, attached to every action run. +# These artifacts can be downloaded from e.g. +# https://github.com/deephaven/deephaven-core/actions/runs/17416308139 +# but the specific URL depends on the build. +# We use this to upload: +# - The C++ install directory +# - The .whl file for the static Python client +# - The .whl file for the ticking Python client +# +# 2. The C++ install directory is cached via actions/cache, and keyed +# by the hash of the cpp-client and proto directories. This allows us +# to skip recompilation if these files don't change. This is an optional +# optimization that might not be very significant. +# +# 3. Github Packages, which is a NuGet-compatible repo accessible from +# https://github.com/orgs/deephaven/packages?repo_name=deephaven-core +# and controlled via the NuGet API. We use this package repository to hold +# the binaries for the C++ dependencies e build (grpc, arrow, etc.). It is +# managed automatically for us by cmake/vcpkg/nuget. This optimization is +# important because building the C++ dependencies takes significant time +# (45 minutes maybe). +#=============================================================================== + +env: + GH_PACKAGES_USERNAME: ${{ github.repository_owner }} + GH_PACKAGES_FEED: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json + DHINSTALL: ${{ github.workspace }}/dhinstall + +jobs: + #============================================================================= + # Identify version of Deephaven Core + #============================================================================= + version_job: + uses: ./.github/workflows/dhc-version.yml + + #============================================================================= + # Build the C++ Client + #============================================================================= + cpp_job: + name: Build C++ client on Windows or fetch from cache + runs-on: windows-2025 + defaults: + run: + shell: cmd + + steps: + - name: Check out this repo + uses: actions/checkout@v6 + + # Try to fetch the whole installation directory (the final product of the + # build) from the Github Cache. The key to this cache is derived + # from the hash of the contents of the files in the cpp-client and proto + # directories. If this fetch is successful, we skip over all the build + # work and just copy this cached install directory to the output artifact + # for this run. + + - name: Restore the cached installation directory, or prepare a new one + id: cache-cpp-install + uses: actions/cache@v5 + with: + path: ${{ env.DHINSTALL }} + key: ${{ runner.os }}-cpp-client-${{ hashFiles('cpp-client/**', 'proto/**') }} + + # [Note: skip this step if cache-cpp-install was successful] + # Check out vcpkg + + - name: Check out vcpkg + id: checkout-vcpkg + if: steps.cache-cpp-install.outputs.cache-hit != 'true' + uses: actions/checkout@v6 + with: + repository: microsoft/vcpkg + path: vcpkg + + # [Note: skip this step if cache-cpp-install was successful] + # Bootstrap vcpkg + + - name: Bootstrap vcpkg + if: steps.cache-cpp-install.outputs.cache-hit != 'true' + run: ${{ github.workspace }}/vcpkg/bootstrap-vcpkg.bat + + # [Note: skip this step if cache-cpp-install was successful] + # Adapted from + # https://learn.microsoft.com/en-us/vcpkg/consume/binary-caching-github-packages?pivots=linux-runner + + - name: "Configure nuget to know how to download (or upload) each cached package that we need (or create)" + if: steps.cache-cpp-install.outputs.cache-hit != 'true' + shell: pwsh + run: | + $nuget_exe = & ./vcpkg/vcpkg.exe fetch nuget + & $nuget_exe sources add ` + -Source "${{ env.GH_PACKAGES_FEED }}" ` + -StorePasswordInClearText ` + -Name GitHubPackages ` + -UserName "${{ env.GH_PACKAGES_USERNAME }}" ` + -Password "${{ secrets.GITHUB_TOKEN }}" + & $nuget_exe setapikey "${{ secrets.GITHUB_TOKEN }}" -Source "${{ env.GH_PACKAGES_FEED }}" + + # [Note: skip this step if cache-cpp-install was successful] + # 1. Invoke vcpkg to: + # a) Try to fetch each of the dependent libraries from the nuget cache + # b) If not found, build them and insert them in the nuget cache + # 2. Configure cmake to prepare for the build/install step, which comes next + + - name: build packages and configure cmake. Cached packages will be found, or stored in github's nuget repo + if: steps.cache-cpp-install.outputs.cache-hit != 'true' + env: + VCPKG_BINARY_SOURCES: "clear;nuget,${{ env.GH_PACKAGES_FEED }},readwrite" + run: cmake -S cpp-client/deephaven -B cpp-client/deephaven/build --toolchain ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_INSTALL_PREFIX=${{ env.DHINSTALL }} -DX_VCPKG_APPLOCAL_DEPS_INSTALL=ON -DVCPKG_TARGET_TRIPLET=x64-windows-release -DVCPKG_OVERLAY_TRIPLETS=cpp-client/deephaven/custom-triplets + + # [Note: skip this step if cache-cpp-install was successful] + # Build our executables and install them in the configured install directory {{ env.DHINSTALL }} + + - name: Run cmake build and install + if: steps.cache-cpp-install.outputs.cache-hit != 'true' + run: cmake --build cpp-client/deephaven/build --config RelWithDebInfo --target install --parallel + + # [Note: skip this step if cache-cpp-install was successful] + + # Upload results as build artifact. Note: if cache-cpp-install was + # then none of the intervening steps were executed. In that case, + # this effectively just copies the cache to the build artifact. + + - name: Upload install directory as build artifact + id: upload-install-directory + uses: actions/upload-artifact@v7 + with: + name: cpp-client-install + path: ${{ env.DHINSTALL }} + + #============================================================================= + # Build the Python Static Client + #============================================================================= + + python_static_job: + name: Build Python static client on Windows + runs-on: windows-2025 + needs: [version_job] + defaults: + run: + shell: cmd + + steps: + - name: Check out this repo + uses: actions/checkout@v6 + + - name: Install Python + uses: actions/setup-python@v6 + with: + python-version: '3.10' + + - name: Install the 'wheel' package + run: pip3 install wheel + + - name: Install requirements-dev.txt + run: | + cd .\py\client + pip3 install -r requirements-dev.txt + + - name: Run setup.py + env: + DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} + run: | + cd .\py\client + python setup.py bdist_wheel + + - name: Upload static .whl file as build artifact, to be used directly or by dependent clients like Python ticking + uses: actions/upload-artifact@v7 + with: + name: py-static-wheel + path: ./py/client/dist/*.whl + + #============================================================================= + # Build the Python Ticking Client + #============================================================================= + + python_ticking_job: + name: Build Python ticking client on Windows + runs-on: windows-2025 + needs: [cpp_job, python_static_job, version_job] + defaults: + run: + shell: cmd + + steps: + - name: Check out this repo + uses: actions/checkout@v6 + + - name: Get the C++ installation as a build artifact + uses: actions/download-artifact@v8 + with: + name: cpp-client-install + path: ${{ env.DHINSTALL }} + + - name: Get the Python static installation as a build artifact + uses: actions/download-artifact@v8 + with: + name: py-static-wheel + path: ./py/client/dist/ + + - name: Install Python + uses: actions/setup-python@v6 + with: + python-version: '3.10' + + - name: Install the 'wheel' and 'cython' packages + run: pip3 install wheel cython + + - name: Install requirements-dev.txt + run: | + cd .\py\client + pip3 install -r requirements-dev.txt + + - name: Install the static client + shell: pwsh + run: | + $wheel = (Get-ChildItem -Path .\py\client\dist\*.whl).FullName + pip3 install --force --no-deps $wheel + + # Pinned to a specific SHA to avoid future surprises + - name: set up PATH to be able to run Visual Studio tools like cl.exe + uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 + + - name: Build cython code + env: + DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} + run: | + cd .\py\client-ticking + python setup.py build_ext -i + + - name: Run setup.py + env: + DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} + run: | + cd .\py\client-ticking + python setup.py bdist_wheel + + - name: Upload ticking .whl file as build artifact + uses: actions/upload-artifact@v7 + with: + name: py-ticking-wheel + path: ./py/client-ticking/dist/*.whl diff --git a/.github/workflows/clients-csharp.yml b/.github/workflows/clients-csharp.yml new file mode 100644 index 00000000000..325124cb783 --- /dev/null +++ b/.github/workflows/clients-csharp.yml @@ -0,0 +1,58 @@ +name: "Build .NET Client on Ubuntu 24.04" +on: + pull_request: + branches: [ 'main', 'rc/v*' ] + push: + branches: [ 'main', 'check/**', 'release/v*' ] +permissions: + packages: write + +env: + GH_PACKAGES_USERNAME: ${{ github.repository_owner }} + GH_PACKAGES_FEED: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json + DHINSTALL: ${{ github.workspace }}/dhinstall + +jobs: + #============================================================================= + # Identify version of Deephaven Core + #============================================================================= + version_job: + uses: ./.github/workflows/dhc-version.yml + + #============================================================================= + # Build the .NET Client + #============================================================================= + dotnet_job: + name: Build .NET client on Ubuntu + runs-on: ubuntu-24.04 + needs: version_job + defaults: + run: + shell: bash + + steps: + - name: Check out this repo + uses: actions/checkout@v6 + + - name: Setup dotnet + uses: actions/setup-dotnet@v5 + with: + dotnet-version: '10.0.x' + + - name: Build Deephaven Core C# Client + run: | + cd csharp/client/Dh_NetClient + dotnet build -c Release + + - name: Create unsigned NuGet package + env: + DHC_VERSION: ${{ needs.version_job.outputs.dhc_version }} + run: | + cd csharp/client/Dh_NetClient + dotnet pack --no-build -c Release /p:Platform="Any CPU" /p:PackageVersion=$DHC_VERSION + + - name: Upload unsigned NuGet package as build artifact + uses: actions/upload-artifact@v7 + with: + name: dotnet-client + path: ./csharp/client/Dh_NetClient/bin/Release/*.nupkg diff --git a/.github/workflows/dhc-version.yml b/.github/workflows/dhc-version.yml new file mode 100644 index 00000000000..913ba6c9b46 --- /dev/null +++ b/.github/workflows/dhc-version.yml @@ -0,0 +1,39 @@ +name: "Determine Deephaven Core version" +on: + workflow_call: + outputs: + dhc_version: + description: "The detected version of Deephaven Core" + value: ${{ jobs.version_job.outputs.dhc_version }} + +jobs: + #============================================================================= + # Identify version of Deephaven Core + #============================================================================= + version_job: + name: Find the Deephaven version by running awk on gradle.properties + runs-on: ubuntu-24.04 + defaults: + run: + shell: bash + + outputs: + dhc_version: ${{ steps.get-dhc-version.outputs.dhc_version }} + + steps: + - name: Check out this repo + uses: actions/checkout@v6 + + - name: Output Deephaven Version + id: get-dhc-version + run: | + DHC_VERSION=$(awk -F= ' + /^[^#]/ { props[$1] = $2 } + END { + v = props["deephavenMajorVersion"] "." props["deephavenMinorVersion"] + if (props["deephavenBaseQualifier"] != "") v = v "-" props["deephavenBaseQualifier"] + print v + } + ' gradle.properties) + echo "dhc_version=$DHC_VERSION" >> $GITHUB_OUTPUT + echo FYI dhc_version is $DHC_VERSION diff --git a/cpp-client/deephaven/CMakeLists.txt b/cpp-client/deephaven/CMakeLists.txt index 041fc1603a1..b764c348a9b 100644 --- a/cpp-client/deephaven/CMakeLists.txt +++ b/cpp-client/deephaven/CMakeLists.txt @@ -52,6 +52,17 @@ if(NOT DHCORE_ONLY) INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} ) + if(VCPKG_TOOLCHAIN AND NOT WIN32) + install(DIRECTORY ${CMAKE_BINARY_DIR}/vcpkg_installed/${VCPKG_TARGET_TRIPLET}/lib/ + DESTINATION lib + FILES_MATCHING + PATTERN "*.so" + PATTERN "*.so.*" + PATTERN "pkgconfig" EXCLUDE + PATTERN "cmake" EXCLUDE + ) + endif() + include(CMakePackageConfigHelpers) write_basic_package_version_file( "${CMAKE_CURRENT_BINARY_DIR}/deephaven/deephavenConfigVersion.cmake" diff --git a/cpp-client/deephaven/custom-triplets/x64-linux-dynamic-release.cmake b/cpp-client/deephaven/custom-triplets/x64-linux-dynamic-release.cmake new file mode 100644 index 00000000000..4921b59b897 --- /dev/null +++ b/cpp-client/deephaven/custom-triplets/x64-linux-dynamic-release.cmake @@ -0,0 +1,14 @@ +# These settings are inspired by [vcpkg repo]/triplets/x64-linux.cmake + +# These settings are the same: +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_CMAKE_SYSTEM_NAME Linux) + +# This setting is changed because we want dynamic library linking +# rather than static. +set(VCPKG_LIBRARY_LINKAGE dynamic) + +# This setting is new because we want to build only the 'release' +# version of the packages, rather the both 'debug' and 'release' +set(VCPKG_BUILD_TYPE release) diff --git a/cpp-client/deephaven/custom-triplets/x64-windows-release b/cpp-client/deephaven/custom-triplets/x64-windows-release new file mode 100644 index 00000000000..49034cbe0f4 --- /dev/null +++ b/cpp-client/deephaven/custom-triplets/x64-windows-release @@ -0,0 +1,10 @@ +# These settings are inspired by [vcpkg repo]/triplets/x64-windows.cmake + +# These settings are the same: +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE dynamic) + +# This setting is new because we want to build only the 'release' +# version of the packages, rather the both 'debug' and 'release' +set(VCPKG_BUILD_TYPE release) From 10b078cd7290af1500860493bac38d216cd0b0ad Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Fri, 1 May 2026 13:43:48 -0400 Subject: [PATCH 02/39] touch a file to trigger a build --- COOKIE.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 COOKIE.txt diff --git a/COOKIE.txt b/COOKIE.txt new file mode 100644 index 00000000000..ce013625030 --- /dev/null +++ b/COOKIE.txt @@ -0,0 +1 @@ +hello From ddc91a1281927a678539007f95750bcc02c47302 Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Fri, 1 May 2026 13:43:55 -0400 Subject: [PATCH 03/39] Revert "touch a file to trigger a build" This reverts commit 10b078cd7290af1500860493bac38d216cd0b0ad. --- COOKIE.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 COOKIE.txt diff --git a/COOKIE.txt b/COOKIE.txt deleted file mode 100644 index ce013625030..00000000000 --- a/COOKIE.txt +++ /dev/null @@ -1 +0,0 @@ -hello From be99f3a4075e7dc36b69fee3f0654fbd120d6d97 Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Fri, 1 May 2026 16:46:19 -0400 Subject: [PATCH 04/39] rename step --- .github/workflows/clients-cpp-python-ubuntu2404.yml | 2 +- .github/workflows/clients-cpp-python-windows.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clients-cpp-python-ubuntu2404.yml b/.github/workflows/clients-cpp-python-ubuntu2404.yml index 97e454d3267..18d91ff19a8 100644 --- a/.github/workflows/clients-cpp-python-ubuntu2404.yml +++ b/.github/workflows/clients-cpp-python-ubuntu2404.yml @@ -129,7 +129,7 @@ jobs: # b) If not found, build them and insert them in the nuget cache # 2. Configure cmake to prepare for the build/install step, which comes next - - name: build packages and configure cmake. Cached packages will be found, or stored in github's nuget repo + - name: "Build packages and configure cmake. Check nupkg cache for existing packages, otherwise build and upload." if: steps.cache-cpp-install.outputs.cache-hit != 'true' env: VCPKG_BINARY_SOURCES: "clear;nuget,${{ env.GH_PACKAGES_FEED }},readwrite" diff --git a/.github/workflows/clients-cpp-python-windows.yml b/.github/workflows/clients-cpp-python-windows.yml index d43bb3d9fbe..b3935e3c891 100644 --- a/.github/workflows/clients-cpp-python-windows.yml +++ b/.github/workflows/clients-cpp-python-windows.yml @@ -115,7 +115,7 @@ jobs: # b) If not found, build them and insert them in the nuget cache # 2. Configure cmake to prepare for the build/install step, which comes next - - name: build packages and configure cmake. Cached packages will be found, or stored in github's nuget repo + - name: "Build packages and configure cmake. Check nupkg cache for existing packages, otherwise build and upload." if: steps.cache-cpp-install.outputs.cache-hit != 'true' env: VCPKG_BINARY_SOURCES: "clear;nuget,${{ env.GH_PACKAGES_FEED }},readwrite" From 11c44b01eea47e76067916eb7c418c789cc260b6 Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 14:54:27 -0400 Subject: [PATCH 05/39] let's see what it feels like when names are shorter --- .github/workflows/clients-csharp.yml | 2 +- .github/workflows/dhc-version.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/clients-csharp.yml b/.github/workflows/clients-csharp.yml index 325124cb783..4883c919830 100644 --- a/.github/workflows/clients-csharp.yml +++ b/.github/workflows/clients-csharp.yml @@ -1,4 +1,4 @@ -name: "Build .NET Client on Ubuntu 24.04" +name: "C# Client-ubu24" on: pull_request: branches: [ 'main', 'rc/v*' ] diff --git a/.github/workflows/dhc-version.yml b/.github/workflows/dhc-version.yml index 913ba6c9b46..251ddaa2fc7 100644 --- a/.github/workflows/dhc-version.yml +++ b/.github/workflows/dhc-version.yml @@ -1,4 +1,4 @@ -name: "Determine Deephaven Core version" +name: "Get DHC version" on: workflow_call: outputs: @@ -8,10 +8,10 @@ on: jobs: #============================================================================= - # Identify version of Deephaven Core + # Identify version of Deephaven Core by running awk on gradle.properties #============================================================================= version_job: - name: Find the Deephaven version by running awk on gradle.properties + name: "Get DHC version" runs-on: ubuntu-24.04 defaults: run: From b7e400e9b6e6cec0e21bfa3d5e9d5a15f56e85ed Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 15:09:11 -0400 Subject: [PATCH 06/39] let's see how this looks now --- .../workflows/clients-cpp-python-windows.yml | 42 +++++++++++-------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/.github/workflows/clients-cpp-python-windows.yml b/.github/workflows/clients-cpp-python-windows.yml index b3935e3c891..7a68a336f48 100644 --- a/.github/workflows/clients-cpp-python-windows.yml +++ b/.github/workflows/clients-cpp-python-windows.yml @@ -1,4 +1,4 @@ -name: "Build C++ and Python Clients on Windows" +name: "C++/Python on Windows" on: pull_request: branches: [ 'main', 'rc/v*' ] @@ -50,7 +50,7 @@ jobs: # Build the C++ Client #============================================================================= cpp_job: - name: Build C++ client on Windows or fetch from cache + name: "Build C++ (or use cache)" runs-on: windows-2025 defaults: run: @@ -93,10 +93,12 @@ jobs: run: ${{ github.workspace }}/vcpkg/bootstrap-vcpkg.bat # [Note: skip this step if cache-cpp-install was successful] + # Configure nuget to know how to download (or upload) each cached + # package that we need (or create) # Adapted from # https://learn.microsoft.com/en-us/vcpkg/consume/binary-caching-github-packages?pivots=linux-runner - - name: "Configure nuget to know how to download (or upload) each cached package that we need (or create)" + - name: "Configure nuget" if: steps.cache-cpp-install.outputs.cache-hit != 'true' shell: pwsh run: | @@ -110,12 +112,13 @@ jobs: & $nuget_exe setapikey "${{ secrets.GITHUB_TOKEN }}" -Source "${{ env.GH_PACKAGES_FEED }}" # [Note: skip this step if cache-cpp-install was successful] + # Build packages and configure cmake. Check nupkg cache for existing + # packages, otherwise build and upload. # 1. Invoke vcpkg to: # a) Try to fetch each of the dependent libraries from the nuget cache # b) If not found, build them and insert them in the nuget cache # 2. Configure cmake to prepare for the build/install step, which comes next - - - name: "Build packages and configure cmake. Check nupkg cache for existing packages, otherwise build and upload." + - name: "Build/cache dependencies" if: steps.cache-cpp-install.outputs.cache-hit != 'true' env: VCPKG_BINARY_SOURCES: "clear;nuget,${{ env.GH_PACKAGES_FEED }},readwrite" @@ -124,7 +127,7 @@ jobs: # [Note: skip this step if cache-cpp-install was successful] # Build our executables and install them in the configured install directory {{ env.DHINSTALL }} - - name: Run cmake build and install + - name: "Build and install" if: steps.cache-cpp-install.outputs.cache-hit != 'true' run: cmake --build cpp-client/deephaven/build --config RelWithDebInfo --target install --parallel @@ -134,7 +137,7 @@ jobs: # then none of the intervening steps were executed. In that case, # this effectively just copies the cache to the build artifact. - - name: Upload install directory as build artifact + - name: Upload install dir id: upload-install-directory uses: actions/upload-artifact@v7 with: @@ -146,7 +149,7 @@ jobs: #============================================================================= python_static_job: - name: Build Python static client on Windows + name: Python static runs-on: windows-2025 needs: [version_job] defaults: @@ -162,7 +165,7 @@ jobs: with: python-version: '3.10' - - name: Install the 'wheel' package + - name: Install 'wheel' package run: pip3 install wheel - name: Install requirements-dev.txt @@ -177,7 +180,8 @@ jobs: cd .\py\client python setup.py bdist_wheel - - name: Upload static .whl file as build artifact, to be used directly or by dependent clients like Python ticking + # Upload .whl file as build artifact, to be used directly or by dependent clients like Python ticking + - name: Upload .whl file uses: actions/upload-artifact@v7 with: name: py-static-wheel @@ -188,7 +192,7 @@ jobs: #============================================================================= python_ticking_job: - name: Build Python ticking client on Windows + name: Python ticking runs-on: windows-2025 needs: [cpp_job, python_static_job, version_job] defaults: @@ -199,13 +203,15 @@ jobs: - name: Check out this repo uses: actions/checkout@v6 - - name: Get the C++ installation as a build artifact + # Get the C++ installation as a build artifact + - name: Fetch C++ installation uses: actions/download-artifact@v8 with: name: cpp-client-install path: ${{ env.DHINSTALL }} - - name: Get the Python static installation as a build artifact + # Get the Python static installation as a build artifact + - name: Fetch Python static installation uses: actions/download-artifact@v8 with: name: py-static-wheel @@ -216,7 +222,7 @@ jobs: with: python-version: '3.10' - - name: Install the 'wheel' and 'cython' packages + - name: "Install the 'wheel' and 'cython' packages" run: pip3 install wheel cython - name: Install requirements-dev.txt @@ -224,17 +230,17 @@ jobs: cd .\py\client pip3 install -r requirements-dev.txt - - name: Install the static client + - name: Install static client shell: pwsh run: | $wheel = (Get-ChildItem -Path .\py\client\dist\*.whl).FullName pip3 install --force --no-deps $wheel # Pinned to a specific SHA to avoid future surprises - - name: set up PATH to be able to run Visual Studio tools like cl.exe + - name: set up PATH for cl.exe uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 - - name: Build cython code + - name: Build our Cython env: DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} run: | @@ -248,7 +254,7 @@ jobs: cd .\py\client-ticking python setup.py bdist_wheel - - name: Upload ticking .whl file as build artifact + - name: Upload .whl file uses: actions/upload-artifact@v7 with: name: py-ticking-wheel From fc8cc8acdd5bcfc5fe952cb552f4f1d0e21d5671 Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 15:17:17 -0400 Subject: [PATCH 07/39] lots more renames. Bring it on!! --- .../clients-cpp-python-ubuntu2404.yml | 38 ++++++++++++------- .../workflows/clients-cpp-python-windows.yml | 10 +++-- .github/workflows/clients-csharp.yml | 2 +- 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/.github/workflows/clients-cpp-python-ubuntu2404.yml b/.github/workflows/clients-cpp-python-ubuntu2404.yml index 18d91ff19a8..29a39066a5a 100644 --- a/.github/workflows/clients-cpp-python-ubuntu2404.yml +++ b/.github/workflows/clients-cpp-python-ubuntu2404.yml @@ -1,4 +1,4 @@ -name: "Build C++ and Python Clients on Ubuntu 24.04" +name: "C++/Python on Ubuntu24" on: pull_request: branches: [ 'main', 'rc/v*' ] @@ -50,7 +50,7 @@ jobs: # Build the C++ Client #============================================================================= cpp_job: - name: Build C++ client on Ubuntu or fetch from cache + name: "Build C++ (or use cache)" runs-on: ubuntu-24.04 defaults: run: @@ -67,7 +67,7 @@ jobs: # work and just copy this cached install directory to the output artifact # for this run. - - name: Restore the cached installation directory, or prepare a new one + - name: Try to restore cached installdir id: cache-cpp-install uses: actions/cache@v5 with: @@ -107,7 +107,10 @@ jobs: # https://learn.microsoft.com/en-us/vcpkg/consume/binary-caching-github-packages?pivots=linux-runner # [Note: skip this step if cache-cpp-install was successful] - - name: "Configure nuget to know how to download (or upload) each cached package that we need (or create)" + # Configure nuget to know how to download (or upload) each cached + # package that we need (or create) + + - name: "Configure nuget" if: steps.cache-cpp-install.outputs.cache-hit != 'true' run: | vcpkg_exe=${{ github.workspace }}/vcpkg/vcpkg @@ -124,21 +127,24 @@ jobs: -Source "${{ env.GH_PACKAGES_FEED }}" # [Note: skip this step if cache-cpp-install was successful] + # Build packages and configure cmake. Check nupkg cache for existing + # packages, otherwise build and upload. # 1. Invoke vcpkg to: # a) Try to fetch each of the dependent libraries from the nuget cache # b) If not found, build them and insert them in the nuget cache # 2. Configure cmake to prepare for the build/install step, which comes next - - name: "Build packages and configure cmake. Check nupkg cache for existing packages, otherwise build and upload." + - name: "Build/cache dependencies" if: steps.cache-cpp-install.outputs.cache-hit != 'true' env: VCPKG_BINARY_SOURCES: "clear;nuget,${{ env.GH_PACKAGES_FEED }},readwrite" run: cmake -S cpp-client/deephaven -B cpp-client/deephaven/build --toolchain ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_INSTALL_PREFIX=${{ env.DHINSTALL }} -DVCPKG_TARGET_TRIPLET=x64-linux-dynamic-release -DVCPKG_OVERLAY_TRIPLETS=cpp-client/deephaven/custom-triplets # [Note: skip this step if cache-cpp-install was successful] - # Build our executables and install them in the configured install directory {{ env.DHINSTALL }} + # Build our executables and install them in the configured install + # directory {{ env.DHINSTALL }} - - name: Run cmake build and install + - name: Build and install if: steps.cache-cpp-install.outputs.cache-hit != 'true' run: cmake --build cpp-client/deephaven/build --config RelWithDebInfo --target install --parallel @@ -146,7 +152,7 @@ jobs: # then none of the intervening steps were executed. In that case, # this effectively just copies the cache to the build artifact. - - name: Upload install directory as build artifact + - name: Upload install dir id: upload-install-directory uses: actions/upload-artifact@v7 with: @@ -158,7 +164,7 @@ jobs: #============================================================================= python_static_job: - name: Build Python static client on Ubuntu + name: Python static runs-on: ubuntu-24.04 needs: [version_job] defaults: @@ -189,7 +195,8 @@ jobs: cd ./py/client python setup.py bdist_wheel - - name: Upload static .whl file as build artifact, to be used directly or by dependent clients like Python ticking + # Upload static .whl file as build artifact, to be used directly or by # dependent clients like Python ticking + - name: Upload .whl file uses: actions/upload-artifact@v7 with: name: py-static-wheel @@ -200,7 +207,7 @@ jobs: #============================================================================= python_ticking_job: - name: Build Python ticking client on Ubuntu + name: Python ticking runs-on: ubuntu-24.04 needs: [cpp_job, python_static_job, version_job] defaults: @@ -211,13 +218,16 @@ jobs: - name: Check out this repo uses: actions/checkout@v6 - - name: Get the C++ installation as a build artifact + # Get the C++ installation as a build artifact + - name: Fetch C++ installation uses: actions/download-artifact@v8 with: name: cpp-client-install path: ${{ env.DHINSTALL }} - - name: Get the Python static installation as a build artifact + + # Get the Python static installation as a build artifact + - name: Fetch Python static installation uses: actions/download-artifact@v8 with: name: py-static-wheel @@ -239,7 +249,7 @@ jobs: - name: Install the static client run: pip3 install --force --no-deps ./py/client/dist/*.whl - - name: Build cython code + - name: Build our Cython code env: DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} CPPFLAGS: -I${{ env.DHINSTALL }}/include diff --git a/.github/workflows/clients-cpp-python-windows.yml b/.github/workflows/clients-cpp-python-windows.yml index 7a68a336f48..c4cf327c19d 100644 --- a/.github/workflows/clients-cpp-python-windows.yml +++ b/.github/workflows/clients-cpp-python-windows.yml @@ -67,7 +67,7 @@ jobs: # work and just copy this cached install directory to the output artifact # for this run. - - name: Restore the cached installation directory, or prepare a new one + - name: Try to restore cached installdir id: cache-cpp-install uses: actions/cache@v5 with: @@ -125,7 +125,8 @@ jobs: run: cmake -S cpp-client/deephaven -B cpp-client/deephaven/build --toolchain ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_INSTALL_PREFIX=${{ env.DHINSTALL }} -DX_VCPKG_APPLOCAL_DEPS_INSTALL=ON -DVCPKG_TARGET_TRIPLET=x64-windows-release -DVCPKG_OVERLAY_TRIPLETS=cpp-client/deephaven/custom-triplets # [Note: skip this step if cache-cpp-install was successful] - # Build our executables and install them in the configured install directory {{ env.DHINSTALL }} + # Build our executables and install them in the configured install + # directory {{ env.DHINSTALL }} - name: "Build and install" if: steps.cache-cpp-install.outputs.cache-hit != 'true' @@ -180,7 +181,8 @@ jobs: cd .\py\client python setup.py bdist_wheel - # Upload .whl file as build artifact, to be used directly or by dependent clients like Python ticking + # Upload .whl file as build artifact, to be used directly or by + # dependent clients like Python ticking - name: Upload .whl file uses: actions/upload-artifact@v7 with: @@ -240,7 +242,7 @@ jobs: - name: set up PATH for cl.exe uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 - - name: Build our Cython + - name: Build our Cython code env: DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} run: | diff --git a/.github/workflows/clients-csharp.yml b/.github/workflows/clients-csharp.yml index 4883c919830..eb34031f535 100644 --- a/.github/workflows/clients-csharp.yml +++ b/.github/workflows/clients-csharp.yml @@ -1,4 +1,4 @@ -name: "C# Client-ubu24" +name: "C# Client on Ubuntu24" on: pull_request: branches: [ 'main', 'rc/v*' ] From 25c7a011647b5744f77d42002231d9984be38c2e Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 15:25:39 -0400 Subject: [PATCH 08/39] try unified --- .github/workflows/clients-cpp-python-ubuntu2404.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clients-cpp-python-ubuntu2404.yml b/.github/workflows/clients-cpp-python-ubuntu2404.yml index 29a39066a5a..28ba6013b0b 100644 --- a/.github/workflows/clients-cpp-python-ubuntu2404.yml +++ b/.github/workflows/clients-cpp-python-ubuntu2404.yml @@ -246,7 +246,7 @@ jobs: cd ./py/client pip3 install -r requirements-dev.txt - - name: Install the static client + - name: Install static client run: pip3 install --force --no-deps ./py/client/dist/*.whl - name: Build our Cython code From c2c33343a1b92ecfbd72edd1c85b6ee132f13626 Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 15:31:48 -0400 Subject: [PATCH 09/39] Let's watch this crash and burn --- .github/workflows/clients-cpp-python.yml | 276 +++++++++++++++++++++++ 1 file changed, 276 insertions(+) create mode 100644 .github/workflows/clients-cpp-python.yml diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml new file mode 100644 index 00000000000..5d504c62ece --- /dev/null +++ b/.github/workflows/clients-cpp-python.yml @@ -0,0 +1,276 @@ +name: "C++/Python" +on: + pull_request: + branches: [ 'main', 'rc/v*' ] + push: + branches: [ 'main', 'check/**', 'release/v*' ] +permissions: + packages: write + +#=============================================================================== +# Background: in this workflow there are three different kinds of caching going +# on: +# +# 1. The outputs are Github Build artifacts, attached to every action run. +# These artifacts can be downloaded from e.g. +# https://github.com/deephaven/deephaven-core/actions/runs/17416308139 +# but the specific URL depends on the build. +# We use this to upload: +# - The C++ install directory +# - The .whl file for the static Python client +# - The .whl file for the ticking Python client +# +# 2. The C++ install directory is cached via actions/cache, and keyed +# by the hash of the cpp-client and proto directories. This allows us +# to skip recompilation if these files don't change. This is an optional +# optimization that might not be very significant. +# +# 3. Github Packages, which is a NuGet-compatible repo accessible from +# https://github.com/orgs/deephaven/packages?repo_name=deephaven-core +# and controlled via the NuGet API. We use this package repository to hold +# the binaries for the C++ dependencies e build (grpc, arrow, etc.). It is +# managed automatically for us by cmake/vcpkg/nuget. This optimization is +# important because building the C++ dependencies takes significant time +# (45 minutes maybe). +#=============================================================================== + +env: + GH_PACKAGES_USERNAME: ${{ github.repository_owner }} + GH_PACKAGES_FEED: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json + DHINSTALL: ${{ github.workspace }}/dhinstall + +jobs: + #============================================================================= + # Identify version of Deephaven Core + #============================================================================= + version_job: + uses: ./.github/workflows/dhc-version.yml + + #============================================================================= + # Build the C++ Client + #============================================================================= + cpp_job: + name: "Build C++ (or use cache)" + runs-on: ubuntu-24.04 + defaults: + run: + shell: bash + + steps: + - name: Check out this repo + uses: actions/checkout@v6 + + # Try to fetch the whole installation directory (the final product of the + # build) from the Github Cache. The key to this cache is derived + # from the hash of the contents of the files in the cpp-client and proto + # directories. If this fetch is successful, we skip over all the build + # work and just copy this cached install directory to the output artifact + # for this run. + + - name: Try to restore cached installdir + id: cache-cpp-install + uses: actions/cache@v5 + with: + path: ${{ env.DHINSTALL }} + key: ${{ runner.os }}-cpp-client-${{ hashFiles('cpp-client/**', 'proto/**') }} + + # [Note: skip this step if cache-cpp-install was successful] + # Check out vcpkg + + - name: Check out vcpkg + id: checkout-vcpkg + if: steps.cache-cpp-install.outputs.cache-hit != 'true' + uses: actions/checkout@v6 + with: + repository: microsoft/vcpkg + path: vcpkg + + # [Note: skip this step if cache-cpp-install was successful] + # Bootstrap vcpkg + + - name: Bootstrap vcpkg + if: steps.cache-cpp-install.outputs.cache-hit != 'true' + run: ${{ github.workspace }}/vcpkg/bootstrap-vcpkg.sh + + # [Note: skip this step if cache-cpp-install was successful] + # Get mono so that we can run nuget. This is because the nuget.exe that + # ships with vcpkg is an old .NET Framework binary (Windows format), and + # only mono knows how to run such a thing on Linux. Some day the maintainers + # will fix this so it runs with the cross-platform "dotnet". + + - name: Get mono so that we can run nuget + if: steps.cache-cpp-install.outputs.cache-hit != 'true' + run: sudo apt install mono-complete + + # [Note: skip this step if cache-cpp-install was successful] + # Adapted from + # https://learn.microsoft.com/en-us/vcpkg/consume/binary-caching-github-packages?pivots=linux-runner + + # [Note: skip this step if cache-cpp-install was successful] + # Configure nuget to know how to download (or upload) each cached + # package that we need (or create) + + - name: "Configure nuget" + if: steps.cache-cpp-install.outputs.cache-hit != 'true' + run: | + vcpkg_exe=${{ github.workspace }}/vcpkg/vcpkg + nuget_dotnet=$($vcpkg_exe fetch nuget | tail -n 1) + mono $nuget_dotnet \ + sources add \ + -Source "${{ env.GH_PACKAGES_FEED }}" \ + -StorePasswordInClearText \ + -Name GitHubPackages \ + -UserName "${{ env.GH_PACKAGES_USERNAME }}" \ + -Password "${{ secrets.GITHUB_TOKEN }}" + mono $nuget_dotnet \ + setapikey "${{ secrets.GITHUB_TOKEN }}" \ + -Source "${{ env.GH_PACKAGES_FEED }}" + + # [Note: skip this step if cache-cpp-install was successful] + # Build packages and configure cmake. Check nupkg cache for existing + # packages, otherwise build and upload. + # 1. Invoke vcpkg to: + # a) Try to fetch each of the dependent libraries from the nuget cache + # b) If not found, build them and insert them in the nuget cache + # 2. Configure cmake to prepare for the build/install step, which comes next + + - name: "Build/cache dependencies" + if: steps.cache-cpp-install.outputs.cache-hit != 'true' + env: + VCPKG_BINARY_SOURCES: "clear;nuget,${{ env.GH_PACKAGES_FEED }},readwrite" + run: cmake -S cpp-client/deephaven -B cpp-client/deephaven/build --toolchain ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_INSTALL_PREFIX=${{ env.DHINSTALL }} -DVCPKG_TARGET_TRIPLET=x64-linux-dynamic-release -DVCPKG_OVERLAY_TRIPLETS=cpp-client/deephaven/custom-triplets + + # [Note: skip this step if cache-cpp-install was successful] + # Build our executables and install them in the configured install + # directory {{ env.DHINSTALL }} + + - name: Build and install + if: steps.cache-cpp-install.outputs.cache-hit != 'true' + run: cmake --build cpp-client/deephaven/build --config RelWithDebInfo --target install --parallel + + # Upload results as build artifact. Note: if cache-cpp-install was + # then none of the intervening steps were executed. In that case, + # this effectively just copies the cache to the build artifact. + + - name: Upload install dir + id: upload-install-directory + uses: actions/upload-artifact@v7 + with: + name: cpp-client-install + path: ${{ env.DHINSTALL }} + + #============================================================================= + # Build the Python Static Client + #============================================================================= + + python_static_job: + name: Python static + runs-on: ubuntu-24.04 + needs: [version_job] + defaults: + run: + shell: bash + + steps: + - name: Check out this repo + uses: actions/checkout@v6 + + - name: Install Python + uses: actions/setup-python@v6 + with: + python-version: '3.10' + + - name: Install the 'wheel' package + run: pip3 install wheel + + - name: Install requirements-dev.txt + run: | + cd ./py/client + pip3 install -r requirements-dev.txt + + - name: Run setup.py + env: + DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} + run: | + cd ./py/client + python setup.py bdist_wheel + + # Upload static .whl file as build artifact, to be used directly or by # dependent clients like Python ticking + - name: Upload .whl file + uses: actions/upload-artifact@v7 + with: + name: py-static-wheel + path: ./py/client/dist/*.whl + + #============================================================================= + # Build the Python Ticking Client + #============================================================================= + + python_ticking_job: + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, windows-2025] + name: Python ticking + runs-on: ${{ matrix.os }} + needs: [cpp_job, python_static_job, version_job] + defaults: + run: + shell: bash + + steps: + - name: Check out this repo + uses: actions/checkout@v6 + + # Get the C++ installation as a build artifact + - name: Fetch C++ installation + uses: actions/download-artifact@v8 + with: + name: cpp-client-install-${{ matrix.os }} + path: ${{ env.DHINSTALL }} + + + # Get the Python static installation as a build artifact + - name: Fetch Python static installation + uses: actions/download-artifact@v8 + with: + name: py-static-wheel + path: ./py/client/dist/ + + - name: Install Python + uses: actions/setup-python@v6 + with: + python-version: '3.10' + + - name: Install the 'wheel' and 'cython' packages + run: pip3 install wheel cython + + - name: Install requirements-dev.txt + run: | + cd ./py/client + pip3 install -r requirements-dev.txt + + - name: Install static client + run: pip3 install --force --no-deps ./py/client/dist/*.whl + + - name: Build our Cython code + env: + DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} + CPPFLAGS: -I${{ env.DHINSTALL }}/include + LDFLAGS: -L${{ env.DHINSTALL }}/lib + run: | + cd ./py/client-ticking + python setup.py build_ext -i + + - name: Run setup.py + env: + DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} + run: | + cd ./py/client-ticking + python setup.py bdist_wheel + + - name: Upload ticking .whl file as build artifact + uses: actions/upload-artifact@v7 + with: + name: py-ticking-wheel-${{ matrix.os }} + path: ./py/client-ticking/dist/*.whl From 300521baed09f38add43e24b661458c46c983eac Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 15:41:55 -0400 Subject: [PATCH 10/39] ok this is going to be UGLY --- .github/workflows/clients-cpp-python.yml | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index 5d504c62ece..5d62f23c50d 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -51,9 +51,14 @@ jobs: #============================================================================= cpp_job: name: "Build C++ (or use cache)" - runs-on: ubuntu-24.04 + strategy: + matrix: + os: [ubuntu-24.04, windows-2022] + fail-fast: false + runs-on: ${{ matrix.os }} defaults: run: + # This gets you Cygwin on Windows shell: bash steps: @@ -92,21 +97,21 @@ jobs: if: steps.cache-cpp-install.outputs.cache-hit != 'true' run: ${{ github.workspace }}/vcpkg/bootstrap-vcpkg.sh + # [Note: Linux only] # [Note: skip this step if cache-cpp-install was successful] # Get mono so that we can run nuget. This is because the nuget.exe that # ships with vcpkg is an old .NET Framework binary (Windows format), and # only mono knows how to run such a thing on Linux. Some day the maintainers # will fix this so it runs with the cross-platform "dotnet". - - name: Get mono so that we can run nuget + - name: Install mono + if: runner.os == 'Linux' if: steps.cache-cpp-install.outputs.cache-hit != 'true' run: sudo apt install mono-complete # [Note: skip this step if cache-cpp-install was successful] # Adapted from # https://learn.microsoft.com/en-us/vcpkg/consume/binary-caching-github-packages?pivots=linux-runner - - # [Note: skip this step if cache-cpp-install was successful] # Configure nuget to know how to download (or upload) each cached # package that we need (or create) @@ -156,11 +161,11 @@ jobs: id: upload-install-directory uses: actions/upload-artifact@v7 with: - name: cpp-client-install + name: cpp-client-install-${{ matrix.os }} path: ${{ env.DHINSTALL }} #============================================================================= - # Build the Python Static Client + # Build the Python Static Client (not OS dependent) #============================================================================= python_static_job: @@ -203,14 +208,14 @@ jobs: path: ./py/client/dist/*.whl #============================================================================= - # Build the Python Ticking Client + # Build the Python Ticking Client (OS dependent) #============================================================================= python_ticking_job: strategy: fail-fast: false matrix: - os: [ubuntu-24.04, windows-2025] + os: [ubuntu-24.04, windows-2025-vs2026] name: Python ticking runs-on: ${{ matrix.os }} needs: [cpp_job, python_static_job, version_job] From 1b42285f234bef74c56a83b5560fc1bc49358edf Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 15:43:44 -0400 Subject: [PATCH 11/39] let's try this again -- I can't see what is what --- .github/workflows/clients-cpp-python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index 5d62f23c50d..b0360b56ec9 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -1,4 +1,4 @@ -name: "C++/Python" +name: "PARTY TIME C++/Python" on: pull_request: branches: [ 'main', 'rc/v*' ] From daf12d8983c02195bcd2f96ade5a9bfcaee1bb22 Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 15:48:25 -0400 Subject: [PATCH 12/39] oops can't have two "ifs" --- .github/workflows/clients-cpp-python.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index b0360b56ec9..3350369d074 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -105,8 +105,7 @@ jobs: # will fix this so it runs with the cross-platform "dotnet". - name: Install mono - if: runner.os == 'Linux' - if: steps.cache-cpp-install.outputs.cache-hit != 'true' + if: runner.os == 'Linux' && steps.cache-cpp-install.outputs.cache-hit != 'true' run: sudo apt install mono-complete # [Note: skip this step if cache-cpp-install was successful] From 310aa4b4c216fe2f0d42837879b65c0ace2b53ea Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 15:53:05 -0400 Subject: [PATCH 13/39] unify windows versions ok? --- .github/workflows/clients-cpp-python.yml | 2 +- .github/workflows/clients-csharp.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index 3350369d074..25930fb7c5a 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -53,7 +53,7 @@ jobs: name: "Build C++ (or use cache)" strategy: matrix: - os: [ubuntu-24.04, windows-2022] + os: [ubuntu-24.04, windows-2025-vs2026] fail-fast: false runs-on: ${{ matrix.os }} defaults: diff --git a/.github/workflows/clients-csharp.yml b/.github/workflows/clients-csharp.yml index eb34031f535..a9200a71f52 100644 --- a/.github/workflows/clients-csharp.yml +++ b/.github/workflows/clients-csharp.yml @@ -20,10 +20,10 @@ jobs: uses: ./.github/workflows/dhc-version.yml #============================================================================= - # Build the .NET Client + # Build the .NET Client (cross-platform, but we use the Ubuntu runner) #============================================================================= dotnet_job: - name: Build .NET client on Ubuntu + name: Build .NET client runs-on: ubuntu-24.04 needs: version_job defaults: From 804fb3b946ebf8d7a2cd07c6a711cea864496f76 Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 16:08:51 -0400 Subject: [PATCH 14/39] not dead yet --- .github/workflows/clients-cpp-python.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index 25930fb7c5a..39d8a770111 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -257,6 +257,11 @@ jobs: - name: Install static client run: pip3 install --force --no-deps ./py/client/dist/*.whl + # Pinned to a specific SHA to avoid future surprises + - name: set up PATH for cl.exe + if: runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 + - name: Build our Cython code env: DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} From c5e765cdfab403010ad037f76f0f6d936e194ee0 Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 16:16:40 -0400 Subject: [PATCH 15/39] if this doesn't work, we will hardcode --- .github/workflows/clients-cpp-python.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index 39d8a770111..c506ae36df5 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -233,7 +233,6 @@ jobs: name: cpp-client-install-${{ matrix.os }} path: ${{ env.DHINSTALL }} - # Get the Python static installation as a build artifact - name: Fetch Python static installation uses: actions/download-artifact@v8 @@ -259,8 +258,9 @@ jobs: # Pinned to a specific SHA to avoid future surprises - name: set up PATH for cl.exe - if: runner.os == 'Windows' - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 + # if: runner.os == 'Windows' + # uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 + uses: ilammy/msvc-dev-cmd@v1 - name: Build our Cython code env: From 2b083d6e02f931bd80c4ae388b51df8ac55c78de Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 16:24:16 -0400 Subject: [PATCH 16/39] HACK JOB 666 --- .github/workflows/clients-cpp-python.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index c506ae36df5..14f15bf5621 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -258,11 +258,11 @@ jobs: # Pinned to a specific SHA to avoid future surprises - name: set up PATH for cl.exe - # if: runner.os == 'Windows' - # uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 - uses: ilammy/msvc-dev-cmd@v1 + if: runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 - - name: Build our Cython code + - name: Build our Cython code (Linux) + if: runner.os == 'Linux' env: DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} CPPFLAGS: -I${{ env.DHINSTALL }}/include @@ -271,6 +271,15 @@ jobs: cd ./py/client-ticking python setup.py build_ext -i + - name: Build our Cython code (Windows) + if: runner.os == 'Windows' + shell: cmd + env: + DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} + run: | + cd .\py\client-ticking + python setup.py build_ext -i + - name: Run setup.py env: DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} From f9f2b4b11f473c7ef85d70be4f0f983b413675cd Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 16:30:42 -0400 Subject: [PATCH 17/39] zamboni time 12456666666 --- .github/workflows/clients-cpp-python-ubuntu2404.yml | 9 +++++---- .github/workflows/clients-cpp-python-windows.yml | 9 +++++---- .github/workflows/clients-cpp-python.yml | 7 +------ 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/.github/workflows/clients-cpp-python-ubuntu2404.yml b/.github/workflows/clients-cpp-python-ubuntu2404.yml index 28ba6013b0b..98899f1ff01 100644 --- a/.github/workflows/clients-cpp-python-ubuntu2404.yml +++ b/.github/workflows/clients-cpp-python-ubuntu2404.yml @@ -1,9 +1,10 @@ name: "C++/Python on Ubuntu24" on: - pull_request: - branches: [ 'main', 'rc/v*' ] - push: - branches: [ 'main', 'check/**', 'release/v*' ] + workflow_dispatch: +# pull_request: +# branches: [ 'main', 'rc/v*' ] +# push: +# branches: [ 'main', 'check/**', 'release/v*' ] permissions: packages: write diff --git a/.github/workflows/clients-cpp-python-windows.yml b/.github/workflows/clients-cpp-python-windows.yml index c4cf327c19d..e0ff4247d6d 100644 --- a/.github/workflows/clients-cpp-python-windows.yml +++ b/.github/workflows/clients-cpp-python-windows.yml @@ -1,9 +1,10 @@ name: "C++/Python on Windows" on: - pull_request: - branches: [ 'main', 'rc/v*' ] - push: - branches: [ 'main', 'check/**', 'release/v*' ] + workflow_dispatch: +# pull_request: +# branches: [ 'main', 'rc/v*' ] +# push: +# branches: [ 'main', 'check/**', 'release/v*' ] permissions: packages: write diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index 14f15bf5621..dc14f6037e0 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -270,6 +270,7 @@ jobs: run: | cd ./py/client-ticking python setup.py build_ext -i + python setup.py bdist_wheel - name: Build our Cython code (Windows) if: runner.os == 'Windows' @@ -279,12 +280,6 @@ jobs: run: | cd .\py\client-ticking python setup.py build_ext -i - - - name: Run setup.py - env: - DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} - run: | - cd ./py/client-ticking python setup.py bdist_wheel - name: Upload ticking .whl file as build artifact From d3f29a22a424f7dfecdbd0cfe2f209df60a6962c Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 17:21:57 -0400 Subject: [PATCH 18/39] tv and ps3 and xbox --- .../clients-cpp-python-ubuntu2404.yml | 273 ------------------ .../workflows/clients-cpp-python-windows.yml | 264 ----------------- cpp-client/CHANGE_COOKIE.txt | 1 + 3 files changed, 1 insertion(+), 537 deletions(-) delete mode 100644 .github/workflows/clients-cpp-python-ubuntu2404.yml delete mode 100644 .github/workflows/clients-cpp-python-windows.yml create mode 100644 cpp-client/CHANGE_COOKIE.txt diff --git a/.github/workflows/clients-cpp-python-ubuntu2404.yml b/.github/workflows/clients-cpp-python-ubuntu2404.yml deleted file mode 100644 index 98899f1ff01..00000000000 --- a/.github/workflows/clients-cpp-python-ubuntu2404.yml +++ /dev/null @@ -1,273 +0,0 @@ -name: "C++/Python on Ubuntu24" -on: - workflow_dispatch: -# pull_request: -# branches: [ 'main', 'rc/v*' ] -# push: -# branches: [ 'main', 'check/**', 'release/v*' ] -permissions: - packages: write - -#=============================================================================== -# Background: in this workflow there are three different kinds of caching going -# on: -# -# 1. The outputs are Github Build artifacts, attached to every action run. -# These artifacts can be downloaded from e.g. -# https://github.com/deephaven/deephaven-core/actions/runs/17416308139 -# but the specific URL depends on the build. -# We use this to upload: -# - The C++ install directory -# - The .whl file for the static Python client -# - The .whl file for the ticking Python client -# -# 2. The C++ install directory is cached via actions/cache, and keyed -# by the hash of the cpp-client and proto directories. This allows us -# to skip recompilation if these files don't change. This is an optional -# optimization that might not be very significant. -# -# 3. Github Packages, which is a NuGet-compatible repo accessible from -# https://github.com/orgs/deephaven/packages?repo_name=deephaven-core -# and controlled via the NuGet API. We use this package repository to hold -# the binaries for the C++ dependencies e build (grpc, arrow, etc.). It is -# managed automatically for us by cmake/vcpkg/nuget. This optimization is -# important because building the C++ dependencies takes significant time -# (45 minutes maybe). -#=============================================================================== - -env: - GH_PACKAGES_USERNAME: ${{ github.repository_owner }} - GH_PACKAGES_FEED: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json - DHINSTALL: ${{ github.workspace }}/dhinstall - -jobs: - #============================================================================= - # Identify version of Deephaven Core - #============================================================================= - version_job: - uses: ./.github/workflows/dhc-version.yml - - #============================================================================= - # Build the C++ Client - #============================================================================= - cpp_job: - name: "Build C++ (or use cache)" - runs-on: ubuntu-24.04 - defaults: - run: - shell: bash - - steps: - - name: Check out this repo - uses: actions/checkout@v6 - - # Try to fetch the whole installation directory (the final product of the - # build) from the Github Cache. The key to this cache is derived - # from the hash of the contents of the files in the cpp-client and proto - # directories. If this fetch is successful, we skip over all the build - # work and just copy this cached install directory to the output artifact - # for this run. - - - name: Try to restore cached installdir - id: cache-cpp-install - uses: actions/cache@v5 - with: - path: ${{ env.DHINSTALL }} - key: ${{ runner.os }}-cpp-client-${{ hashFiles('cpp-client/**', 'proto/**') }} - - # [Note: skip this step if cache-cpp-install was successful] - # Check out vcpkg - - - name: Check out vcpkg - id: checkout-vcpkg - if: steps.cache-cpp-install.outputs.cache-hit != 'true' - uses: actions/checkout@v6 - with: - repository: microsoft/vcpkg - path: vcpkg - - # [Note: skip this step if cache-cpp-install was successful] - # Bootstrap vcpkg - - - name: Bootstrap vcpkg - if: steps.cache-cpp-install.outputs.cache-hit != 'true' - run: ${{ github.workspace }}/vcpkg/bootstrap-vcpkg.sh - - # [Note: skip this step if cache-cpp-install was successful] - # Get mono so that we can run nuget. This is because the nuget.exe that - # ships with vcpkg is an old .NET Framework binary (Windows format), and - # only mono knows how to run such a thing on Linux. Some day the maintainers - # will fix this so it runs with the cross-platform "dotnet". - - - name: Get mono so that we can run nuget - if: steps.cache-cpp-install.outputs.cache-hit != 'true' - run: sudo apt install mono-complete - - # [Note: skip this step if cache-cpp-install was successful] - # Adapted from - # https://learn.microsoft.com/en-us/vcpkg/consume/binary-caching-github-packages?pivots=linux-runner - - # [Note: skip this step if cache-cpp-install was successful] - # Configure nuget to know how to download (or upload) each cached - # package that we need (or create) - - - name: "Configure nuget" - if: steps.cache-cpp-install.outputs.cache-hit != 'true' - run: | - vcpkg_exe=${{ github.workspace }}/vcpkg/vcpkg - nuget_dotnet=$($vcpkg_exe fetch nuget | tail -n 1) - mono $nuget_dotnet \ - sources add \ - -Source "${{ env.GH_PACKAGES_FEED }}" \ - -StorePasswordInClearText \ - -Name GitHubPackages \ - -UserName "${{ env.GH_PACKAGES_USERNAME }}" \ - -Password "${{ secrets.GITHUB_TOKEN }}" - mono $nuget_dotnet \ - setapikey "${{ secrets.GITHUB_TOKEN }}" \ - -Source "${{ env.GH_PACKAGES_FEED }}" - - # [Note: skip this step if cache-cpp-install was successful] - # Build packages and configure cmake. Check nupkg cache for existing - # packages, otherwise build and upload. - # 1. Invoke vcpkg to: - # a) Try to fetch each of the dependent libraries from the nuget cache - # b) If not found, build them and insert them in the nuget cache - # 2. Configure cmake to prepare for the build/install step, which comes next - - - name: "Build/cache dependencies" - if: steps.cache-cpp-install.outputs.cache-hit != 'true' - env: - VCPKG_BINARY_SOURCES: "clear;nuget,${{ env.GH_PACKAGES_FEED }},readwrite" - run: cmake -S cpp-client/deephaven -B cpp-client/deephaven/build --toolchain ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_INSTALL_PREFIX=${{ env.DHINSTALL }} -DVCPKG_TARGET_TRIPLET=x64-linux-dynamic-release -DVCPKG_OVERLAY_TRIPLETS=cpp-client/deephaven/custom-triplets - - # [Note: skip this step if cache-cpp-install was successful] - # Build our executables and install them in the configured install - # directory {{ env.DHINSTALL }} - - - name: Build and install - if: steps.cache-cpp-install.outputs.cache-hit != 'true' - run: cmake --build cpp-client/deephaven/build --config RelWithDebInfo --target install --parallel - - # Upload results as build artifact. Note: if cache-cpp-install was - # then none of the intervening steps were executed. In that case, - # this effectively just copies the cache to the build artifact. - - - name: Upload install dir - id: upload-install-directory - uses: actions/upload-artifact@v7 - with: - name: cpp-client-install - path: ${{ env.DHINSTALL }} - - #============================================================================= - # Build the Python Static Client - #============================================================================= - - python_static_job: - name: Python static - runs-on: ubuntu-24.04 - needs: [version_job] - defaults: - run: - shell: bash - - steps: - - name: Check out this repo - uses: actions/checkout@v6 - - - name: Install Python - uses: actions/setup-python@v6 - with: - python-version: '3.10' - - - name: Install the 'wheel' package - run: pip3 install wheel - - - name: Install requirements-dev.txt - run: | - cd ./py/client - pip3 install -r requirements-dev.txt - - - name: Run setup.py - env: - DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} - run: | - cd ./py/client - python setup.py bdist_wheel - - # Upload static .whl file as build artifact, to be used directly or by # dependent clients like Python ticking - - name: Upload .whl file - uses: actions/upload-artifact@v7 - with: - name: py-static-wheel - path: ./py/client/dist/*.whl - - #============================================================================= - # Build the Python Ticking Client - #============================================================================= - - python_ticking_job: - name: Python ticking - runs-on: ubuntu-24.04 - needs: [cpp_job, python_static_job, version_job] - defaults: - run: - shell: bash - - steps: - - name: Check out this repo - uses: actions/checkout@v6 - - # Get the C++ installation as a build artifact - - name: Fetch C++ installation - uses: actions/download-artifact@v8 - with: - name: cpp-client-install - path: ${{ env.DHINSTALL }} - - - # Get the Python static installation as a build artifact - - name: Fetch Python static installation - uses: actions/download-artifact@v8 - with: - name: py-static-wheel - path: ./py/client/dist/ - - - name: Install Python - uses: actions/setup-python@v6 - with: - python-version: '3.10' - - - name: Install the 'wheel' and 'cython' packages - run: pip3 install wheel cython - - - name: Install requirements-dev.txt - run: | - cd ./py/client - pip3 install -r requirements-dev.txt - - - name: Install static client - run: pip3 install --force --no-deps ./py/client/dist/*.whl - - - name: Build our Cython code - env: - DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} - CPPFLAGS: -I${{ env.DHINSTALL }}/include - LDFLAGS: -L${{ env.DHINSTALL }}/lib - run: | - cd ./py/client-ticking - python setup.py build_ext -i - - - name: Run setup.py - env: - DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} - run: | - cd ./py/client-ticking - python setup.py bdist_wheel - - - name: Upload ticking .whl file as build artifact - uses: actions/upload-artifact@v7 - with: - name: py-ticking-wheel - path: ./py/client-ticking/dist/*.whl diff --git a/.github/workflows/clients-cpp-python-windows.yml b/.github/workflows/clients-cpp-python-windows.yml deleted file mode 100644 index e0ff4247d6d..00000000000 --- a/.github/workflows/clients-cpp-python-windows.yml +++ /dev/null @@ -1,264 +0,0 @@ -name: "C++/Python on Windows" -on: - workflow_dispatch: -# pull_request: -# branches: [ 'main', 'rc/v*' ] -# push: -# branches: [ 'main', 'check/**', 'release/v*' ] -permissions: - packages: write - -#=============================================================================== -# Background: in this workflow there are three different kinds of caching going -# on: -# -# 1. The outputs are Github Build artifacts, attached to every action run. -# These artifacts can be downloaded from e.g. -# https://github.com/deephaven/deephaven-core/actions/runs/17416308139 -# but the specific URL depends on the build. -# We use this to upload: -# - The C++ install directory -# - The .whl file for the static Python client -# - The .whl file for the ticking Python client -# -# 2. The C++ install directory is cached via actions/cache, and keyed -# by the hash of the cpp-client and proto directories. This allows us -# to skip recompilation if these files don't change. This is an optional -# optimization that might not be very significant. -# -# 3. Github Packages, which is a NuGet-compatible repo accessible from -# https://github.com/orgs/deephaven/packages?repo_name=deephaven-core -# and controlled via the NuGet API. We use this package repository to hold -# the binaries for the C++ dependencies e build (grpc, arrow, etc.). It is -# managed automatically for us by cmake/vcpkg/nuget. This optimization is -# important because building the C++ dependencies takes significant time -# (45 minutes maybe). -#=============================================================================== - -env: - GH_PACKAGES_USERNAME: ${{ github.repository_owner }} - GH_PACKAGES_FEED: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json - DHINSTALL: ${{ github.workspace }}/dhinstall - -jobs: - #============================================================================= - # Identify version of Deephaven Core - #============================================================================= - version_job: - uses: ./.github/workflows/dhc-version.yml - - #============================================================================= - # Build the C++ Client - #============================================================================= - cpp_job: - name: "Build C++ (or use cache)" - runs-on: windows-2025 - defaults: - run: - shell: cmd - - steps: - - name: Check out this repo - uses: actions/checkout@v6 - - # Try to fetch the whole installation directory (the final product of the - # build) from the Github Cache. The key to this cache is derived - # from the hash of the contents of the files in the cpp-client and proto - # directories. If this fetch is successful, we skip over all the build - # work and just copy this cached install directory to the output artifact - # for this run. - - - name: Try to restore cached installdir - id: cache-cpp-install - uses: actions/cache@v5 - with: - path: ${{ env.DHINSTALL }} - key: ${{ runner.os }}-cpp-client-${{ hashFiles('cpp-client/**', 'proto/**') }} - - # [Note: skip this step if cache-cpp-install was successful] - # Check out vcpkg - - - name: Check out vcpkg - id: checkout-vcpkg - if: steps.cache-cpp-install.outputs.cache-hit != 'true' - uses: actions/checkout@v6 - with: - repository: microsoft/vcpkg - path: vcpkg - - # [Note: skip this step if cache-cpp-install was successful] - # Bootstrap vcpkg - - - name: Bootstrap vcpkg - if: steps.cache-cpp-install.outputs.cache-hit != 'true' - run: ${{ github.workspace }}/vcpkg/bootstrap-vcpkg.bat - - # [Note: skip this step if cache-cpp-install was successful] - # Configure nuget to know how to download (or upload) each cached - # package that we need (or create) - # Adapted from - # https://learn.microsoft.com/en-us/vcpkg/consume/binary-caching-github-packages?pivots=linux-runner - - - name: "Configure nuget" - if: steps.cache-cpp-install.outputs.cache-hit != 'true' - shell: pwsh - run: | - $nuget_exe = & ./vcpkg/vcpkg.exe fetch nuget - & $nuget_exe sources add ` - -Source "${{ env.GH_PACKAGES_FEED }}" ` - -StorePasswordInClearText ` - -Name GitHubPackages ` - -UserName "${{ env.GH_PACKAGES_USERNAME }}" ` - -Password "${{ secrets.GITHUB_TOKEN }}" - & $nuget_exe setapikey "${{ secrets.GITHUB_TOKEN }}" -Source "${{ env.GH_PACKAGES_FEED }}" - - # [Note: skip this step if cache-cpp-install was successful] - # Build packages and configure cmake. Check nupkg cache for existing - # packages, otherwise build and upload. - # 1. Invoke vcpkg to: - # a) Try to fetch each of the dependent libraries from the nuget cache - # b) If not found, build them and insert them in the nuget cache - # 2. Configure cmake to prepare for the build/install step, which comes next - - name: "Build/cache dependencies" - if: steps.cache-cpp-install.outputs.cache-hit != 'true' - env: - VCPKG_BINARY_SOURCES: "clear;nuget,${{ env.GH_PACKAGES_FEED }},readwrite" - run: cmake -S cpp-client/deephaven -B cpp-client/deephaven/build --toolchain ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_INSTALL_PREFIX=${{ env.DHINSTALL }} -DX_VCPKG_APPLOCAL_DEPS_INSTALL=ON -DVCPKG_TARGET_TRIPLET=x64-windows-release -DVCPKG_OVERLAY_TRIPLETS=cpp-client/deephaven/custom-triplets - - # [Note: skip this step if cache-cpp-install was successful] - # Build our executables and install them in the configured install - # directory {{ env.DHINSTALL }} - - - name: "Build and install" - if: steps.cache-cpp-install.outputs.cache-hit != 'true' - run: cmake --build cpp-client/deephaven/build --config RelWithDebInfo --target install --parallel - - # [Note: skip this step if cache-cpp-install was successful] - - # Upload results as build artifact. Note: if cache-cpp-install was - # then none of the intervening steps were executed. In that case, - # this effectively just copies the cache to the build artifact. - - - name: Upload install dir - id: upload-install-directory - uses: actions/upload-artifact@v7 - with: - name: cpp-client-install - path: ${{ env.DHINSTALL }} - - #============================================================================= - # Build the Python Static Client - #============================================================================= - - python_static_job: - name: Python static - runs-on: windows-2025 - needs: [version_job] - defaults: - run: - shell: cmd - - steps: - - name: Check out this repo - uses: actions/checkout@v6 - - - name: Install Python - uses: actions/setup-python@v6 - with: - python-version: '3.10' - - - name: Install 'wheel' package - run: pip3 install wheel - - - name: Install requirements-dev.txt - run: | - cd .\py\client - pip3 install -r requirements-dev.txt - - - name: Run setup.py - env: - DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} - run: | - cd .\py\client - python setup.py bdist_wheel - - # Upload .whl file as build artifact, to be used directly or by - # dependent clients like Python ticking - - name: Upload .whl file - uses: actions/upload-artifact@v7 - with: - name: py-static-wheel - path: ./py/client/dist/*.whl - - #============================================================================= - # Build the Python Ticking Client - #============================================================================= - - python_ticking_job: - name: Python ticking - runs-on: windows-2025 - needs: [cpp_job, python_static_job, version_job] - defaults: - run: - shell: cmd - - steps: - - name: Check out this repo - uses: actions/checkout@v6 - - # Get the C++ installation as a build artifact - - name: Fetch C++ installation - uses: actions/download-artifact@v8 - with: - name: cpp-client-install - path: ${{ env.DHINSTALL }} - - # Get the Python static installation as a build artifact - - name: Fetch Python static installation - uses: actions/download-artifact@v8 - with: - name: py-static-wheel - path: ./py/client/dist/ - - - name: Install Python - uses: actions/setup-python@v6 - with: - python-version: '3.10' - - - name: "Install the 'wheel' and 'cython' packages" - run: pip3 install wheel cython - - - name: Install requirements-dev.txt - run: | - cd .\py\client - pip3 install -r requirements-dev.txt - - - name: Install static client - shell: pwsh - run: | - $wheel = (Get-ChildItem -Path .\py\client\dist\*.whl).FullName - pip3 install --force --no-deps $wheel - - # Pinned to a specific SHA to avoid future surprises - - name: set up PATH for cl.exe - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 - - - name: Build our Cython code - env: - DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} - run: | - cd .\py\client-ticking - python setup.py build_ext -i - - - name: Run setup.py - env: - DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} - run: | - cd .\py\client-ticking - python setup.py bdist_wheel - - - name: Upload .whl file - uses: actions/upload-artifact@v7 - with: - name: py-ticking-wheel - path: ./py/client-ticking/dist/*.whl diff --git a/cpp-client/CHANGE_COOKIE.txt b/cpp-client/CHANGE_COOKIE.txt new file mode 100644 index 00000000000..6232bf844ec --- /dev/null +++ b/cpp-client/CHANGE_COOKIE.txt @@ -0,0 +1 @@ +taint the C++ build to cause it to run again or whatever From 5177b2cd4649c262da523a156c7ca0f26196f337 Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 17:28:03 -0400 Subject: [PATCH 19/39] try this hell --- .github/workflows/clients-cpp-python.yml | 3 ++- .github/workflows/clients-csharp.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index dc14f6037e0..d421b2b2136 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -37,7 +37,8 @@ permissions: env: GH_PACKAGES_USERNAME: ${{ github.repository_owner }} GH_PACKAGES_FEED: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json - DHINSTALL: ${{ github.workspace }}/dhinstall + # DHINSTALL: ${{ github.workspace }}/dhinstall + # DHINSTALL: ./dhinstall jobs: #============================================================================= diff --git a/.github/workflows/clients-csharp.yml b/.github/workflows/clients-csharp.yml index a9200a71f52..4771d48991f 100644 --- a/.github/workflows/clients-csharp.yml +++ b/.github/workflows/clients-csharp.yml @@ -1,4 +1,4 @@ -name: "C# Client on Ubuntu24" +name: "C# Client" on: pull_request: branches: [ 'main', 'rc/v*' ] From 7528fdb3dcacd271ac4373a524972092b6438cd4 Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 17:28:38 -0400 Subject: [PATCH 20/39] no no trhis --- .github/workflows/clients-cpp-python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index d421b2b2136..3c9ed284e1e 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -38,7 +38,7 @@ env: GH_PACKAGES_USERNAME: ${{ github.repository_owner }} GH_PACKAGES_FEED: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json # DHINSTALL: ${{ github.workspace }}/dhinstall - # DHINSTALL: ./dhinstall + DHINSTALL: ./dhinstall jobs: #============================================================================= From dd1f5e9e6ba9a10ec5a0dc7121fcebb2f56af07b Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 17:33:34 -0400 Subject: [PATCH 21/39] $20,000 / night --- .github/workflows/clients-cpp-python.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index 3c9ed284e1e..e24b1527482 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -37,8 +37,7 @@ permissions: env: GH_PACKAGES_USERNAME: ${{ github.repository_owner }} GH_PACKAGES_FEED: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json - # DHINSTALL: ${{ github.workspace }}/dhinstall - DHINSTALL: ./dhinstall + DHINSTALL: ${{ github.workspace }}/dhinstall jobs: #============================================================================= @@ -94,10 +93,15 @@ jobs: # [Note: skip this step if cache-cpp-install was successful] # Bootstrap vcpkg - - name: Bootstrap vcpkg - if: steps.cache-cpp-install.outputs.cache-hit != 'true' + - name: Bootstrap vcpkg (Linux) + if: runner.os == 'Linux' && steps.cache-cpp-install.outputs.cache-hit != 'true' run: ${{ github.workspace }}/vcpkg/bootstrap-vcpkg.sh + - name: Bootstrap vcpkg (Windows) + shell: cmd + if: runner.os == 'Windows' && steps.cache-cpp-install.outputs.cache-hit != 'true' + run: ${{ github.workspace }}/vcpkg/bootstrap-vcpkg.bat + # [Note: Linux only] # [Note: skip this step if cache-cpp-install was successful] # Get mono so that we can run nuget. This is because the nuget.exe that From 9a3ac71a0c5eefad655d168161a120e950c81077 Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 17:36:40 -0400 Subject: [PATCH 22/39] one femto-step at a time --- .github/workflows/clients-cpp-python.yml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index e24b1527482..d5209e7345c 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -119,8 +119,8 @@ jobs: # Configure nuget to know how to download (or upload) each cached # package that we need (or create) - - name: "Configure nuget" - if: steps.cache-cpp-install.outputs.cache-hit != 'true' + - name: "Configure nuget (Linux)" + if: runner.os == 'Linux' && steps.cache-cpp-install.outputs.cache-hit != 'true' run: | vcpkg_exe=${{ github.workspace }}/vcpkg/vcpkg nuget_dotnet=$($vcpkg_exe fetch nuget | tail -n 1) @@ -135,6 +135,19 @@ jobs: setapikey "${{ secrets.GITHUB_TOKEN }}" \ -Source "${{ env.GH_PACKAGES_FEED }}" + - name: "Configure nuget (Windows)" + if: runner.os == 'Windows' && steps.cache-cpp-install.outputs.cache-hit != 'true' + shell: pwsh + run: | + $nuget_exe = & ./vcpkg/vcpkg.exe fetch nuget + & $nuget_exe sources add ` + -Source "${{ env.GH_PACKAGES_FEED }}" ` + -StorePasswordInClearText ` + -Name GitHubPackages ` + -UserName "${{ env.GH_PACKAGES_USERNAME }}" ` + -Password "${{ secrets.GITHUB_TOKEN }}" + & $nuget_exe setapikey "${{ secrets.GITHUB_TOKEN }}" -Source "${{ env.GH_PACKAGES_FEED }}" + # [Note: skip this step if cache-cpp-install was successful] # Build packages and configure cmake. Check nupkg cache for existing # packages, otherwise build and upload. From 9fddbc39e3664a38730ace01cd942d1faf8a9c05 Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 17:45:55 -0400 Subject: [PATCH 23/39] what a hellish mess --- .github/workflows/clients-cpp-python.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index d5209e7345c..32f5d8a2ce1 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -1,4 +1,4 @@ -name: "PARTY TIME C++/Python" +name: "C++/Python" on: pull_request: branches: [ 'main', 'rc/v*' ] @@ -53,7 +53,12 @@ jobs: name: "Build C++ (or use cache)" strategy: matrix: - os: [ubuntu-24.04, windows-2025-vs2026] + include: + - os: ubuntu-24.04 + target_triplet: x64-linux-dynamic-release + - os: windows-2025-vs2026 + target-triplet: x64-windows-release + fail-fast: false runs-on: ${{ matrix.os }} defaults: @@ -90,6 +95,10 @@ jobs: repository: microsoft/vcpkg path: vcpkg + - name: Bootstrap vcpkg (test) + if: runner.os == 'Linux' && steps.cache-cpp-install.outputs.cache-hit != 'true' + run: "$GITHUB_WORKSPACE/vcpkg/bootstrap-vcpkg.sh" + # [Note: skip this step if cache-cpp-install was successful] # Bootstrap vcpkg @@ -160,7 +169,7 @@ jobs: if: steps.cache-cpp-install.outputs.cache-hit != 'true' env: VCPKG_BINARY_SOURCES: "clear;nuget,${{ env.GH_PACKAGES_FEED }},readwrite" - run: cmake -S cpp-client/deephaven -B cpp-client/deephaven/build --toolchain ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_INSTALL_PREFIX=${{ env.DHINSTALL }} -DVCPKG_TARGET_TRIPLET=x64-linux-dynamic-release -DVCPKG_OVERLAY_TRIPLETS=cpp-client/deephaven/custom-triplets + run: cmake -S cpp-client/deephaven -B cpp-client/deephaven/build --toolchain "$GITHUB_WORKSPACE/vcpkg/scripts/buildsystems/vcpkg.cmake" -DCMAKE_INSTALL_PREFIX=${{ env.DHINSTALL }} -DVCPKG_TARGET_TRIPLET="$TARGET_TRIPLET" -DVCPKG_OVERLAY_TRIPLETS=cpp-client/deephaven/custom-triplets # [Note: skip this step if cache-cpp-install was successful] # Build our executables and install them in the configured install From 39588159ecb5f4af37a7376018317869b8eaa19c Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 17:47:56 -0400 Subject: [PATCH 24/39] gakkk --- .github/workflows/clients-cpp-python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index 32f5d8a2ce1..490d799a939 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -57,7 +57,7 @@ jobs: - os: ubuntu-24.04 target_triplet: x64-linux-dynamic-release - os: windows-2025-vs2026 - target-triplet: x64-windows-release + target_triplet: x64-windows-release fail-fast: false runs-on: ${{ matrix.os }} From e27b1326b2e960f6930ce06d933b77dcecfab4f1 Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 17:56:01 -0400 Subject: [PATCH 25/39] blah --- .github/workflows/clients-cpp-python.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index 490d799a939..08ed9fb0b26 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -58,7 +58,6 @@ jobs: target_triplet: x64-linux-dynamic-release - os: windows-2025-vs2026 target_triplet: x64-windows-release - fail-fast: false runs-on: ${{ matrix.os }} defaults: @@ -169,7 +168,7 @@ jobs: if: steps.cache-cpp-install.outputs.cache-hit != 'true' env: VCPKG_BINARY_SOURCES: "clear;nuget,${{ env.GH_PACKAGES_FEED }},readwrite" - run: cmake -S cpp-client/deephaven -B cpp-client/deephaven/build --toolchain "$GITHUB_WORKSPACE/vcpkg/scripts/buildsystems/vcpkg.cmake" -DCMAKE_INSTALL_PREFIX=${{ env.DHINSTALL }} -DVCPKG_TARGET_TRIPLET="$TARGET_TRIPLET" -DVCPKG_OVERLAY_TRIPLETS=cpp-client/deephaven/custom-triplets + run: cmake -S cpp-client/deephaven -B cpp-client/deephaven/build --toolchain "$GITHUB_WORKSPACE/vcpkg/scripts/buildsystems/vcpkg.cmake" -DCMAKE_INSTALL_PREFIX=${{ env.DHINSTALL }} -DVCPKG_TARGET_TRIPLET="${{ matrix.target_triplet }}" -DVCPKG_OVERLAY_TRIPLETS=cpp-client/deephaven/custom-triplets # [Note: skip this step if cache-cpp-install was successful] # Build our executables and install them in the configured install From 3df64a06eca1c5f8301e40fb6f03160298b5e71e Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 18:01:26 -0400 Subject: [PATCH 26/39] super blah --- .github/workflows/clients-cpp-python.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index 08ed9fb0b26..a863102991b 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -62,7 +62,7 @@ jobs: runs-on: ${{ matrix.os }} defaults: run: - # This gets you Cygwin on Windows + # This gets you Git Bash on Windows shell: bash steps: @@ -95,7 +95,7 @@ jobs: path: vcpkg - name: Bootstrap vcpkg (test) - if: runner.os == 'Linux' && steps.cache-cpp-install.outputs.cache-hit != 'true' + if: steps.cache-cpp-install.outputs.cache-hit != 'true' run: "$GITHUB_WORKSPACE/vcpkg/bootstrap-vcpkg.sh" # [Note: skip this step if cache-cpp-install was successful] From c3c2de0eb518130662182974fc4648536c150899 Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 18:02:50 -0400 Subject: [PATCH 27/39] ok confirm this works --- .github/workflows/clients-cpp-python.yml | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index a863102991b..4e1b047f5d6 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -94,21 +94,11 @@ jobs: repository: microsoft/vcpkg path: vcpkg - - name: Bootstrap vcpkg (test) - if: steps.cache-cpp-install.outputs.cache-hit != 'true' - run: "$GITHUB_WORKSPACE/vcpkg/bootstrap-vcpkg.sh" - # [Note: skip this step if cache-cpp-install was successful] # Bootstrap vcpkg - - - name: Bootstrap vcpkg (Linux) - if: runner.os == 'Linux' && steps.cache-cpp-install.outputs.cache-hit != 'true' - run: ${{ github.workspace }}/vcpkg/bootstrap-vcpkg.sh - - - name: Bootstrap vcpkg (Windows) - shell: cmd - if: runner.os == 'Windows' && steps.cache-cpp-install.outputs.cache-hit != 'true' - run: ${{ github.workspace }}/vcpkg/bootstrap-vcpkg.bat + - name: Bootstrap vcpkg + if: steps.cache-cpp-install.outputs.cache-hit != 'true' + run: "$GITHUB_WORKSPACE/vcpkg/bootstrap-vcpkg.sh" # [Note: Linux only] # [Note: skip this step if cache-cpp-install was successful] From 34911a7fc9cfe437cbdc02e897fd8693e5026c27 Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 18:10:34 -0400 Subject: [PATCH 28/39] godspeed --- .github/workflows/clients-cpp-python.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index 4e1b047f5d6..611f943dc1e 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -94,6 +94,13 @@ jobs: repository: microsoft/vcpkg path: vcpkg + - name: Zamboni Time + run: | + nuget_tool=$("$GITHUB_WORKSPACE/vcpkg/vcpkg" fetch nuget | tail -n 1) + echo "Path: $nuget_tool" + file "$nuget_tool" + ls -la "$nuget_tool" + # [Note: skip this step if cache-cpp-install was successful] # Bootstrap vcpkg - name: Bootstrap vcpkg From 2d3e76131d3630d0414c36f5f4179149fa2c8d9a Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 18:11:57 -0400 Subject: [PATCH 29/39] zamboni time 2 --- .github/workflows/clients-cpp-python.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index 611f943dc1e..2b57cf469dc 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -94,6 +94,12 @@ jobs: repository: microsoft/vcpkg path: vcpkg + # [Note: skip this step if cache-cpp-install was successful] + # Bootstrap vcpkg + - name: Bootstrap vcpkg + if: steps.cache-cpp-install.outputs.cache-hit != 'true' + run: "$GITHUB_WORKSPACE/vcpkg/bootstrap-vcpkg.sh" + - name: Zamboni Time run: | nuget_tool=$("$GITHUB_WORKSPACE/vcpkg/vcpkg" fetch nuget | tail -n 1) @@ -101,12 +107,6 @@ jobs: file "$nuget_tool" ls -la "$nuget_tool" - # [Note: skip this step if cache-cpp-install was successful] - # Bootstrap vcpkg - - name: Bootstrap vcpkg - if: steps.cache-cpp-install.outputs.cache-hit != 'true' - run: "$GITHUB_WORKSPACE/vcpkg/bootstrap-vcpkg.sh" - # [Note: Linux only] # [Note: skip this step if cache-cpp-install was successful] # Get mono so that we can run nuget. This is because the nuget.exe that From 37b5b8e1a2f3273056f1ddca5a5d087871fd33eb Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 18:14:28 -0400 Subject: [PATCH 30/39] why am I sad --- .github/workflows/clients-cpp-python.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index 2b57cf469dc..72ba59c6fc2 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -102,6 +102,8 @@ jobs: - name: Zamboni Time run: | + echo ls -l "$GITHUB_WORKSPACE/vcpkg/vcpkg" + ls -l "$GITHUB_WORKSPACE/vcpkg/vcpkg" nuget_tool=$("$GITHUB_WORKSPACE/vcpkg/vcpkg" fetch nuget | tail -n 1) echo "Path: $nuget_tool" file "$nuget_tool" From af37011c27feeb9860d11434665bbd7ee23859c9 Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 18:16:14 -0400 Subject: [PATCH 31/39] i'll be honest this is pissing me off --- .github/workflows/clients-cpp-python.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index 72ba59c6fc2..bfb0ac9da45 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -102,7 +102,9 @@ jobs: - name: Zamboni Time run: | - echo ls -l "$GITHUB_WORKSPACE/vcpkg/vcpkg" + echo "$GITHUB_WORKSPACE" + ls -l "$GITHUB_WORKSPACE" + ls -l "$GITHUB_WORKSPACE/vcpkg" ls -l "$GITHUB_WORKSPACE/vcpkg/vcpkg" nuget_tool=$("$GITHUB_WORKSPACE/vcpkg/vcpkg" fetch nuget | tail -n 1) echo "Path: $nuget_tool" From a23011a6190f0d07c467170eab970c3a8bbd40f4 Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 18:16:30 -0400 Subject: [PATCH 32/39] thi s too --- .github/workflows/clients-cpp-python.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index bfb0ac9da45..3f095f7d5f3 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -106,6 +106,7 @@ jobs: ls -l "$GITHUB_WORKSPACE" ls -l "$GITHUB_WORKSPACE/vcpkg" ls -l "$GITHUB_WORKSPACE/vcpkg/vcpkg" + $GITHUB_WORKSPACE/vcpkg/vcpkg nuget_tool=$("$GITHUB_WORKSPACE/vcpkg/vcpkg" fetch nuget | tail -n 1) echo "Path: $nuget_tool" file "$nuget_tool" From eba194ff9dda8a656e165723214d5e4c34a019a4 Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 18:17:10 -0400 Subject: [PATCH 33/39] might as well go all the way --- .github/workflows/clients-cpp-python.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index 3f095f7d5f3..b6067479e90 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -107,6 +107,7 @@ jobs: ls -l "$GITHUB_WORKSPACE/vcpkg" ls -l "$GITHUB_WORKSPACE/vcpkg/vcpkg" $GITHUB_WORKSPACE/vcpkg/vcpkg + $GITHUB_WORKSPACE/vcpkg/vcpkg fetch nuget nuget_tool=$("$GITHUB_WORKSPACE/vcpkg/vcpkg" fetch nuget | tail -n 1) echo "Path: $nuget_tool" file "$nuget_tool" From c1e61f0ea1cf28370f37be42ed4d7aaa7a6f2f75 Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 18:19:18 -0400 Subject: [PATCH 34/39] the final ultimatum --- .github/workflows/clients-cpp-python.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index b6067479e90..d05dd1b7f50 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -102,12 +102,8 @@ jobs: - name: Zamboni Time run: | - echo "$GITHUB_WORKSPACE" - ls -l "$GITHUB_WORKSPACE" - ls -l "$GITHUB_WORKSPACE/vcpkg" - ls -l "$GITHUB_WORKSPACE/vcpkg/vcpkg" - $GITHUB_WORKSPACE/vcpkg/vcpkg - $GITHUB_WORKSPACE/vcpkg/vcpkg fetch nuget + echo "$GITHUB_WORKSPACE/vcpkg/vcpkg" fetch nuget + "$GITHUB_WORKSPACE/vcpkg/vcpkg" fetch nuget nuget_tool=$("$GITHUB_WORKSPACE/vcpkg/vcpkg" fetch nuget | tail -n 1) echo "Path: $nuget_tool" file "$nuget_tool" From 1a07d2880d3962b7fe5e19ad54a6040fc4417d6c Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 18:27:54 -0400 Subject: [PATCH 35/39] now try this version --- .github/workflows/clients-cpp-python.yml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index d05dd1b7f50..7950768306c 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -96,25 +96,17 @@ jobs: # [Note: skip this step if cache-cpp-install was successful] # Bootstrap vcpkg + - name: Bootstrap vcpkg if: steps.cache-cpp-install.outputs.cache-hit != 'true' run: "$GITHUB_WORKSPACE/vcpkg/bootstrap-vcpkg.sh" - - name: Zamboni Time - run: | - echo "$GITHUB_WORKSPACE/vcpkg/vcpkg" fetch nuget - "$GITHUB_WORKSPACE/vcpkg/vcpkg" fetch nuget - nuget_tool=$("$GITHUB_WORKSPACE/vcpkg/vcpkg" fetch nuget | tail -n 1) - echo "Path: $nuget_tool" - file "$nuget_tool" - ls -la "$nuget_tool" - # [Note: Linux only] # [Note: skip this step if cache-cpp-install was successful] # Get mono so that we can run nuget. This is because the nuget.exe that # ships with vcpkg is an old .NET Framework binary (Windows format), and - # only mono knows how to run such a thing on Linux. Some day the maintainers - # will fix this so it runs with the cross-platform "dotnet". + # only mono knows how to run such a thing on Linux. Some day the + # maintainers will fix this so it runs with the cross-platform "dotnet". - name: Install mono if: runner.os == 'Linux' && steps.cache-cpp-install.outputs.cache-hit != 'true' @@ -167,7 +159,7 @@ jobs: if: steps.cache-cpp-install.outputs.cache-hit != 'true' env: VCPKG_BINARY_SOURCES: "clear;nuget,${{ env.GH_PACKAGES_FEED }},readwrite" - run: cmake -S cpp-client/deephaven -B cpp-client/deephaven/build --toolchain "$GITHUB_WORKSPACE/vcpkg/scripts/buildsystems/vcpkg.cmake" -DCMAKE_INSTALL_PREFIX=${{ env.DHINSTALL }} -DVCPKG_TARGET_TRIPLET="${{ matrix.target_triplet }}" -DVCPKG_OVERLAY_TRIPLETS=cpp-client/deephaven/custom-triplets + run: cmake -S cpp-client/deephaven -B cpp-client/deephaven/build --toolchain "$GITHUB_WORKSPACE/vcpkg/scripts/buildsystems/vcpkg.cmake" -DCMAKE_INSTALL_PREFIX="$DHINSTALL" -DVCPKG_TARGET_TRIPLET="${{ matrix.target_triplet }}" -DVCPKG_OVERLAY_TRIPLETS=cpp-client/deephaven/custom-triplets # [Note: skip this step if cache-cpp-install was successful] # Build our executables and install them in the configured install From 41c05aa04536661366ebaf213564534bb27e6ee1 Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 18:50:41 -0400 Subject: [PATCH 36/39] arg one more painful change --- .github/workflows/clients-cpp-python.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index 7950768306c..98280e2921e 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -5,7 +5,7 @@ on: push: branches: [ 'main', 'check/**', 'release/v*' ] permissions: - packages: write + contents: read #=============================================================================== # Background: in this workflow there are three different kinds of caching going @@ -51,6 +51,9 @@ jobs: #============================================================================= cpp_job: name: "Build C++ (or use cache)" + permissions: + contents: read + packages: write strategy: matrix: include: @@ -108,9 +111,9 @@ jobs: # only mono knows how to run such a thing on Linux. Some day the # maintainers will fix this so it runs with the cross-platform "dotnet". - - name: Install mono + - name: Install mono (Linux) if: runner.os == 'Linux' && steps.cache-cpp-install.outputs.cache-hit != 'true' - run: sudo apt install mono-complete + run: sudo apt install -y mono-complete # [Note: skip this step if cache-cpp-install was successful] # Adapted from @@ -170,8 +173,8 @@ jobs: run: cmake --build cpp-client/deephaven/build --config RelWithDebInfo --target install --parallel # Upload results as build artifact. Note: if cache-cpp-install was - # then none of the intervening steps were executed. In that case, - # this effectively just copies the cache to the build artifact. + # successful, then none of the intervening steps were executed. In that + # case, this effectively just copies the cache to the build artifact. - name: Upload install dir id: upload-install-directory From fe922c8846c415e7b8b6d9585ef7caaedf2d2e60 Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 19:34:13 -0400 Subject: [PATCH 37/39] This pain, no name --- .github/workflows/clients-cpp-python.yml | 12 ++++++++++++ cpp-client/deephaven/vcpkg-configuration.json | 11 ++--------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index 98280e2921e..862fbbcbe65 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -86,6 +86,17 @@ jobs: path: ${{ env.DHINSTALL }} key: ${{ runner.os }}-cpp-client-${{ hashFiles('cpp-client/**', 'proto/**') }} + # Make sure we check out the same version of the vcpkg tool that we + # have configured as our registry baseline. This (we suppose) will help + # improve the problem we have with our package hash drift. + + - name: Extract vcpkg baseline + id: vcpkg-baseline + run: | + baseline=$(jq -r '.["default-registry"].baseline' vcpkg-configuration.json) + echo "baseline=$baseline" + echo "baseline=$baseline" >> "$GITHUB_OUTPUT" + # [Note: skip this step if cache-cpp-install was successful] # Check out vcpkg @@ -95,6 +106,7 @@ jobs: uses: actions/checkout@v6 with: repository: microsoft/vcpkg + ref: ${{ steps.vcpkg-baseline.outputs.baseline }} path: vcpkg # [Note: skip this step if cache-cpp-install was successful] diff --git a/cpp-client/deephaven/vcpkg-configuration.json b/cpp-client/deephaven/vcpkg-configuration.json index 733e6283f80..1b3ca5f58dd 100644 --- a/cpp-client/deephaven/vcpkg-configuration.json +++ b/cpp-client/deephaven/vcpkg-configuration.json @@ -1,14 +1,7 @@ { "default-registry": { "kind": "git", - "baseline": "d2bf92d8880153596e7ba15a2d672d4d77aa9d30", + "baseline": "443dda0f876acb0baf87a907e6d64e25a1184c16", "repository": "https://github.com/microsoft/vcpkg" - }, - "registries": [ - { - "kind": "artifact", - "location": "https://github.com/microsoft/vcpkg-ce-catalog/archive/refs/heads/main.zip", - "name": "microsoft" - } - ] + } } From dbf5d2a621f4aea4b891cab6972b9c07b7d4b79f Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 19:35:23 -0400 Subject: [PATCH 38/39] NO --- .github/workflows/clients-cpp-python.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index 862fbbcbe65..e69760231a0 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -93,9 +93,9 @@ jobs: - name: Extract vcpkg baseline id: vcpkg-baseline run: | - baseline=$(jq -r '.["default-registry"].baseline' vcpkg-configuration.json) - echo "baseline=$baseline" - echo "baseline=$baseline" >> "$GITHUB_OUTPUT" + baseline=$(jq -r '.["default-registry"].baseline' vcpkg-configuration.json) + echo "baseline=$baseline" + echo "baseline=$baseline" >> "$GITHUB_OUTPUT" # [Note: skip this step if cache-cpp-install was successful] # Check out vcpkg From 98bb00d53f4bb04a4d8fe34e946430f4f2fdf64f Mon Sep 17 00:00:00 2001 From: Corey Kosak Date: Wed, 6 May 2026 19:36:51 -0400 Subject: [PATCH 39/39] REALLY NO --- .github/workflows/clients-cpp-python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml index e69760231a0..b8c33979f87 100644 --- a/.github/workflows/clients-cpp-python.yml +++ b/.github/workflows/clients-cpp-python.yml @@ -93,7 +93,7 @@ jobs: - name: Extract vcpkg baseline id: vcpkg-baseline run: | - baseline=$(jq -r '.["default-registry"].baseline' vcpkg-configuration.json) + baseline=$(jq -r '.["default-registry"].baseline' ./cpp-client/deephaven/vcpkg-configuration.json) echo "baseline=$baseline" echo "baseline=$baseline" >> "$GITHUB_OUTPUT"