diff --git a/.github/workflows/clients-cpp-python.yml b/.github/workflows/clients-cpp-python.yml new file mode 100644 index 00000000000..b8c33979f87 --- /dev/null +++ b/.github/workflows/clients-cpp-python.yml @@ -0,0 +1,321 @@ +name: "C++/Python" +on: + pull_request: + branches: [ 'main', 'rc/v*' ] + push: + branches: [ 'main', 'check/**', 'release/v*' ] +permissions: + contents: read + +#=============================================================================== +# Background: in this workflow there are three different kinds of caching going +# on: +# +# 1. The outputs are Github Build artifacts, attached to every action run. +# These artifacts can be downloaded from e.g. +# https://github.com/deephaven/deephaven-core/actions/runs/17416308139 +# but the specific URL depends on the build. +# We use this to upload: +# - The C++ install directory +# - The .whl file for the static Python client +# - The .whl file for the ticking Python client +# +# 2. The C++ install directory is cached via actions/cache, and keyed +# by the hash of the cpp-client and proto directories. This allows us +# to skip recompilation if these files don't change. This is an optional +# optimization that might not be very significant. +# +# 3. Github Packages, which is a NuGet-compatible repo accessible from +# https://github.com/orgs/deephaven/packages?repo_name=deephaven-core +# and controlled via the NuGet API. We use this package repository to hold +# the binaries for the C++ dependencies e build (grpc, arrow, etc.). It is +# managed automatically for us by cmake/vcpkg/nuget. This optimization is +# important because building the C++ dependencies takes significant time +# (45 minutes maybe). +#=============================================================================== + +env: + GH_PACKAGES_USERNAME: ${{ github.repository_owner }} + GH_PACKAGES_FEED: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json + DHINSTALL: ${{ github.workspace }}/dhinstall + +jobs: + #============================================================================= + # Identify version of Deephaven Core + #============================================================================= + version_job: + uses: ./.github/workflows/dhc-version.yml + + #============================================================================= + # Build the C++ Client + #============================================================================= + cpp_job: + name: "Build C++ (or use cache)" + permissions: + contents: read + packages: write + strategy: + matrix: + include: + - os: ubuntu-24.04 + target_triplet: x64-linux-dynamic-release + - os: windows-2025-vs2026 + target_triplet: x64-windows-release + fail-fast: false + runs-on: ${{ matrix.os }} + defaults: + run: + # This gets you Git Bash on Windows + shell: bash + + steps: + - name: Check out this repo + uses: actions/checkout@v6 + + # Try to fetch the whole installation directory (the final product of the + # build) from the Github Cache. The key to this cache is derived + # from the hash of the contents of the files in the cpp-client and proto + # directories. If this fetch is successful, we skip over all the build + # work and just copy this cached install directory to the output artifact + # for this run. + + - name: Try to restore cached installdir + id: cache-cpp-install + uses: actions/cache@v5 + with: + path: ${{ env.DHINSTALL }} + key: ${{ runner.os }}-cpp-client-${{ hashFiles('cpp-client/**', 'proto/**') }} + + # Make sure we check out the same version of the vcpkg tool that we + # have configured as our registry baseline. This (we suppose) will help + # improve the problem we have with our package hash drift. + + - name: Extract vcpkg baseline + id: vcpkg-baseline + run: | + baseline=$(jq -r '.["default-registry"].baseline' ./cpp-client/deephaven/vcpkg-configuration.json) + echo "baseline=$baseline" + echo "baseline=$baseline" >> "$GITHUB_OUTPUT" + + # [Note: skip this step if cache-cpp-install was successful] + # Check out vcpkg + + - name: Check out vcpkg + id: checkout-vcpkg + if: steps.cache-cpp-install.outputs.cache-hit != 'true' + uses: actions/checkout@v6 + with: + repository: microsoft/vcpkg + ref: ${{ steps.vcpkg-baseline.outputs.baseline }} + path: vcpkg + + # [Note: skip this step if cache-cpp-install was successful] + # Bootstrap vcpkg + + - name: Bootstrap vcpkg + if: steps.cache-cpp-install.outputs.cache-hit != 'true' + run: "$GITHUB_WORKSPACE/vcpkg/bootstrap-vcpkg.sh" + + # [Note: Linux only] + # [Note: skip this step if cache-cpp-install was successful] + # Get mono so that we can run nuget. This is because the nuget.exe that + # ships with vcpkg is an old .NET Framework binary (Windows format), and + # only mono knows how to run such a thing on Linux. Some day the + # maintainers will fix this so it runs with the cross-platform "dotnet". + + - name: Install mono (Linux) + if: runner.os == 'Linux' && steps.cache-cpp-install.outputs.cache-hit != 'true' + run: sudo apt install -y mono-complete + + # [Note: skip this step if cache-cpp-install was successful] + # Adapted from + # https://learn.microsoft.com/en-us/vcpkg/consume/binary-caching-github-packages?pivots=linux-runner + # Configure nuget to know how to download (or upload) each cached + # package that we need (or create) + + - name: "Configure nuget (Linux)" + if: runner.os == 'Linux' && steps.cache-cpp-install.outputs.cache-hit != 'true' + run: | + vcpkg_exe=${{ github.workspace }}/vcpkg/vcpkg + nuget_dotnet=$($vcpkg_exe fetch nuget | tail -n 1) + mono $nuget_dotnet \ + sources add \ + -Source "${{ env.GH_PACKAGES_FEED }}" \ + -StorePasswordInClearText \ + -Name GitHubPackages \ + -UserName "${{ env.GH_PACKAGES_USERNAME }}" \ + -Password "${{ secrets.GITHUB_TOKEN }}" + mono $nuget_dotnet \ + setapikey "${{ secrets.GITHUB_TOKEN }}" \ + -Source "${{ env.GH_PACKAGES_FEED }}" + + - name: "Configure nuget (Windows)" + if: runner.os == 'Windows' && steps.cache-cpp-install.outputs.cache-hit != 'true' + shell: pwsh + run: | + $nuget_exe = & ./vcpkg/vcpkg.exe fetch nuget + & $nuget_exe sources add ` + -Source "${{ env.GH_PACKAGES_FEED }}" ` + -StorePasswordInClearText ` + -Name GitHubPackages ` + -UserName "${{ env.GH_PACKAGES_USERNAME }}" ` + -Password "${{ secrets.GITHUB_TOKEN }}" + & $nuget_exe setapikey "${{ secrets.GITHUB_TOKEN }}" -Source "${{ env.GH_PACKAGES_FEED }}" + + # [Note: skip this step if cache-cpp-install was successful] + # Build packages and configure cmake. Check nupkg cache for existing + # packages, otherwise build and upload. + # 1. Invoke vcpkg to: + # a) Try to fetch each of the dependent libraries from the nuget cache + # b) If not found, build them and insert them in the nuget cache + # 2. Configure cmake to prepare for the build/install step, which comes next + + - name: "Build/cache dependencies" + if: steps.cache-cpp-install.outputs.cache-hit != 'true' + env: + VCPKG_BINARY_SOURCES: "clear;nuget,${{ env.GH_PACKAGES_FEED }},readwrite" + run: cmake -S cpp-client/deephaven -B cpp-client/deephaven/build --toolchain "$GITHUB_WORKSPACE/vcpkg/scripts/buildsystems/vcpkg.cmake" -DCMAKE_INSTALL_PREFIX="$DHINSTALL" -DVCPKG_TARGET_TRIPLET="${{ matrix.target_triplet }}" -DVCPKG_OVERLAY_TRIPLETS=cpp-client/deephaven/custom-triplets + + # [Note: skip this step if cache-cpp-install was successful] + # Build our executables and install them in the configured install + # directory {{ env.DHINSTALL }} + + - name: Build and install + if: steps.cache-cpp-install.outputs.cache-hit != 'true' + run: cmake --build cpp-client/deephaven/build --config RelWithDebInfo --target install --parallel + + # Upload results as build artifact. Note: if cache-cpp-install was + # successful, then none of the intervening steps were executed. In that + # case, this effectively just copies the cache to the build artifact. + + - name: Upload install dir + id: upload-install-directory + uses: actions/upload-artifact@v7 + with: + name: cpp-client-install-${{ matrix.os }} + path: ${{ env.DHINSTALL }} + + #============================================================================= + # Build the Python Static Client (not OS dependent) + #============================================================================= + + python_static_job: + name: Python static + runs-on: ubuntu-24.04 + needs: [version_job] + defaults: + run: + shell: bash + + steps: + - name: Check out this repo + uses: actions/checkout@v6 + + - name: Install Python + uses: actions/setup-python@v6 + with: + python-version: '3.10' + + - name: Install the 'wheel' package + run: pip3 install wheel + + - name: Install requirements-dev.txt + run: | + cd ./py/client + pip3 install -r requirements-dev.txt + + - name: Run setup.py + env: + DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} + run: | + cd ./py/client + python setup.py bdist_wheel + + # Upload static .whl file as build artifact, to be used directly or by # dependent clients like Python ticking + - name: Upload .whl file + uses: actions/upload-artifact@v7 + with: + name: py-static-wheel + path: ./py/client/dist/*.whl + + #============================================================================= + # Build the Python Ticking Client (OS dependent) + #============================================================================= + + python_ticking_job: + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, windows-2025-vs2026] + name: Python ticking + runs-on: ${{ matrix.os }} + needs: [cpp_job, python_static_job, version_job] + defaults: + run: + shell: bash + + steps: + - name: Check out this repo + uses: actions/checkout@v6 + + # Get the C++ installation as a build artifact + - name: Fetch C++ installation + uses: actions/download-artifact@v8 + with: + name: cpp-client-install-${{ matrix.os }} + path: ${{ env.DHINSTALL }} + + # Get the Python static installation as a build artifact + - name: Fetch Python static installation + uses: actions/download-artifact@v8 + with: + name: py-static-wheel + path: ./py/client/dist/ + + - name: Install Python + uses: actions/setup-python@v6 + with: + python-version: '3.10' + + - name: Install the 'wheel' and 'cython' packages + run: pip3 install wheel cython + + - name: Install requirements-dev.txt + run: | + cd ./py/client + pip3 install -r requirements-dev.txt + + - name: Install static client + run: pip3 install --force --no-deps ./py/client/dist/*.whl + + # Pinned to a specific SHA to avoid future surprises + - name: set up PATH for cl.exe + if: runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 + + - name: Build our Cython code (Linux) + if: runner.os == 'Linux' + env: + DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} + CPPFLAGS: -I${{ env.DHINSTALL }}/include + LDFLAGS: -L${{ env.DHINSTALL }}/lib + run: | + cd ./py/client-ticking + python setup.py build_ext -i + python setup.py bdist_wheel + + - name: Build our Cython code (Windows) + if: runner.os == 'Windows' + shell: cmd + env: + DEEPHAVEN_VERSION: ${{ needs.version_job.outputs.dhc_version }} + run: | + cd .\py\client-ticking + python setup.py build_ext -i + python setup.py bdist_wheel + + - name: Upload ticking .whl file as build artifact + uses: actions/upload-artifact@v7 + with: + name: py-ticking-wheel-${{ matrix.os }} + path: ./py/client-ticking/dist/*.whl diff --git a/.github/workflows/clients-csharp.yml b/.github/workflows/clients-csharp.yml new file mode 100644 index 00000000000..4771d48991f --- /dev/null +++ b/.github/workflows/clients-csharp.yml @@ -0,0 +1,58 @@ +name: "C# Client" +on: + pull_request: + branches: [ 'main', 'rc/v*' ] + push: + branches: [ 'main', 'check/**', 'release/v*' ] +permissions: + packages: write + +env: + GH_PACKAGES_USERNAME: ${{ github.repository_owner }} + GH_PACKAGES_FEED: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json + DHINSTALL: ${{ github.workspace }}/dhinstall + +jobs: + #============================================================================= + # Identify version of Deephaven Core + #============================================================================= + version_job: + uses: ./.github/workflows/dhc-version.yml + + #============================================================================= + # Build the .NET Client (cross-platform, but we use the Ubuntu runner) + #============================================================================= + dotnet_job: + name: Build .NET client + runs-on: ubuntu-24.04 + needs: version_job + defaults: + run: + shell: bash + + steps: + - name: Check out this repo + uses: actions/checkout@v6 + + - name: Setup dotnet + uses: actions/setup-dotnet@v5 + with: + dotnet-version: '10.0.x' + + - name: Build Deephaven Core C# Client + run: | + cd csharp/client/Dh_NetClient + dotnet build -c Release + + - name: Create unsigned NuGet package + env: + DHC_VERSION: ${{ needs.version_job.outputs.dhc_version }} + run: | + cd csharp/client/Dh_NetClient + dotnet pack --no-build -c Release /p:Platform="Any CPU" /p:PackageVersion=$DHC_VERSION + + - name: Upload unsigned NuGet package as build artifact + uses: actions/upload-artifact@v7 + with: + name: dotnet-client + path: ./csharp/client/Dh_NetClient/bin/Release/*.nupkg diff --git a/.github/workflows/dhc-version.yml b/.github/workflows/dhc-version.yml new file mode 100644 index 00000000000..251ddaa2fc7 --- /dev/null +++ b/.github/workflows/dhc-version.yml @@ -0,0 +1,39 @@ +name: "Get DHC version" +on: + workflow_call: + outputs: + dhc_version: + description: "The detected version of Deephaven Core" + value: ${{ jobs.version_job.outputs.dhc_version }} + +jobs: + #============================================================================= + # Identify version of Deephaven Core by running awk on gradle.properties + #============================================================================= + version_job: + name: "Get DHC version" + runs-on: ubuntu-24.04 + defaults: + run: + shell: bash + + outputs: + dhc_version: ${{ steps.get-dhc-version.outputs.dhc_version }} + + steps: + - name: Check out this repo + uses: actions/checkout@v6 + + - name: Output Deephaven Version + id: get-dhc-version + run: | + DHC_VERSION=$(awk -F= ' + /^[^#]/ { props[$1] = $2 } + END { + v = props["deephavenMajorVersion"] "." props["deephavenMinorVersion"] + if (props["deephavenBaseQualifier"] != "") v = v "-" props["deephavenBaseQualifier"] + print v + } + ' gradle.properties) + echo "dhc_version=$DHC_VERSION" >> $GITHUB_OUTPUT + echo FYI dhc_version is $DHC_VERSION diff --git a/cpp-client/CHANGE_COOKIE.txt b/cpp-client/CHANGE_COOKIE.txt new file mode 100644 index 00000000000..6232bf844ec --- /dev/null +++ b/cpp-client/CHANGE_COOKIE.txt @@ -0,0 +1 @@ +taint the C++ build to cause it to run again or whatever diff --git a/cpp-client/deephaven/CMakeLists.txt b/cpp-client/deephaven/CMakeLists.txt index 041fc1603a1..b764c348a9b 100644 --- a/cpp-client/deephaven/CMakeLists.txt +++ b/cpp-client/deephaven/CMakeLists.txt @@ -52,6 +52,17 @@ if(NOT DHCORE_ONLY) INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} ) + if(VCPKG_TOOLCHAIN AND NOT WIN32) + install(DIRECTORY ${CMAKE_BINARY_DIR}/vcpkg_installed/${VCPKG_TARGET_TRIPLET}/lib/ + DESTINATION lib + FILES_MATCHING + PATTERN "*.so" + PATTERN "*.so.*" + PATTERN "pkgconfig" EXCLUDE + PATTERN "cmake" EXCLUDE + ) + endif() + include(CMakePackageConfigHelpers) write_basic_package_version_file( "${CMAKE_CURRENT_BINARY_DIR}/deephaven/deephavenConfigVersion.cmake" diff --git a/cpp-client/deephaven/custom-triplets/x64-linux-dynamic-release.cmake b/cpp-client/deephaven/custom-triplets/x64-linux-dynamic-release.cmake new file mode 100644 index 00000000000..4921b59b897 --- /dev/null +++ b/cpp-client/deephaven/custom-triplets/x64-linux-dynamic-release.cmake @@ -0,0 +1,14 @@ +# These settings are inspired by [vcpkg repo]/triplets/x64-linux.cmake + +# These settings are the same: +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_CMAKE_SYSTEM_NAME Linux) + +# This setting is changed because we want dynamic library linking +# rather than static. +set(VCPKG_LIBRARY_LINKAGE dynamic) + +# This setting is new because we want to build only the 'release' +# version of the packages, rather the both 'debug' and 'release' +set(VCPKG_BUILD_TYPE release) diff --git a/cpp-client/deephaven/custom-triplets/x64-windows-release b/cpp-client/deephaven/custom-triplets/x64-windows-release new file mode 100644 index 00000000000..49034cbe0f4 --- /dev/null +++ b/cpp-client/deephaven/custom-triplets/x64-windows-release @@ -0,0 +1,10 @@ +# These settings are inspired by [vcpkg repo]/triplets/x64-windows.cmake + +# These settings are the same: +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE dynamic) + +# This setting is new because we want to build only the 'release' +# version of the packages, rather the both 'debug' and 'release' +set(VCPKG_BUILD_TYPE release) diff --git a/cpp-client/deephaven/vcpkg-configuration.json b/cpp-client/deephaven/vcpkg-configuration.json index 733e6283f80..1b3ca5f58dd 100644 --- a/cpp-client/deephaven/vcpkg-configuration.json +++ b/cpp-client/deephaven/vcpkg-configuration.json @@ -1,14 +1,7 @@ { "default-registry": { "kind": "git", - "baseline": "d2bf92d8880153596e7ba15a2d672d4d77aa9d30", + "baseline": "443dda0f876acb0baf87a907e6d64e25a1184c16", "repository": "https://github.com/microsoft/vcpkg" - }, - "registries": [ - { - "kind": "artifact", - "location": "https://github.com/microsoft/vcpkg-ce-catalog/archive/refs/heads/main.zip", - "name": "microsoft" - } - ] + } }