Wheels: ADIOS2 v2.10.1 (#1554)

ax3l · vicentebolea · web-flow · commit 9a0b519ef96e · 2024-10-08T10:04:43.000-07:00
* Wheels: ADIOS2 v2.9.2 Build wheels with ADIOS2 v2.9.2, which fixes c-blosc2 issues with null-blocks. * C-Blosc2 v2.11.1 * ADIOS2: Remove installed FindBlosc2.cmake Again * [Patch] C-Blosc2 Static * No Patch: Static C-Blosc2 * Windows: More Blosc2 Updates * Windows: keep FindBlosc2.cmake * Win: Try ADIOS2 v2.10.0-rc1 * ADIOS2 Windows: Disable libFFI * Print CMake Version * ADIOS2: `release_210` Branch * Use Vincente's patch Co-authored-by: Vicente Bolea <vicente.bolea@gmail.com> * ADIOS2: v2.10.1 * Windows: Update ADIOS2 v2.10.1 URL * Update GH Actions * Python 3.8 -> 3.13 * cibuildwheel: 2.16.2 -> 2.21.2 * `-DADIOS2_USE_Campaign=OFF` Issues: - ornladios/ADIOS2#4360 - ornladios/ADIOS2#4148 * Win: Compile Release Install was already Release * Window: Fix unzipped ADIOS2 path * `actions/upload-artifact@v4` Names * `macos-11` -> `macos-12` --------- Co-authored-by: Vicente Bolea <vicente.bolea@gmail.com>
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -40,15 +40,15 @@ jobs:
               CMAKE_GENERATOR: "Visual Studio 16 2019"
               CMAKE_GENERATOR_PLATFORM: "Win32"
 
-          - os: macos-11
+          - os: macos-12
             arch: "x86_64"
             env:
               MACOSX_DEPLOYMENT_TARGET: 11.0
 
           # Apple Silicon M1/arm64/aarch64 builds:
           #   https://cibuildwheel.readthedocs.io/en/stable/faq/#apple-silicon
           #   https://github.com/pypa/cibuildwheel/pull/704
-          - os: macos-11
+          - os: macos-12
             arch: "arm64"
             env:
               CMAKE_OSX_ARCHITECTURES: "arm64"
@@ -60,31 +60,31 @@ jobs:
           #   https://developer.apple.com/documentation/apple-silicon/building-a-universal-macos-binary
           # ADIOS1 tricky to build and HDF5 even with CMake as well (as of 1.12)
           # We could build them twice and use `lipo` to combine the lib artifacts.
-          #- os: macos-11
+          #- os: macos-12
           #  arch: "universal2"
           #  env:
           #    CMAKE_OSX_ARCHITECTURES: "arm64;x86_64"
           #    MACOSX_DEPLOYMENT_TARGET: 11.0
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         path: 'src'
         ref: '0.15.2'
 
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         path: 'src/.github/'
 
-    - uses: actions/setup-python@v1
+    - uses: actions/setup-python@v5
       name: Install Python
       with:
-        python-version: '3.8'
+        python-version: '3.9'
 
     - name: Install cibuildwheel
       run: |
         python -m pip install --upgrade pip setuptools wheel
-        python -m pip install cibuildwheel==2.16.2
+        python -m pip install cibuildwheel==2.21.2
 
     # Hack: cmake --trace-expand
     #- name: Download Patch 2/2
@@ -106,15 +106,15 @@ jobs:
         #     https://github.com/pypa/manylinux/issues/899
         CIBW_SKIP: "pp*-manylinux*"
         CIBW_ARCHS: "${{ matrix.arch }}"
-        CIBW_PROJECT_REQUIRES_PYTHON: ">=3.8"
+        CIBW_PROJECT_REQUIRES_PYTHON: ">=3.9"
         # Install dependencies
         CIBW_BEFORE_BUILD_LINUX: bash -x .github/library_builders.sh
         CIBW_BEFORE_BUILD_MACOS: bash -x .github/library_builders.sh
         CIBW_BEFORE_BUILD_WINDOWS: 'cmd /E:ON /V:ON /C .github\library_builders.bat'
         # for the openPMD-api build, CMake shall search for
         # static dependencies of HDF5 and ADIOS1 (see setup.py)
         CIBW_ENVIRONMENT: HDF5_USE_STATIC_LIBRARIES='ON' ZLIB_USE_STATIC_LIBS='ON' ADIOS_USE_STATIC_LIBS='ON' openPMD_CMAKE_openPMD_USE_HDF5='ON' openPMD_CMAKE_openPMD_USE_ADIOS2='ON'
-        CIBW_ENVIRONMENT_WINDOWS: HDF5_USE_STATIC_LIBRARIES='ON' ZLIB_USE_STATIC_LIBS='ON' openPMD_CMAKE_openPMD_USE_HDF5='ON' openPMD_CMAKE_openPMD_USE_ADIOS2='ON' CMAKE_PREFIX_PATH='C:/Program Files (x86)/ADIOS2;C:/Program Files (x86)/blosc;C:/Program Files (x86)/blosc2;C:/Program Files (x86)/HDF5;C:/Program Files (x86)/ZFP;C:/Program Files (x86)/zlib'
+        CIBW_ENVIRONMENT_WINDOWS: HDF5_USE_STATIC_LIBRARIES='ON' ZLIB_USE_STATIC_LIBS='ON' openPMD_CMAKE_openPMD_USE_HDF5='ON' openPMD_CMAKE_openPMD_USE_ADIOS2='ON' CMAKE_PREFIX_PATH='C:/Program Files (x86)/ADIOS2;C:/Program Files (x86)/blosc2;C:/Program Files (x86)/HDF5;C:/Program Files (x86)/ZFP;C:/Program Files (x86)/zlib'
         # C++17 support in macOS 10.13+ (partial) and 10.14+ (std::visit) and 10.15+ (std::filesystem::path)
         #   https://cibuildwheel.readthedocs.io/en/stable/cpp_standards/#macos-and-deployment-target-versions
         # arm64/aarch64 (M1/M2) requires 11.0+
@@ -132,11 +132,12 @@ jobs:
         python setup.py sdist --dist-dir ../wheelhouse
         python -m cibuildwheel --output-dir ../wheelhouse
 
-    - uses: actions/upload-artifact@v1
+    - uses: actions/upload-artifact@v4
       name: Publish as GitHub artifact
       with:
-        name: wheels
+        name: wheels-${{ matrix.os }}-${{ matrix.arch }}
         path: ./wheelhouse
+        overwrite: true
 
     - name: Publish on pypi.org
       if: github.event_name == 'push' && github.repository == 'openPMD/openPMD-api' && github.ref == 'refs/heads/wheels'
diff --git a/.travis.yml b/.travis.yml
@@ -10,7 +10,7 @@ env:
   global:
     - OPENPMD_GIT_REF="0.15.2"
 
-    - CIBW_PROJECT_REQUIRES_PYTHON=">=3.8"
+    - CIBW_PROJECT_REQUIRES_PYTHON=">=3.9"
     # Install dependencies on Linux and OSX
     - CIBW_BEFORE_BUILD="bash -x .github/library_builders.sh"
     # for the openPMD-api build, CMake shall search for
@@ -28,7 +28,7 @@ jobs:
       arch: arm64
       dist: focal
       env:
-        - CIBW_BUILD="cp38-manylinux_aarch64 cp39-manylinux_aarch64"
+        - CIBW_BUILD="cp39-manylinux_aarch64"
     - services: docker
       arch: arm64
       dist: focal
@@ -38,7 +38,7 @@ jobs:
       arch: arm64
       dist: focal
       env:
-        - CIBW_BUILD="cp312-manylinux_aarch64"
+        - CIBW_BUILD="cp312-manylinux_aarch64 cp313-manylinux_aarch64"
     - services: docker
       arch: arm64
       dist: focal
@@ -48,13 +48,13 @@ jobs:
       arch: arm64
       dist: focal
       env:
-        - CIBW_BUILD="pp38-manylinux_aarch64 pp39-manylinux_aarch64"
+        - CIBW_BUILD="pp39-manylinux_aarch64"
     # +musllinux
     - services: docker
       arch: arm64
       dist: focal
       env:
-        - CIBW_BUILD="cp38-musllinux_aarch64 cp39-musllinux_aarch64"
+        - CIBW_BUILD="cp39-musllinux_aarch64"
     - services: docker
       arch: arm64
       dist: focal
@@ -64,14 +64,9 @@ jobs:
       arch: arm64
       dist: focal
       env:
-        - CIBW_BUILD="cp312-musllinux_aarch64"
+        - CIBW_BUILD="cp312-musllinux_aarch64 cp313-musllinux_aarch64"
 
     # perform a linux PPC64LE build
-    - services: docker
-      arch: ppc64le
-      dist: focal
-      env:
-        - CIBW_BUILD="cp38-manylinux_ppc64le"
     - services: docker
       arch: ppc64le
       dist: focal
@@ -96,7 +91,7 @@ jobs:
       arch: ppc64le
       dist: focal
       env:
-        - CIBW_BUILD="cp38-musllinux_ppc64le"
+        - CIBW_BUILD="cp313-manylinux_ppc64le"
     - services: docker
       arch: ppc64le
       dist: focal
@@ -117,6 +112,11 @@ jobs:
       dist: focal
       env:
         - CIBW_BUILD="cp312-musllinux_ppc64le"
+    - services: docker
+      arch: ppc64le
+      dist: focal
+      env:
+        - CIBW_BUILD="cp313-musllinux_ppc64le"
 
     # perform a linux S390X build
     # blocked by https://github.com/GTkorvo/dill/issues/15
@@ -136,7 +136,7 @@ install:
   - git clone --branch ${OPENPMD_GIT_REF} --depth 1 https://github.com/openPMD/openPMD-api.git src
   - cp library_builders.sh src/.github/
   - python -m pip install --upgrade pip setuptools wheel
-  - python -m pip install cibuildwheel==2.16.2
+  - python -m pip install cibuildwheel==2.21.2
   # twine & cryptography: see
   #   https://github.com/scikit-build/cmake-python-distributions/blob/4730aeee240917303f293dffc89a8d8d5a4787c4/requirements-deploy.txt
   #   https://github.com/pyca/cryptography/issues/6086
diff --git a/library_builders.bat b/library_builders.bat
@@ -17,21 +17,23 @@ exit /b 0
 
 :build_adios2
   if exist adios2-stamp exit /b 0
-  ::curl -sLo adios2-2.9.1.zip ^
-  ::  https://github.com/ornladios/ADIOS2/archive/v2.9.1.zip
-  curl -sLo adios2-2.9.1.zip ^
-    https://github.com/ax3l/ADIOS2/archive/refs/heads/release-2.9.1-bp-wheels.zip
-  powershell Expand-Archive adios2-2.9.1.zip -DestinationPath dep-adios2
+  curl -sLo adios2-2.10.1.zip ^
+    https://github.com/ornladios/ADIOS2/archive/v2.10.1.zip
+  powershell Expand-Archive adios2-2.10.1.zip -DestinationPath dep-adios2
 
-  cmake -S dep-adios2/ADIOS2-release-2.9.1-bp-wheels -B build-adios2 ^
+  cmake --version
+
+  cmake -S dep-adios2/ADIOS2-2.10.1 -B build-adios2 ^
     -DCMAKE_BUILD_TYPE=Release  ^
+    -DCMAKE_DISABLE_FIND_PACKAGE_LibFFI=TRUE  ^
     -DBUILD_SHARED_LIBS=OFF     ^
     -DBUILD_TESTING=OFF         ^
     -DADIOS2_USE_MPI=OFF        ^
     -DADIOS2_BUILD_EXAMPLES=OFF ^
     -DADIOS2_Blosc2_PREFER_SHARED=OFF ^
     -DADIOS2_USE_Blosc2=ON      ^
     -DADIOS2_USE_BZip2=OFF      ^
+    -DADIOS2_USE_Campaign=OFF   ^
     -DADIOS2_USE_Fortran=OFF    ^
     -DADIOS2_USE_HDF5=OFF       ^
     -DADIOS2_USE_MHS=OFF        ^
@@ -45,12 +47,17 @@ exit /b 0
 :: TODO: Could NOT find HDF5 (missing: HDF5_LIBRARIES C)
 ::  -DADIOS2_USE_HDF5=ON
 
-  cmake --build build-adios2 --parallel %CPU_COUNT%
+  cmake --build build-adios2 --config Release --parallel %CPU_COUNT%
   if errorlevel 1 exit 1
 
   cmake --build build-adios2 --target install --config Release
   if errorlevel 1 exit 1
 
+  :: CMake Config package of C-Blosc 2.10.1+ only
+  :: https://github.com/ornladios/ADIOS2/issues/3903
+  :: rmdir /s /q "%BUILD_PREFIX%/ADIOS2/lib/cmake/adios2/FindBlosc2.cmake"
+  :: if errorlevel 1 exit 1
+
   rmdir /s /q build-adios2
   if errorlevel 1 exit 1
 
@@ -61,19 +68,16 @@ exit /b 0
 :build_blosc2
   if exist blosc2-stamp exit /b 0
 
-  curl -sLo blosc2-2.10.2.zip ^
-    https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.10.2.zip
-  powershell Expand-Archive blosc2-2.10.2.zip -DestinationPath dep-blosc2
+  curl -sLo blosc2-2.11.1.zip ^
+    https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.11.1.zip
+  powershell Expand-Archive blosc2-2.11.1.zip -DestinationPath dep-blosc2
 
-  :: Fix Threads search in Blosc2Config.cmake
-  :: https://github.com/Blosc/c-blosc2/pull/549
-  curl -sLo blosc2-threads.patch ^
-    https://github.com/Blosc/c-blosc2/pull/549.patch
-  python -m patch -p 1 -d dep-blosc2/c-blosc2-2.10.2 blosc2-threads.patch
-  if errorlevel 1 exit 1
+  cmake --version
 
-  cmake -S dep-blosc2/c-blosc2-2.10.2 -B build-blosc2 ^
+  cmake -S dep-blosc2/c-blosc2-2.11.1 -B build-blosc2 ^
     -DCMAKE_BUILD_TYPE=Release  ^
+    -DCMAKE_INSTALL_PREFIX=%BUILD_PREFIX%/blosc2  ^
+    -DCMAKE_POSITION_INDEPENDENT_CODE=ON   ^
     -DBUILD_SHARED=OFF          ^
     -DBUILD_STATIC=ON           ^
     -DBUILD_BENCHMARKS=OFF      ^
@@ -88,7 +92,7 @@ exit /b 0
 ::    -DZLIB_USE_STATIC_LIBS=ON
   if errorlevel 1 exit 1
 
-  cmake --build build-blosc2 --parallel %CPU_COUNT%
+  cmake --build build-blosc2 --config Release --parallel %CPU_COUNT%
   if errorlevel 1 exit 1
 
   cmake --build build-blosc2 --target install --config Release
@@ -126,7 +130,7 @@ exit /b 0
     -DCMAKE_INSTALL_PREFIX=%BUILD_PREFIX%/HDF5
   if errorlevel 1 exit 1
 
-  cmake --build build-hdf5 --parallel %CPU_COUNT%
+  cmake --build build-hdf5 --config Release --parallel %CPU_COUNT%
   if errorlevel 1 exit 1
 
   cmake --build build-hdf5 --target install --config Release
@@ -156,7 +160,7 @@ exit /b 0
 
   if errorlevel 1 exit 1
 
-  cmake --build build-zfp --parallel %CPU_COUNT%
+  cmake --build build-zfp --config Release --parallel %CPU_COUNT%
   if errorlevel 1 exit 1
 
   cmake --build build-zfp --target install --config Release
@@ -183,7 +187,7 @@ exit /b 0
 :: Manually-specified variables were not used by the project:
 ::   CMAKE_BUILD_TYPE
 
-  cmake --build build-zlib --parallel %CPU_COUNT%
+  cmake --build build-zlib --config Release --parallel %CPU_COUNT%
   if errorlevel 1 exit 1
 
   cmake --build build-zlib --target install --config Release
diff --git a/library_builders.sh b/library_builders.sh
@@ -103,10 +103,8 @@ function build_adios1 {
 function build_adios2 {
     if [ -e adios2-stamp ]; then return; fi
 
-    #curl -sLo adios2-2.9.1.tar.gz \
-    #    https://github.com/ornladios/ADIOS2/archive/v2.9.1.tar.gz
-    curl -sLo adios2-2.9.1.tar.gz \
-        https://github.com/ax3l/ADIOS2/archive/refs/heads/release-2.9.1-bp-wheels.tar.gz
+    curl -sLo adios2-2.10.1.tar.gz \
+        https://github.com/ornladios/ADIOS2/archive/v2.10.1.tar.gz
     file adios2*.tar.gz
     tar -xzf adios2*.tar.gz
     rm adios2*.tar.gz
@@ -124,11 +122,13 @@ function build_adios2 {
         -DADIOS2_Blosc2_PREFER_SHARED=OFF         \
         -DADIOS2_USE_BZip2=OFF                    \
         -DADIOS2_USE_Blosc2=ON                    \
+        -DADIOS2_USE_Campaign=OFF                 \
         -DADIOS2_USE_Fortran=OFF                  \
         -DADIOS2_USE_HDF5=OFF                     \
         -DADIOS2_USE_MHS=OFF                      \
         -DADIOS2_USE_MPI=OFF                      \
         -DADIOS2_USE_PNG=OFF                      \
+        -DADIOS2_USE_Sodium=OFF                   \
         -DADIOS2_USE_SST=ON                       \
         -DADIOS2_USE_ZFP=ON                       \
         -DADIOS2_RUN_INSTALL_TEST=OFF             \
@@ -143,6 +143,7 @@ function build_adios2 {
     make install
 
     # CMake Config package of C-Blosc 2.10.1+ only
+    # https://github.com/ornladios/ADIOS2/issues/3903
     rm -rf ${BUILD_PREFIX}/lib*/cmake/adios2/FindBlosc2.cmake
 
     cd -
@@ -200,8 +201,8 @@ function build_blosc {
 function build_blosc2 {
     if [ -e blosc-stamp2 ]; then return; fi
 
-    curl -sLo blosc2-v2.10.2.tar.gz \
-        https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.10.2.tar.gz
+    curl -sLo blosc2-v2.11.1.tar.gz \
+        https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.11.1.tar.gz
     file blosc2*.tar.gz
     tar -xzf blosc2*.tar.gz
     rm blosc2*.tar.gz
