security stuff

.github/dependabot.yml

@@ -28,9 +28,19 @@ updates:
       - dependency-name: "datafusion"
     cooldown:
       default-days: 7
+    groups:
+      minor-and-patch:
+        update-types:
+          - "minor"
+          - "patch"
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
       interval: "weekly"
     cooldown:
       default-days: 7
+    groups:
+      actions-minor-patch:
+        update-types:
+          - "minor"
+          - "patch"

.github/workflows/codeql.yml

@@ -32,13 +32,18 @@ permissions:
 
 jobs:
   analyze:
-    name: Analyze Actions
+    name: Analyze (${{ matrix.language }})
     runs-on: ubuntu-slim
     permissions:
       contents: read
       security-events: write
       packages: read
 
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'actions', 'python' ]
+
     steps:
     - name: Checkout repository
       uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
@@ -48,9 +53,9 @@ jobs:
     - name: Initialize CodeQL
       uses: github/codeql-action/init@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4.35.2
       with:
-        languages: actions
+        languages: ${{ matrix.language }}
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4.35.2
       with:
-        category: "/language:actions"
+        category: "/language:${{ matrix.language }}"

.github/workflows/dependency-review.yml

@@ -0,0 +1,83 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: "Dependency Review"
+
+on:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  dependency-review:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+        with:
+          persist-credentials: false
+
+      - name: Dependency Review
+        uses: actions/dependency-review-action@2031cfc080254a8a887f58cffee85186f0e49e48 # v4.9.0
+        with:
+          fail-on-severity: high
+          # ASF 3rd Party License Policy: https://www.apache.org/legal/resolved.html
+          #
+          # Category A — auto-allowed (listed below):
+          #   Apache-like, BSD, MIT, PSF, CC0, etc.
+          #
+          # Category B — blocked by allow-list, requires manual review:
+          #   CDDL-1.0, CDDL-1.1, CPL-1.0,
+          #   EPL-1.0, EPL-2.0,
+          #   IPL-1.0,
+          #   MPL-1.0, MPL-1.1, MPL-2.0,
+          #   SPL-1.0,
+          #   OSL-3.0,
+          #   CC-BY-2.5, CC-BY-3.0, CC-BY-4.0,
+          #   CC-BY-SA-2.5, CC-BY-SA-3.0, CC-BY-SA-4.0 (unmodified media only)
+          #   Permitted in binary form only, with appropriate labeling.
+          #
+          # Category X — always blocked (never allow):
+          #   GPL-1.0/2.0/3.0, AGPL-1.0/3.0, LGPL-2.0/2.1/3.0,
+          #   SSPL-1.0, BUSL-1.1,
+          #   CC-BY-NC-*, BSD-4-Clause, QPL-1.0, Sleepycat,
+          #   CPOL-1.02, NPL-1.0/1.1, JSON, APSL-2.0
+          #
+          allow-licenses: >-
+            Apache-2.0, Apache-1.1,
+            MIT, MIT-0,
+            ISC,
+            BSD-2-Clause, BSD-3-Clause,
+            PSF-2.0, Python-2.0,
+            BSL-1.0,
+            Unlicense,
+            0BSD,
+            Zlib,
+            CC0-1.0,
+            CC-PDDC,
+            AFL-3.0,
+            MS-PL,
+            UPL-1.0,
+            NCSA,
+            W3C,
+            PostgreSQL,
+            HPND,
+            MulanPSL-2.0,
+            BlueOak-1.0.0,
+            ZPL-2.0

.github/workflows/nightly-pypi-build.yml

@@ -73,10 +73,9 @@ jobs:
     environment:
       name: testpypi
       url: https://test.pypi.org/p/pyiceberg
-
     permissions:
-      id-token: write  # IMPORTANT: mandatory for trusted publishing
-
+      id-token: write    # OIDC token for Trusted Publishing
+      attestations: write  # PEP 740 build attestations
     steps:
     - name: Download all the artifacts
       uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
@@ -91,6 +90,7 @@ jobs:
       uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
       with:
         repository-url: https://test.pypi.org/legacy/
+        attestations: true
         skip-existing: true
         verbose: true
     - name: Display error message on publish failure

.github/workflows/pypi-build-artifacts.yml

@@ -31,7 +31,7 @@ permissions:
 
 jobs:
   pypi-build-artifacts:
-    name: Build artifacts for PyPi on ${{ matrix.os }}
+    name: Build artifacts for PyPI on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
       max-parallel: 15

.github/workflows/python-ci.yml

@@ -45,6 +45,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
+env:
+  UV_LOCKED: 1  # All uv commands enforce --locked in CI (no re-resolution)
+
 jobs:
   lint-and-unit-test:
     runs-on: ubuntu-latest
@@ -67,8 +70,6 @@ jobs:
         enable-cache: true
     - name: Install system dependencies
       run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
-    - name: Check uv.lock is up to date
-      run: uv lock --check
     - name: Install
       run: make install
     - name: Run linters

.github/workflows/python-publish-pypi.yml

@@ -0,0 +1,76 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Publishes the final release to PyPI with PEP 740 build attestations via
+# Trusted Publishing. Run after the vote passes and SVN release is promoted.
+#
+# The RC pre-release is published automatically by python-release.yml.
+#
+# Prerequisites (one-time setup):
+#   1. Create a "pypi" environment in GitHub repo settings with required reviewers.
+#   2. Configure Trusted Publishing on PyPI:
+#      https://pypi.org/manage/project/pyiceberg/settings/publishing/
+#      - Owner: apache
+#      - Repository: iceberg-python
+#      - Workflow: python-publish-pypi.yml
+#      - Environment: pypi
+
+name: "Publish Release to PyPI"
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Release version (e.g., 0.8.0)'
+        type: string
+        required: true
+
+permissions:
+  contents: read
+
+jobs:
+  publish-pypi:
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/pyiceberg/${{ inputs.version }}
+    permissions:
+      id-token: write   # OIDC token for Trusted Publishing + Sigstore signing
+      attestations: write  # PEP 740 build attestations stored on PyPI
+    steps:
+      - name: Download release artifacts from Apache SVN
+        env:
+          VERSION: ${{ inputs.version }}
+        run: |
+          SVN_URL="https://dist.apache.org/repos/dist/release/iceberg/pyiceberg-${VERSION}"
+
+          echo "Downloading from $SVN_URL..."
+          svn export --non-interactive "$SVN_URL" svn-artifacts/
+
+          mkdir -p dist/
+          cp svn-artifacts/pyiceberg-*.whl svn-artifacts/pyiceberg-*.tar.gz dist/
+
+          echo "Artifacts to publish:"
+          ls -lah dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
+        with:
+          attestations: true
+          verbose: true

.github/workflows/python-release-docs.yml

@@ -33,7 +33,6 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: write
-
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
         with:

.github/workflows/python-release.yml

@@ -17,6 +17,15 @@
 # under the License.
 #
 
+# Prerequisites (one-time setup):
+#   1. Create a "pypi" environment in GitHub repo settings with required reviewers.
+#   2. Configure Trusted Publishing on PyPI for RC publishes:
+#      https://pypi.org/manage/project/pyiceberg/settings/publishing/
+#      - Owner: apache
+#      - Repository: iceberg-python
+#      - Workflow: python-release.yml
+#      - Environment: pypi
+
 name: "Python Build Release Candidate"
 
 on:
@@ -138,11 +147,36 @@ jobs:
     with:
       version: ${{ needs.validate-inputs.outputs.VERSION }}rc${{ needs.validate-inputs.outputs.RC }}
 
-  # PyPi
+  # PyPI
   pypi-build-artifacts:
     needs:
       - validate-inputs
       - validate-library-version
     uses: ./.github/workflows/pypi-build-artifacts.yml
     with:
       version: ${{ needs.validate-inputs.outputs.VERSION }}rc${{ needs.validate-inputs.outputs.RC }}
+
+  # Publish RC pre-release to PyPI with PEP 740 attestations.
+  # Gated by the "pypi" environment (requires reviewer approval).
+  publish-rc-to-pypi:
+    needs:
+      - validate-inputs
+      - pypi-build-artifacts
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/pyiceberg/${{ needs.validate-inputs.outputs.VERSION }}rc${{ needs.validate-inputs.outputs.RC }}
+    permissions:
+      id-token: write   # OIDC token for Trusted Publishing + Sigstore signing
+      attestations: write  # PEP 740 build attestations stored on PyPI
+    steps:
+      - name: Download merged PyPI artifacts
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+        with:
+          name: "pypi-release-candidate-${{ needs.validate-inputs.outputs.VERSION }}rc${{ needs.validate-inputs.outputs.RC }}"
+          path: dist/
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
+        with:
+          attestations: true
+          verbose: true

.github/workflows/scorecard.yml

@@ -0,0 +1,54 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: "OpenSSF Scorecard"
+
+on:
+  push:
+    branches: ["main"]
+  schedule:
+    - cron: '30 2 * * 1' # Weekly on Monday at 02:30 UTC
+
+permissions: {}
+
+jobs:
+  scorecard:
+    name: Scorecard analysis
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write # Upload SARIF results
+      id-token: write        # Publish results
+      contents: read         # Read repo
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+        with:
+          persist-credentials: false
+
+      - name: Run OpenSSF Scorecard
+        uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          publish_results: true
+
+      - name: Upload Scorecard results to Security tab
+        uses: github/codeql-action/upload-sarif@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4.35.2
+        with:
+          sarif_file: results.sarif