From 71a50b086a66476a5153b312c906117c58b58c9f Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sat, 2 May 2026 12:56:48 -0700 Subject: [PATCH 1/6] security stuff Co-authored-by: Copilot --- .github/dependabot.yml | 10 +++ .github/workflows/codeql.yml | 11 ++- .github/workflows/dependency-review.yml | 85 +++++++++++++++++++++++ .github/workflows/nightly-pypi-build.yml | 6 +- .github/workflows/python-ci.yml | 5 +- .github/workflows/python-publish-pypi.yml | 76 ++++++++++++++++++++ .github/workflows/python-release-docs.yml | 1 - .github/workflows/python-release.yml | 34 +++++++++ .github/workflows/scorecard.yml | 54 ++++++++++++++ SECURITY.md | 31 +++++++++ mkdocs/docs/how-to-release.md | 51 ++++---------- uv.lock | 24 +++---- 12 files changed, 329 insertions(+), 59 deletions(-) create mode 100644 .github/workflows/dependency-review.yml create mode 100644 .github/workflows/python-publish-pypi.yml create mode 100644 .github/workflows/scorecard.yml create mode 100644 SECURITY.md diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4207b660d4..98ed914d39 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -28,9 +28,19 @@ updates: - dependency-name: "datafusion" cooldown: default-days: 7 + groups: + minor-and-patch: + update-types: + - "minor" + - "patch" - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" cooldown: default-days: 7 + groups: + actions-minor-patch: + update-types: + - "minor" + - "patch" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 7e9c8208c8..6057949b9b 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -32,13 +32,18 @@ permissions: jobs: analyze: - name: Analyze Actions + name: Analyze (${{ matrix.language }}) runs-on: ubuntu-slim permissions: contents: read security-events: write packages: read + strategy: + fail-fast: false + matrix: + language: [ 'actions', 'python' ] + steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -48,9 +53,9 @@ jobs: - name: Initialize CodeQL uses: github/codeql-action/init@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4.35.2 with: - languages: actions + languages: ${{ matrix.language }} - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4.35.2 with: - category: "/language:actions" + category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml new file mode 100644 index 0000000000..4ad01cc815 --- /dev/null +++ b/.github/workflows/dependency-review.yml @@ -0,0 +1,85 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: "Dependency Review" + +on: + pull_request: + +permissions: + contents: read + +jobs: + dependency-review: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - name: Dependency Review + uses: actions/dependency-review-action@2031cfc080254a8a887f58cffee85186f0e49e48 # v4.9.0 + with: + fail-on-severity: high + # ASF 3rd Party License Policy: https://www.apache.org/legal/resolved.html + # + # Category A — auto-allowed (listed below): + # Apache-like, BSD, MIT, PSF, CC0, etc. + # + # Category B — blocked by allow-list, requires manual review: + # CDDL-1.0, CDDL-1.1, CPL-1.0, + # EPL-1.0, EPL-2.0, + # IPL-1.0, + # MPL-1.0, MPL-1.1, MPL-2.0, + # SPL-1.0, + # OSL-3.0, + # CC-BY-2.5, CC-BY-3.0, CC-BY-4.0, + # CC-BY-SA-2.5, CC-BY-SA-3.0, CC-BY-SA-4.0 (unmodified media only) + # Permitted in binary form only, with appropriate labeling. + # + # Category X — always blocked (never allow): + # GPL-1.0/2.0/3.0, AGPL-1.0/3.0, LGPL-2.0/2.1/3.0, + # SSPL-1.0, BUSL-1.1, + # CC-BY-NC-*, BSD-4-Clause, QPL-1.0, Sleepycat, + # CPOL-1.02, NPL-1.0/1.1, JSON, APSL-2.0 + # + allow-licenses: >- + Apache-2.0, Apache-1.1, + MIT, MIT-0, + ISC, + BSD-2-Clause, BSD-3-Clause, + PSF-2.0, Python-2.0, + BSL-1.0, + Unlicense, + 0BSD, + Zlib, + CC0-1.0, + CC-PDDC, + ECL-2.0, + AFL-3.0, + MS-PL, + UPL-1.0, + NCSA, + W3C, + PostgreSQL, + HPND, + MulanPSL-2.0, + BlueOak-1.0.0, + Artistic-2.0, + Zope-2.0 diff --git a/.github/workflows/nightly-pypi-build.yml b/.github/workflows/nightly-pypi-build.yml index 647c3b7bae..dec5bbea8e 100644 --- a/.github/workflows/nightly-pypi-build.yml +++ b/.github/workflows/nightly-pypi-build.yml @@ -73,10 +73,9 @@ jobs: environment: name: testpypi url: https://test.pypi.org/p/pyiceberg - permissions: - id-token: write # IMPORTANT: mandatory for trusted publishing - + id-token: write # OIDC token for Trusted Publishing + attestations: write # PEP 740 build attestations steps: - name: Download all the artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -91,6 +90,7 @@ jobs: uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 with: repository-url: https://test.pypi.org/legacy/ + attestations: true skip-existing: true verbose: true - name: Display error message on publish failure diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index b780d20b28..1f5a55b6e4 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -45,6 +45,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +env: + UV_LOCKED: 1 # All uv commands enforce --locked in CI (no re-resolution) + jobs: lint-and-unit-test: runs-on: ubuntu-latest @@ -67,8 +70,6 @@ jobs: enable-cache: true - name: Install system dependencies run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos - - name: Check uv.lock is up to date - run: uv lock --check - name: Install run: make install - name: Run linters diff --git a/.github/workflows/python-publish-pypi.yml b/.github/workflows/python-publish-pypi.yml new file mode 100644 index 0000000000..204ee5e412 --- /dev/null +++ b/.github/workflows/python-publish-pypi.yml @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Publishes the final release to PyPI with PEP 740 build attestations via +# Trusted Publishing. Run after the vote passes and SVN release is promoted. +# +# The RC pre-release is published automatically by python-release.yml. +# +# Prerequisites (one-time setup): +# 1. Create a "pypi" environment in GitHub repo settings with required reviewers. +# 2. Configure Trusted Publishing on PyPI: +# https://pypi.org/manage/project/pyiceberg/settings/publishing/ +# - Owner: apache +# - Repository: iceberg-python +# - Workflow: python-publish-pypi.yml +# - Environment: pypi + +name: "Publish Release to PyPI" + +on: + workflow_dispatch: + inputs: + version: + description: 'Release version (e.g., 0.8.0)' + type: string + required: true + +permissions: + contents: read + +jobs: + publish-pypi: + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/pyiceberg/${{ inputs.version }} + permissions: + id-token: write # OIDC token for Trusted Publishing + Sigstore signing + attestations: write # PEP 740 build attestations stored on PyPI + steps: + - name: Download release artifacts from Apache SVN + env: + VERSION: ${{ inputs.version }} + run: | + SVN_URL="https://dist.apache.org/repos/dist/release/iceberg/pyiceberg-${VERSION}" + + echo "Downloading from $SVN_URL..." + svn export --non-interactive "$SVN_URL" svn-artifacts/ + + mkdir -p dist/ + cp svn-artifacts/pyiceberg-*.whl svn-artifacts/pyiceberg-*.tar.gz dist/ + + echo "Artifacts to publish:" + ls -lah dist/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + with: + attestations: true + verbose: true diff --git a/.github/workflows/python-release-docs.yml b/.github/workflows/python-release-docs.yml index ec50dd4e15..324f45963d 100644 --- a/.github/workflows/python-release-docs.yml +++ b/.github/workflows/python-release-docs.yml @@ -33,7 +33,6 @@ jobs: runs-on: ubuntu-latest permissions: contents: write - steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index 4aa9073164..789205a479 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -17,6 +17,15 @@ # under the License. # +# Prerequisites (one-time setup): +# 1. Create a "pypi" environment in GitHub repo settings with required reviewers. +# 2. Configure Trusted Publishing on PyPI for RC publishes: +# https://pypi.org/manage/project/pyiceberg/settings/publishing/ +# - Owner: apache +# - Repository: iceberg-python +# - Workflow: python-release.yml +# - Environment: pypi + name: "Python Build Release Candidate" on: @@ -146,3 +155,28 @@ jobs: uses: ./.github/workflows/pypi-build-artifacts.yml with: version: ${{ needs.validate-inputs.outputs.VERSION }}rc${{ needs.validate-inputs.outputs.RC }} + + # Publish RC pre-release to PyPI with PEP 740 attestations. + # Gated by the "pypi" environment (requires reviewer approval). + publish-rc-to-pypi: + needs: + - validate-inputs + - pypi-build-artifacts + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/pyiceberg/${{ needs.validate-inputs.outputs.VERSION }}rc${{ needs.validate-inputs.outputs.RC }} + permissions: + id-token: write # OIDC token for Trusted Publishing + Sigstore signing + attestations: write # PEP 740 build attestations stored on PyPI + steps: + - name: Download merged PyPI artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: "pypi-release-candidate-${{ needs.validate-inputs.outputs.VERSION }}rc${{ needs.validate-inputs.outputs.RC }}" + path: dist/ + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + with: + attestations: true + verbose: true diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml new file mode 100644 index 0000000000..18ce8db255 --- /dev/null +++ b/.github/workflows/scorecard.yml @@ -0,0 +1,54 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: "OpenSSF Scorecard" + +on: + push: + branches: ["main"] + schedule: + - cron: '30 2 * * 1' # Weekly on Monday at 02:30 UTC + +permissions: {} + +jobs: + scorecard: + name: Scorecard analysis + runs-on: ubuntu-latest + permissions: + security-events: write # Upload SARIF results + id-token: write # Publish results + contents: read # Read repo + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - name: Run OpenSSF Scorecard + uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1 + with: + results_file: results.sarif + results_format: sarif + publish_results: true + + - name: Upload Scorecard results to Security tab + uses: github/codeql-action/upload-sarif@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4.35.2 + with: + sarif_file: results.sarif diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000..1cec018e11 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,31 @@ +# Security Policy + +## Reporting a Vulnerability + +Apache Iceberg uses the standard [Apache Security Process](https://www.apache.org/security/) +for reporting and handling security vulnerabilities. + +**Please do NOT create public GitHub issues for security vulnerabilities.** + +To report a vulnerability, send an email to [security@apache.org](mailto:security@apache.org) +with the following information: + +- Description of the vulnerability +- Steps to reproduce the issue +- Affected versions +- Any potential mitigations you have identified + +The Apache Security Team will acknowledge your report and work with the project +maintainers to address the issue. You can expect an initial response within 48 hours. + +For more details, see: + +## Supported Versions + +Security fixes are applied to the latest released version of PyIceberg. +We do not backport fixes to older minor versions. + +| Version | Supported | +| ------- | ------------------ | +| Latest | :white_check_mark: | +| < Latest | :x: | diff --git a/mkdocs/docs/how-to-release.md b/mkdocs/docs/how-to-release.md index 01a1b4d08e..4d7c55a7b0 100644 --- a/mkdocs/docs/how-to-release.md +++ b/mkdocs/docs/how-to-release.md @@ -35,8 +35,10 @@ This guide outlines the process for releasing PyIceberg in accordance with the [ * Permission to upload artifacts to the [Apache development distribution](https://dist.apache.org/repos/dist/dev/iceberg/) (requires Apache Committer access). * Permission to upload artifacts to the [Apache release distribution](https://dist.apache.org/repos/dist/release/iceberg/) (requires Apache PMC access). * PyPI Access - * The `twine` package must be installed for uploading releases to PyPi. - * A PyPI account with publishing permissions for the [pyiceberg project](https://pypi.org/project/pyiceberg/). + * A `pypi` [environment](https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment) must be configured in the GitHub repo with required reviewers. + * PyPI [Trusted Publishing](https://docs.pypi.org/trusted-publishers/) must be configured with **two** publisher entries: + * `python-release.yml` with environment `pypi` (for RC pre-releases) + * `python-publish-pypi.yml` with environment `pypi` (for final releases) ## Preparing for a Release @@ -208,27 +210,11 @@ svn delete https://dist.apache.org/repos/dist/dev/iceberg/pyiceberg- - -!!! note - `twine` might require an PyPi API token. - - - -```bash -: "${VERSION:?ERROR: VERSION is not set or is empty}" -: "${RC:?ERROR: RC is not set or is empty}" - -twine upload pypi-release-candidate-${VERSION}rc${RC}/* -``` +This **won't** bump the version for everyone that hasn't pinned their version, since it is set to an RC [pre-release and those are ignored](https://packaging.python.org/en/latest/guides/distributing-packages-using-setuptools/#pre-release-versioning). Verify the artifact is uploaded to [PyPi](https://pypi.org/project/pyiceberg/#history). @@ -346,26 +332,15 @@ svn delete https://dist.apache.org/repos/dist/release/iceberg/pyiceberg- - -!!! note - `twine` might require an PyPi API token. - - - -```bash -: "${VERSION:?ERROR: VERSION is not set or is empty}" - -svn checkout https://dist.apache.org/repos/dist/release/iceberg/pyiceberg-${VERSION} /tmp/iceberg-dist-release/pyiceberg-${VERSION} +1. Go to **Actions** → **Publish Release to PyPI** → **Run workflow** +2. Enter the `version` (e.g., `0.8.0`) +3. Approve the deployment in the `pypi` environment -cd /tmp/iceberg-dist-release/pyiceberg-${VERSION} +The workflow downloads the clean-versioned artifacts (e.g., `pyiceberg-0.8.0-*.whl`) from the [Apache SVN release dist](https://dist.apache.org/repos/dist/release/iceberg/) and publishes them to PyPI with [PEP 740 build attestations](https://peps.python.org/pep-0740/). -twine upload pyiceberg-*.whl pyiceberg-*.tar.gz -``` - -Verify the artifact is uploaded to [PyPi](https://pypi.org/project/pyiceberg/#history). +Verify the artifact is available on [PyPi](https://pypi.org/project/pyiceberg/#history). ## Post Release diff --git a/uv.lock b/uv.lock index bf0bd0174d..e573700718 100644 --- a/uv.lock +++ b/uv.lock @@ -27,7 +27,7 @@ wheels = [ [[package]] name = "aiobotocore" -version = "3.6.0" +version = "3.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -39,9 +39,9 @@ dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.11'" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/28/52/4689a0c2ddced3888c687a84b820da37da6abda5ec52d96a7f11f22ff6ec/aiobotocore-3.6.0.tar.gz", hash = "sha256:78e6b95b258bcdde8d681d6851f88f337ed30b2d6611fae45366e6bf7488feb3", size = 123104, upload-time = "2026-05-01T20:33:31.109Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e6/89/9533b377e9412013cc43a539d81bc5f8feeb4b6830643821ad612f78b09b/aiobotocore-3.5.0.tar.gz", hash = "sha256:d45d1c4659ad0e48b694a5aa4ff18829100386f7de96c8d146ec7757a6f12918", size = 123061, upload-time = "2026-04-21T07:25:26.993Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/72/9a/17033aaafb880932eec0b0054a8e1cf201cb74501dfb0518c6b39fc5b258/aiobotocore-3.6.0-py3-none-any.whl", hash = "sha256:bf1c2e1d8492d98c9b63b330723a44b41a5d402afd19eaca704e107b91669c09", size = 88259, upload-time = "2026-05-01T20:33:29.213Z" }, + { url = "https://files.pythonhosted.org/packages/2d/05/6eeeadef45c24630af0ceae4d038b883e9a394786300529286ba8cc1e62d/aiobotocore-3.5.0-py3-none-any.whl", hash = "sha256:49ce35bb8b96b85d3251c2cbbb2ed7a028dc0cb0d0d0801f9ccca1ccd0d41ded", size = 88281, upload-time = "2026-04-21T07:25:25.258Z" }, ] [[package]] @@ -565,30 +565,30 @@ wheels = [ [[package]] name = "boto3" -version = "1.42.96" +version = "1.42.91" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, { name = "jmespath" }, { name = "s3transfer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a6/2d/69fb3acd50bab83fb295c167d33c4b653faeb5fb0f42bfca4d9b69d6fb68/boto3-1.42.96.tar.gz", hash = "sha256:b38a9e4a3fbbee9017252576f1379780d0a5814768676c08df2f539d31fcdd68", size = 113203, upload-time = "2026-04-24T19:47:18.677Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/c0/98b8cec7ca22dde776df48c58940ae1abc425593959b7226e270760d726f/boto3-1.42.91.tar.gz", hash = "sha256:03d70532b17f7f84df37ca7e8c21553280454dea53ae12b15d1cfef9b16fcb8a", size = 113181, upload-time = "2026-04-17T19:31:06.251Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2b/9d/b3f617d011c42eb804d993103b8fa9acdce153e181a3042f58bfe33d7cb4/boto3-1.42.96-py3-none-any.whl", hash = "sha256:2f4566da2c209a98bdbfc874d813ef231c84ad24e4f815e9bc91de5f63351a24", size = 140557, upload-time = "2026-04-24T19:47:15.824Z" }, + { url = "https://files.pythonhosted.org/packages/02/29/faba6521257c34085cc9b439ef98235b581772580f417fa3629728007270/boto3-1.42.91-py3-none-any.whl", hash = "sha256:04e72071cde022951ce7f81bd9933c90095ab8923e8ced61c8dacfe9edac0f5c", size = 140553, upload-time = "2026-04-17T19:31:02.57Z" }, ] [[package]] name = "botocore" -version = "1.42.97" +version = "1.42.91" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jmespath" }, { name = "python-dateutil" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c6/95/c37edb602948fad2253ffd1bb3dba5b938645bd1845ee4160350136a0f41/botocore-1.42.97.tar.gz", hash = "sha256:5c0bb00e32d16ff6d278cc8c9e10dc3672d9c1d569031635ac3c908a60de8310", size = 15269348, upload-time = "2026-04-27T20:39:05.625Z" } +sdist = { url = "https://files.pythonhosted.org/packages/21/bc/a4b7c46471c2e789ad8c4c7acfd7f302fdb481d93ff870f441249b924ae6/botocore-1.42.91.tar.gz", hash = "sha256:d252e27bc454afdbf5ed3dc617aa423f2c855c081e98b7963093399483ecc698", size = 15213010, upload-time = "2026-04-17T19:30:50.793Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e3/d2/8e025ba1a4e257879af72d06913272311af79673d82fa2581a351b924317/botocore-1.42.97-py3-none-any.whl", hash = "sha256:77d2c8ce1bc592d3fbd7c01c35836f4a5b0cac2ca03ccdf6ffc60faa16b5fadc", size = 14950367, upload-time = "2026-04-27T20:39:01.261Z" }, + { url = "https://files.pythonhosted.org/packages/b1/fc/24cc0a47c824f13933e210e9ad034b4fba22f7185b8d904c0fbf5a3b2be8/botocore-1.42.91-py3-none-any.whl", hash = "sha256:7a28c3cc6bfab5724ad18899d52402b776a0de7d87fa20c3c5270bcaaf199ce8", size = 14897344, upload-time = "2026-04-17T19:30:44.245Z" }, ] [[package]] @@ -2643,7 +2643,7 @@ wheels = [ [[package]] name = "jupyterlab" -version = "4.5.7" +version = "4.5.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "async-lru" }, @@ -2661,9 +2661,9 @@ dependencies = [ { name = "tornado" }, { name = "traitlets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2b/22/8440ec827762146e7cdecf04335bd348795899d29dc6ae82238707353a2c/jupyterlab-4.5.7.tar.gz", hash = "sha256:55a9822c4754da305f41e113452c68383e214dcf96de760146af89ce5d5117b0", size = 23992763, upload-time = "2026-04-29T16:43:51.328Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/d5/730628e03fff2e8a8e8ccdaedde1489ab1309f9a4fa2536248884e30b7c7/jupyterlab-4.5.6.tar.gz", hash = "sha256:642fe2cfe7f0f5922a8a558ba7a0d246c7bc133b708dfe43f7b3a826d163cf42", size = 23970670, upload-time = "2026-03-11T14:17:04.531Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/aa/537b8f7d80e799af19af35fb3ddfc970b951088a13c57dd9387dcfbb7f61/jupyterlab-4.5.7-py3-none-any.whl", hash = "sha256:fba4cb0e2c44a52859669d8c98b45de029d5e515f8407bf8534d2a8fc5f0964d", size = 12450123, upload-time = "2026-04-29T16:43:46.639Z" }, + { url = "https://files.pythonhosted.org/packages/e1/1b/dad6fdcc658ed7af26fdf3841e7394072c9549a8b896c381ab49dd11e2d9/jupyterlab-4.5.6-py3-none-any.whl", hash = "sha256:d6b3dac883aa4d9993348e0f8e95b24624f75099aed64eab6a4351a9cdd1e580", size = 12447124, upload-time = "2026-03-11T14:17:00.229Z" }, ] [[package]] From 3036dec29eb5e067e650a52be118cc185bbc311a Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sat, 2 May 2026 13:00:28 -0700 Subject: [PATCH 2/6] RAT --- SECURITY.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/SECURITY.md b/SECURITY.md index 1cec018e11..78f3ee6de8 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,3 +1,20 @@ + + # Security Policy ## Reporting a Vulnerability From 8b5c9ec02758c93ab514e90a68f3c4f1b59dac59 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sat, 2 May 2026 13:03:38 -0700 Subject: [PATCH 3/6] fix allow licenses Co-authored-by: Copilot --- .github/workflows/dependency-review.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 4ad01cc815..5d378bcb7e 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -71,7 +71,6 @@ jobs: Zlib, CC0-1.0, CC-PDDC, - ECL-2.0, AFL-3.0, MS-PL, UPL-1.0, @@ -81,5 +80,4 @@ jobs: HPND, MulanPSL-2.0, BlueOak-1.0.0, - Artistic-2.0, - Zope-2.0 + ZPL-2.0 From 024ea6318703fdb59b8c5dfcd3e89374c31cdbee Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sat, 2 May 2026 13:06:43 -0700 Subject: [PATCH 4/6] upgrade jupyterlab --- uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index e573700718..a7199177ba 100644 --- a/uv.lock +++ b/uv.lock @@ -2643,7 +2643,7 @@ wheels = [ [[package]] name = "jupyterlab" -version = "4.5.6" +version = "4.5.7" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "async-lru" }, @@ -2661,9 +2661,9 @@ dependencies = [ { name = "tornado" }, { name = "traitlets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ac/d5/730628e03fff2e8a8e8ccdaedde1489ab1309f9a4fa2536248884e30b7c7/jupyterlab-4.5.6.tar.gz", hash = "sha256:642fe2cfe7f0f5922a8a558ba7a0d246c7bc133b708dfe43f7b3a826d163cf42", size = 23970670, upload-time = "2026-03-11T14:17:04.531Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2b/22/8440ec827762146e7cdecf04335bd348795899d29dc6ae82238707353a2c/jupyterlab-4.5.7.tar.gz", hash = "sha256:55a9822c4754da305f41e113452c68383e214dcf96de760146af89ce5d5117b0", size = 23992763, upload-time = "2026-04-29T16:43:51.328Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/1b/dad6fdcc658ed7af26fdf3841e7394072c9549a8b896c381ab49dd11e2d9/jupyterlab-4.5.6-py3-none-any.whl", hash = "sha256:d6b3dac883aa4d9993348e0f8e95b24624f75099aed64eab6a4351a9cdd1e580", size = 12447124, upload-time = "2026-03-11T14:17:00.229Z" }, + { url = "https://files.pythonhosted.org/packages/3d/aa/537b8f7d80e799af19af35fb3ddfc970b951088a13c57dd9387dcfbb7f61/jupyterlab-4.5.7-py3-none-any.whl", hash = "sha256:fba4cb0e2c44a52859669d8c98b45de029d5e515f8407bf8534d2a8fc5f0964d", size = 12450123, upload-time = "2026-04-29T16:43:46.639Z" }, ] [[package]] From 516e136651acc9ff4ae1c82dba0fcb1c0d78fcd7 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sat, 2 May 2026 13:11:02 -0700 Subject: [PATCH 5/6] casing PyPI Co-authored-by: Copilot --- .github/workflows/pypi-build-artifacts.yml | 2 +- .github/workflows/python-release.yml | 2 +- mkdocs/docs/how-to-release.md | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pypi-build-artifacts.yml b/.github/workflows/pypi-build-artifacts.yml index 86ae161459..65177607d5 100644 --- a/.github/workflows/pypi-build-artifacts.yml +++ b/.github/workflows/pypi-build-artifacts.yml @@ -31,7 +31,7 @@ permissions: jobs: pypi-build-artifacts: - name: Build artifacts for PyPi on ${{ matrix.os }} + name: Build artifacts for PyPI on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: max-parallel: 15 diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index 789205a479..f51087099d 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -147,7 +147,7 @@ jobs: with: version: ${{ needs.validate-inputs.outputs.VERSION }}rc${{ needs.validate-inputs.outputs.RC }} - # PyPi + # PyPI pypi-build-artifacts: needs: - validate-inputs diff --git a/mkdocs/docs/how-to-release.md b/mkdocs/docs/how-to-release.md index 4d7c55a7b0..c3d55b3dda 100644 --- a/mkdocs/docs/how-to-release.md +++ b/mkdocs/docs/how-to-release.md @@ -134,7 +134,7 @@ This action will generate artifacts that will include both source distribution ( This action will generate two final artifacts: * `svn-release-candidate-${VERSION}rc${RC}` for SVN -* `pypi-release-candidate-${VERSION}rc${RC}` for PyPi +* `pypi-release-candidate-${VERSION}rc${RC}` for PyPI If `gh` is available, watch the GitHub Action progress using: @@ -208,7 +208,7 @@ Clean up old RC artifacts: svn delete https://dist.apache.org/repos/dist/dev/iceberg/pyiceberg- -m "Remove old RC artifacts" ``` -#### Upload to PyPi +#### Upload to PyPI The [`Python Build Release Candidate` workflow](https://github.com/apache/iceberg-python/actions/workflows/python-release.yml) includes a `publish-rc-to-pypi` job that automatically publishes the RC artifacts to PyPI with [PEP 740 build attestations](https://peps.python.org/pep-0740/) after the build completes. This job is gated by the `pypi` environment, which requires reviewer approval before publishing. @@ -216,7 +216,7 @@ Once the build artifacts are ready, approve the deployment in the `pypi` environ This **won't** bump the version for everyone that hasn't pinned their version, since it is set to an RC [pre-release and those are ignored](https://packaging.python.org/en/latest/guides/distributing-packages-using-setuptools/#pre-release-versioning). -Verify the artifact is uploaded to [PyPi](https://pypi.org/project/pyiceberg/#history). +Verify the artifact is uploaded to [PyPI](https://pypi.org/project/pyiceberg/#history). ## Vote @@ -330,7 +330,7 @@ We only want to host the latest release. Clean up old release artifacts: svn delete https://dist.apache.org/repos/dist/release/iceberg/pyiceberg- -m "Remove old release artifacts" ``` -### Upload the accepted release to PyPi +### Upload the accepted release to PyPI Run the [`Publish Release to PyPI` workflow](https://github.com/apache/iceberg-python/actions/workflows/python-publish-pypi.yml) to publish the final release: @@ -338,9 +338,9 @@ Run the [`Publish Release to PyPI` workflow](https://github.com/apache/iceberg-p 2. Enter the `version` (e.g., `0.8.0`) 3. Approve the deployment in the `pypi` environment -The workflow downloads the clean-versioned artifacts (e.g., `pyiceberg-0.8.0-*.whl`) from the [Apache SVN release dist](https://dist.apache.org/repos/dist/release/iceberg/) and publishes them to PyPI with [PEP 740 build attestations](https://peps.python.org/pep-0740/). +The workflow downloads the clean-versioned artifacts (e.g., `pyiceberg-0.8.0-*.whl` and `pyiceberg-0.8.0.tar.gz`) from the [Apache SVN release dist](https://dist.apache.org/repos/dist/release/iceberg/) and publishes them to PyPI with [PEP 740 build attestations](https://peps.python.org/pep-0740/). -Verify the artifact is available on [PyPi](https://pypi.org/project/pyiceberg/#history). +Verify the artifact is available on [PyPI](https://pypi.org/project/pyiceberg/#history). ## Post Release From 052a4d35b478a7be31f275955601cab62a3d758f Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sat, 2 May 2026 13:18:25 -0700 Subject: [PATCH 6/6] svn Co-authored-by: Copilot --- mkdocs/docs/how-to-release.md | 24 ++++++++++++++++++------ mkdocs/docs/verify-release.md | 2 +- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/mkdocs/docs/how-to-release.md b/mkdocs/docs/how-to-release.md index c3d55b3dda..01cc7960b2 100644 --- a/mkdocs/docs/how-to-release.md +++ b/mkdocs/docs/how-to-release.md @@ -195,7 +195,9 @@ Now, upload the files from the same directory: : "${VERSION_WITH_RC:?ERROR: VERSION_WITH_RC is not set or is empty}" : "${RC:?ERROR: RC is not set or is empty}" -svn import "svn-release-candidate-${VERSION}rc${RC}" "https://dist.apache.org/repos/dist/dev/iceberg/pyiceberg-${VERSION_WITH_RC}" -m "PyIceberg ${VERSION_WITH_RC}" +svn import "svn-release-candidate-${VERSION}rc${RC}" \ + "https://dist.apache.org/repos/dist/dev/iceberg/pyiceberg-${VERSION_WITH_RC}" \ + -m "PyIceberg ${VERSION_WITH_RC}" ``` Verify the artifact is uploaded to [https://dist.apache.org/repos/dist/dev/iceberg](https://dist.apache.org/repos/dist/dev/iceberg/). @@ -205,7 +207,11 @@ Verify the artifact is uploaded to [https://dist.apache.org/repos/dist/dev/icebe Clean up old RC artifacts: ```bash -svn delete https://dist.apache.org/repos/dist/dev/iceberg/pyiceberg- -m "Remove old RC artifacts" +export OLD_RC_VERSION="" # e.g., 0.8.0rc1 +: "${OLD_RC_VERSION:?ERROR: OLD_RC_VERSION is not set or is empty}" + +svn delete "https://dist.apache.org/repos/dist/dev/iceberg/pyiceberg-${OLD_RC_VERSION}" \ + -m "Remove old RC artifacts" ``` #### Upload to PyPI @@ -317,7 +323,8 @@ Kind regards, export SVN_DEV_DIR_VERSIONED="https://dist.apache.org/repos/dist/dev/iceberg/pyiceberg-${VERSION_WITH_RC}" export SVN_RELEASE_DIR_VERSIONED="https://dist.apache.org/repos/dist/release/iceberg/pyiceberg-${VERSION}" -svn mv ${SVN_DEV_DIR_VERSIONED} ${SVN_RELEASE_DIR_VERSIONED} -m "PyIceberg: Add release ${VERSION}" +svn mv "${SVN_DEV_DIR_VERSIONED}" "${SVN_RELEASE_DIR_VERSIONED}" \ + -m "PyIceberg: Add release ${VERSION}" ``` Verify the artifact is uploaded to [https://dist.apache.org/repos/dist/release/iceberg](https://dist.apache.org/repos/dist/release/iceberg/). @@ -327,7 +334,11 @@ Verify the artifact is uploaded to [https://dist.apache.org/repos/dist/release/i We only want to host the latest release. Clean up old release artifacts: ```bash -svn delete https://dist.apache.org/repos/dist/release/iceberg/pyiceberg- -m "Remove old release artifacts" +export OLD_RELEASE_VERSION="" # e.g., 0.7.0 +: "${OLD_RELEASE_VERSION:?ERROR: OLD_RELEASE_VERSION is not set or is empty}" + +svn delete "https://dist.apache.org/repos/dist/release/iceberg/pyiceberg-${OLD_RELEASE_VERSION}" \ + -m "Remove old release artifacts" ``` ### Upload the accepted release to PyPI @@ -399,12 +410,13 @@ To install gpg on a M1 based Mac, a couple of additional steps are required: > KEYS # append a newline gpg --list-sigs >> KEYS # append signatures gpg --armor --export >> KEYS # append public key block -svn commit -m "add key for " # this requires Iceberg PMC privileges +svn commit -m "add key for " # this requires Iceberg PMC privileges ``` diff --git a/mkdocs/docs/verify-release.md b/mkdocs/docs/verify-release.md index 1844a9bc85..7f8eed576b 100644 --- a/mkdocs/docs/verify-release.md +++ b/mkdocs/docs/verify-release.md @@ -59,7 +59,7 @@ export PYICEBERG_VERIFICATION_DIR=/tmp/pyiceberg/${PYICEBERG_VERSION} Next, verify the `.asc` file. ```sh -svn checkout https://dist.apache.org/repos/dist/dev/iceberg/pyiceberg-$PYICEBERG_VERSION/ ${PYICEBERG_VERIFICATION_DIR} +svn export https://dist.apache.org/repos/dist/dev/iceberg/pyiceberg-$PYICEBERG_VERSION/ ${PYICEBERG_VERIFICATION_DIR} cd ${PYICEBERG_VERIFICATION_DIR}