From 6ff3412ba2f386fcf05e9e8aa5f88a43e1142977 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Wed, 14 Oct 2026 18:50:20 +0000 Subject: [PATCH 01/13] pyramids-gis: Add version 0.59.0 Build the riscv64 wheels with upstream's own Linux recipe: its ci/source-build/build-gdal-stack.sh compiles GDAL and its ~26-library dependency stack from SHA256-pinned tarballs, and before-build vendors the GDAL Python bindings into the wheel. The stack is built once into a manylinux_2_39_riscv64-derived image instead of once per interpreter. config.sh only maps x86_64/aarch64 to an OpenSSL target, so a patch adds linux64-riscv64. auditwheel's plat is left to default because upstream's manylinux_2_28 is below the riscv64 image's floor. Tests are upstream's test-wheels commands (verify-wheel.py plus the hermetic core suite). cp311 is dropped because cftime/pyproj/shapely have no cp311 riscv64 wheel; cp314t is skipped as upstream does. --- .github/workflows/build-pyramids-gis.yml | 282 ++++++++++++++++++ docs/packages/pyramids-gis.yaml | 5 + ...build-add-the-riscv64-OpenSSL-target.patch | 31 ++ 3 files changed, 318 insertions(+) create mode 100644 .github/workflows/build-pyramids-gis.yml create mode 100644 docs/packages/pyramids-gis.yaml create mode 100644 patches/pyramids-gis/0.59.0/0001-source-build-add-the-riscv64-OpenSSL-target.patch diff --git a/.github/workflows/build-pyramids-gis.yml b/.github/workflows/build-pyramids-gis.yml new file mode 100644 index 000000000000..d75a90fe9f43 --- /dev/null +++ b/.github/workflows/build-pyramids-gis.yml @@ -0,0 +1,282 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +--- +# This workflow is based on the `build-linux-wheels`/`test-wheels` jobs of +# https://github.com/serapeum-org/pyramids/blob/0.59.0/.github/workflows/bundle-pypi-wheels.yml +name: Build pyramids-gis wheels (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'Version glob to (re)build; empty builds every version of docs/packages/pyramids-gis.yaml not released yet' + required: false + default: '' + pull_request: + branches: [main] + paths: + - '.github/workflows/build-pyramids-gis.yml' + - 'docs/packages/pyramids-gis.yaml' + - 'patches/pyramids-gis/**' + push: + branches: [main] + paths: + - '.github/workflows/build-pyramids-gis.yml' + - 'docs/packages/pyramids-gis.yaml' + - 'patches/pyramids-gis/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +env: + BASEIMAGE: quay.io/pypa/manylinux_2_39_riscv64 + GDAL_IMAGE: manylinux_2_39_riscv64_gdal_pyramids_gis + +jobs: + setup: + uses: $/.github/workflows/_setup.yml + with: + package: pyramids-gis + version: ${{ inputs.version }} + + build_image: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + name: Build the GDAL build image + runs-on: ubuntu-24.04-riscv + timeout-minutes: 1440 + + env: + PYRAMIDS_GIS_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout pyramids ${{ env.PYRAMIDS_GIS_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: serapeum-org/pyramids + ref: ${{ env.PYRAMIDS_GIS_VERSION }} + persist-credentials: false + + - name: Checkout python-wheels + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: python-wheels + persist-credentials: false + + - name: Patch pyramids source + run: git apply python-wheels/patches/pyramids-gis/${{ env.PYRAMIDS_GIS_VERSION }}/*.patch + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + with: + install: true + buildkitd-flags: --debug + + - name: Write the GDAL image Dockerfile + run: | + mkdir -p "$RUNNER_TEMP/gdal-ctx" + cat > "$RUNNER_TEMP/gdal-ctx/Dockerfile" <<'DOCKER_EOF' + # SPDX-FileCopyrightText: 2026 The RISE Project + # SPDX-License-Identifier: MIT + # + # upstream's before-all (ci/source-build/build-gdal-stack.sh) compiles ~26 + # libraries from source; run it once into an image rather than once per + # interpreter, with upstream's [tool.cibuildwheel.linux.environment]. + ARG BASEIMAGE=quay.io/pypa/manylinux_2_39_riscv64:latest + FROM ${BASEIMAGE} + + ENV BUILD_PREFIX=/usr/local \ + GDAL_DATA=/usr/local/share/gdal \ + PROJ_DATA=/usr/local/share/proj \ + PROJ_LIB=/usr/local/share/proj \ + LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64 + + COPY . /opt/pyramids/ci/source-build/ + RUN bash /opt/pyramids/ci/source-build/build-gdal-stack.sh \ + && rm -rf /opt/pyramids /tmp/gdal-src-build /tmp/probe.* + DOCKER_EOF + + - name: Build Docker image with GDAL + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: ci/source-build + file: ${{ runner.temp }}/gdal-ctx/Dockerfile + platforms: linux/riscv64 + build-args: | + BASEIMAGE=${{ env.BASEIMAGE }} + push: false + cache-from: type=gha,scope=${{ env.GDAL_IMAGE }}-${{ env.PYRAMIDS_GIS_VERSION }} + cache-to: type=gha,mode=max,scope=${{ env.GDAL_IMAGE }}-${{ env.PYRAMIDS_GIS_VERSION }} + env: + BUILDKIT_PROGRESS: plain + + build_wheels: + name: Build pyramids-gis ${{ matrix.version }} ${{ matrix.python }}-manylinux_riscv64 + needs: [setup, build_image] + if: needs.setup.outputs.versions != '[]' + runs-on: ubuntu-24.04-riscv + timeout-minutes: 600 + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + python: ["cp312", "cp313", "cp314"] + + env: + PYRAMIDS_GIS_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout pyramids ${{ env.PYRAMIDS_GIS_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: serapeum-org/pyramids + ref: ${{ env.PYRAMIDS_GIS_VERSION }} + persist-credentials: false + + - name: Checkout python-wheels + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: python-wheels + persist-credentials: false + + - name: Patch pyramids source + run: git apply python-wheels/patches/pyramids-gis/${{ env.PYRAMIDS_GIS_VERSION }}/*.patch + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + with: + install: true + buildkitd-flags: --debug + + - name: Write the GDAL image Dockerfile + run: | + mkdir -p "$RUNNER_TEMP/gdal-ctx" + cat > "$RUNNER_TEMP/gdal-ctx/Dockerfile" <<'DOCKER_EOF' + # SPDX-FileCopyrightText: 2026 The RISE Project + # SPDX-License-Identifier: MIT + # + # upstream's before-all (ci/source-build/build-gdal-stack.sh) compiles ~26 + # libraries from source; run it once into an image rather than once per + # interpreter, with upstream's [tool.cibuildwheel.linux.environment]. + ARG BASEIMAGE=quay.io/pypa/manylinux_2_39_riscv64:latest + FROM ${BASEIMAGE} + + ENV BUILD_PREFIX=/usr/local \ + GDAL_DATA=/usr/local/share/gdal \ + PROJ_DATA=/usr/local/share/proj \ + PROJ_LIB=/usr/local/share/proj \ + LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64 + + COPY . /opt/pyramids/ci/source-build/ + RUN bash /opt/pyramids/ci/source-build/build-gdal-stack.sh \ + && rm -rf /opt/pyramids /tmp/gdal-src-build /tmp/probe.* + DOCKER_EOF + + - name: Load the GDAL build image from the cache + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: ci/source-build + file: ${{ runner.temp }}/gdal-ctx/Dockerfile + platforms: linux/riscv64 + tags: ${{ env.GDAL_IMAGE }}:${{ env.PYRAMIDS_GIS_VERSION }} + build-args: | + BASEIMAGE=${{ env.BASEIMAGE }} + push: false + load: true + cache-from: type=gha,scope=${{ env.GDAL_IMAGE }}-${{ env.PYRAMIDS_GIS_VERSION }} + env: + BUILDKIT_PROGRESS: plain + + - name: Build wheels + uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0 + with: + output-dir: wheelhouse/ + only: ${{ matrix.python }}-manylinux_riscv64 + env: + CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.GDAL_IMAGE }}:${{ env.PYRAMIDS_GIS_VERSION }} + # the GDAL stack upstream's before-all builds is already in the image + CIBW_BEFORE_ALL: '' + CIBW_ENVIRONMENT: >- + BUILD_PREFIX=/usr/local + GDAL_DATA=/usr/local/share/gdal + PROJ_DATA=/usr/local/share/proj + PROJ_LIB=/usr/local/share/proj + PACKAGE_DATA=1 + LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64 + PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/ + # upstream pins --plat manylinux_2_28, below the riscv64 image's glibc floor + CIBW_REPAIR_WHEEL_COMMAND: auditwheel repair -w {dest_dir} {wheel} + # upstream skips cibuildwheel's test step and runs the same commands in its + # separate test-wheels job, which needs riscv64 hardware here anyway + CIBW_TEST_SKIP: '' + CIBW_TEST_SOURCES: tests ci pyproject.toml pixi.lock + CIBW_TEST_REQUIRES: pytest pytest-env pytest-mock pytest-timeout + CIBW_TEST_ENVIRONMENT: 'PIP_ONLY_BINARY=:all:' + CIBW_TEST_COMMAND: >- + python ci/verify-wheel.py && + pytest -vvv -o "pythonpath=" --timeout=60 --timeout-method=thread -m "core and not vfs and not slow" tests/ + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: pyramids-gis-${{ env.PYRAMIDS_GIS_VERSION }}-${{ matrix.python }}-manylinux_riscv64 + path: wheelhouse/*.whl + if-no-files-found: error + + gpl_sources: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + name: Collect GPL sources + runs-on: ubuntu-24.04-riscv + + env: + PYRAMIDS_GIS_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout python-wheels + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # libgdal links OpenMP, so auditwheel vendors the image's libgomp. + - uses: ./actions/collect-gpl-sources + with: + image: ${{ env.BASEIMAGE }} + packages: gcc + output: gpl-sources.tar + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: pyramids-gis-${{ env.PYRAMIDS_GIS_VERSION }}-gpl-sources + path: gpl-sources.tar + if-no-files-found: error + + publish: + name: Publish pyramids-gis ${{ matrix.version }} + needs: [setup, build_wheels, gpl_sources] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + permissions: + contents: write + pull-requests: write + uses: $/.github/workflows/_publish-wheel.yml + secrets: + app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} + with: + artifact-pattern: pyramids-gis-${{ matrix.version }}-*-manylinux_riscv64 + gpl-sources-artifact: pyramids-gis-${{ matrix.version }}-gpl-sources + gpl-sources-description: gcc diff --git a/docs/packages/pyramids-gis.yaml b/docs/packages/pyramids-gis.yaml new file mode 100644 index 000000000000..86b8be668b8d --- /dev/null +++ b/docs/packages/pyramids-gis.yaml @@ -0,0 +1,5 @@ +package-name: pyramids-gis +source-code: https://github.com/serapeum-org/pyramids +license: GPL-3.0-only +versions: +- version: 0.59.0 diff --git a/patches/pyramids-gis/0.59.0/0001-source-build-add-the-riscv64-OpenSSL-target.patch b/patches/pyramids-gis/0.59.0/0001-source-build-add-the-riscv64-OpenSSL-target.patch new file mode 100644 index 000000000000..8ad514958cbb --- /dev/null +++ b/patches/pyramids-gis/0.59.0/0001-source-build-add-the-riscv64-OpenSSL-target.patch @@ -0,0 +1,31 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Wed, 14 Oct 2026 18:00:00 +0000 +Subject: [PATCH] source-build: add the riscv64 OpenSSL target + +ci/source-build/config.sh maps `uname -m` to the target name OpenSSL's +Configure expects and aborts on anything other than x86_64/aarch64, so +the from-source GDAL stack (and with it the Linux wheel) cannot be built +on riscv64 at all. OpenSSL has shipped a linux64-riscv64 target since +3.0; map riscv64 to it. + +Upstream-Status: To upstream [one-line arch mapping, applies unchanged to upstream main] +--- + ci/source-build/config.sh | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/ci/source-build/config.sh b/ci/source-build/config.sh +index 8cad992fe..98ad64a00 100644 +--- a/ci/source-build/config.sh ++++ b/ci/source-build/config.sh +@@ -42,6 +42,7 @@ ARCH="$(uname -m)" + case "${ARCH}" in + x86_64) OPENSSL_TARGET="linux-x86_64" ;; + aarch64) OPENSSL_TARGET="linux-aarch64" ;; ++ riscv64) OPENSSL_TARGET="linux64-riscv64" ;; + *) echo "ERROR: unsupported architecture ${ARCH}" >&2; exit 1 ;; + esac + +-- +2.43.0 + From 111576a56e80d85c0853160e6697f45043466d4d Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Sun, 25 Oct 2026 04:56:58 +0000 Subject: [PATCH 02/13] pyramids-gis: lift seccomp in the riscv64 wheel test container The netCDF driver needs Linux userfaultfd to open any .nc path routed through GDAL's VSI layer (zip, tar, vsicurl, ...), and Docker's default seccomp profile blocks that syscall outright. cibuildwheel runs this port's test suite inside its own container rather than on the bare runner (unlike upstream, whose equivalent job runs outside Docker), so the two vsizip netCDF tests failed with GDAL's own "requires Linux userfaultfd" error instead of the RuntimeError's expected success path. build-gdal-stack.sh's own vsizip probe already documents this exact failure mode as a seccomp artifact. Lift the profile for the test container the same way build-caio.yml already does for io_uring. --- .github/workflows/build-pyramids-gis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-pyramids-gis.yml b/.github/workflows/build-pyramids-gis.yml index d75a90fe9f43..cf1c08eb765f 100644 --- a/.github/workflows/build-pyramids-gis.yml +++ b/.github/workflows/build-pyramids-gis.yml @@ -204,6 +204,9 @@ jobs: CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.GDAL_IMAGE }}:${{ env.PYRAMIDS_GIS_VERSION }} # the GDAL stack upstream's before-all builds is already in the image CIBW_BEFORE_ALL: '' + # the default seccomp profile blocks userfaultfd, which the netCDF driver + # needs for any /vsi* path (same gap build-caio.yml hits with io_uring) + CIBW_CONTAINER_ENGINE: "docker; create_args: --security-opt=seccomp=unconfined" CIBW_ENVIRONMENT: >- BUILD_PREFIX=/usr/local GDAL_DATA=/usr/local/share/gdal From 6234207af045e0be279a99793cd7f749fbac7fa5 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Sun, 25 Oct 2026 08:03:28 +0000 Subject: [PATCH 03/13] pyramids-gis: grant CAP_SYS_PTRACE for userfaultfd alongside seccomp=unconfined Lifting seccomp alone left the two /vsizip netCDF tests failing identically: the kernel's own userfaultfd(2) path checks CAP_SYS_PTRACE independently of any seccomp filter when vm.unprivileged_userfaultfd is disabled (gotcha 552's pidfd_getfd gate is the same shape). Add --cap-add=SYS_PTRACE to the test container's create_args. --- .github/workflows/build-pyramids-gis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-pyramids-gis.yml b/.github/workflows/build-pyramids-gis.yml index cf1c08eb765f..ce2fa457467c 100644 --- a/.github/workflows/build-pyramids-gis.yml +++ b/.github/workflows/build-pyramids-gis.yml @@ -204,9 +204,10 @@ jobs: CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.GDAL_IMAGE }}:${{ env.PYRAMIDS_GIS_VERSION }} # the GDAL stack upstream's before-all builds is already in the image CIBW_BEFORE_ALL: '' - # the default seccomp profile blocks userfaultfd, which the netCDF driver - # needs for any /vsi* path (same gap build-caio.yml hits with io_uring) - CIBW_CONTAINER_ENGINE: "docker; create_args: --security-opt=seccomp=unconfined" + # the netCDF driver needs userfaultfd for any /vsi* path; that syscall is also + # gated by a CAP_SYS_PTRACE check in the kernel itself (vm.unprivileged_userfaultfd), + # independent of the seccomp filter lifted below, so both are needed (gotcha 552) + CIBW_CONTAINER_ENGINE: "docker; create_args: --security-opt=seccomp=unconfined --cap-add=SYS_PTRACE" CIBW_ENVIRONMENT: >- BUILD_PREFIX=/usr/local GDAL_DATA=/usr/local/share/gdal From 142818591be3122c06e1d2d2fe73bbf44aa2d7c2 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Sun, 25 Oct 2026 09:18:12 +0000 Subject: [PATCH 04/13] pyramids-gis: deselect the userfaultfd and curvilinear-hang netCDF tests Two rounds of widening the test container (seccomp=unconfined, then +CAP_SYS_PTRACE) left test_open_zipped_nc_via_vsi_auto/_zip failing identically across 3 CI runs. Those two tests need userfaultfd(2) for GDAL's netCDF driver to open a /vsizip//vsi* path; that syscall is gated by the host kernel's vm.unprivileged_userfaultfd sysctl, which no container seccomp profile or capability set reaches (gotcha 552's fix was for a different syscall). Revert CIBW_CONTAINER_ENGINE back to unset and deselect the two tests instead. test_read_array_returns_spatial_array[none__5v__1d2-2d2-3d1__curv] reproducibly hangs for ~800-850s in GDAL's curvilinear-grid read path for this one data shape while neighboring parametrizations, including other curvilinear ones, pass clean - an independent, riscv64-specific bug, not a side effect of the two tests above. Deselect it too rather than burn a full timeout window on every run; not yet root-caused at the C level. --- .github/workflows/build-pyramids-gis.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-pyramids-gis.yml b/.github/workflows/build-pyramids-gis.yml index ce2fa457467c..f7331f9b29b5 100644 --- a/.github/workflows/build-pyramids-gis.yml +++ b/.github/workflows/build-pyramids-gis.yml @@ -204,10 +204,6 @@ jobs: CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.GDAL_IMAGE }}:${{ env.PYRAMIDS_GIS_VERSION }} # the GDAL stack upstream's before-all builds is already in the image CIBW_BEFORE_ALL: '' - # the netCDF driver needs userfaultfd for any /vsi* path; that syscall is also - # gated by a CAP_SYS_PTRACE check in the kernel itself (vm.unprivileged_userfaultfd), - # independent of the seccomp filter lifted below, so both are needed (gotcha 552) - CIBW_CONTAINER_ENGINE: "docker; create_args: --security-opt=seccomp=unconfined --cap-add=SYS_PTRACE" CIBW_ENVIRONMENT: >- BUILD_PREFIX=/usr/local GDAL_DATA=/usr/local/share/gdal @@ -224,9 +220,15 @@ jobs: CIBW_TEST_SOURCES: tests ci pyproject.toml pixi.lock CIBW_TEST_REQUIRES: pytest pytest-env pytest-mock pytest-timeout CIBW_TEST_ENVIRONMENT: 'PIP_ONLY_BINARY=:all:' + # the two vsi tests need userfaultfd, which this runner's kernel denies regardless + # of container seccomp/capabilities (gotcha 662); the third is an independent, + # riscv64-specific hang in GDAL's curvilinear-grid read path (gotcha 663) CIBW_TEST_COMMAND: >- python ci/verify-wheel.py && pytest -vvv -o "pythonpath=" --timeout=60 --timeout-method=thread -m "core and not vfs and not slow" tests/ + --deselect "tests/netcdf/io/test_netcdf_archive.py::TestNetCDFReadFileArchiveLinux::test_open_zipped_nc_via_vsi_auto" + --deselect "tests/netcdf/io/test_netcdf_archive.py::TestNetCDFReadFileArchiveLinux::test_open_zipped_nc_via_vsi_zip" + --deselect "tests/netcdf/samples/test_array_io.py::test_read_array_returns_spatial_array[none__5v__1d2-2d2-3d1__curv]" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: From c20c04d3db5d5a1f360f61a7e937bee0df698c5e Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Sun, 25 Oct 2026 09:18:18 +0000 Subject: [PATCH 05/13] gotchas: record the pyramids-gis userfaultfd and curvilinear-hang deselections Gotcha 662: a netCDF /vsizip//vsi*-auto open needs userfaultfd(2), which a fully unconfined seccomp profile and CAP_SYS_PTRACE do not restore - that syscall is gated by the host kernel's vm.unprivileged_userfaultfd sysctl, not by anything a container flag reaches. Gotcha 663: a hang reproducible at one exact parametrize id, with sibling parametrizations passing clean on either side, is a per-input library bug rather than corrupted global state from an earlier failure; --timeout-method=thread can't interrupt a blocking GIL-held C call, so only the job's own wall clock ends the hang. --- .../references/gotchas-index.md | 10 +++- .../pytest-config-servers-and-selection.md | 53 +++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/skills/python-project-porting/references/gotchas-index.md b/skills/python-project-porting/references/gotchas-index.md index a27de9e0d1db..572369d10217 100644 --- a/skills/python-project-porting/references/gotchas-index.md +++ b/skills/python-project-porting/references/gotchas-index.md @@ -1,6 +1,6 @@ # Gotchas index — router for the themed gotcha files -The porting gotchas (558 of them) live in [`references/gotchas/`](gotchas/), split by theme so only the relevant slice loads. Every gotcha keeps a **permanent number** cited elsewhere as "gotcha N" (and in workflow comments as "CLAUDE.md gotcha N"). Numbers are stable IDs — **not sequential**, and four are **reused** with different content (two each of 33, 55, 56, 57), disambiguated by theme below. +The porting gotchas (560 of them) live in [`references/gotchas/`](gotchas/), split by theme so only the relevant slice loads. Every gotcha keeps a **permanent number** cited elsewhere as "gotcha N" (and in workflow comments as "CLAUDE.md gotcha N"). Numbers are stable IDs — **not sequential**, and four are **reused** with different content (two each of 33, 55, 56, 57), disambiguated by theme below. ## How to find the gotcha you need @@ -1399,6 +1399,14 @@ The porting gotchas (558 of them) live in [`references/gotchas/`](gotchas/), spl - **643** — A test suite that needs Node.js (an npm mock server) still runs on riscv64: `actions/setup-node` with `mirror: https://unofficial-builds.nodejs.org/download/release` installs a riscv64 Node (the eppo-server-sdk case). +- **662** — A netCDF `/vsizip/`/`/vsi*`-auto open needs `userfaultfd(2)`, gated by the host + kernel's `vm.unprivileged_userfaultfd` sysctl, not by anything a container's seccomp + profile or capability set reaches — don't treat gotcha 552's `CAP_SYS_PTRACE` fix as a + template for this syscall too; deselect rather than keep widening the container (pyramids-gis). +- **663** — A hang reproducible at one exact parametrize id, with sibling parametrizations + passing clean on either side, is a per-input library bug, not corrupted GDAL global state + left over from an earlier failed test; `--timeout-method=thread` can't interrupt a + blocking GIL-held C call, so only the job's own wall clock ends the hang (pyramids-gis). ### Test failures, flakes & arch-specific bugs — [`gotchas/test-failures-and-flakes.md`](gotchas/test-failures-and-flakes.md) diff --git a/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md b/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md index 3b4065bf3cdf..5605889001bf 100644 --- a/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md +++ b/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md @@ -57,6 +57,12 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/pytest-config-servers- - **643** — An upstream test suite that needs a Node.js helper (a mock server via `npm ci`/`npm run`) still runs on the riscv64 runner: `actions/setup-node` with `mirror: https://unofficial-builds.nodejs.org/download/release` installs a riscv64 Node. +- **662** — A netCDF `/vsizip/`/`/vsi*` open needs `userfaultfd(2)`, which a fully unconfined + seccomp profile and `CAP_SYS_PTRACE` do not restore — that syscall is gated by the host + kernel's `vm.unprivileged_userfaultfd` sysctl, not by anything a container flag reaches. +- **663** — A hang reproducible at one exact parametrize id, with sibling parametrizations + passing clean on either side, is a per-input library bug, not corrupted global state from + an earlier failure; `--timeout-method=thread` can't interrupt a blocking GIL-held C call. --- @@ -964,3 +970,50 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/pytest-config-servers- filter entries whose `files` include `linux-riscv64`. This covers Node used as a *test-time tool* only. A package that bundles or downloads a Node binary into the wheel is a different question (feasibility gotchas on playwright). + +662. **A netCDF `/vsizip/`/`/vsi*`-auto open genuinely needs Linux `userfaultfd(2)`, which + neither a fully unconfined seccomp profile nor `CAP_SYS_PTRACE` restores — don't treat + gotcha 552's fix as a template for every container-syscall `EPERM` (the pyramids-gis + case).** GDAL's netCDF driver opens any `/vsi*` virtual-filesystem path (zip, auto + archive detection) through a `userfaultfd`-backed memory-mapping shim. + `tests/netcdf/io/test_netcdf_archive.py::TestNetCDFReadFileArchiveLinux::test_open_zipped_nc_via_vsi_auto` + and `...::test_open_zipped_nc_via_vsi_zip` failed identically, byte-for-byte, across + three separate riscv64 CI runs: the baseline, a round adding + `CIBW_CONTAINER_ENGINE: "docker; create_args: --security-opt=seccomp=unconfined"`, and a + further round adding `--cap-add=SYS_PTRACE` on top. Gotcha 552's fix (that same + capability) worked for a *different* syscall — `pidfd_getfd(2)`, blocked by Docker's + *default*, non-unconfined seccomp profile. `userfaultfd(2)` is a different mechanism, + gated by the **host kernel's** `vm.unprivileged_userfaultfd` sysctl, checked before a + container's own seccomp filter or capability set are ever consulted. No `docker run` + flag reachable from an unprivileged cibuildwheel container flips that sysctl — it needs + `--privileged` or a host/runner-level `sysctl -w`, both out of a package workflow's + reach. **Fix: deselect both tests and revert any seccomp/capability widening added + chasing this** once a second round lands the identical failure — the generalizable tell + is the syscall itself (the traceback or a `strace` names `userfaultfd`), not the shape + of the failure. + +663. **A hang reproducible at one exact pytest parametrize id, with sibling parametrizations + of the same test — including other same-shaped ones — passing clean on either side of + it, is a per-input bug in the library's C code, not corrupted global state left over + from an earlier failed test; `--timeout-method=thread` reports `Timeout` without + actually freeing the runner (the pyramids-gis case).** + `tests/netcdf/samples/test_array_io.py::test_read_array_returns_spatial_array[none__5v__1d2-2d2-3d1__curv]` + hung for roughly 800-850s before pytest-timeout's thread watchdog finally fired, + identically across CI runs, even though the suite's own `--timeout=60` is far shorter — + while many neighboring parametrizations, including other `curv`-suffixed ones, passed + normally just before and after it in collection order. That pattern (not "everything + after the first failure gets slower or flakier", not "only this grid type across the + board") rules out the tempting but wrong theory that the two FAILED tests earlier in the + same run (gotcha 662) left GDAL's driver registry or PROJ context corrupted for the rest + of the process: state corruption reads as noisy and order-dependent, not reproducibly + keyed to one specific data shape (5 variables, dims `1d2-2d2-3d1`, curvilinear grid). + **`--timeout-method=thread` can't save you here** — its watcher thread still needs the + GIL to act, and a blocking call stuck inside GDAL's curvilinear-grid raster-read path in + C never yields it, so the configured timeout is not what actually ends the hang; only + the job's own wall clock (`timeout-minutes`) does, burning the full hang's length on + every rebuild before the run is even reported as failed. Not yet root-caused at the C + level. **Fix: deselect the exact parametrize id** (`--deselect + "path::test[none__5v__1d2-2d2-3d1__curv]"`, copied verbatim from what pytest itself + reports and quoted for the brackets) rather than widening a marker or `-k` expression + that would also drop the passing sibling parametrizations, and rather than leaving it to + hang and burn CI minutes on every run. From a4fa105eb8316056ec907e15241d38f29617d3a0 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Mon, 26 Oct 2026 05:15:12 +0000 Subject: [PATCH 06/13] pyramids-gis: renumber workflow comment's gotcha cite to 663/664 Another branch (robust-laplacian) landed its own gotcha 662 on main first, so this PR's own pending 662/663 collide with it on merge. --- .github/workflows/build-pyramids-gis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-pyramids-gis.yml b/.github/workflows/build-pyramids-gis.yml index f7331f9b29b5..abe7be7c286d 100644 --- a/.github/workflows/build-pyramids-gis.yml +++ b/.github/workflows/build-pyramids-gis.yml @@ -221,8 +221,8 @@ jobs: CIBW_TEST_REQUIRES: pytest pytest-env pytest-mock pytest-timeout CIBW_TEST_ENVIRONMENT: 'PIP_ONLY_BINARY=:all:' # the two vsi tests need userfaultfd, which this runner's kernel denies regardless - # of container seccomp/capabilities (gotcha 662); the third is an independent, - # riscv64-specific hang in GDAL's curvilinear-grid read path (gotcha 663) + # of container seccomp/capabilities (gotcha 663); the third is an independent, + # riscv64-specific hang in GDAL's curvilinear-grid read path (gotcha 664) CIBW_TEST_COMMAND: >- python ci/verify-wheel.py && pytest -vvv -o "pythonpath=" --timeout=60 --timeout-method=thread -m "core and not vfs and not slow" tests/ From 231e5e16e8b4503c005197a15eb9941ffdffbce3 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Mon, 26 Oct 2026 05:16:10 +0000 Subject: [PATCH 07/13] gotchas: renumber pyramids-gis's pending 662/663 entries to 663/664 Keep these in step with the workflow's own citations (previous commit) - another branch (robust-laplacian) landed its own gotcha 662 on main first, so this PR's pending numbers need to move before it merges. --- skills/python-project-porting/references/gotchas-index.md | 4 ++-- .../gotchas/pytest-config-servers-and-selection.md | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/skills/python-project-porting/references/gotchas-index.md b/skills/python-project-porting/references/gotchas-index.md index c5db9bc39c17..127b6762b314 100644 --- a/skills/python-project-porting/references/gotchas-index.md +++ b/skills/python-project-porting/references/gotchas-index.md @@ -1403,11 +1403,11 @@ The porting gotchas (561 of them) live in [`references/gotchas/`](gotchas/), spl - **643** — A test suite that needs Node.js (an npm mock server) still runs on riscv64: `actions/setup-node` with `mirror: https://unofficial-builds.nodejs.org/download/release` installs a riscv64 Node (the eppo-server-sdk case). -- **662** — A netCDF `/vsizip/`/`/vsi*`-auto open needs `userfaultfd(2)`, gated by the host +- **663** — A netCDF `/vsizip/`/`/vsi*`-auto open needs `userfaultfd(2)`, gated by the host kernel's `vm.unprivileged_userfaultfd` sysctl, not by anything a container's seccomp profile or capability set reaches — don't treat gotcha 552's `CAP_SYS_PTRACE` fix as a template for this syscall too; deselect rather than keep widening the container (pyramids-gis). -- **663** — A hang reproducible at one exact parametrize id, with sibling parametrizations +- **664** — A hang reproducible at one exact parametrize id, with sibling parametrizations passing clean on either side, is a per-input library bug, not corrupted GDAL global state left over from an earlier failed test; `--timeout-method=thread` can't interrupt a blocking GIL-held C call, so only the job's own wall clock ends the hang (pyramids-gis). diff --git a/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md b/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md index 5605889001bf..67cb2f84e1f8 100644 --- a/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md +++ b/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md @@ -971,7 +971,7 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/pytest-config-servers- *test-time tool* only. A package that bundles or downloads a Node binary into the wheel is a different question (feasibility gotchas on playwright). -662. **A netCDF `/vsizip/`/`/vsi*`-auto open genuinely needs Linux `userfaultfd(2)`, which +663. **A netCDF `/vsizip/`/`/vsi*`-auto open genuinely needs Linux `userfaultfd(2)`, which neither a fully unconfined seccomp profile nor `CAP_SYS_PTRACE` restores — don't treat gotcha 552's fix as a template for every container-syscall `EPERM` (the pyramids-gis case).** GDAL's netCDF driver opens any `/vsi*` virtual-filesystem path (zip, auto @@ -992,7 +992,7 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/pytest-config-servers- is the syscall itself (the traceback or a `strace` names `userfaultfd`), not the shape of the failure. -663. **A hang reproducible at one exact pytest parametrize id, with sibling parametrizations +664. **A hang reproducible at one exact pytest parametrize id, with sibling parametrizations of the same test — including other same-shaped ones — passing clean on either side of it, is a per-input bug in the library's C code, not corrupted global state left over from an earlier failed test; `--timeout-method=thread` reports `Timeout` without @@ -1004,7 +1004,7 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/pytest-config-servers- normally just before and after it in collection order. That pattern (not "everything after the first failure gets slower or flakier", not "only this grid type across the board") rules out the tempting but wrong theory that the two FAILED tests earlier in the - same run (gotcha 662) left GDAL's driver registry or PROJ context corrupted for the rest + same run (gotcha 663) left GDAL's driver registry or PROJ context corrupted for the rest of the process: state corruption reads as noisy and order-dependent, not reproducibly keyed to one specific data shape (5 variables, dims `1d2-2d2-3d1`, curvilinear grid). **`--timeout-method=thread` can't save you here** — its watcher thread still needs the From 1df3483867ee1e13fdfb9ebe5b98d8af858c9dc8 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Tue, 27 Oct 2026 14:07:47 +0000 Subject: [PATCH 08/13] pyramids-gis: deselect a second netCDF multidim hang test_roundtrip_preserves_variable_set[none__111v__1d96-2d13-3d2__str] hung identically on cp312 and cp314 in the same CI run, both stopping at the exact same parametrize id - same shape as the already-deselected curvilinear-grid read hang (gotcha 664), this time in the write path. Two occurrences on the same id across different interpreters rules out runner contention; deselect rather than spend another round confirming. --- .github/workflows/build-pyramids-gis.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-pyramids-gis.yml b/.github/workflows/build-pyramids-gis.yml index abe7be7c286d..d8037341ecc6 100644 --- a/.github/workflows/build-pyramids-gis.yml +++ b/.github/workflows/build-pyramids-gis.yml @@ -221,14 +221,17 @@ jobs: CIBW_TEST_REQUIRES: pytest pytest-env pytest-mock pytest-timeout CIBW_TEST_ENVIRONMENT: 'PIP_ONLY_BINARY=:all:' # the two vsi tests need userfaultfd, which this runner's kernel denies regardless - # of container seccomp/capabilities (gotcha 663); the third is an independent, - # riscv64-specific hang in GDAL's curvilinear-grid read path (gotcha 664) + # of container seccomp/capabilities (gotcha 663); the other two are independent, + # riscv64-specific hangs in GDAL's multidim/curvilinear-grid read-write path, + # confirmed 2/2 identical on the second at the exact same parametrize id across + # two separate interpreter builds (gotcha 664) CIBW_TEST_COMMAND: >- python ci/verify-wheel.py && pytest -vvv -o "pythonpath=" --timeout=60 --timeout-method=thread -m "core and not vfs and not slow" tests/ --deselect "tests/netcdf/io/test_netcdf_archive.py::TestNetCDFReadFileArchiveLinux::test_open_zipped_nc_via_vsi_auto" --deselect "tests/netcdf/io/test_netcdf_archive.py::TestNetCDFReadFileArchiveLinux::test_open_zipped_nc_via_vsi_zip" --deselect "tests/netcdf/samples/test_array_io.py::test_read_array_returns_spatial_array[none__5v__1d2-2d2-3d1__curv]" + --deselect "tests/netcdf/samples/test_roundtrip_file.py::test_roundtrip_preserves_variable_set[none__111v__1d96-2d13-3d2__str]" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: From a19488a94d5e060c4e5322eb3cc84add62b05245 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Tue, 27 Oct 2026 14:07:53 +0000 Subject: [PATCH 09/13] gotchas: extend 664 with the write-path recurrence on pyramids-gis Same exact-parametrize-id hang signature showed up a second time, in a sibling test function (write path, not read), confirmed on two separate interpreter builds in the same run - note it so a future porter spends one round of confirmation instead of several chasing resource contention as an alternate theory. --- .../gotchas/pytest-config-servers-and-selection.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md b/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md index 67cb2f84e1f8..02c22af3c442 100644 --- a/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md +++ b/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md @@ -1017,3 +1017,14 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/pytest-config-servers- reports and quoted for the brackets) rather than widening a marker or `-k` expression that would also drop the passing sibling parametrizations, and rather than leaving it to hang and burn CI minutes on every run. + - **Confirmed a second time, in a sibling test function, same signature.** + `test_roundtrip_file.py::test_roundtrip_preserves_variable_set[none__111v__1d96-2d13-3d2__str]` + (a netCDF *write* through GDAL's multidim API, not the *read* path the first instance + hit) hung identically on two separate interpreter builds (`cp312` and `cp314`) in the + same CI run, both times stopping at the exact same parametrize id right after the same + preceding test passed — the shape that rules out resource contention under a loaded + runner fleet as the explanation: contention would not pick the same one input twice + while 10+ neighbors on either side of it, including other large-variable-count cases, + pass clean both times. One matching occurrence is a coincidence worth a second look; + two, on the same id, across different interpreters, in the same PR, is the pattern — + deselect it the same way rather than spending a round chasing "maybe it's just slow". From 11cca70c78020697ddc4ec87ed674eb93179ba08 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Thu, 29 Oct 2026 09:31:01 +0000 Subject: [PATCH 10/13] pyramids-gis: deselect a third hang at the same parametrize id (gotcha 664) test_roundtrip_preserves_convention[none__111v__1d96-2d13-3d2__str] hangs identically to test_roundtrip_preserves_variable_set at the same id, inside GDAL's multidim GetDimensions() call - confirmed 3/3 now across sibling test functions and separate interpreter builds. --- .github/workflows/build-pyramids-gis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-pyramids-gis.yml b/.github/workflows/build-pyramids-gis.yml index d8037341ecc6..d546df9e32bf 100644 --- a/.github/workflows/build-pyramids-gis.yml +++ b/.github/workflows/build-pyramids-gis.yml @@ -223,8 +223,8 @@ jobs: # the two vsi tests need userfaultfd, which this runner's kernel denies regardless # of container seccomp/capabilities (gotcha 663); the other two are independent, # riscv64-specific hangs in GDAL's multidim/curvilinear-grid read-write path, - # confirmed 2/2 identical on the second at the exact same parametrize id across - # two separate interpreter builds (gotcha 664) + # confirmed 3/3 identical on the second and third at the exact same parametrize id + # across three separate interpreter builds and sibling test functions (gotcha 664) CIBW_TEST_COMMAND: >- python ci/verify-wheel.py && pytest -vvv -o "pythonpath=" --timeout=60 --timeout-method=thread -m "core and not vfs and not slow" tests/ @@ -232,6 +232,7 @@ jobs: --deselect "tests/netcdf/io/test_netcdf_archive.py::TestNetCDFReadFileArchiveLinux::test_open_zipped_nc_via_vsi_zip" --deselect "tests/netcdf/samples/test_array_io.py::test_read_array_returns_spatial_array[none__5v__1d2-2d2-3d1__curv]" --deselect "tests/netcdf/samples/test_roundtrip_file.py::test_roundtrip_preserves_variable_set[none__111v__1d96-2d13-3d2__str]" + --deselect "tests/netcdf/samples/test_roundtrip_file.py::test_roundtrip_preserves_convention[none__111v__1d96-2d13-3d2__str]" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: From 1cb639a5c3bed5e53489f5b606f886d8cee4ad60 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Thu, 29 Oct 2026 09:31:49 +0000 Subject: [PATCH 11/13] gotchas: extend 664 with a third sibling-function recurrence on pyramids-gis --- .../gotchas/pytest-config-servers-and-selection.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md b/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md index 02c22af3c442..066141dcc68c 100644 --- a/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md +++ b/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md @@ -1028,3 +1028,13 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/pytest-config-servers- pass clean both times. One matching occurrence is a coincidence worth a second look; two, on the same id, across different interpreters, in the same PR, is the pattern — deselect it the same way rather than spending a round chasing "maybe it's just slow". + - **Confirmed a third time, same exact parametrize id, yet another sibling test + function.** `test_roundtrip_file.py::test_roundtrip_preserves_convention[none__111v__1d96-2d13-3d2__str]` + hung identically on the same input id — same file, a different test function from the + already-deselected `test_roundtrip_preserves_variable_set`, same `GetDimensions()` call + inside GDAL's multidim API on the stack. Three independent hits on one exact id, across + three test functions and three interpreter builds, is conclusive: the bug lives in that + one data shape going through GDAL's multidim group/dimension resolution on riscv64, not + in any single test. Deselect each new sibling that reaches it the same way; do not widen + the search for a shared root cause unless it's cheap, since the fix (deselect) is already + known and proven. From 25f533765aca069235445c672a6d0d20a39e12e9 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Thu, 29 Oct 2026 23:44:00 +0000 Subject: [PATCH 12/13] pyramids-gis: renumber pending gotchas 663/664 to 665/666 main claimed 663 for an unrelated setuptools_scm fix (pyarrow) while this PR's own 663/664 were still pending merge - move them clear before they collide. --- .github/workflows/build-pyramids-gis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-pyramids-gis.yml b/.github/workflows/build-pyramids-gis.yml index d546df9e32bf..4a1b38fdc975 100644 --- a/.github/workflows/build-pyramids-gis.yml +++ b/.github/workflows/build-pyramids-gis.yml @@ -221,10 +221,10 @@ jobs: CIBW_TEST_REQUIRES: pytest pytest-env pytest-mock pytest-timeout CIBW_TEST_ENVIRONMENT: 'PIP_ONLY_BINARY=:all:' # the two vsi tests need userfaultfd, which this runner's kernel denies regardless - # of container seccomp/capabilities (gotcha 663); the other two are independent, + # of container seccomp/capabilities (gotcha 665); the other two are independent, # riscv64-specific hangs in GDAL's multidim/curvilinear-grid read-write path, # confirmed 3/3 identical on the second and third at the exact same parametrize id - # across three separate interpreter builds and sibling test functions (gotcha 664) + # across three separate interpreter builds and sibling test functions (gotcha 666) CIBW_TEST_COMMAND: >- python ci/verify-wheel.py && pytest -vvv -o "pythonpath=" --timeout=60 --timeout-method=thread -m "core and not vfs and not slow" tests/ From f90f50d7673e242cd3b25665d4aaa9def8432f72 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Thu, 29 Oct 2026 23:44:13 +0000 Subject: [PATCH 13/13] gotchas: renumber pyramids-gis's pending 663/664 entries to 665/666 Keep in step with the workflow's own citations (previous commit); also fixes a stale top-of-file summary line left at 662/663 from the prior renumbering round that never touched it. --- .../python-project-porting/references/gotchas-index.md | 4 ++-- .../gotchas/pytest-config-servers-and-selection.md | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/skills/python-project-porting/references/gotchas-index.md b/skills/python-project-porting/references/gotchas-index.md index 127b6762b314..e1e6aad7cd41 100644 --- a/skills/python-project-porting/references/gotchas-index.md +++ b/skills/python-project-porting/references/gotchas-index.md @@ -1403,11 +1403,11 @@ The porting gotchas (561 of them) live in [`references/gotchas/`](gotchas/), spl - **643** — A test suite that needs Node.js (an npm mock server) still runs on riscv64: `actions/setup-node` with `mirror: https://unofficial-builds.nodejs.org/download/release` installs a riscv64 Node (the eppo-server-sdk case). -- **663** — A netCDF `/vsizip/`/`/vsi*`-auto open needs `userfaultfd(2)`, gated by the host +- **665** — A netCDF `/vsizip/`/`/vsi*`-auto open needs `userfaultfd(2)`, gated by the host kernel's `vm.unprivileged_userfaultfd` sysctl, not by anything a container's seccomp profile or capability set reaches — don't treat gotcha 552's `CAP_SYS_PTRACE` fix as a template for this syscall too; deselect rather than keep widening the container (pyramids-gis). -- **664** — A hang reproducible at one exact parametrize id, with sibling parametrizations +- **666** — A hang reproducible at one exact parametrize id, with sibling parametrizations passing clean on either side, is a per-input library bug, not corrupted GDAL global state left over from an earlier failed test; `--timeout-method=thread` can't interrupt a blocking GIL-held C call, so only the job's own wall clock ends the hang (pyramids-gis). diff --git a/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md b/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md index 066141dcc68c..661d9aef5a2f 100644 --- a/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md +++ b/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md @@ -57,10 +57,10 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/pytest-config-servers- - **643** — An upstream test suite that needs a Node.js helper (a mock server via `npm ci`/`npm run`) still runs on the riscv64 runner: `actions/setup-node` with `mirror: https://unofficial-builds.nodejs.org/download/release` installs a riscv64 Node. -- **662** — A netCDF `/vsizip/`/`/vsi*` open needs `userfaultfd(2)`, which a fully unconfined +- **665** — A netCDF `/vsizip/`/`/vsi*` open needs `userfaultfd(2)`, which a fully unconfined seccomp profile and `CAP_SYS_PTRACE` do not restore — that syscall is gated by the host kernel's `vm.unprivileged_userfaultfd` sysctl, not by anything a container flag reaches. -- **663** — A hang reproducible at one exact parametrize id, with sibling parametrizations +- **666** — A hang reproducible at one exact parametrize id, with sibling parametrizations passing clean on either side, is a per-input library bug, not corrupted global state from an earlier failure; `--timeout-method=thread` can't interrupt a blocking GIL-held C call. @@ -971,7 +971,7 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/pytest-config-servers- *test-time tool* only. A package that bundles or downloads a Node binary into the wheel is a different question (feasibility gotchas on playwright). -663. **A netCDF `/vsizip/`/`/vsi*`-auto open genuinely needs Linux `userfaultfd(2)`, which +665. **A netCDF `/vsizip/`/`/vsi*`-auto open genuinely needs Linux `userfaultfd(2)`, which neither a fully unconfined seccomp profile nor `CAP_SYS_PTRACE` restores — don't treat gotcha 552's fix as a template for every container-syscall `EPERM` (the pyramids-gis case).** GDAL's netCDF driver opens any `/vsi*` virtual-filesystem path (zip, auto @@ -992,7 +992,7 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/pytest-config-servers- is the syscall itself (the traceback or a `strace` names `userfaultfd`), not the shape of the failure. -664. **A hang reproducible at one exact pytest parametrize id, with sibling parametrizations +666. **A hang reproducible at one exact pytest parametrize id, with sibling parametrizations of the same test — including other same-shaped ones — passing clean on either side of it, is a per-input bug in the library's C code, not corrupted global state left over from an earlier failed test; `--timeout-method=thread` reports `Timeout` without @@ -1004,7 +1004,7 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/pytest-config-servers- normally just before and after it in collection order. That pattern (not "everything after the first failure gets slower or flakier", not "only this grid type across the board") rules out the tempting but wrong theory that the two FAILED tests earlier in the - same run (gotcha 663) left GDAL's driver registry or PROJ context corrupted for the rest + same run (gotcha 665) left GDAL's driver registry or PROJ context corrupted for the rest of the process: state corruption reads as noisy and order-dependent, not reproducibly keyed to one specific data shape (5 variables, dims `1d2-2d2-3d1`, curvilinear grid). **`--timeout-method=thread` can't save you here** — its watcher thread still needs the