From 27e3c938e07b1a9ee860da2c59a3eecb7e993a76 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Thu, 15 Oct 2026 10:59:30 +0000 Subject: [PATCH 1/6] pyarrow: add 16.1.0 for apache-beam's pyarrow<17.0.0 pin apache-beam 2.54.0-2.61.0 (the window apache-flink needs for its cimport'd Cython bindings) all pin pyarrow<17.0.0; our registry only carried 25.0.1/26.0.0. 16.1.0 shares the existing bundled-Arrow-C++ build (env-var driven, same as 25.0.1/26.0.0); the build matrix drops cp313/cp314/cp314t for this version since it predates cp313's release by ~5 months and upstream itself never shipped past cp312 for it. --- .github/workflows/build-pyarrow.yml | 9 +++++++++ docs/packages/pyarrow.yaml | 1 + 2 files changed, 10 insertions(+) diff --git a/.github/workflows/build-pyarrow.yml b/.github/workflows/build-pyarrow.yml index 43c0b6219050..e273c9aa20c0 100644 --- a/.github/workflows/build-pyarrow.yml +++ b/.github/workflows/build-pyarrow.yml @@ -77,6 +77,15 @@ jobs: - cp313-cp313 - cp314-cp314 - cp314-cp314t + exclude: + # 16.1.0 predates cp313 (released Oct 2024, ~5 months after this + # release) and never shipped a wheel past cp312 upstream either. + - version: "16.1.0" + python: cp313-cp313 + - version: "16.1.0" + python: cp314-cp314 + - version: "16.1.0" + python: cp314-cp314t env: PYARROW_VERSION: ${{ matrix.version }} diff --git a/docs/packages/pyarrow.yaml b/docs/packages/pyarrow.yaml index 95b97b7635b5..948bfb0b19eb 100644 --- a/docs/packages/pyarrow.yaml +++ b/docs/packages/pyarrow.yaml @@ -2,6 +2,7 @@ package-name: pyarrow source-code: https://github.com/apache/arrow license: Apache-2.0 versions: +- version: 16.1.0 - version: 25.0.1 tag: pyarrow-v25.0.1-20260902125649 files: From 333f257040e4803047c93dc2af37b23211f71d4a Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Sun, 25 Oct 2026 18:00:11 +0000 Subject: [PATCH 2/6] pyarrow: fix bundled gflags configure under CMake >= 4 ARROW_DEPENDENCY_SOURCE=BUNDLED builds gflags via ExternalProject_Add using its vendored cmake_minimum_required(VERSION 2.8.12), below CMake 4's removed-compatibility floor of 3.5. The riscv64 image's pipx-installed cmake is >= 4, so the gflags_ep configure step hard-errors before any other bundled dependency gets a chance to build. Export CMAKE_POLICY_VERSION_MINIMUM=3.5 for the whole build: CMake reads it from the environment with the same effect as the -D cache flag, so it reaches every nested cmake invocation ExternalProject_Add makes without needing to patch Arrow's own CMakeLists.txt. --- .github/workflows/build-pyarrow.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/build-pyarrow.yml b/.github/workflows/build-pyarrow.yml index e273c9aa20c0..fc39c607b23f 100644 --- a/.github/workflows/build-pyarrow.yml +++ b/.github/workflows/build-pyarrow.yml @@ -144,6 +144,14 @@ jobs: DIST=/tmp/arrow-dist BUILD=/tmp/arrow-build + # BUNDLED thirdparty deps are built via ExternalProject_Add with their own + # vendored cmake_minimum_required (e.g. gflags' is 2.8.12), below CMake 4's + # floor of 3.5. The image's cmake (pipx-installed, >= 4) hard-errors on that + # instead of warning. We don't control the ExternalProject configure line, but + # CMake also reads CMAKE_POLICY_VERSION_MINIMUM from the environment, which + # reaches every nested cmake invocation this build makes (gotcha 257). + export CMAKE_POLICY_VERSION_MINIMUM=3.5 + # ninja and OpenSSL headers aren't in the base image dnf install -y --setopt=install_weak_deps=False ninja-build openssl-devel From 77b935d3096bb5b337377ce12c915398a50d39dc Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Wed, 28 Oct 2026 11:01:21 +0000 Subject: [PATCH 3/6] pyarrow: force-include for 16.1.0's bundled Abseil Arrow 16.1.0 pins Abseil 20211102.0 for its BUNDLED absl_ep. That release's absl/strings/internal/str_format/extension.h forward-declares `enum class FormatConversionChar : uint8_t` and `enum class FormatConversionCharSet : uint64_t` without including , relying on libstdc++ headers to pull it in transitively. Since GCC 13, libstdc++ no longer does, and manylinux_2_39_riscv64 ships GCC 14, so both declarations fail and str_format's extension.cc and float_conversion.cc cascade into "FormatConversionCharSet was not declared" / "no member named conversion_char". Abseil added the include upstream in later LTS releases; 25.0.1/26.0.0 pin 20250127.0 and never hit it. Pass -DCMAKE_CXX_FLAGS="-include cstdint": Arrow copies CMAKE_CXX_FLAGS into EP_CXX_FLAGS, which every bundled ExternalProject receives through EP_COMMON_CMAKE_ARGS, so absl_ep (and gRPC/ORC built on top of it) get the header without patching Arrow or Abseil. Same remedy as build-aimrocks.yml uses for the same libstdc++ change. --- .github/workflows/build-pyarrow.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-pyarrow.yml b/.github/workflows/build-pyarrow.yml index fc39c607b23f..c9ad33175df8 100644 --- a/.github/workflows/build-pyarrow.yml +++ b/.github/workflows/build-pyarrow.yml @@ -159,6 +159,8 @@ jobs: pushd "$BUILD" # The image ships OpenSSL shared-only, so force it shared (the rest stay static). # UNITY_BUILD off: bundled c-ares isn't unity-clean and upstream never compiles it. + # `-include cstdint`: older bundled Abseil (16.1.0 pins 20211102.0) relies on + # arriving transitively, which GCC 13+'s libstdc++ no longer does; EP_CXX_FLAGS hands it on. cmake \ -DARROW_DEPENDENCY_SOURCE=BUNDLED \ -DARROW_DEPENDENCY_USE_SHARED=OFF \ @@ -195,6 +197,7 @@ jobs: -DARROW_RPATH_ORIGIN=ON \ -DARROW_USE_CCACHE=ON \ -DCMAKE_BUILD_TYPE=release \ + -DCMAKE_CXX_FLAGS="-include cstdint" \ -DCMAKE_INSTALL_PREFIX="$DIST" \ -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_UNITY_BUILD=OFF \ From aac9d3e6fc9a02a7c58d1b63b3e0da1f12d7a724 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Thu, 29 Oct 2026 08:38:24 +0000 Subject: [PATCH 4/6] pyarrow: cap setuptools below 82 for 16.1.0's pkg_resources import pyarrow 16.1.0's setup.py does `import pkg_resources` at module scope (and uses pkg_resources.resource_filename in build_ext). setuptools 82.0.0 removed pkg_resources, and the unpinned preinstall resolved to 84.2.1, so `build --no-isolation` died in get_requires_for_build_wheel with ModuleNotFoundError. Cap at <82 (81.0.0 still ships it), as asyncpg does (gotcha 29). --- .github/workflows/build-pyarrow.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-pyarrow.yml b/.github/workflows/build-pyarrow.yml index c9ad33175df8..2e06226ad1ba 100644 --- a/.github/workflows/build-pyarrow.yml +++ b/.github/workflows/build-pyarrow.yml @@ -244,7 +244,8 @@ jobs: # numpy/libcst: riscv64 wheels only in our registry "$pybin" -m pip install -U --extra-index-url "$PIP_EXTRA_INDEX_URL" \ --only-binary=numpy,libcst "numpy>=2.0.0" "libcst>=1.8.6" - "$pybin" -m pip install -U "cython>=3.1" scikit-build-core setuptools_scm wheel build + # 16.1.0's setup.py imports pkg_resources, removed in setuptools 82 (gotcha 29) + "$pybin" -m pip install -U "cython>=3.1" scikit-build-core setuptools_scm wheel build "setuptools<82" mkdir -p /output pushd "$ARROW_SRC/python" "$pybin" -m build --wheel --no-isolation \ From b5394c4ef7e8d4f684b376b3628758ebf67ea695 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Thu, 29 Oct 2026 23:41:45 +0000 Subject: [PATCH 5/6] pyarrow: cap setuptools_scm below 10 for 16.1.0's absolute write_to 16.1.0's setup.py sets use_scm_version's write_to to os.path.join(setup_dir, 'pyarrow/_generated_version.py') with an abspath'd setup_dir. setuptools_scm 10.1 added _sanitize_relative_path to its build_py hook and rejects that with "Version file path must be relative"; 9.2.2 accepts an absolute write_to under root by design. --- .github/workflows/build-pyarrow.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-pyarrow.yml b/.github/workflows/build-pyarrow.yml index 2e06226ad1ba..0fbc2d7d25c9 100644 --- a/.github/workflows/build-pyarrow.yml +++ b/.github/workflows/build-pyarrow.yml @@ -244,8 +244,9 @@ jobs: # numpy/libcst: riscv64 wheels only in our registry "$pybin" -m pip install -U --extra-index-url "$PIP_EXTRA_INDEX_URL" \ --only-binary=numpy,libcst "numpy>=2.0.0" "libcst>=1.8.6" - # 16.1.0's setup.py imports pkg_resources, removed in setuptools 82 (gotcha 29) - "$pybin" -m pip install -U "cython>=3.1" scikit-build-core setuptools_scm wheel build "setuptools<82" + # 16.1.0's setup.py imports pkg_resources, removed in setuptools 82 (gotcha 29), + # and passes an absolute write_to, rejected by setuptools_scm >= 10.1 (gotcha 663) + "$pybin" -m pip install -U "cython>=3.1" scikit-build-core "setuptools_scm<10" wheel build "setuptools<82" mkdir -p /output pushd "$ARROW_SRC/python" "$pybin" -m build --wheel --no-isolation \ From 61b744ddc9fceae0c01e1ef124ed7ae7ce32d566 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Fri, 30 Oct 2026 16:33:38 +0000 Subject: [PATCH 6/6] pyarrow: cap cython below 3.1 for 16.1.0's nogil-strict table.pxi Cython 3.1 (May 2025) tightened with-nogil GIL-safety checking; pyarrow/table.pxi's `options.unify_schemas = promote_options != "none"` inside a nogil block is a Python string compare that only Cython's older, more lenient check let through. Upstream's own build-system.requires floor for 16.1.0 is cython>=0.29.31 -- release-era Cython was 3.0.x. Scoped to 16.1.0 alone: the matrix already excludes cp313/cp314/cp314t for it, and those are the only legs that need 3.1's free-threading support. --- .github/workflows/build-pyarrow.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-pyarrow.yml b/.github/workflows/build-pyarrow.yml index 0fbc2d7d25c9..5e502ec2bbfd 100644 --- a/.github/workflows/build-pyarrow.yml +++ b/.github/workflows/build-pyarrow.yml @@ -246,7 +246,20 @@ jobs: --only-binary=numpy,libcst "numpy>=2.0.0" "libcst>=1.8.6" # 16.1.0's setup.py imports pkg_resources, removed in setuptools 82 (gotcha 29), # and passes an absolute write_to, rejected by setuptools_scm >= 10.1 (gotcha 663) - "$pybin" -m pip install -U "cython>=3.1" scikit-build-core "setuptools_scm<10" wheel build "setuptools<82" + CYTHON_SPEC="cython>=3.1" + if [ "$ARROW_VERSION" = "16.1.0" ]; then + # Cython 3.1 (May 2025, ~a year after this release) tightened `with nogil:` + # GIL-safety checking; pyarrow/table.pxi's `options.unify_schemas = + # promote_options != "none"` inside a nogil block is a Python string + # compare that only Cython's older, more lenient check let through + # (gotcha 23's shape). Upstream's own build-system.requires floor for + # 16.1.0 is just cython>=0.29.31 -- release-era Cython was 3.0.x, so cap + # below the breaking version rather than patch pyarrow's source. Safe to + # scope to this version alone: the matrix excludes cp313/cp314/cp314t for + # it above, and those are the only legs that need 3.1's free-threading support. + CYTHON_SPEC="cython>=3,<3.1" + fi + "$pybin" -m pip install -U "$CYTHON_SPEC" scikit-build-core "setuptools_scm<10" wheel build "setuptools<82" mkdir -p /output pushd "$ARROW_SRC/python" "$pybin" -m build --wheel --no-isolation \