diff --git a/.github/workflows/build-pyarrow.yml b/.github/workflows/build-pyarrow.yml index 43c0b6219050..35775441dd39 100644 --- a/.github/workflows/build-pyarrow.yml +++ b/.github/workflows/build-pyarrow.yml @@ -77,6 +77,15 @@ jobs: - cp313-cp313 - cp314-cp314 - cp314-cp314t + exclude: + # 16.1.0 predates cp313 (released Oct 2024, ~5 months after this + # release) and never shipped a wheel past cp312 upstream either. + - version: "16.1.0" + python: cp313-cp313 + - version: "16.1.0" + python: cp314-cp314 + - version: "16.1.0" + python: cp314-cp314t env: PYARROW_VERSION: ${{ matrix.version }} @@ -135,6 +144,14 @@ jobs: DIST=/tmp/arrow-dist BUILD=/tmp/arrow-build + # BUNDLED thirdparty deps are built via ExternalProject_Add with their own + # vendored cmake_minimum_required (e.g. gflags' is 2.8.12), below CMake 4's + # floor of 3.5. The image's cmake (pipx-installed, >= 4) hard-errors on that + # instead of warning. We don't control the ExternalProject configure line, but + # CMake also reads CMAKE_POLICY_VERSION_MINIMUM from the environment, which + # reaches every nested cmake invocation this build makes (gotcha 257). + export CMAKE_POLICY_VERSION_MINIMUM=3.5 + # ninja and OpenSSL headers aren't in the base image dnf install -y --setopt=install_weak_deps=False ninja-build openssl-devel @@ -142,6 +159,8 @@ jobs: pushd "$BUILD" # The image ships OpenSSL shared-only, so force it shared (the rest stay static). # UNITY_BUILD off: bundled c-ares isn't unity-clean and upstream never compiles it. + # `-include cstdint`: older bundled Abseil (16.1.0 pins 20211102.0) relies on + # arriving transitively, which GCC 13+'s libstdc++ no longer does; EP_CXX_FLAGS hands it on. cmake \ -DARROW_DEPENDENCY_SOURCE=BUNDLED \ -DARROW_DEPENDENCY_USE_SHARED=OFF \ @@ -178,6 +197,7 @@ jobs: -DARROW_RPATH_ORIGIN=ON \ -DARROW_USE_CCACHE=ON \ -DCMAKE_BUILD_TYPE=release \ + -DCMAKE_CXX_FLAGS="-include cstdint" \ -DCMAKE_INSTALL_PREFIX="$DIST" \ -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_UNITY_BUILD=OFF \ @@ -224,7 +244,22 @@ jobs: # numpy/libcst: riscv64 wheels only in our registry "$pybin" -m pip install -U --extra-index-url "$PIP_EXTRA_INDEX_URL" \ --only-binary=numpy,libcst "numpy>=2.0.0" "libcst>=1.8.6" - "$pybin" -m pip install -U "cython>=3.1" scikit-build-core setuptools_scm wheel build + # 16.1.0's setup.py imports pkg_resources, removed in setuptools 82 (gotcha 29), + # and passes an absolute write_to, rejected by setuptools_scm >= 10.1 (gotcha 663) + CYTHON_SPEC="cython>=3.1" + if [ "$ARROW_VERSION" = "16.1.0" ]; then + # Cython 3.1 (May 2025, ~a year after this release) tightened `with nogil:` + # GIL-safety checking; pyarrow/table.pxi's `options.unify_schemas = + # promote_options != "none"` inside a nogil block is a Python string + # compare that only Cython's older, more lenient check let through + # (gotcha 23's shape). Upstream's own build-system.requires floor for + # 16.1.0 is just cython>=0.29.31 -- release-era Cython was 3.0.x, so cap + # below the breaking version rather than patch pyarrow's source. Safe to + # scope to this version alone: the matrix excludes cp313/cp314/cp314t for + # it above, and those are the only legs that need 3.1's free-threading support. + CYTHON_SPEC="cython>=3,<3.1" + fi + "$pybin" -m pip install -U "$CYTHON_SPEC" scikit-build-core "setuptools_scm<10" wheel build "setuptools<82" mkdir -p /output pushd "$ARROW_SRC/python" "$pybin" -m build --wheel --no-isolation \ @@ -250,7 +285,7 @@ jobs: docker run -i --rm \ -v "$PWD/arrow:/arrow" \ -v "$PWD/output:/output" \ - -e PIP_EXTRA_INDEX_URL \ + -e PIP_EXTRA_INDEX_URL -e PYARROW_VERSION \ -e ARROW_DATASET -e ARROW_ACERO -e ARROW_PARQUET \ -e PARQUET_REQUIRE_ENCRYPTION -e ARROW_ORC -e ARROW_SUBSTRAIT \ -e ARROW_FLIGHT -e ARROW_S3 -e ARROW_GCS -e ARROW_AZURE \ @@ -311,12 +346,54 @@ jobs: # hypothesis is only-binary too: upstream ships no cp314t riscv64 wheel (it builds # riscv64 abi3-only, and abi3t starts at 3.15), so without this pip falls back to its # sdist and the Rust build dies on a triple rustup does not have. Ours comes from the registry. + PYTEST_SPEC="pytest" + if [ "$PYARROW_VERSION" = "16.1.0" ]; then + # 16.1.0's bundled conftest.py defines pytest_ignore_collect(path, config), + # the legacy (path: py.path.local) hookimpl signature. pytest's own policy + # escalates a RemovedInN warning to a hard error starting at major version N: + # 8.4.0 only warns (PytestRemovedIn9Warning) and collection still passes, but + # 9.0.3 raises that same warning as an error during plugin registration -- + # confirmed by running a two-line repro plugin under both. <9.1 (tried first) + # was not low enough; the escalation already happens at 9.0. Release-era + # pytest was comfortably below both (gotcha 23's shape, in the test deps + # rather than the build tool). + PYTEST_SPEC="pytest<9" + fi "$pybin" -m pip install -U --only-binary=numpy,cffi,hypothesis \ --extra-index-url "$PIP_EXTRA_INDEX_URL" \ - pytest hypothesis cffi pytz packaging "numpy>=2.0.0" + "$PYTEST_SPEC" hypothesis cffi pytz packaging "numpy>=2.0.0" + + PANDAS_SPEC="pandas" + PYTEST_K="" + if [ "$PYARROW_VERSION" = "16.1.0" ]; then + # pandas 3.0 made string-backed dtypes the default for columns that used + # to come back as plain object dtype (and changed pa.array(Categorical)'s + # inferred value type to large_string); 16.1.0's pandas-compat code and its + # own test suite's expected values both predate that and assume object/ + # string, so an unconstrained `pip install pandas` (today's 3.1.0) breaks + # ~42 of the suite's tests on dtype-equality assertions alone -- confirmed + # from the CI log (test_backwards_compatible_index_naming et al. failing + # with dtype='object' vs dtype='str', test_dictionary_from_pandas with + # string vs large_string). Same release-era-drift shape as the cython/ + # pytest caps above, this time in a floating *test* dependency. + PANDAS_SPEC="pandas<3" + # test_memory.py hardcodes `should_have_jemalloc = sys.platform == "linux"` + # (removed in later pyarrow releases, e.g. it's gone by 25.0.1) -- it never + # checks whether *this* build actually enabled jemalloc. We build with + # ARROW_JEMALLOC=OFF (see the env block above: "jemalloc is page-size + # fragile"), so these 4 tests fail here on every Linux build that disables + # it, regardless of pandas: confirmed from the CI log, each failing on the + # exact jemalloc/mimalloc assertion (ArrowNotImplementedError, "Unsupported + # backend 'jemalloc'", "'jemalloc' in ['mimalloc', 'system']", an + # allocation byte-count tied to the allocator). Excluded by test function + # name via -k (not --deselect: --pyargs resolves test files under the + # interpreter's own site-packages path, which varies per matrix.python, so + # there is no single static path --deselect could name). + PYTEST_K="not (test_env_var or test_specific_memory_pools or test_supported_memory_backends or test_table_uses_memory_pool)" + fi # pandas may lack a free-threaded riscv64 wheel; skip its tests if so if "$pybin" -m pip install -U --only-binary=pandas \ - --extra-index-url "$PIP_EXTRA_INDEX_URL" pandas; then + --extra-index-url "$PIP_EXTRA_INDEX_URL" "$PANDAS_SPEC"; then export PYARROW_TEST_PANDAS=ON else echo "WARNING: pandas has no riscv64 wheel for ${{ matrix.python }}; disabling pandas tests." @@ -324,7 +401,11 @@ jobs: fi "$pybin" -c 'import pyarrow; pyarrow.create_library_symlinks()' - "$pybin" -m pytest -r s --pyargs pyarrow + if [ -n "$PYTEST_K" ]; then + "$pybin" -m pytest -r s --pyargs pyarrow -k "$PYTEST_K" + else + "$pybin" -m pytest -r s --pyargs pyarrow + fi PYARROW_TEST_EOF publish: diff --git a/docs/packages/pyarrow.yaml b/docs/packages/pyarrow.yaml index 95b97b7635b5..948bfb0b19eb 100644 --- a/docs/packages/pyarrow.yaml +++ b/docs/packages/pyarrow.yaml @@ -2,6 +2,7 @@ package-name: pyarrow source-code: https://github.com/apache/arrow license: Apache-2.0 versions: +- version: 16.1.0 - version: 25.0.1 tag: pyarrow-v25.0.1-20260902125649 files: