diff --git a/.flake8 b/.flake8 index 617214e..0d31d68 100644 --- a/.flake8 +++ b/.flake8 @@ -5,7 +5,5 @@ extend-ignore = E203, # E203 whitespace before ':' (to be compatible with black) per-file-ignores = - suitesparse_graphblas/io/binary.py:C408, - suitesparse_graphblas/tests/test_io.py:E721, # The api package re-exports its submodules for `from suitesparse_graphblas import api` suitesparse_graphblas/api/__init__.py:F401, diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b18fd29..5fa64ac 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,3 +4,12 @@ updates: directory: '/' schedule: interval: 'weekly' + # Wait a week after a release before proposing it, so a compromised or + # broken release has time to be noticed and yanked. + cooldown: + default-days: 7 + # One PR per week for all action bumps instead of one PR per action. + groups: + github-actions: + patterns: + - '*' diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 777414d..a6f536b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -6,8 +6,7 @@ on: workflow_dispatch: permissions: - pages: write - id-token: write + contents: read concurrency: group: pages @@ -17,33 +16,38 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Build package in Docker run: | docker build --target psg \ - --build-arg SUITESPARSE=v$(cat GB_VERSION.txt) \ + --build-arg SUITESPARSE="v$(cat GB_VERSION.txt)" \ --build-arg VERSION=99.0.0.0 \ --build-arg COMPACT=1 \ -t psg-docs . - name: Generate docs with pdoc run: | - docker run --rm -v ${{ github.workspace }}/site:/out psg-docs \ + docker run --rm -v "$GITHUB_WORKSPACE/site:/out" psg-docs \ bash -c "pip install --break-system-packages -q pdoc && \ pdoc --output-directory /out --no-show-source --docformat numpy \ suitesparse_graphblas.api" - - uses: actions/upload-pages-artifact@v3 + - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 with: path: site/ deploy: needs: build runs-on: ubuntu-latest + permissions: + pages: write # to deploy to GitHub Pages + id-token: write # to prove the deployment comes from this workflow environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} steps: - - uses: actions/deploy-pages@v4 + - uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1 id: deployment diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 63644a1..b05ed8a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -9,16 +9,21 @@ on: permissions: contents: read +# Cancel superseded runs of a PR; never cancel branch pushes. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: pre-commit: name: pre-commit-hooks runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.11" - - uses: pre-commit/action@v3.0.1 + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 07815f1..0d79922 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,12 +5,22 @@ on: branches: [ main ] pull_request: +permissions: + contents: read + +# Cancel superseded runs of a PR; never cancel runs on main. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: test: runs-on: ${{ matrix.os }} defaults: run: - shell: bash -l {0} + # -e: a custom shell gets no errexit, so without it a multi-command step + # (e.g. "Test") would only report the exit status of its last command. + shell: bash -el {0} strategy: fail-fast: false matrix: @@ -19,14 +29,17 @@ jobs: # os: ["ubuntu-latest"] # source: ["source"] # A trailing "t" means the free-threaded (no-GIL) build of that version. - python-version: ["3.11", "3.12", "3.13", "3.14", "3.14t"] + # TODO: add "3.15t" once conda-forge builds cffi and numpy for cp315t. + python-version: ["3.11", "3.12", "3.13", "3.14", "3.14t", "3.15"] steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false - name: Compose conda environment file + env: + PYVER: ${{ matrix.python-version }} run: | # conda-forge ships two ABIs for CPython 3.13+ ("cp314" and "cp314t"), # and `python=3.14` alone lets the solver pick either one: it resolved @@ -34,35 +47,48 @@ jobs: # The `python-gil` and `python-freethreading` metapackages pin # `python_abi` to one or the other, so ask for the one this matrix # entry wants instead of leaving it to the solver. - pyver="${{ matrix.python-version }}" + pyver=$PYVER if [[ $pyver == *t ]]; then variant=python-freethreading pyver=${pyver%t} else variant=python-gil fi - cp continuous_integration/environment.yml environment-ci.yml + channels=conda-forge,nodefaults + if [[ $pyver == 3.15 ]]; then + # TODO: drop this branch once Python 3.15.0 final is on conda-forge. + # Until then the 3.15 release candidates need `_python_rc`, which only + # the `python_rc` label provides. The label must rank *below* + # conda-forge: it also holds older `python` rc builds (but no 3.15), and + # strict channel priority would then hide conda-forge's 3.15 rc. + channels=conda-forge,conda-forge/label/python_rc,nodefaults + awk '{ print } /^ - conda-forge$/ { print " - conda-forge/label/python_rc" }' \ + continuous_integration/environment.yml > environment-ci.yml + else + cp continuous_integration/environment.yml environment-ci.yml + fi + echo "CONDA_CHANNELS=$channels" >> "$GITHUB_ENV" echo " - python=$pyver" >> environment-ci.yml echo " - $variant" >> environment-ci.yml cat environment-ci.yml - name: Conda - uses: conda-incubator/setup-miniconda@v3 + uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 with: auto-update-conda: true environment-file: environment-ci.yml - channels: conda-forge,nodefaults + channels: ${{ env.CONDA_CHANNELS }} channel-priority: strict activate-environment: suitesparse-graphblas auto-activate-base: false - name: GraphBLAS (from conda-forge) if: (contains(matrix.source, 'conda-forge')) run: | - conda install graphblas=$(cat GB_VERSION.txt) + conda install "graphblas=$(cat GB_VERSION.txt)" - name: GraphBLAS (from source) if: (contains(matrix.source, 'source')) run: | # From release (also works with beta versions) - GRAPHBLAS_PREFIX=${CONDA_PREFIX} bash suitesparse.sh refs/tags/$(cat GB_VERSION.txt).0 + GRAPHBLAS_PREFIX="${CONDA_PREFIX}" bash suitesparse.sh "refs/tags/$(cat GB_VERSION.txt).0" # From tag # curl -L https://github.com/DrTimothyAldenDavis/GraphBLAS/archive/refs/tags/v$(cat GB_VERSION.txt).tar.gz | tar xzf - @@ -84,18 +110,18 @@ jobs: - name: Configure coverage run: | # Cython's coverage plugin (see [tool.coverage.run] in pyproject.toml) - # needs coverage's C tracer. conda-forge builds coverage without its C - # extension for free-threaded Python, and conda resolves `python=3.14` - # to the free-threaded build (3.14t) on some platforms. There, loading - # the plugin raises ImportError and every `coverage run` dies before it - # runs anything, so run the tests directly instead. + # needs coverage's C tracer. Where conda-forge has no compiled build of + # coverage for a Python (3.15, for now), it installs a pure-Python build + # that lacks the tracer. There, loading the plugin raises ImportError and + # every `coverage run` dies before it runs anything, so run the tests + # directly instead. if python -c "import Cython.Coverage" 2>/dev/null; then coverage erase - echo "COV=coverage run -a --branch" >> $GITHUB_ENV + echo "COV=coverage run -a --branch" >> "$GITHUB_ENV" else echo "Cython coverage plugin unusable here; running without coverage" python -VV - echo "COV=python" >> $GITHUB_ENV + echo "COV=python" >> "$GITHUB_ENV" fi - name: Test env: diff --git a/.github/workflows/update_graphblas.yml b/.github/workflows/update_graphblas.yml index 532072f..f5c0fb6 100644 --- a/.github/workflows/update_graphblas.yml +++ b/.github/workflows/update_graphblas.yml @@ -1,12 +1,7 @@ # Checks for latest SuiteSparse:GraphBLAS version on GitHub and creates a PR to update the version used by this repo. name: Check for GraphBLAS updates -# In addition to permissions below, must explicitly allow GitHub Actions to create pull requests. -# This setting can be found in a repository's settings under Actions > General > Workflow permissions. -# https://github.com/peter-evans/create-pull-request#workflow-permissions -permissions: - contents: write - pull-requests: write +permissions: {} on: # Note: Workflow must run at least once to appear in workflows list. @@ -26,24 +21,30 @@ jobs: name: Check for GraphBLAS update if: github.repository == 'GraphBLAS/python-suitesparse-graphblas' || github.repository == 'alugowski/python-suitesparse-graphblas' runs-on: ubuntu-latest + # In addition to these permissions, must explicitly allow GitHub Actions to create pull requests. + # This setting can be found in a repository's settings under Actions > General > Workflow permissions. + # https://github.com/peter-evans/create-pull-request#workflow-permissions + permissions: + contents: write + pull-requests: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.11" - name: Get latest GraphBLAS version run: | python latest_suitesparse_graphblas_version.py > GB_VERSION.txt - echo "GB_VERSION=$(cat GB_VERSION.txt)" >> $GITHUB_ENV + echo "GB_VERSION=$(cat GB_VERSION.txt)" >> "$GITHUB_ENV" shell: bash - name: Create Pull Request - uses: peter-evans/create-pull-request@v8 + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 with: # See documentation: https://github.com/peter-evans/create-pull-request # Action behavior: https://github.com/peter-evans/create-pull-request#action-behaviour diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index a35412b..07caebe 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -21,20 +21,21 @@ on: pull_request: +# Cancel superseded runs of a PR only. Keying on `github.ref` would also let a +# push to main cancel a running release or manual (workflow_dispatch) upload. concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: true permissions: - # For PyPI Trusted Publisher - id-token: write + contents: read jobs: build_sdist: name: Build SDist runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false @@ -45,7 +46,7 @@ jobs: - name: Check metadata run: pipx run twine check dist/* - - uses: actions/upload-artifact@v7 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: sdist path: dist/*.tar.gz @@ -94,7 +95,7 @@ jobs: cibw_archs: "arm64" steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false @@ -103,20 +104,20 @@ jobs: if: github.event_name == 'push' || github.event_name == 'pull_request' # Ask suitesparse.sh to compile in the fastest way possible and provide a GB version to build run: | - echo "SUITESPARSE_FASTEST_BUILD=1" >> $GITHUB_ENV + echo "SUITESPARSE_FASTEST_BUILD=1" >> "$GITHUB_ENV" shell: bash - name: Setup GraphBLAS version from GB_VERSION.txt # Use GraphBLAS version specified in GB_VERSION.txt unless specified in a git tag (next workflow step). # Git tag method required for uploads to PyPI. if: github.event_name != 'release' && github.event.inputs.upload_dest != 'PyPI' - run: echo "GB_VERSION_REF=refs/tags/$(cat GB_VERSION.txt).0" >> $GITHUB_ENV + run: echo "GB_VERSION_REF=refs/tags/$(cat GB_VERSION.txt).0" >> "$GITHUB_ENV" shell: bash - name: Setup GraphBLAS version from git tag if: ${{ startsWith(github.ref, 'refs/tags/') }} # If this is a tagged ref, like a release, then use the tag for the graphblas version - run: echo "GB_VERSION_REF=${{ github.ref }}" >> $GITHUB_ENV + run: echo "GB_VERSION_REF=$GITHUB_REF" >> "$GITHUB_ENV" shell: bash - name: Install tools (macOS) @@ -129,21 +130,19 @@ jobs: run: | brew fetch --retry coreutils && brew install coreutils brew fetch --retry libomp && brew reinstall libomp - echo MACOSX_DEPLOYMENT_TARGET=$(otool -l $(brew --prefix libomp)/lib/libomp.dylib | grep minos | awk '{print $2}') >> $GITHUB_ENV + echo "MACOSX_DEPLOYMENT_TARGET=$(otool -l "$(brew --prefix libomp)/lib/libomp.dylib" | grep minos | awk '{print $2}')" >> "$GITHUB_ENV" - - uses: pypa/cibuildwheel@v3.4 + - uses: pypa/cibuildwheel@e090b81e30c4d855ea63bf4b6e59204c09a101ae # v4.2.1 with: output-dir: wheelhouse env: # very verbose CIBW_BUILD_VERBOSITY: 3 - # cibuildwheel v3: PyPy no longer built by default; enable explicitly. - # Free-threaded wheels: cibuildwheel only gates cp313t behind - # "cpython-freethreading" (see its selector.py), so cp314t is already - # built and tested here by default. cp313t stays off because cffi - # publishes no cp313t wheels, so enabling it would build cffi from - # source in every wheel job. + # PyPy is not built by default (since cibuildwheel v3); enable explicitly. + # Free-threaded wheels (cp314t, cp315t) are built and tested by default. + # cibuildwheel v4 dropped cp313t (and its "cpython-freethreading" + # enable option) entirely. CIBW_ENABLE: "pypy" # Build SuiteSparse @@ -171,60 +170,67 @@ jobs: # Use delvewheel on Windows. # This copies graphblas.dll into the wheel. "repair" in cibuildwheel parlance includes copying any shared # libraries from the build host into the wheel to make the wheel self-contained. - # Cibuildwheel includes tools for this for Linux and macOS, and they recommend delvewheel for Windows. - # Note: Currently using a workaround: --no-mangle instead of stripping graphblas.dll + # cibuildwheel v4 installs delvewheel and uses it by default; this overrides its command so it can + # find graphblas.dll and to apply a workaround: --no-mangle instead of stripping graphblas.dll # see https://github.com/adang1345/delvewheel/issues/33 - CIBW_BEFORE_BUILD_WINDOWS: "pip install delvewheel" CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair --add-path \"C:\\GraphBLAS\\bin\" --no-mangle \"libgomp-1.dll;libgcc_s_seh-1.dll\" -w {dest_dir} {wheel}" # make cibuildwheel install test dependencies from pyproject.toml CIBW_TEST_EXTRAS: "test" - # run tests - CIBW_TEST_COMMAND: "pytest --pyargs suitesparse_graphblas -s -k test_print_jit_config && pytest -v --pyargs suitesparse_graphblas" + # run tests. They run from a temporary directory, so pyproject.toml's pytest + # config doesn't apply; -W error restores its `filterwarnings = ["error"]`. + CIBW_TEST_COMMAND: "pytest --pyargs suitesparse_graphblas -s -k test_print_jit_config && pytest -v -W error --pyargs suitesparse_graphblas" - CIBW_TEST_SKIP: ${{ matrix.cibw_test_skip }} - - - uses: actions/upload-artifact@v7 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: wheels-${{ matrix.os }}-${{ matrix.cibw_archs }}${{ matrix.arch_note}} path: wheelhouse/*.whl if-no-files-found: error - upload_all: + # Uploads use PyPI Trusted Publishing (no API token). Each index trusts this workflow + # (wheels.yml) only when it runs in the GitHub environment registered with it. + upload_pypi: name: Upload to PyPI needs: [build_wheels, build_sdist] runs-on: ubuntu-latest - # only upload releases to PyPI - if: github.repository == 'GraphBLAS/python-suitesparse-graphblas' && ((github.event_name == 'release' && github.event.action == 'published') || (github.event_name == 'workflow_dispatch' && github.event.inputs.upload_dest != 'No Upload')) - + # only upload releases (or an explicit manual request) to PyPI + if: github.repository == 'GraphBLAS/python-suitesparse-graphblas' && ((github.event_name == 'release' && github.event.action == 'published') || (github.event_name == 'workflow_dispatch' && github.event.inputs.upload_dest == 'PyPI')) + environment: + name: pypi + url: https://pypi.org/p/suitesparse-graphblas + permissions: + id-token: write # for Trusted Publishing (and its attestations) steps: - - uses: actions/setup-python@v6 - with: - python-version: "3.x" - - - uses: actions/download-artifact@v8 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: dist merge-multiple: true - # Upload to PyPI - - uses: pypa/gh-action-pypi-publish@v1.13.0 - name: Upload to PyPI - if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.upload_dest == 'PyPI') + - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 with: # PyPI does not allow replacing a file. Without this flag the entire action fails if even a single duplicate exists. skip-existing: true - verbose: true - password: ${{ secrets.PYPI_TOKEN }} - # Upload to Test PyPI - - uses: pypa/gh-action-pypi-publish@v1.13.0 - name: Upload to Test PyPI - if: github.event_name == 'workflow_dispatch' && github.event.inputs.upload_dest == 'Test PyPI' + upload_test_pypi: + name: Upload to Test PyPI + needs: [build_wheels, build_sdist] + runs-on: ubuntu-latest + if: github.repository == 'GraphBLAS/python-suitesparse-graphblas' && github.event_name == 'workflow_dispatch' && github.event.inputs.upload_dest == 'Test PyPI' + environment: + name: test-pypi + url: https://test.pypi.org/p/suitesparse-graphblas + permissions: + id-token: write # for Trusted Publishing (and its attestations) + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: + path: dist + merge-multiple: true + + - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 + with: + repository-url: https://test.pypi.org/legacy/ # PyPI does not allow replacing a file. Without this flag the entire action fails if even a single duplicate exists. skip-existing: true - verbose: true - repository-url: https://test.pypi.org/legacy/ diff --git a/.gitignore b/.gitignore index d079cfe..7d098cf 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,10 @@ wheelhouse # Wheel building stuff GraphBLAS-*/ +# Left behind by create_headers.py +suitesparse_graphblas/GraphBLAS-orig.h +suitesparse_graphblas/GraphBLAS-processed.h + # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 292c81c..2b3decc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,7 +41,7 @@ repos: - id: autoflake args: [--in-place] - repo: https://github.com/pycqa/isort - rev: 9.0.1 + rev: 9.0.2 hooks: - id: isort - repo: https://github.com/asottile/pyupgrade @@ -54,15 +54,15 @@ repos: hooks: - id: black - repo: https://github.com/PyCQA/flake8 - rev: 7.3.0 + rev: 7.4.1 hooks: - id: flake8 args: ["--config=.flake8"] additional_dependencies: &flake8_dependencies # These versions need updated manually - - flake8==7.3.0 + - flake8==7.4.1 - flake8-comprehensions==3.17.0 - - flake8-bugbear==25.11.29 + - flake8-bugbear==26.9.30 # - flake8-simplify==0.22.0 - repo: https://github.com/asottile/yesqa rev: v1.5.0 @@ -88,11 +88,24 @@ repos: - id: python-no-eval - id: python-no-log-warn - id: text-unicode-replacement-char + - repo: https://github.com/rhysd/actionlint + rev: v1.7.12 + hooks: + - id: actionlint + - repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: v1.30.1 + hooks: + - id: zizmor - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.38.0 + rev: 0.38.2 hooks: - id: check-dependabot - id: check-github-workflows + - repo: https://github.com/pysentry/pysentry-pre-commit + rev: v0.5.0 + hooks: + - id: pysentry + args: ["--format=human", "--fail-on=low", "--sources=pypa,osv,pypi"] - repo: meta hooks: - id: check-hooks-apply diff --git a/CLAUDE.md b/AGENTS.md similarity index 60% rename from CLAUDE.md rename to AGENTS.md index ef34ae4..1e93373 100644 --- a/CLAUDE.md +++ b/AGENTS.md @@ -1,16 +1,17 @@ -# CLAUDE.md +# AGENTS.md -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. +This file provides guidance to AI coding agents (e.g., Claude Code) when working with code in this repository. ## What this package is `suitesparse-graphblas` is a low-level Python CFFI binding around the C library [SuiteSparse:GraphBLAS](https://github.com/DrTimothyAldenDavis/GraphBLAS). It exposes the -raw `ffi` and `lib` symbols and, on top of them, a small **functional API** in -`suitesparse_graphblas.matrix`, `suitesparse_graphblas.vector`, and -`suitesparse_graphblas.scalar`. These three modules are the **main entry point for users -of this library directly** — they are module-level functions operating on opaque CFFI -handles, e.g. `A = matrix.new(lib.GrB_BOOL, 3, 3); matrix.set_bool(A, True, 2, 2)`. Higher-level +raw `ffi` and `lib` symbols and, on top of them, a **functional API** in the +`suitesparse_graphblas.api` package (`matrix`, `vector`, `scalar`, plus `binaryop`, `monoid`, +`semiring`, `descriptor`, `iterator`, …). `matrix`, `vector`, and `scalar` (also re-exported as +`suitesparse_graphblas.matrix` etc. for backward compatibility) are the **main entry point for +users of this library directly** — they are module-level functions operating on opaque CFFI +handles, e.g. `A = matrix.matrix_new(lib.GrB_BOOL, 3, 3); matrix.set_bool(A, True, 2, 2)`. Higher-level syntax wrappers ([python-graphblas](https://github.com/python-graphblas/python-graphblas) and [pygraphblas](https://github.com/Graphegon/pygraphblas)) build on top of this same package for users who want a more Pythonic, OO-style interface. @@ -131,25 +132,62 @@ The package re-exports `ffi`/`lib` from the compiled extension and adds only thi - `libget(name)` — fallback that retries a `GrB_*` lookup as `GxB_*` when SuiteSparse moves a symbol between standard and extension namespaces. - `burble` — context manager / global toggle for `GxB_BURBLE` diagnostic output. -- `matrix.py`, `vector.py`, `scalar.py` — **the functional API** of the package, and the - primary user-facing surface for code that uses `suitesparse-graphblas` directly. Each - module follows the same convention: `.new(...)` returns an `ffi.gc`-managed +- `api/matrix.py`, `api/vector.py`, `api/scalar.py` (and the other `api/` modules) — + **the functional API** of the package, and the primary user-facing surface for code that + uses `suitesparse-graphblas` directly. `__init__.py` re-exports `matrix`, `vector`, + `scalar`, and `iterator` at the bottom of the file (to avoid a circular import) so + `suitesparse_graphblas.matrix` keeps working. Each module follows the same convention: + `._new(...)` (e.g. `matrix.matrix_new`) returns an `ffi.gc`-managed cdata handle (`GrB_Matrix*`, `GrB_Vector*`, or `GxB_Scalar*`) and every other function takes that handle as its first argument and routes errors through `check_status`. The design is deliberately functional rather than class-based so the same handles can be passed through higher-level wrappers without object-identity friction. When adding features to this package, this is the layer where new user-facing helpers belong. -- `io/serialize.py`, `io/binary.py` — supporting I/O helpers (compressed serialize / - deserialize, binary format read/write). `matrix.py` and `vector.py` already re-export - `serialize` / `deserialize` from `io/serialize.py` so callers can reach them as - `matrix.serialize(A)` etc. +- `api/io/serialize.py`, `api/io/binary.py` — supporting I/O helpers (compressed serialize / + deserialize, binary format read/write), also importable as `suitesparse_graphblas.io`. + `api/matrix.py` and `api/vector.py` already re-export `serialize` / `deserialize` from + `api/io/serialize.py` so callers can reach them as `matrix.serialize(A)` etc. ### 4. `utils.pyx` and free-threading -The Cython module exists primarily to (a) call `GxB_init` with NumPy's allocators by -casting the cffi function pointer through `uintptr_t` into a real C function pointer, and -(b) wrap NumPy buffers around GraphBLAS-allocated memory with `claim_buffer` / -`unclaim_buffer`, transferring ownership of the underlying allocation to NumPy. +The Cython module exists to (a) call `GxB_init` with NumPy's allocators by casting the cffi +function pointer through `uintptr_t` into a real C function pointer, and (b) move buffers +between GraphBLAS and NumPy without copying. Its module docstring is the reference for (b). + +Memory that changes owner must be freed by the allocator that allocated it. GraphBLAS frees +with the allocator of the buffer's *arena* (arena 0 is NumPy's `PyDataMem_*` allocator for +`memory_manager="numpy"`, libc's for `"c"`; `GxB_arena_init` adds more). NumPy frees with +the array's memory handler (NEP 49), or with libc `free` if it has none. Don't assume any +two of these agree: with NumPy ≥ 2.5, NumPy's allocator is `PyMem_RawMalloc`, which is +mimalloc on free-threaded Python ≥ 3.15 and gains debug hooks under `python -X dev`. So, +per arena, `utils.pyx` asks GraphBLAS for the arena's functions and uses NumPy's default +handler if they are NumPy's, else its own handler that calls them +(`suitesparse_graphblas_arena`): + +- GraphBLAS → NumPy: `claim_buffer` / `claim_buffer_2d` attach that handler. +- NumPy → GraphBLAS: use `give_buffer` (a context manager) around the GraphBLAS call that + takes ownership (`GxB_*_pack_*`, `GxB_*_load` with `GrB_DEFAULT + arena`, ...). It hands + over the array itself only if `can_unclaim_buffer` (owns its data, writeable, and its + handler is the arena's), otherwise a copy in `empty(...)` memory, and it commits only if + GraphBLAS set the pointer to NULL. Never hand GraphBLAS libc `malloc` memory or arrays of + unknown provenance; `api/io/binary.py::binread` shows the pattern. +- `arena` defaults to the *global* data arena. GraphBLAS itself uses the data arena of the + Context engaged on the calling thread, which can't be queried, so code that must be right + under any Context names the arena: `serialize_*` call `GxB_*_serialize_arena`, and + `binread` uses the data arena of the matrix it just created. `GxB_*_unpack_*` and + `GxB_*_export_*` move the object to that arena before handing out its arrays, so the arena + an object was made in says nothing about them. + +`tests/test_memory.py` checks all of this (both managers, a libc arena as an explicit, +global, and Context arena, threads, `binread`) in a subprocess under `PYTHONMALLOC=debug`, +which turns a mismatch into an abort on any Python. Its buffers are over 1024 bytes on +purpose: NumPy caches smaller freed blocks, which hides a wrong free. `conftest.py` also +makes NumPy warn, failing the test, whenever it frees an array that owns its data but has +no memory handler; that catches a missing handler even where the allocators happen to agree. + +When claiming a buffer from `GxB_*_unload`, use the arena in the returned `handling` +(`handling - GrB_DEFAULT`), and never claim one with `handling >= GxB_IS_READONLY`: whoever +loaded it read-only still owns it. The file is marked `freethreading_compatible=True`. The package does nothing special for free-threading itself — correctness depends on SuiteSparse:GraphBLAS being thread-safe, @@ -160,7 +198,14 @@ which it is required to be. - Black, isort, flake8 (config in `.flake8`, line length 100, double quotes), pyupgrade (`--py311-plus`), autoflake, shellcheck — all wired through `pre-commit`. Run `pre-commit run --all-files` before pushing. +- `pre-commit` also lints GitHub Actions workflows (actionlint, zizmor) and scans + dependencies for known vulnerabilities (pysentry). actionlint runs shellcheck on `run:` + scripts only when `shellcheck` is on `PATH` — it is on the CI runner, so quote `$(...)` + and `"$GITHUB_ENV"` even if it passes locally. Don't expand `${{ ... }}` contexts inside + `run:` scripts (zizmor's template-injection audit); pass them through `env:` instead. - `pre-commit` also blocks direct commits to `main`. +- pytest is strict (see `[tool.pytest.ini_options]`): warnings are errors, markers must be + registered, and xfails must fail. - Python ≥ 3.11. NumPy ≥ 2.0 is required at build time (CFFI extension), ≥ 1.24 at runtime. - Generated headers (`suitesparse_graphblas.h`, `suitesparse_graphblas_no_complex.h`, `source.c`) are checked in and **must be regenerated via `create_headers.py`** rather diff --git a/pyproject.toml b/pyproject.toml index 15dee86..81a34e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,7 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: Free Threading :: 2 - Beta", "Intended Audience :: Developers", @@ -100,7 +101,7 @@ dirty_template = "{tag}+{ccount}.g{sha}.dirty" [tool.black] line-length = 100 -target-version = ["py311", "py312", "py313", "py314"] +target-version = ["py311", "py312", "py313", "py314", "py315"] [tool.isort] sections = ["FUTURE", "STDLIB", "THIRDPARTY", "FIRSTPARTY", "LOCALFOLDER"] @@ -132,7 +133,16 @@ exclude_lines = [ [tool.pytest.ini_options] testpaths = ["suitesparse_graphblas"] -addopts = "--doctest-modules" +addopts = [ + "--doctest-modules", + "--strict-config", # Error on unknown or misspelled config options + "--strict-markers", # Error on unregistered markers + "-ra", # Summarize all non-passing tests (skips, xfails, ...) at the end +] +xfail_strict = true +# Turn warnings into errors. On free-threaded builds this also catches +# "The global interpreter lock (GIL) has been enabled to load module ...". +filterwarnings = ["error"] # NOTE: When running in Docker, use -w /tmp and --pyargs to avoid the source # directory shadowing the installed package (compiled extensions aren't in the -# source tree). See CLAUDE.md for the full Docker test commands. +# source tree). See AGENTS.md for the full Docker test commands. diff --git a/setup.py b/setup.py index 860cd3a..c554dd9 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,12 @@ except ImportError: use_cython = False -define_macros = [("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION")] +define_macros = [ + ("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION"), + # utils.pyx needs `PyArrayObject_fields.mem_handler`, added in NumPy 1.22. The NumPy + # version targeted by default depends on the NumPy being built against (1.19 for 2.0). + ("NPY_TARGET_VERSION", "NPY_1_22_API_VERSION"), +] # /d2FH4- flag needed only for early Python 3.8 builds on Windows. # See https://cibuildwheel.readthedocs.io/en/stable/faq/ diff --git a/suitesparse_graphblas/__init__.py b/suitesparse_graphblas/__init__.py index 7d08d23..48eac6c 100644 --- a/suitesparse_graphblas/__init__.py +++ b/suitesparse_graphblas/__init__.py @@ -83,10 +83,12 @@ def initialize(*, blocking=False, memory_manager="numpy"): Whether to call init with GrB_BLOCKING or GrB_NONBLOCKING. Default is False. memory_manager : {'numpy', 'c'}, optional - Choose which malloc/free functions to use. 'numpy' uses numpy's - allocators, which makes it safe to perform zero-copy to and from numpy, - and allows Python to track memory usage via tracemalloc (if enabled). - 'c' uses the default allocators. Default is 'numpy'. + Choose which malloc/free functions GraphBLAS uses. 'numpy' uses NumPy's + allocator, so arrays NumPy allocated can be handed to GraphBLAS without + copying, and Python can track memory usage via tracemalloc (if enabled). + 'c' uses the C library's. Either way, `suitesparse_graphblas.utils` moves + buffers between GraphBLAS and NumPy safely, copying when it must. + Default is 'numpy'. The global variable `suitesparse_graphblas.is_initialized` indicates whether GraphBLAS has been initialized. @@ -96,11 +98,15 @@ def initialize(*, blocking=False, memory_manager="numpy"): blocking = lib.GrB_BLOCKING if blocking else lib.GrB_NONBLOCKING memory_manager = memory_manager.lower() if memory_manager == "numpy": - utils.call_gxb_init(ffi, lib, blocking) + info = utils.call_gxb_init(ffi, lib, blocking) elif memory_manager == "c": - lib.GrB_init(blocking) + info = lib.GrB_init(blocking) else: raise ValueError(f'memory_manager argument must be "numpy" or "c"; got: {memory_manager!r}') + if info != lib.GrB_SUCCESS: + raise _error_code_lookup.get(info, RuntimeError)( + f"Failed to initialize GraphBLAS (info={info})" + ) # See: https://github.com/GraphBLAS/python-suitesparse-graphblas/issues/40 for attr in dir(lib): getattr(lib, attr) diff --git a/suitesparse_graphblas/api/io/binary.py b/suitesparse_graphblas/api/io/binary.py index 4124061..974ecf0 100644 --- a/suitesparse_graphblas/api/io/binary.py +++ b/suitesparse_graphblas/api/io/binary.py @@ -1,35 +1,11 @@ -import sys -from ctypes.util import find_library +from contextlib import ExitStack from pathlib import Path -from cffi import FFI +import numpy as np from suitesparse_graphblas import __version__, check_status, ffi, lib from suitesparse_graphblas.api import matrix - -stdffi = FFI() -stdffi.cdef(""" -void *malloc(size_t size); -""") -if sys.platform == "win32": - stdlib = stdffi.dlopen("ucrtbase") -else: - stdlib = stdffi.dlopen(find_library("c")) - -# When "packing" a matrix the owner of the memory buffer is transfered -# to SuiteSparse, which then becomes responsible for freeing it. cffi -# wisely does not allow you to do this without declaring and calling -# malloc directly. When SuiteSparse moves over to a more formal -# memory manager with the cuda work, this will likely change and have -# to be replaceable with a allocator common to numpy, cuda, and here. -# Maybe PyDataMem_NEW? - - -def readinto_new_buffer(f, typ, size, allocator=stdlib.malloc): - buff = ffi.cast(typ, allocator(size)) - f.readinto(ffi.buffer(buff, size)) - return buff - +from suitesparse_graphblas.utils import empty, give_buffer GRB_HEADER_LEN = 512 NULL = ffi.NULL @@ -90,6 +66,12 @@ def readinto_new_buffer(f, typ, size, allocator=stdlib.malloc): def binwrite(A, filename, comments=None, opener=Path.open): + """Write Matrix ``A`` to ``filename`` in SuiteSparse:GraphBLAS's binary format. + + ``opener`` opens the file for writing (e.g. ``gzip.open`` to compress it). To avoid a + copy, ``A`` is unpacked while it is written and packed again afterwards (even if writing + fails), so other threads must not use ``A`` meanwhile. + """ if isinstance(filename, str): filename = Path(filename) @@ -261,198 +243,45 @@ def binwrite(A, filename, comments=None, opener=Path.open): header_content = header_template.format(**vars) header = f"{header_content: <{GRB_HEADER_LEN}}".encode("ascii") - with opener(filename, "wb") as f: - fwrite = f.write - fwrite(header) - fwrite(buff(impl, sizeof("uint64_t"))) - fwrite(buff(format, sizeof("GxB_Format_Value"))) - fwrite(buff(sparsity_status, sizeof("int32_t"))) - fwrite(buff(sparsity_control, sizeof("int32_t"))) - fwrite(buff(hyper_switch, sizeof("double"))) - fwrite(buff(bitmap_switch, sizeof("double"))) - fwrite(buff(nrows, Isize)) - fwrite(buff(ncols, Isize)) - fwrite(buff(nvec, Isize)) - fwrite(buff(nvals, Isize)) - fwrite(buff(typecode, sizeof("int32_t"))) - fwrite(buff(typesize, sizeof("size_t"))) - fwrite(buff(is_iso, sizeof("bool"))) - - Tsize = typesize[0] - iso = is_iso[0] - - if is_hyper: - fwrite(buff(Ap[0], (nvec[0] + 1) * Isize)) - fwrite(buff(Ah[0], nvec[0] * Isize)) - fwrite(buff(Ai[0], nvals[0] * Isize)) - Axsize = Tsize if iso else nvals[0] * Tsize - elif is_sparse: - fwrite(buff(Ap[0], (nvec[0] + 1) * Isize)) - fwrite(buff(Ai[0], nvals[0] * Isize)) - Axsize = Tsize if iso else nvals[0] * Tsize - elif is_bitmap: - fwrite(buff(Ab[0], nrows[0] * ncols[0] * ffi.sizeof("int8_t"))) - Axsize = Tsize if iso else nrows[0] * ncols[0] * Tsize - else: - Axsize = Tsize if iso else nrows[0] * ncols[0] * Tsize - - fwrite(buff(Ax[0], Axsize)) - - if by_col and is_hyper: - check_status( - A, - lib.GxB_Matrix_pack_HyperCSC( - A[0], - Ap, - Ah, - Ai, - Ax, - Ap_size[0], - Ah_size[0], - Ai_size[0], - Ax_size[0], - is_iso[0], - nvec[0], - is_jumbled[0], - NULL, - ), - ) - - elif by_row and is_hyper: - check_status( - A, - lib.GxB_Matrix_pack_HyperCSR( - A[0], - Ap, - Ah, - Ai, - Ax, - Ap_size[0], - Ah_size[0], - Ai_size[0], - Ax_size[0], - is_iso[0], - nvec[0], - is_jumbled[0], - NULL, - ), - ) - - elif by_col and is_sparse: - check_status( - A, - lib.GxB_Matrix_pack_CSC( - A[0], Ap, Ai, Ax, Ap_size[0], Ai_size[0], Ax_size[0], is_iso[0], is_jumbled[0], NULL - ), - ) - - elif by_row and is_sparse: - check_status( - A, - lib.GxB_Matrix_pack_CSR( - A[0], Ap, Ai, Ax, Ap_size[0], Ai_size[0], Ax_size[0], is_iso[0], is_jumbled[0], NULL - ), - ) - - elif by_col and is_bitmap: - check_status( - A, - lib.GxB_Matrix_pack_BitmapC( - A[0], Ab, Ax, Ab_size[0], Ax_size[0], is_iso[0], nvals[0], NULL - ), - ) - - elif by_row and is_bitmap: - check_status( - A, - lib.GxB_Matrix_pack_BitmapR( - A[0], Ab, Ax, Ab_size[0], Ax_size[0], is_iso[0], nvals[0], NULL - ), - ) - - elif by_col and is_full: - check_status(A, lib.GxB_Matrix_pack_FullC(A[0], Ax, Ax_size[0], is_iso[0], NULL)) - - elif by_row and is_full: - check_status(A, lib.GxB_Matrix_pack_FullR(A[0], Ax, Ax_size[0], is_iso[0], NULL)) - else: - raise TypeError("This should hever happen") - - -def binread(filename, opener=Path.open): - if isinstance(filename, str): - filename = Path(filename) - - with opener(filename, "rb") as f: - fread = f.read - - fread(GRB_HEADER_LEN) - impl = frombuff("uint64_t*", fread(sizeof("uint64_t"))) - - assert impl[0] == lib.GxB_IMPLEMENTATION - - format = frombuff("GxB_Format_Value*", fread(sizeof("GxB_Format_Value"))) - sparsity_status = frombuff("int32_t*", fread(sizeof("int32_t"))) - sparsity_control = frombuff("int32_t*", fread(sizeof("int32_t"))) - hyper_switch = frombuff("double*", fread(sizeof("double"))) - bitmap_switch = frombuff("double*", fread(sizeof("double"))) - nrows = frombuff("GrB_Index*", fread(Isize)) - ncols = frombuff("GrB_Index*", fread(Isize)) - nvec = frombuff("GrB_Index*", fread(Isize)) - nvals = frombuff("GrB_Index*", fread(Isize)) - typecode = frombuff("int32_t*", fread(sizeof("int32_t"))) - typesize = frombuff("size_t*", fread(sizeof("size_t"))) - is_iso = frombuff("bool*", fread(sizeof("bool"))) - is_jumbled = ffi.new("bool*", 0) - - by_row = format[0] == lib.GxB_BY_ROW - by_col = format[0] == lib.GxB_BY_COL - - is_hyper = sparsity_status[0] == lib.GxB_HYPERSPARSE - is_sparse = sparsity_status[0] == lib.GxB_SPARSE - is_bitmap = sparsity_status[0] == lib.GxB_BITMAP - is_full = sparsity_status[0] == lib.GxB_FULL - - atype = _ss_codetypes[typecode[0]] - - Ap = ffinew("GrB_Index**") - Ai = ffinew("GrB_Index**") - Ah = ffinew("GrB_Index**") - Ax = ffinew("void**") - Ab = ffinew("int8_t**") - - Ap_size = ffinew("GrB_Index*") - Ai_size = ffinew("GrB_Index*") - Ah_size = ffinew("GrB_Index*") - Ax_size = ffinew("GrB_Index*") - Ab_size = ffinew("GrB_Index*") - - if is_hyper: - Ap_size[0] = (nvec[0] + 1) * Isize - Ah_size[0] = nvec[0] * Isize - Ai_size[0] = nvals[0] * Isize - Ax_size[0] = nvals[0] * typesize[0] - - Ap[0] = readinto_new_buffer(f, "GrB_Index*", Ap_size[0]) - Ah[0] = readinto_new_buffer(f, "GrB_Index*", Ah_size[0]) - Ai[0] = readinto_new_buffer(f, "GrB_Index*", Ai_size[0]) - elif is_sparse: - Ap_size[0] = (nvec[0] + 1) * Isize - Ai_size[0] = nvals[0] * Isize - Ax_size[0] = nvals[0] * typesize[0] - Ap[0] = readinto_new_buffer(f, "GrB_Index*", Ap_size[0]) - Ai[0] = readinto_new_buffer(f, "GrB_Index*", Ai_size[0]) - elif is_bitmap: - Ab_size[0] = nrows[0] * ncols[0] * ffi.sizeof("int8_t") - Ax_size[0] = nrows[0] * ncols[0] * typesize[0] - Ab[0] = readinto_new_buffer(f, "int8_t*", Ab_size[0]) - elif is_full: - Ax_size[0] = nrows[0] * ncols[0] * typesize[0] - - Ax[0] = readinto_new_buffer(f, "uint8_t*", typesize[0] if is_iso[0] else Ax_size[0]) - - A = matrix.matrix_new(atype, nrows[0], ncols[0]) - + try: + with opener(filename, "wb") as f: + fwrite = f.write + fwrite(header) + fwrite(buff(impl, sizeof("uint64_t"))) + fwrite(buff(format, sizeof("GxB_Format_Value"))) + fwrite(buff(sparsity_status, sizeof("int32_t"))) + fwrite(buff(sparsity_control, sizeof("int32_t"))) + fwrite(buff(hyper_switch, sizeof("double"))) + fwrite(buff(bitmap_switch, sizeof("double"))) + fwrite(buff(nrows, Isize)) + fwrite(buff(ncols, Isize)) + fwrite(buff(nvec, Isize)) + fwrite(buff(nvals, Isize)) + fwrite(buff(typecode, sizeof("int32_t"))) + fwrite(buff(typesize, sizeof("size_t"))) + fwrite(buff(is_iso, sizeof("bool"))) + + Tsize = typesize[0] + iso = is_iso[0] + + if is_hyper: + fwrite(buff(Ap[0], (nvec[0] + 1) * Isize)) + fwrite(buff(Ah[0], nvec[0] * Isize)) + fwrite(buff(Ai[0], nvals[0] * Isize)) + Axsize = Tsize if iso else nvals[0] * Tsize + elif is_sparse: + fwrite(buff(Ap[0], (nvec[0] + 1) * Isize)) + fwrite(buff(Ai[0], nvals[0] * Isize)) + Axsize = Tsize if iso else nvals[0] * Tsize + elif is_bitmap: + fwrite(buff(Ab[0], nrows[0] * ncols[0] * ffi.sizeof("int8_t"))) + Axsize = Tsize if iso else nrows[0] * ncols[0] * Tsize + else: + Axsize = Tsize if iso else nrows[0] * ncols[0] * Tsize + + fwrite(buff(Ax[0], Axsize)) + finally: + # The matrix was unpacked to write its arrays, so give them back even if that failed if by_col and is_hyper: check_status( A, @@ -549,7 +378,101 @@ def binread(filename, opener=Path.open): elif by_row and is_full: check_status(A, lib.GxB_Matrix_pack_FullR(A[0], Ax, Ax_size[0], is_iso[0], NULL)) else: - raise TypeError("Unknown format {format[0]}") + raise TypeError("This should never happen") + + +def binread(filename, opener=Path.open): + """Read a Matrix from ``filename``, written by ``binwrite``. + + ``opener`` opens the file for reading (e.g. ``gzip.open``). + """ + if isinstance(filename, str): + filename = Path(filename) + + with opener(filename, "rb") as f: + fread = f.read + + fread(GRB_HEADER_LEN) + impl = frombuff("uint64_t*", fread(sizeof("uint64_t"))) + + assert impl[0] == lib.GxB_IMPLEMENTATION + + format = frombuff("GxB_Format_Value*", fread(sizeof("GxB_Format_Value"))) + sparsity_status = frombuff("int32_t*", fread(sizeof("int32_t"))) + sparsity_control = frombuff("int32_t*", fread(sizeof("int32_t"))) + hyper_switch = frombuff("double*", fread(sizeof("double"))) + bitmap_switch = frombuff("double*", fread(sizeof("double"))) + nrows = frombuff("GrB_Index*", fread(Isize)) + ncols = frombuff("GrB_Index*", fread(Isize)) + nvec = frombuff("GrB_Index*", fread(Isize)) + nvals = frombuff("GrB_Index*", fread(Isize)) + typecode = frombuff("int32_t*", fread(sizeof("int32_t"))) + typesize = frombuff("size_t*", fread(sizeof("size_t"))) + is_iso = frombuff("bool*", fread(sizeof("bool"))) + + by_row = format[0] == lib.GxB_BY_ROW + by_col = format[0] == lib.GxB_BY_COL + + is_hyper = sparsity_status[0] == lib.GxB_HYPERSPARSE + is_sparse = sparsity_status[0] == lib.GxB_SPARSE + is_bitmap = sparsity_status[0] == lib.GxB_BITMAP + is_full = sparsity_status[0] == lib.GxB_FULL + + atype = _ss_codetypes[typecode[0]] + + if not (by_row or by_col): + raise TypeError(f"Unknown format {format[0]}") + + A = matrix.matrix_new(atype, nrows[0], ncols[0]) + # GraphBLAS will own the buffers we read into, and takes them to be from the data + # arena that new matrices use + arena = ffinew("int32_t*") + check_status(A, lib.GrB_Matrix_get_INT32(A[0], arena, lib.GxB_ARENA_DATA)) + arena = arena[0] + buffers = [] # (array, ctype), in the order the file and `GxB_Matrix_pack_*` have them + + def read(nbytes, ctype): + array = empty(nbytes, np.uint8, arena=arena) + if f.readinto(array) != nbytes: + raise EOFError(f"{filename} is truncated") + buffers.append((array, ctype)) + + if is_hyper: + pack = lib.GxB_Matrix_pack_HyperCSC if by_col else lib.GxB_Matrix_pack_HyperCSR + read((nvec[0] + 1) * Isize, "GrB_Index*") # Ap + read(nvec[0] * Isize, "GrB_Index*") # Ah + read(nvals[0] * Isize, "GrB_Index*") # Ai + nx = nvals[0] + args = [nvec[0], False] # False: not jumbled + elif is_sparse: + pack = lib.GxB_Matrix_pack_CSC if by_col else lib.GxB_Matrix_pack_CSR + read((nvec[0] + 1) * Isize, "GrB_Index*") # Ap + read(nvals[0] * Isize, "GrB_Index*") # Ai + nx = nvals[0] + args = [False] # not jumbled + elif is_bitmap: + pack = lib.GxB_Matrix_pack_BitmapC if by_col else lib.GxB_Matrix_pack_BitmapR + read(nrows[0] * ncols[0], "int8_t*") # Ab + nx = nrows[0] * ncols[0] + args = [nvals[0]] + elif is_full: + pack = lib.GxB_Matrix_pack_FullC if by_col else lib.GxB_Matrix_pack_FullR + nx = nrows[0] * ncols[0] + args = [] + else: + raise TypeError(f"Unknown sparsity status {sparsity_status[0]}") + # An iso matrix stores a single value, and GraphBLAS must be told its true size + read(typesize[0] if is_iso[0] else nx * typesize[0], "void*") # Ax + + with ExitStack() as stack: + # The buffers become GraphBLAS's only if the pack succeeds + given = [ + stack.enter_context(give_buffer(array, ctype, copy=False, arena=arena)) + for array, ctype in buffers + ] + ptrs = [ptr for ptr, _, _ in given] + sizes = [nbytes for _, nbytes, _ in given] + check_status(A, pack(A[0], *ptrs, *sizes, is_iso[0], *args, NULL)) matrix.matrix_set_sparsity_control(A, sparsity_control[0]) matrix.matrix_set_hyper_switch(A, hyper_switch[0]) diff --git a/suitesparse_graphblas/api/io/serialize.py b/suitesparse_graphblas/api/io/serialize.py index bc7c9a5..c6aefde 100644 --- a/suitesparse_graphblas/api/io/serialize.py +++ b/suitesparse_graphblas/api/io/serialize.py @@ -1,6 +1,7 @@ import numpy as np from suitesparse_graphblas import check_status, ffi, lib +from suitesparse_graphblas.api.global_options import global_get_int32 from suitesparse_graphblas.utils import claim_buffer @@ -56,10 +57,12 @@ def serialize_matrix(A, compression=lib.GxB_COMPRESSION_DEFAULT, level=None, *, desc = get_serialize_desc(compression, level, nthreads) data_ptr = ffi.new("void**") size_ptr = ffi.new("GrB_Index*") - check_status( - A, lib.GxB_Matrix_serialize(data_ptr, size_ptr, A[0], ffi.NULL if desc is None else desc) - ) - return claim_buffer(ffi, data_ptr[0], size_ptr[0], np.dtype(np.uint8)) + # Name the arena for the result, which is otherwise that of the engaged Context (if + # any), so that `claim_buffer` knows which allocator must free it + arena = global_get_int32(lib.GxB_ARENA_DATA) + desc = ffi.NULL if desc is None else desc + check_status(A, lib.GxB_Matrix_serialize_arena(data_ptr, size_ptr, A[0], arena, desc)) + return claim_buffer(ffi, data_ptr[0], size_ptr[0], np.dtype(np.uint8), arena) def serialize_vector(v, compression=lib.GxB_COMPRESSION_DEFAULT, level=None, *, nthreads=None): @@ -79,10 +82,10 @@ def serialize_vector(v, compression=lib.GxB_COMPRESSION_DEFAULT, level=None, *, desc = get_serialize_desc(compression, level, nthreads) data_ptr = ffi.new("void**") size_ptr = ffi.new("GrB_Index*") - check_status( - v, lib.GxB_Vector_serialize(data_ptr, size_ptr, v[0], ffi.NULL if desc is None else desc) - ) - return claim_buffer(ffi, data_ptr[0], size_ptr[0], np.dtype(np.uint8)) + arena = global_get_int32(lib.GxB_ARENA_DATA) # see `serialize_matrix` + desc = ffi.NULL if desc is None else desc + check_status(v, lib.GxB_Vector_serialize_arena(data_ptr, size_ptr, v[0], arena, desc)) + return claim_buffer(ffi, data_ptr[0], size_ptr[0], np.dtype(np.uint8), arena) def deserialize_matrix(data, *, free=True, nthreads=None): diff --git a/suitesparse_graphblas/conftest.py b/suitesparse_graphblas/conftest.py index b6c5ead..4e25e1d 100644 --- a/suitesparse_graphblas/conftest.py +++ b/suitesparse_graphblas/conftest.py @@ -2,7 +2,28 @@ from suitesparse_graphblas import initialize +try: + from numpy._core._multiarray_umath import _set_numpy_warn_if_no_mem_policy +except ImportError: # pragma: no cover + try: # NumPy < 2 + from numpy.core._multiarray_umath import _set_numpy_warn_if_no_mem_policy + except ImportError: # private, so it may go away + _set_numpy_warn_if_no_mem_policy = None + @pytest.fixture(scope="session", autouse=True) def intialize_suitesparse_graphblas(): initialize() + + +@pytest.fixture(scope="session", autouse=True) +def warn_if_no_memory_handler(): + # NumPy frees the data of an array that owns it but has no memory handler with libc + # `free`, which is only right by luck (see utils.pyx). Make NumPy warn when it does; + # the warning fails the test (see `filterwarnings` in pyproject.toml). + if _set_numpy_warn_if_no_mem_policy is None: # pragma: no cover + yield + return + old = _set_numpy_warn_if_no_mem_policy(True) + yield + _set_numpy_warn_if_no_mem_policy(old) diff --git a/suitesparse_graphblas/tests/test_doctest.py b/suitesparse_graphblas/tests/test_doctest.py index 8b2b456..8718558 100644 --- a/suitesparse_graphblas/tests/test_doctest.py +++ b/suitesparse_graphblas/tests/test_doctest.py @@ -1,41 +1,23 @@ -def test_run_doctests(): - import doctest +import doctest +import importlib +import pkgutil + +from suitesparse_graphblas import api, utils - from suitesparse_graphblas.api import ( - binaryop, - container, - context, - descriptor, - global_options, - grb_type, - indexbinaryop, - indexunaryop, - iterator, - matrix, - monoid, - scalar, - selectop, - semiring, - unaryop, - vector, - ) - for mod in ( - matrix, - vector, - scalar, - iterator, - context, - global_options, - grb_type, - unaryop, - binaryop, - indexunaryop, - indexbinaryop, - monoid, - semiring, - descriptor, - selectop, - container, - ): - doctest.testmod(mod, optionflags=doctest.ELLIPSIS, raise_on_error=True) +def test_run_doctests(): + # Every module of the functional API, found rather than listed so none is missed: the + # wheel builds' test command does not collect doctests, so they run only through here + attempted = 0 + names = sorted(info.name for info in pkgutil.walk_packages(api.__path__, f"{api.__name__}.")) + for name in names: + mod = importlib.import_module(name) + _, tried = doctest.testmod(mod, optionflags=doctest.ELLIPSIS, raise_on_error=True) + attempted += tried + assert attempted > 0 + + # `testmod` doesn't find these: Cython functions, or wrapped by `contextmanager` + runner = doctest.DebugRunner(optionflags=doctest.ELLIPSIS) # raises on failure + for test in doctest.DocTestFinder().find(utils.give_buffer, "give_buffer", globs={}): + runner.run(test) + assert runner.tries > 0 diff --git a/suitesparse_graphblas/tests/test_io.py b/suitesparse_graphblas/tests/test_io.py index a06afb9..2819a2c 100644 --- a/suitesparse_graphblas/tests/test_io.py +++ b/suitesparse_graphblas/tests/test_io.py @@ -198,3 +198,12 @@ def test_matrix_binfile_read_write(tmp_path): ) assert is_eq[0] + + +def test_binwrite_failure_keeps_matrix(tmp_path): + # `binwrite` unpacks the matrix to write its arrays, and must put them back regardless + A = matrix.matrix_new(lib.GrB_INT64, 2, 2) + check_status(A, lib.GrB_Matrix_setElement_INT64(A[0], 1, 0, 0)) + with pytest.raises(OSError): + binary.binwrite(A, tmp_path / "missing_directory" / "matrix.binfile") + assert matrix.matrix_nvals(A) == 1 diff --git a/suitesparse_graphblas/tests/test_memory.py b/suitesparse_graphblas/tests/test_memory.py new file mode 100644 index 0000000..ed70352 --- /dev/null +++ b/suitesparse_graphblas/tests/test_memory.py @@ -0,0 +1,361 @@ +import os +import subprocess +import sys +import textwrap + +import numpy as np +import pytest + +from suitesparse_graphblas import check_status, exceptions, ffi, lib, matrix, vector +from suitesparse_graphblas.utils import ( + can_unclaim_buffer, + claim_buffer, + claim_buffer_2d, + empty, + give_buffer, + unclaim_buffer, +) + +# Move memory between GraphBLAS and NumPy every supported way, then free all of it. +# `binary` is not tested on Windows; see test_io.py. +SCRIPT = textwrap.dedent(""" + import gc + import sys + import threading + from pathlib import Path + + import numpy as np + + import suitesparse_graphblas as ssgb + from suitesparse_graphblas import check_status, ffi, lib, matrix, vector + from suitesparse_graphblas.api import context + from suitesparse_graphblas.utils import can_unclaim_buffer, claim_buffer, empty, give_buffer + + memory_manager, tmpdir = sys.argv[1], Path(sys.argv[2]) + ssgb.initialize(memory_manager=memory_manager) + # NumPy warns (here, raises) when it frees an array without a memory handler (see the + # subprocess environment), but from deallocation, where it can only report it + unraisable = [] + sys.unraisablehook = unraisable.append + int64 = np.dtype(np.int64) + # NumPy keeps freed blocks under 1024 bytes for reuse, which would hide a wrong free + N = 1000 + + + def load(array, **kwargs): + # NumPy -> GraphBLAS, the way to use `give_buffer` + v = vector.vector_new(lib.GrB_INT64) + with give_buffer(array, **kwargs) as (X, nbytes, arena): + info = lib.GxB_Vector_load( + v[0], X, lib.GrB_INT64, array.size, nbytes, lib.GrB_DEFAULT + arena, ffi.NULL + ) + check_status(v, info) + assert vector.vector_nvals(v) == array.size + return v + + + def unload(v): + # GraphBLAS -> NumPy, the way to use `claim_buffer` + X = ffi.new("void**") + n = ffi.new("uint64_t*") + handling = ffi.new("int*") + info = lib.GxB_Vector_unload( + v[0], X, ffi.new("GrB_Type*"), n, ffi.new("uint64_t*"), handling, ffi.NULL + ) + check_status(v, info) + assert handling[0] < lib.GxB_IS_READONLY # else the data is not GraphBLAS's to give + return claim_buffer(ffi, X[0], n[0], int64, arena=handling[0] - lib.GrB_DEFAULT) + + + def roundtrip(**kwargs): + # NumPy -> GraphBLAS -> NumPy: without a copy (if "numpy"), with one, and from `empty` + ones = empty(N, int64, **kwargs) + ones[:] = 1 + for array in [np.arange(N), np.arange(2 * N)[::2], ones]: + total = array.sum() + assert unload(load(array, **kwargs)).sum() == total + v = load(np.arange(N), **kwargs) + data = vector.serialize(v, lib.GxB_COMPRESSION_NONE) + assert data.nbytes > N * 8 + assert vector.vector_nvals(vector.deserialize(data)) == N + + + def main(): + array = np.arange(N) + assert can_unclaim_buffer(array) == (memory_manager == "numpy") + load(array) + assert array.flags.owndata == (memory_manager != "numpy") # was it copied? + roundtrip() + + # GraphBLAS has nothing to give for an empty array, so NumPy allocates it + assert claim_buffer(ffi, ffi.NULL, 0, int64).shape == (0,) + + # The same array as two arguments: GraphBLAS must get two buffers + array = np.arange(N, dtype=np.uint64) + A = matrix.matrix_new(lib.GrB_UINT64, 1, N) + with ( + give_buffer(np.array([0, N], np.uint64), "GrB_Index*") as (Ap, Ap_size, _), + give_buffer(array, "GrB_Index*") as (Aj, Aj_size, _), + give_buffer(array) as (Ax, Ax_size, _), + ): + info = lib.GxB_Matrix_pack_CSR( + A[0], Ap, Aj, Ax, Ap_size, Aj_size, Ax_size, False, False, ffi.NULL + ) + check_status(A, info) + assert matrix.matrix_nvals(A) == N + + # Many threads at once (free-threaded Python runs them in parallel) + def roundtrips(): + for _ in range(20): + roundtrip() + + threads = [threading.Thread(target=roundtrips) for _ in range(8)] + for t in threads: + t.start() + for t in threads: + t.join() + + if sys.platform == "win32": + return + from suitesparse_graphblas.io import binary + + # `binread` hands its buffers to GraphBLAS; an iso matrix stores a single value, and + # making it non-iso later must not write past it + for sparsity in [lib.GxB_HYPERSPARSE, lib.GxB_SPARSE, lib.GxB_BITMAP, lib.GxB_FULL]: + A = matrix.matrix_new(lib.GrB_INT64, 30, 30) + info = lib.GrB_Matrix_assign_INT64( + A[0], ffi.NULL, ffi.NULL, 7, lib.GrB_ALL, 30, lib.GrB_ALL, 30, ffi.NULL + ) + check_status(A, info) + matrix.matrix_set_sparsity_control(A, sparsity) + binary.binwrite(A, tmpdir / "iso.binfile") + B = binary.binread(tmpdir / "iso.binfile") + matrix.set_int64(B, 99, 0, 0) + check_status(B, lib.GrB_Matrix_wait(B[0], lib.GrB_MATERIALIZE)) + assert matrix.matrix_nvals(B) == 900 + + # Another arena, using libc's allocator. The last one, since GraphBLAS means to give + # CUDA an arena of its own (the first 10.4.0 betas reserved arena 1). It needs a realloc: + # without one, GraphBLAS (10.4.0 to 10.5.1) frees the arena's memory with arena 0's + # free when it grows a buffer. + k = lib.GxB_NARENAS - 1 + import cffi + + std = cffi.FFI() + std.cdef("void *malloc(size_t); void *calloc(size_t, size_t);") + std.cdef("void *realloc(void *, size_t); void free(void *);") + libc = std.dlopen(None) + funcs = [ + ffi.cast(sig, int(std.cast("uintptr_t", std.addressof(libc, name)))) + for name, sig in [ + ("malloc", "void *(*)(size_t)"), + ("calloc", "void *(*)(size_t, size_t)"), + ("realloc", "void *(*)(void *, size_t)"), + ("free", "void (*)(void *)"), + ] + ] + assert lib.GxB_arena_init(k, *funcs) == lib.GrB_SUCCESS + array = empty(N, int64, arena=k) + assert can_unclaim_buffer(array, arena=k) and not can_unclaim_buffer(array, arena=0) + assert not can_unclaim_buffer(np.arange(N), arena=k) + roundtrip(arena=k) + + def others(): + # GraphBLAS -> NumPy and NumPy -> GraphBLAS in functions that have no `arena` + roundtrip() + binary.binwrite(A, tmpdir / "arena.binfile") + assert matrix.matrix_nvals(binary.binread(tmpdir / "arena.binfile")) == 900 + + # ...as the global data arena, which is then the default + assert lib.GrB_Global_set_INT32(lib.GrB_GLOBAL, k, lib.GxB_ARENA_DATA) == lib.GrB_SUCCESS + assert can_unclaim_buffer(empty(N)) and not can_unclaim_buffer(np.arange(N)) + others() + assert lib.GrB_Global_set_INT32(lib.GrB_GLOBAL, 0, lib.GxB_ARENA_DATA) == lib.GrB_SUCCESS + + # ...and as the data arena of an engaged Context, which GraphBLAS doesn't reveal + ctx = context.context_new() + check_status(ctx, lib.GxB_Context_set_INT(ctx[0], k, lib.GxB_ARENA_DATA)) + context.context_engage(ctx) + others() + context.context_disengage(ctx) + + + main() + gc.collect() + assert not unraisable, [str(u.exc_value) for u in unraisable] + """) + + +@pytest.mark.parametrize("memory_manager", ["numpy", "c"]) +def test_allocators_match(memory_manager, tmp_path): + """Memory must be freed by the allocator that allocated it, whoever ends up owning it. + + A mismatch is silent wherever both allocators happen to be libc's, which used to be + everywhere. They differ on free-threaded Python 3.15, and Python's debug hooks make + them differ on any Python (given NumPy >=2.5), so this test doesn't depend on which + Python runs it. It needs its own process: a mismatch aborts the interpreter, and + GraphBLAS can only be initialized once. + """ + proc = subprocess.run( + [ + sys.executable, + "-W", + "error::RuntimeWarning", + "-c", + SCRIPT, + memory_manager, + str(tmp_path), + ], + env={**os.environ, "PYTHONMALLOC": "debug", "NUMPY_WARN_IF_NO_MEM_POLICY": "1"}, + capture_output=True, + text=True, + timeout=300, + ) + assert proc.returncode == 0, proc.stderr + + +def load(array, **kwargs): + v = vector.vector_new(lib.GrB_INT64) + with give_buffer(array, **kwargs) as (X, nbytes, arena): + info = lib.GxB_Vector_load( + v[0], X, lib.GrB_INT64, array.size, nbytes, lib.GrB_DEFAULT + arena, ffi.NULL + ) + check_status(v, info) + return v + + +def test_claim_buffer(): + int64 = np.dtype(np.int64) + assert claim_buffer(ffi, ffi.NULL, 0, int64).shape == (0,) + assert claim_buffer_2d(ffi, ffi.NULL, 0, 0, 5, int64, True).shape == (0, 5) + with pytest.raises(ValueError, match="NULL pointer"): + claim_buffer(ffi, ffi.NULL, 3, int64) + with pytest.raises(ValueError, match="NULL pointer"): + claim_buffer_2d(ffi, ffi.NULL, 6, 2, 3, int64, True) + + +def test_can_unclaim_buffer(): + array = np.arange(10) + assert can_unclaim_buffer(array) + assert can_unclaim_buffer(matrix.serialize(matrix.matrix_new(lib.GrB_BOOL, 2, 2))) + assert can_unclaim_buffer(empty(3)) + # GraphBLAS can only take ownership from an owner, and only if it may change the data + assert not can_unclaim_buffer(array[:5]) + assert not can_unclaim_buffer(np.frombuffer(b"1234", np.uint8)) + readonly = np.arange(10) + readonly.flags.writeable = False + assert not can_unclaim_buffer(readonly) + assert not can_unclaim_buffer(np.array([None, 1])) + assert not can_unclaim_buffer([1, 2, 3]) + for arena in [8, -1, 2**40]: + with pytest.raises(ValueError, match="range"): + can_unclaim_buffer(array, arena=arena) + with pytest.raises(ValueError, match="arena 5 has not been initialized"): + can_unclaim_buffer(array, arena=5) + + +def test_unclaim_buffer(): + def handover(array): + # The low-level way: hand over (after checking `can_unclaim_buffer`), then unclaim + v = vector.vector_new(lib.GrB_INT64) + X = ffi.new("void**", ffi.cast("void*", array.ctypes.data)) + info = lib.GxB_Vector_load( + v[0], X, lib.GrB_INT64, array.size, array.nbytes, lib.GrB_DEFAULT, ffi.NULL + ) + check_status(v, info) + return v + + array = np.arange(10) + assert can_unclaim_buffer(array) + v = handover(array) + unclaim_buffer(array) + assert not array.flags.owndata + assert not array.flags.writeable + assert vector.vector_nvals(v) == 10 + + # GraphBLAS should not have been given this, but it now has it, so NumPy must let go + array = np.arange(10) + array.flags.writeable = False + assert not can_unclaim_buffer(array) + v = handover(array) + with pytest.raises(ValueError, match="it is read-only. Check `can_unclaim_buffer`"): + unclaim_buffer(array) + assert not array.flags.owndata + assert vector.vector_nvals(v) == 10 + + for array in [np.arange(10)[::2], np.frombuffer(bytearray(8), np.uint8)]: + with pytest.raises(ValueError, match="does not own its data"): + unclaim_buffer(array) + assert array.flags.writeable # unchanged + + +def test_empty(): + a = empty((2, 3), np.int32) + assert a.shape == (2, 3) + assert a.dtype == np.int32 + assert a.flags.c_contiguous + assert a.flags.owndata + a[...] = 1 + assert a.sum() == 6 + f = empty([2, 3], order="F") + assert f.dtype == np.float64 + assert f.flags.f_contiguous + assert not f.flags.c_contiguous + assert empty(4).shape == empty(np.int64(4)).shape == (4,) + assert empty(()).shape == () + assert empty((0, 5)).size == 0 + with pytest.raises(ValueError, match="negative"): + empty(-1) + with pytest.raises(TypeError, match="sequence of ints"): + empty(1.5) + with pytest.raises(TypeError, match="Python objects"): + empty(3, object) + with pytest.raises(ValueError, match="order"): + empty(3, order="A") + + +def test_give_buffer(): + # Zero-copy: the array no longer owns the data GraphBLAS took + array = np.arange(5) + v = load(array) + assert not array.flags.owndata + assert not array.flags.writeable + x = ffi.new("int64_t*") + check_status(v, lib.GrB_Vector_extractElement_INT64(x, v[0], 4)) + assert x[0] == 4 + + # A copy, so the array is unchanged + for array, kwargs in [(np.arange(10)[::2], {}), (np.arange(5), {"copy": True})]: + v = load(array, **kwargs) + assert array.flags.writeable + check_status(v, lib.GrB_Vector_extractElement_INT64(x, v[0], 4)) + assert x[0] == array[4] + with pytest.raises(ValueError, match="without copying: it does not own its data"): + load(np.arange(10)[::2], copy=False) + + # GraphBLAS didn't take ownership, so the array keeps it + array = np.arange(5) + v = vector.vector_new(lib.GrB_INT64) + with pytest.raises(exceptions.InvalidValue): + with give_buffer(array) as (X, nbytes, arena): + assert not array.flags.owndata # so that it can't be given twice + info = lib.GxB_Vector_load( # claims more entries than fit + v[0], X, lib.GrB_INT64, array.size + 1, nbytes, lib.GrB_DEFAULT, ffi.NULL + ) + check_status(v, info) + assert array.flags.owndata + assert array.flags.writeable + + # The same array twice: the second is a copy, or an error if it may not be + array = np.arange(5) + with give_buffer(array) as (X, _, _), give_buffer(array) as (Y, _, _): + assert X[0] != Y[0] + with pytest.raises(ValueError, match="already being given"): + with give_buffer(array), give_buffer(array, copy=False): + pass + assert array.flags.owndata + + with pytest.raises(TypeError, match="Python objects"): + load(np.array([None, 1])) + with pytest.raises(TypeError, match="copy"): + load(np.arange(5), copy="yes") diff --git a/suitesparse_graphblas/utils.pxd b/suitesparse_graphblas/utils.pxd index 69a279e..05da4ff 100644 --- a/suitesparse_graphblas/utils.pxd +++ b/suitesparse_graphblas/utils.pxd @@ -1,4 +1,5 @@ -from libc.stdint cimport uint64_t +from cpython.ref cimport PyObject +from libc.stdint cimport int32_t, uint8_t from numpy cimport dtype as dtype_t from numpy cimport ndarray, npy_intp @@ -9,6 +10,22 @@ cdef extern from "numpy/arrayobject.h" nogil: void *PyDataMem_NEW_ZEROED(size_t nmemb, size_t size) void *PyDataMem_RENEW(void *ptr, size_t size) void PyDataMem_FREE(void *ptr) + # The memory handler (a PyCapsule) whose allocator matches the functions above + PyObject *PyDataMem_DefaultHandler + # NumPy memory handlers (NEP 49); like `mem_handler`, these need + # NPY_TARGET_VERSION >= NPY_1_22_API_VERSION (see setup.py) + ctypedef struct PyDataMemAllocator: + void *ctx + void *(*malloc)(void *ctx, size_t size) noexcept nogil + void *(*calloc)(void *ctx, size_t nelem, size_t elsize) noexcept nogil + void *(*realloc)(void *ctx, void *ptr, size_t new_size) noexcept nogil + void (*free)(void *ctx, void *ptr, size_t size) noexcept nogil + ctypedef struct PyDataMem_Handler: + char name[127] + uint8_t version + PyDataMemAllocator allocator + ctypedef struct PyArrayObject_fields: + PyObject *mem_handler # These are available in newer Cython versions void PyArray_ENABLEFLAGS(ndarray array, int flags) void PyArray_CLEARFLAGS(ndarray array, int flags) @@ -21,20 +38,32 @@ ctypedef enum GrB_Mode: GrB_NONBLOCKING GrB_BLOCKING -ctypedef uint64_t (*GxB_init)( +# GrB_Info is a C enum (int); errors are negative +ctypedef int (*GxB_init)( GrB_Mode, void *(*user_malloc_function)(size_t), void *(*user_calloc_function)(size_t, size_t), void *(*user_realloc_function)(void *, size_t), void (*user_free_function)(void *), ) +# GrB_Global_get_INT32 +ctypedef int (*global_get_int32_t)(void *, int32_t *, int) noexcept nogil cpdef int call_gxb_init(object ffi, object lib, int mode) -cpdef ndarray claim_buffer(object ffi, object cdata, size_t size, dtype_t dtype) +cpdef ndarray claim_buffer(object ffi, object cdata, size_t size, dtype_t dtype, object arena=*) cpdef ndarray claim_buffer_2d( - object ffi, object cdata, size_t cdata_size, size_t nrows, size_t ncols, dtype_t dtype, bint is_c_order + object ffi, + object cdata, + size_t cdata_size, + size_t nrows, + size_t ncols, + dtype_t dtype, + bint is_c_order, + object arena=*, ) -cpdef unclaim_buffer(ndarray array) +cpdef bint can_unclaim_buffer(object array, object arena=*) except -1 + +cpdef unclaim_buffer(ndarray array, object arena=*) diff --git a/suitesparse_graphblas/utils.pyx b/suitesparse_graphblas/utils.pyx index 5c9ae99..7124cdc 100644 --- a/suitesparse_graphblas/utils.pyx +++ b/suitesparse_graphblas/utils.pyx @@ -4,15 +4,226 @@ # libraries are required to be thread-safe, so things should "just work". # Of course, users writing multithreaded code can find many creative ways # to fail, but python-suitesparse-graphblas shouldn't crash. +"""Move memory between GraphBLAS and NumPy without copying. + +Memory must be freed by the allocator that allocated it. GraphBLAS frees the memory it +owns with the allocator of the *arena* the memory belongs to: arena 0 is set up by +``suitesparse_graphblas.initialize`` (NumPy's allocator for ``memory_manager="numpy"``, +libc's for ``"c"``), and ``GxB_arena_init`` can add more. NumPy frees the data of an +array with the array's memory handler (NEP 49). So: + +- GraphBLAS -> NumPy: ``claim_buffer`` and ``claim_buffer_2d`` wrap memory GraphBLAS + allocated in an array whose memory handler frees it with the same arena's allocator. +- NumPy -> GraphBLAS: ``give_buffer`` hands the data of an array to a GraphBLAS function + that takes ownership of it, such as ``GxB_Matrix_pack_CSR`` or ``GxB_Vector_load`` + (copying it first if GraphBLAS can't own it). ``can_unclaim_buffer`` and + ``unclaim_buffer`` are the lower-level pieces. +- ``empty`` makes an array that GraphBLAS can always take ownership of. + +Each function takes an optional ``arena``. It defaults to GraphBLAS's global data arena +(``GxB_ARENA_DATA`` of ``GrB_GLOBAL``; 0 unless changed), which is where GraphBLAS +allocates, and where it expects memory it is given to be from, unless the calling thread +has engaged a ``GxB_Context`` with another data arena. GraphBLAS can't be asked which +Context is engaged, so pass ``arena`` in that case. + +Which arena a buffer from GraphBLAS is in: ``GxB_*_unpack_*`` and ``GxB_*_export_*`` first +move the object to the arena above, so their buffers are in it whatever arena the object +was made in; ``GxB_*_unload`` reports each buffer's arena (``handling - GrB_DEFAULT``). +""" +import operator +import threading +from contextlib import contextmanager + import numpy as np + +from suitesparse_graphblas._graphblas import ffi as _ffi +from suitesparse_graphblas._graphblas import lib as _lib + +from cpython.pycapsule cimport PyCapsule_GetPointer, PyCapsule_New +from cpython.pyport cimport PY_SSIZE_T_MAX from cpython.ref cimport Py_INCREF -from libc.stdint cimport uintptr_t +from libc.stdint cimport SIZE_MAX, int32_t, uintptr_t +from libc.stdio cimport snprintf +from libc.string cimport memset from numpy cimport NPY_ARRAY_F_CONTIGUOUS, NPY_ARRAY_OWNDATA, NPY_ARRAY_WRITEABLE from numpy cimport dtype as dtype_t -from numpy cimport import_array, ndarray, npy_intp +from numpy cimport PyArray_CHKFLAGS, PyArray_DATA, import_array, ndarray, npy_intp import_array() + +cdef enum: + NARENAS = 8 # GxB_NARENAS + +if _lib.GxB_NARENAS > NARENAS: # pragma: no cover + raise ImportError(f"Unsupported GxB_NARENAS: {_lib.GxB_NARENAS}") + +ctypedef void *(*malloc_t)(size_t) noexcept nogil +ctypedef void *(*calloc_t)(size_t, size_t) noexcept nogil +ctypedef void *(*realloc_t)(void *, size_t) noexcept nogil +ctypedef void (*free_t)(void *) noexcept nogil + +ctypedef struct Arena: + malloc_t malloc + calloc_t calloc # may be NULL + realloc_t realloc # may be NULL + free_t free + +# The allocator of each GraphBLAS arena, and a NumPy memory handler that uses it +cdef Arena arenas[NARENAS] +cdef PyDataMem_Handler arena_handlers[NARENAS] +# The NumPy memory handler (a PyCapsule) for the memory of each arena, made on first use. +# It is NumPy's default handler if the arena uses NumPy's allocator. +cdef list _handlers = [None] * NARENAS +_handlers_lock = threading.Lock() + + +cdef uintptr_t address(object cdata) except? 0: + # The address in a cffi pointer. Cython isn't compiled against GraphBLAS, so this is how + # it gets at what cffi's `lib` has: cast the result to the C type and use it directly. + return int(_ffi.cast("uintptr_t", cdata)) + + +# `GrB_Global_get_INT32(GrB_GLOBAL, ...)`, called directly rather than through cffi because +# it is on the path of every function below +cdef global_get_int32_t global_get_int32 = address( + _ffi.addressof(_lib, "GrB_Global_get_INT32") +) +cdef void *GrB_GLOBAL = address(_lib.GrB_GLOBAL) +cdef int GxB_ARENA_DATA = _lib.GxB_ARENA_DATA + + +cdef void *arena_malloc(void *ctx, size_t size) noexcept nogil: + return (ctx).malloc(size) + + +cdef void *arena_calloc(void *ctx, size_t nelem, size_t elsize) noexcept nogil: + cdef Arena *arena = ctx + cdef void *ptr + if arena.calloc != NULL: + return arena.calloc(nelem, elsize) + if elsize != 0 and nelem > SIZE_MAX // elsize: + return NULL + ptr = arena.malloc(nelem * elsize) + if ptr != NULL: + memset(ptr, 0, nelem * elsize) + return ptr + + +cdef void *arena_realloc(void *ctx, void *ptr, size_t new_size) noexcept nogil: + cdef Arena *arena = ctx + if arena.realloc == NULL: # optional for GraphBLAS arenas; NumPy raises MemoryError + return NULL + return arena.realloc(ptr, new_size) + + +cdef void arena_free(void *ctx, void *ptr, size_t size) noexcept nogil: + (ctx).free(ptr) + + +cdef int _resolve_arena(object arena) except -1: + cdef int32_t k + cdef int info + if arena is None: + info = global_get_int32(GrB_GLOBAL, &k, GxB_ARENA_DATA) + if info != 0: # GrB_SUCCESS + raise RuntimeError( + f"Unable to get GraphBLAS's data arena (info={info}); is GraphBLAS initialized?" + ) + return k + index = operator.index(arena) + if not 0 <= index < NARENAS: + raise ValueError(f"arena must be in range({NARENAS}); got {arena!r}") + return index + + +cdef object _handler(int k): + # The NumPy memory handler for memory from arena `k` + handler = _handlers[k] + if handler is None: + with _handlers_lock: + handler = _handlers[k] + if handler is None: + handler = _handlers[k] = _new_handler(k) + return handler + + +cdef object _new_handler(int k): + cdef uintptr_t funcs[4] + flag = _ffi.new("int*") + info = _lib.GxB_arena_initialized(flag, k) + if info != _lib.GrB_SUCCESS: + raise RuntimeError( + f"Unable to query GraphBLAS arena {k} (info={info}); is GraphBLAS initialized?" + ) + if not flag[0]: + raise ValueError(f"GraphBLAS arena {k} has not been initialized (see GxB_arena_init)") + func = _ffi.new("void**") + for i, field in enumerate( + [_lib.GxB_ARENA_MALLOC, _lib.GxB_ARENA_CALLOC, _lib.GxB_ARENA_REALLOC, _lib.GxB_ARENA_FREE] + ): + info = _lib.GrB_Global_get_VOID(_lib.GrB_GLOBAL, func, field + k) + if info != _lib.GrB_SUCCESS: + raise RuntimeError(f"Unable to get the allocator of GraphBLAS arena {k} (info={info})") + funcs[i] = address(func[0]) + arenas[k].malloc = funcs[0] + arenas[k].calloc = funcs[1] + arenas[k].realloc = funcs[2] + arenas[k].free = funcs[3] + if funcs[3] == PyDataMem_FREE: + # NumPy's allocator (memory_manager="numpy"): use NumPy's own handler, so these + # arrays are like any other NumPy array (including for `can_unclaim_buffer`). + return PyDataMem_DefaultHandler + snprintf(arena_handlers[k].name, sizeof(arena_handlers[k].name), "suitesparse_graphblas_arena%d", k) + arena_handlers[k].version = 1 + arena_handlers[k].allocator.ctx = &arenas[k] + arena_handlers[k].allocator.malloc = arena_malloc + arena_handlers[k].allocator.calloc = arena_calloc + arena_handlers[k].allocator.realloc = arena_realloc + arena_handlers[k].allocator.free = arena_free + return PyCapsule_New(&arena_handlers[k], "mem_handler", NULL) + + +cdef str _handler_name(object handler): + return (PyCapsule_GetPointer(handler, "mem_handler")).name.decode() + + +cdef inline void own_data(ndarray array, object handler): + # Make `array`, which was wrapped around existing data, own (and eventually free) that + # data with `handler`. Without a handler, NumPy frees with libc `free`, which is wrong + # unless the data is from libc `malloc`. + cdef PyArrayObject_fields *fields = array + if fields.mem_handler == NULL: # else NumPy allocated the data (it was given NULL) + Py_INCREF(handler) + fields.mem_handler = handler + PyArray_ENABLEFLAGS(array, NPY_ARRAY_OWNDATA) + + +cdef str _why_not_ownable(object array, object handler): + # Why GraphBLAS can't take ownership of the data of `array` (to free with `handler`), + # or None if it can. + cdef ndarray arr + cdef PyObject *mem_handler + if not isinstance(array, ndarray): + return f"it is a {type(array).__name__}, not a NumPy array" + arr = array + if not PyArray_CHKFLAGS(arr, NPY_ARRAY_OWNDATA): + return "it does not own its data (e.g., it is a view, or is already being given)" + if not PyArray_CHKFLAGS(arr, NPY_ARRAY_WRITEABLE): + return "it is read-only" + if arr.dtype.hasobject: + return "it holds Python objects" + mem_handler = (arr).mem_handler + if mem_handler == handler: + return None + if mem_handler == NULL: + return "it has no NumPy memory handler, so its data came from an unknown allocator" + return ( + f"its data came from NumPy memory handler {_handler_name(mem_handler)!r}, " + f"but GraphBLAS will free it as {_handler_name(handler)!r}" + ) + + cpdef int call_gxb_init(object ffi, object lib, int mode): # We need to call `GxB_init`, but we didn't compile Cython against GraphBLAS. So, we get it from cffi. # Step 1: ffi.addressof(lib, "GxB_init") @@ -25,27 +236,42 @@ cpdef int call_gxb_init(object ffi, object lib, int mode): # Return type: uintptr_t in Cython. Cast Python int to Cython integer for pointers. # Step 5: (...) # Return: function pointer in Cython! - cdef GxB_init func = int(ffi.cast("uintptr_t", ffi.addressof(lib, "GxB_init"))) return func(mode, PyDataMem_NEW, PyDataMem_NEW_ZEROED, PyDataMem_RENEW, PyDataMem_FREE) -cpdef ndarray claim_buffer(object ffi, object cdata, size_t size, dtype_t dtype): +cpdef ndarray claim_buffer(object ffi, object cdata, size_t size, dtype_t dtype, object arena=None): + """Return a 1-d array that owns ``size`` elements of GraphBLAS-allocated memory at ``cdata``. + + ``arena`` is the GraphBLAS arena that allocated the memory (default: the global data + arena); NumPy will free the memory with that arena's allocator. + """ cdef: npy_intp dims = size uintptr_t ptr = int(ffi.cast("uintptr_t", cdata)) ndarray array + handler = _handler(_resolve_arena(arena)) + if ptr == 0 and size != 0: + raise ValueError(f"Unable to claim {size} elements from a NULL pointer") Py_INCREF(dtype) array = PyArray_NewFromDescr( ndarray, dtype, 1, &dims, NULL, ptr, NPY_ARRAY_WRITEABLE, NULL ) - PyArray_ENABLEFLAGS(array, NPY_ARRAY_OWNDATA) + own_data(array, handler) return array cpdef ndarray claim_buffer_2d( - object ffi, object cdata, size_t cdata_size, size_t nrows, size_t ncols, dtype_t dtype, bint is_c_order + object ffi, + object cdata, + size_t cdata_size, + size_t nrows, + size_t ncols, + dtype_t dtype, + bint is_c_order, + object arena=None, ): + """Like ``claim_buffer``, but return a 2-d ``nrows`` by ``ncols`` array.""" cdef: size_t size = nrows * ncols ndarray array @@ -53,7 +279,10 @@ cpdef ndarray claim_buffer_2d( npy_intp dims[2] int flags = NPY_ARRAY_WRITEABLE if cdata_size == size: + handler = _handler(_resolve_arena(arena)) ptr = int(ffi.cast("uintptr_t", cdata)) + if ptr == 0 and size != 0: + raise ValueError(f"Unable to claim {size} elements from a NULL pointer") dims[0] = nrows dims[1] = ncols if not is_c_order: @@ -62,9 +291,9 @@ cpdef ndarray claim_buffer_2d( array = PyArray_NewFromDescr( ndarray, dtype, 2, dims, NULL, ptr, flags, NULL ) - PyArray_ENABLEFLAGS(array, NPY_ARRAY_OWNDATA) + own_data(array, handler) elif cdata_size > size: # pragma: no cover - array = claim_buffer(ffi, cdata, cdata_size, dtype) + array = claim_buffer(ffi, cdata, cdata_size, dtype, arena) if is_c_order: array = array[:size].reshape((nrows, ncols)) else: @@ -77,5 +306,164 @@ cpdef ndarray claim_buffer_2d( return array -cpdef unclaim_buffer(ndarray array): - PyArray_CLEARFLAGS(array, NPY_ARRAY_OWNDATA | NPY_ARRAY_WRITEABLE) +cpdef bint can_unclaim_buffer(object array, object arena=None) except -1: + """Whether GraphBLAS may take ownership of the data of ``array``. + + True only if ``array`` owns its data, is writeable, does not hold Python objects, and + its data came from the allocator of the GraphBLAS arena (default: the global data + arena) that will free it. So views never qualify, nor do arrays from a custom NumPy + memory handler, nor (with ``memory_manager="c"``) arrays NumPy allocated. Arrays from + ``claim_buffer``, ``empty``, and (with ``memory_manager="numpy"``) NumPy's default + allocator do. Prefer ``give_buffer``, which also copies when this is False. + + This cannot see other references to the data: GraphBLAS may free or reallocate it at + any time after the handover, so no view of ``array`` (nor memoryview or cffi pointer + into it) may outlive the handover. + """ + if not isinstance(array, ndarray) or not PyArray_CHKFLAGS(array, NPY_ARRAY_OWNDATA): + return False + return _why_not_ownable(array, _handler(_resolve_arena(arena))) is None + + +cpdef unclaim_buffer(ndarray array, object arena=None): + """Make ``array`` a read-only view of its data, which GraphBLAS has taken ownership of. + + Call this *after* a GraphBLAS function took ownership, and only for an array that + passed ``can_unclaim_buffer`` beforehand. Prefer ``give_buffer``, which does both. + + Raises ValueError if GraphBLAS can't own the data (see ``can_unclaim_buffer``). Since + GraphBLAS already has it and will free it, ``array`` stops owning the data even then, + so that NumPy does not free it too. + """ + why_not = _why_not_ownable(array, _handler(_resolve_arena(arena))) + if why_not is None: + PyArray_CLEARFLAGS(array, NPY_ARRAY_OWNDATA | NPY_ARRAY_WRITEABLE) + return + PyArray_CLEARFLAGS(array, NPY_ARRAY_OWNDATA) + raise ValueError( + f"GraphBLAS cannot safely own the data of this array: {why_not}. Check " + "`can_unclaim_buffer` before handing data to GraphBLAS, or use `give_buffer`." + ) + + +def empty(shape, dtype=float, order="C", arena=None): + """Return a new, uninitialized array whose data GraphBLAS can take ownership of. + + Like ``numpy.empty``, but allocated by the allocator of the GraphBLAS ``arena`` + (default: the global data arena) whatever the ``memory_manager`` or NumPy memory + handler in use, so ``can_unclaim_buffer`` is True for it. + """ + cdef: + int k = _resolve_arena(arena) + void *ptr + ndarray array + ndarray dims + handler = _handler(k) + dtype = np.dtype(dtype) + if dtype.hasobject: + raise TypeError("GraphBLAS cannot own arrays that hold Python objects") + if order not in {"C", "F"}: + raise ValueError(f"order must be 'C' or 'F'; got {order!r}") + try: + shape = (operator.index(shape),) + except TypeError: + try: + shape = tuple([operator.index(dim) for dim in shape]) + except TypeError: + raise TypeError(f"shape must be an int or a sequence of ints; got {shape!r}") from None + nbytes = dtype.itemsize + for dim in shape: + if dim < 0: + raise ValueError("negative dimensions are not allowed") + nbytes *= dim + if nbytes > PY_SSIZE_T_MAX: + raise ValueError("array is too big") + dims = np.array(shape, dtype=np.intp) + ptr = arenas[k].malloc(nbytes or 1) # NumPy never allocates 0 bytes either + if ptr == NULL: + raise MemoryError(f"Unable to allocate {nbytes} bytes") + Py_INCREF(dtype) + try: + array = PyArray_NewFromDescr( + ndarray, + dtype, + len(shape), + PyArray_DATA(dims), + NULL, + ptr, + NPY_ARRAY_WRITEABLE | (NPY_ARRAY_F_CONTIGUOUS if order == "F" else 0), + NULL, + ) + except BaseException: + arenas[k].free(ptr) + raise + own_data(array, handler) + return array + + +@contextmanager +def give_buffer(array, ctype="void*", *, copy=None, arena=None): + """Hand the data of ``array`` to a GraphBLAS function that takes ownership of it. + + Yields ``(ptr, nbytes, arena)``: ``ptr`` is a new ``ctype *`` that points to the data, + to pass to, e.g., ``GxB_Matrix_pack_CSR`` (with ``nbytes`` as the size) or to + ``GxB_Vector_load`` (with handling ``GrB_DEFAULT + arena``). GraphBLAS sets ``*ptr`` + to NULL when it takes ownership; then, on exit, the array handed over becomes a + read-only view of data it no longer owns. Otherwise (e.g., the call failed) nothing + changes. + + copy : bool, optional + None (default): hand over the data of ``array`` itself if GraphBLAS can own it + (see ``can_unclaim_buffer``), else a copy. True: always hand over a copy, leaving + ``array`` unchanged. False: never copy; raise ValueError if GraphBLAS can't own it. + arena : int, optional + The GraphBLAS arena that will own the data (default: the global data arena). + + An array is handed over only once: to give the same array as two arguments of one + call, nest two ``give_buffer`` and the second gets a copy. Giving one array from two + threads at once is a race, like any other change to an array. + + Examples + -------- + >>> import numpy as np + >>> from suitesparse_graphblas import check_status, ffi, lib, vector + >>> from suitesparse_graphblas.utils import give_buffer + >>> v = vector.vector_new(lib.GrB_INT64) + >>> values = np.arange(3) + >>> with give_buffer(values) as (X, nbytes, arena): + ... info = lib.GxB_Vector_load( + ... v[0], X, lib.GrB_INT64, values.size, nbytes, lib.GrB_DEFAULT + arena, ffi.NULL + ... ) + ... check_status(v, info) + >>> vector.vector_nvals(v), values.flags.owndata + (3, False) + """ + cdef int k = _resolve_arena(arena) + cdef ndarray buf + if copy is not None and not isinstance(copy, bool): + raise TypeError(f"copy must be None, True, or False; got {copy!r}") + handler = _handler(k) + array = np.asarray(array) + if array.dtype.hasobject: + raise TypeError("GraphBLAS cannot own arrays that hold Python objects") + why_not = None if copy else _why_not_ownable(array, handler) + if copy is False and why_not is not None: + raise ValueError( + f"GraphBLAS cannot take ownership of the data of this array without copying: {why_not}" + ) + if copy or why_not is not None: + order = "F" if array.flags.f_contiguous and not array.flags.c_contiguous else "C" + buf = empty(array.shape, array.dtype, order, k) + np.copyto(buf, array, casting="no") + else: + buf = array + ptr = _ffi.new(f"{ctype}*", _ffi.cast(ctype, PyArray_DATA(buf))) + # `buf` stops owning its data for now, so that giving the same array again copies it + PyArray_CLEARFLAGS(buf, NPY_ARRAY_OWNDATA) + try: + yield ptr, buf.nbytes, k + finally: + if ptr[0] == _ffi.NULL: # GraphBLAS owns the data now + PyArray_CLEARFLAGS(buf, NPY_ARRAY_WRITEABLE) + else: + PyArray_ENABLEFLAGS(buf, NPY_ARRAY_OWNDATA)