From 23470a2b2bd10686cc230d61f06972eb458f73e0 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Thu, 8 Oct 2026 13:34:17 -0500 Subject: [PATCH] Build Pyodide (WebAssembly) wheels Add a pyodide job to wheels.yml that builds and tests cp313 and cp314 pyemscripten wheels (PEP 783) for PyPI. GraphBLAS is built as a static library in before-build (SUITESPARSE_EMSCRIPTEN=1), and EMCC_CFLAGS=-O2 replaces Pyodide's -Oz link, which never finishes. cp315 is skipped until Pyodide 315 is final. Also: - suitesparse.sh stops at the first failed step and uses the CMake option names GraphBLAS honors (GRAPHBLAS_COMPACT, GRAPHBLAS_JITINIT); the old names were silently ignored. - Tests: explicit int64 where GraphBLAS reads GrB_INT64 (NumPy's default int is int32 on wasm32); skip the subprocess and OpenMP tests on emscripten; lzma is optional. - Wheels no longer ship build-only .c/.h files (the sdist keeps them). - README and AGENTS.md notes on Pyodide; Emscripten classifier. --- .github/workflows/wheels.yml | 28 +++++++++++- AGENTS.md | 15 +++++++ README.md | 14 ++++++ pyproject.toml | 6 +++ suitesparse.sh | 47 +++++++++++++++------ suitesparse_graphblas/tests/test_io.py | 11 ++++- suitesparse_graphblas/tests/test_memory.py | 22 ++++++---- suitesparse_graphblas/tests/test_package.py | 5 +++ suitesparse_graphblas/utils.pyx | 2 +- 9 files changed, 124 insertions(+), 26 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 07caebe..ead4e0b 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -94,6 +94,17 @@ jobs: - os: macos-14 cibw_archs: "arm64" + # WebAssembly for Pyodide (PEP 783 `pyemscripten_*` wheels). GraphBLAS is built in + # before-build (see CIBW_*_PYODIDE below). + - os: ubuntu-latest + cibw_platform: "pyodide" + cibw_archs: "wasm32" + arch_note: "pyodide" + # Pyodide for 3.15 is an alpha, and building for it names the cffi module with + # the build machine's abi3 suffix (e.g., `_graphblas.abi3-x86_64-linux-gnu.so`), + # which Pyodide can't import. + cibw_skip: "cp315-*" + steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -145,11 +156,25 @@ jobs: # enable option) entirely. CIBW_ENABLE: "pypy" + CIBW_PLATFORM: ${{ matrix.cibw_platform || 'auto' }} + # Build SuiteSparse CIBW_BEFORE_ALL: bash suitesparse.sh ${{ env.GB_VERSION_REF }} CIBW_BEFORE_BUILD_LINUX: ${{ matrix.cibw_before_build_linux }} + # Pyodide: cibuildwheel sets up Emscripten for each Python version (each Pyodide + # version has its own) after before-all, so build GraphBLAS in before-build instead. + # SUITESPARSE_EMSCRIPTEN=1 asks suitesparse.sh for a static library to link into the + # extension module. The empty before-all overrides the one above. + # EMCC_CFLAGS=-O2: Pyodide links with -Oz, and binaryen's wasm-opt -Oz had not + # finished with GraphBLAS after 30 CPU-minutes; -O2 takes about a minute. emcc + # appends EMCC_CFLAGS to every command, so its -O2 is the one that counts (also for + # compiling GraphBLAS, which is then smaller and no slower). + CIBW_BEFORE_ALL_PYODIDE: "" + CIBW_BEFORE_BUILD_PYODIDE: bash suitesparse.sh ${{ env.GB_VERSION_REF }} + CIBW_ENVIRONMENT_PYODIDE: SUITESPARSE_EMSCRIPTEN=1 GRAPHBLAS_PREFIX=/tmp/graphblas-wasm GraphBLAS_ROOT=/tmp/graphblas-wasm EMCC_CFLAGS=-O2 + CIBW_ENVIRONMENT_PASS_LINUX: SUITESPARSE_FASTEST_BUILD # CMAKE_GNUtoMS=ON asks suitesparse.sh to build libraries in MSVC style on Windows. @@ -180,7 +205,8 @@ jobs: # run tests. They run from a temporary directory, so pyproject.toml's pytest # config doesn't apply; -W error restores its `filterwarnings = ["error"]`. - CIBW_TEST_COMMAND: "pytest --pyargs suitesparse_graphblas -s -k test_print_jit_config && pytest -v -W error --pyargs suitesparse_graphblas" + # `python -m pytest`, because the `pytest` script doesn't work in a Pyodide venv. + CIBW_TEST_COMMAND: "python -m pytest --pyargs suitesparse_graphblas -s -k test_print_jit_config && python -m pytest -v -W error --pyargs suitesparse_graphblas" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: diff --git a/AGENTS.md b/AGENTS.md index 1e93373..aeaf36c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -38,6 +38,21 @@ source instead, run `bash suitesparse.sh refs/tags/$(cat GB_VERSION.txt).0`. Tha honors `SUITESPARSE_FAST_BUILD` / `SUITESPARSE_FASTEST_BUILD` env vars to disable many type specializations for much faster local builds. +### WebAssembly (Pyodide) + +`wheels.yml` builds `pyemscripten` wheels for Pyodide with cibuildwheel +(`CIBW_PLATFORM=pyodide`). With `SUITESPARSE_EMSCRIPTEN=1`, `suitesparse.sh` builds GraphBLAS +as a static library (no OpenMP: Pyodide has no threads) with `emcmake` and Pyodide's compiler +flags, so `emcc` and `pyodide` must be on `PATH`; cibuildwheel provides them only in +before-build, once per Python. Two things that native builds forgive break on wasm32: + +- Calling a C function through a pointer of the wrong type traps ("function signature + mismatch"), so each function pointer type in `utils.pxd`/`utils.pyx` must match the + function's C declaration exactly, return type included (10.5.1.0 typed `GxB_init` as + returning `uint64_t`, which fails on import there). +- NumPy's default integer is int32, so give `dtype=np.int64` to arrays that GraphBLAS reads as + `GrB_INT64`. + ## Tests, lint, and other commands Testing requires the compiled CFFI extension and the SuiteSparse:GraphBLAS C library, so diff --git a/README.md b/README.md index 2d15f15..5334701 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,20 @@ or conda install -c conda-forge python-suitesparse-graphblas ``` +### In the browser (Pyodide) +PyPI also has WebAssembly wheels for [Pyodide](https://pyodide.org) 314 (Python 3.14) and +0.29.4 or later (Python 3.13), the `pyemscripten` platform of +[PEP 783](https://peps.python.org/pep-0783/). So, in Pyodide or JupyterLite +(jupyterlite-pyodide-kernel 0.8 uses Pyodide 314): + +```python +import micropip +await micropip.install("suitesparse-graphblas") +``` + +Older versions of Pyodide don't recognize these wheels: micropip says it can't find a pure +Python 3 wheel. GraphBLAS runs single-threaded in Pyodide, which has no threads (so no OpenMP). + ## Installation from source If you wish to link against your own copy of SuiteSparse:GraphBLAS you may build from source. diff --git a/pyproject.toml b/pyproject.toml index 81a34e6..501d5f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ keywords = [ ] classifiers = [ "Development Status :: 5 - Production/Stable", + "Environment :: WebAssembly :: Emscripten", "Operating System :: MacOS :: MacOS X", "Operating System :: POSIX :: Linux", "Operating System :: Microsoft :: Windows", @@ -94,6 +95,11 @@ packages = [ 'suitesparse_graphblas.io', ] +# Build inputs that MANIFEST.in puts in the sdist. Nothing reads them at runtime, so keep them +# out of wheels (`utils.c` alone is about 1 MB, which matters most for Pyodide's downloads). +[tool.setuptools.exclude-package-data] +suitesparse_graphblas = ["*.c", "*.h"] + [tool.setuptools-git-versioning] enabled = true dev_template = "{tag}+{ccount}.g{sha}" diff --git a/suitesparse.sh b/suitesparse.sh index aab7fd3..18edb20 100755 --- a/suitesparse.sh +++ b/suitesparse.sh @@ -33,7 +33,12 @@ cmake_params=() # invisible until someone measures it: conda-forge's graphblas 10.5.0 shipped # that way on osx-arm64 while still depending on llvm-openmp. The matching # runtime check on the built wheel is tests/test_package.py::test_openmp. -cmake_params+=(-DSUITESPARSE_USE_OPENMP=ON) +if [ -n "${SUITESPARSE_EMSCRIPTEN}" ]; then + # ...except for WebAssembly, which has no threads in Pyodide + cmake_params+=(-DSUITESPARSE_USE_OPENMP=OFF) +else + cmake_params+=(-DSUITESPARSE_USE_OPENMP=ON) +fi cmake_params+=(-DSUITESPARSE_USE_STRICT=ON) # STRICT makes any requested-but-missing feature fatal, and SuiteSparsePolicy # defaults both of these to ON, so they must be turned off explicitly or the @@ -66,17 +71,35 @@ if [ -n "${CMAKE_GNUtoMS}" ]; then cmake_params+=(-DCMAKE_STATIC_LIBRARY_PREFIX=) fi +cmake_cmd=(cmake) +if [ -n "${SUITESPARSE_EMSCRIPTEN}" ]; then + # WebAssembly for Pyodide (emcc and pyodide must be on PATH, as in cibuildwheel's + # before-build). Build a static library to link into the extension module, with Pyodide's + # compiler flags (-fPIC and its exception handling ABI). Its -Oz is overridden by + # CMake's Release flags (-O3), or by EMCC_CFLAGS if that sets one (see wheels.yml). + cmake_cmd=(emcmake cmake) + CFLAGS="$(pyodide config get cflags)" || exit 1 + export CFLAGS + cmake_params+=(-DBUILD_SHARED_LIBS=OFF -DBUILD_STATIC_LIBS=ON) + # No CPU features to detect + cmake_params+=(-DGBNCPUFEAT=1) +fi + if [ -n "${GRAPHBLAS_PREFIX}" ]; then echo "GRAPHBLAS_PREFIX=${GRAPHBLAS_PREFIX}" cmake_params+=(-DCMAKE_INSTALL_PREFIX="${GRAPHBLAS_PREFIX}") fi +# Start clean: this may run more than once (e.g., per Python version for Pyodide) +rm -rf "GraphBLAS-${VERSION}" curl -L "https://github.com/DrTimothyAldenDavis/GraphBLAS/archive/refs/tags/v${VERSION}.tar.gz" | tar xzf - cd "GraphBLAS-${VERSION}/build" || exit # Disable optimizing some rarely-used types for significantly faster builds and significantly smaller wheel size. # Also the build with all types enabled sometimes stalls on GitHub Actions. Probably due to exceeded resource limits. # These can still be used, they'll just have reduced performance (AFAIK similar to UDTs). +# GraphBLAS's own GB_control.h (as of 10.5.1) already disables INT8, INT16, UINT8 and UINT16, so +# the types left without FactoryKernels here are those four plus FC32, FC64 and UINT32. # shellcheck disable=SC2129 # echo "#define GxB_NO_BOOL 1" >> ../Source/GB_control.h # # echo "#define GxB_NO_FP32 1" >> ../Source/GB_control.h # @@ -132,12 +155,8 @@ if [ -n "${SUITESPARSE_FASTEST_BUILD}" ]; then echo "#define GxB_NO_UINT64 1" >> ../Source/GB_control.h echo "#define GxB_NO_UINT8 1" >> ../Source/GB_control.h - # Setting COMPACT probably makes setting config in GB_control.h above unnecessary - cmake_params+=(-DCOMPACT=1) - # Also no JIT for the fastest possible build - cmake_params+=(-DNJIT=1) - # Disable all Source/Generated2 kernels. For workflow development only. - cmake_params+=(-DCMAKE_CUDA_DEV=1) + # No FactoryKernels at all, which probably makes the GB_control.h settings above unnecessary + cmake_params+=(-DGRAPHBLAS_COMPACT=ON) fi if [ -n "${CMAKE_GNUtoMS}" ]; then @@ -145,24 +164,26 @@ if [ -n "${CMAKE_GNUtoMS}" ]; then echo "Skipping JIT on Windows for now because it fails to build." cmake_params+=(-DGRAPHBLAS_USE_JIT=OFF) else - # Use `-DJITINIT=2` so that the JIT functionality is available, but disabled by default. + # Use `GRAPHBLAS_JITINIT=2` so that the JIT functionality is available, but disabled by default. # Level 2, "run", means that pre-JIT kernels may be used, which does not require a compiler at runtime. - cmake_params+=(-DJITINIT=2) + cmake_params+=(-DGRAPHBLAS_JITINIT=2) # Disable JIT here too to not segfault in tests cmake_params+=(-DGRAPHBLAS_USE_JIT=OFF) fi # some platforms require sudo for installation, some don't have sudo at all -if [ "$(uname)" == "Darwin" ]; then +if [ "$(uname)" == "Darwin" ] && [ -z "${SUITESPARSE_EMSCRIPTEN}" ]; then SUDO=sudo else SUDO="" fi -cmake .. -DCMAKE_BUILD_TYPE=Release -G 'Unix Makefiles' "${cmake_params[@]}" -make -j"$NPROC" -$SUDO make install +# Stop at the first failure. Carrying on would hide it until the wheel fails to link, or let +# the wheel link against a library left by an earlier run (e.g., for another Python). +"${cmake_cmd[@]}" .. -DCMAKE_BUILD_TYPE=Release -G 'Unix Makefiles' "${cmake_params[@]}" || exit 1 +make -j"$NPROC" || exit 1 +$SUDO make install || exit 1 if [ -n "${CMAKE_GNUtoMS}" ]; then if [ -z "${GRAPHBLAS_PREFIX}" ]; then diff --git a/suitesparse_graphblas/tests/test_io.py b/suitesparse_graphblas/tests/test_io.py index 2819a2c..1134314 100644 --- a/suitesparse_graphblas/tests/test_io.py +++ b/suitesparse_graphblas/tests/test_io.py @@ -1,6 +1,5 @@ import bz2 import gzip -import lzma import platform from pathlib import Path @@ -21,6 +20,11 @@ vector, ) +try: + import lzma +except ImportError: # Pyodide before 314 has it as a separate package + lzma = None + if platform.system() == "Windows": pytest.skip("skipping windows-only tests", allow_module_level=True) @@ -136,7 +140,10 @@ def test_serialize_vector(): def test_matrix_binfile_read_write(tmp_path): - for opener in (Path.open, gzip.open, bz2.open, lzma.open): + openers = [Path.open, gzip.open, bz2.open] + if lzma is not None: + openers.append(lzma.open) + for opener in openers: for format in (lib.GxB_BY_ROW, lib.GxB_BY_COL): for T in grb_types: for sparsity in (lib.GxB_HYPERSPARSE, lib.GxB_SPARSE, lib.GxB_BITMAP, lib.GxB_FULL): diff --git a/suitesparse_graphblas/tests/test_memory.py b/suitesparse_graphblas/tests/test_memory.py index ed70352..d5d26c3 100644 --- a/suitesparse_graphblas/tests/test_memory.py +++ b/suitesparse_graphblas/tests/test_memory.py @@ -71,17 +71,17 @@ def roundtrip(**kwargs): # NumPy -> GraphBLAS -> NumPy: without a copy (if "numpy"), with one, and from `empty` ones = empty(N, int64, **kwargs) ones[:] = 1 - for array in [np.arange(N), np.arange(2 * N)[::2], ones]: + for array in [np.arange(N, dtype=int64), np.arange(2 * N, dtype=int64)[::2], ones]: total = array.sum() assert unload(load(array, **kwargs)).sum() == total - v = load(np.arange(N), **kwargs) + v = load(np.arange(N, dtype=int64), **kwargs) data = vector.serialize(v, lib.GxB_COMPRESSION_NONE) assert data.nbytes > N * 8 assert vector.vector_nvals(vector.deserialize(data)) == N def main(): - array = np.arange(N) + array = np.arange(N, dtype=int64) assert can_unclaim_buffer(array) == (memory_manager == "numpy") load(array) assert array.flags.owndata == (memory_manager != "numpy") # was it copied? @@ -186,6 +186,7 @@ def others(): """) +@pytest.mark.skipif(sys.platform == "emscripten", reason="needs a subprocess") @pytest.mark.parametrize("memory_manager", ["numpy", "c"]) def test_allocators_match(memory_manager, tmp_path): """Memory must be freed by the allocator that allocated it, whoever ends up owning it. @@ -265,7 +266,7 @@ def handover(array): check_status(v, info) return v - array = np.arange(10) + array = np.arange(10, dtype=np.int64) assert can_unclaim_buffer(array) v = handover(array) unclaim_buffer(array) @@ -274,7 +275,7 @@ def handover(array): assert vector.vector_nvals(v) == 10 # GraphBLAS should not have been given this, but it now has it, so NumPy must let go - array = np.arange(10) + array = np.arange(10, dtype=np.int64) array.flags.writeable = False assert not can_unclaim_buffer(array) v = handover(array) @@ -316,7 +317,7 @@ def test_empty(): def test_give_buffer(): # Zero-copy: the array no longer owns the data GraphBLAS took - array = np.arange(5) + array = np.arange(5, dtype=np.int64) v = load(array) assert not array.flags.owndata assert not array.flags.writeable @@ -325,7 +326,10 @@ def test_give_buffer(): assert x[0] == 4 # A copy, so the array is unchanged - for array, kwargs in [(np.arange(10)[::2], {}), (np.arange(5), {"copy": True})]: + for array, kwargs in [ + (np.arange(10, dtype=np.int64)[::2], {}), + (np.arange(5, dtype=np.int64), {"copy": True}), + ]: v = load(array, **kwargs) assert array.flags.writeable check_status(v, lib.GrB_Vector_extractElement_INT64(x, v[0], 4)) @@ -334,7 +338,7 @@ def test_give_buffer(): load(np.arange(10)[::2], copy=False) # GraphBLAS didn't take ownership, so the array keeps it - array = np.arange(5) + array = np.arange(5, dtype=np.int64) v = vector.vector_new(lib.GrB_INT64) with pytest.raises(exceptions.InvalidValue): with give_buffer(array) as (X, nbytes, arena): @@ -347,7 +351,7 @@ def test_give_buffer(): assert array.flags.writeable # The same array twice: the second is a copy, or an error if it may not be - array = np.arange(5) + array = np.arange(5, dtype=np.int64) with give_buffer(array) as (X, _, _), give_buffer(array) as (Y, _, _): assert X[0] != Y[0] with pytest.raises(ValueError, match="already being given"): diff --git a/suitesparse_graphblas/tests/test_package.py b/suitesparse_graphblas/tests/test_package.py index b0e2d32..99563cb 100644 --- a/suitesparse_graphblas/tests/test_package.py +++ b/suitesparse_graphblas/tests/test_package.py @@ -1,3 +1,5 @@ +import sys + import pytest import suitesparse_graphblas @@ -16,6 +18,9 @@ def test_version(): assert version > [9, 4, 4, 0] +@pytest.mark.skipif( + sys.platform == "emscripten", reason="WebAssembly (Pyodide) has no threads, so no OpenMP" +) def test_openmp(): # GraphBLAS only warns at configure time when OpenMP is missing and still # builds a working, but serial, library, so a serial build is invisible diff --git a/suitesparse_graphblas/utils.pyx b/suitesparse_graphblas/utils.pyx index 7124cdc..1330f7d 100644 --- a/suitesparse_graphblas/utils.pyx +++ b/suitesparse_graphblas/utils.pyx @@ -429,7 +429,7 @@ def give_buffer(array, ctype="void*", *, copy=None, arena=None): >>> from suitesparse_graphblas import check_status, ffi, lib, vector >>> from suitesparse_graphblas.utils import give_buffer >>> v = vector.vector_new(lib.GrB_INT64) - >>> values = np.arange(3) + >>> values = np.arange(3, dtype=np.int64) >>> with give_buffer(values) as (X, nbytes, arena): ... info = lib.GxB_Vector_load( ... v[0], X, lib.GrB_INT64, values.size, nbytes, lib.GrB_DEFAULT + arena, ffi.NULL