DataFrame/Arrow ingest, to_polars(), PRODUCT quantization, vector SQL options, example 27 (#19)

.github/workflows/test-python-bindings.yml

@@ -369,7 +369,7 @@ jobs:
           # to_dataframe() tests skipped themselves in this job while it
           # reported green. The guard step below now fails the build if any
           # test skips for a missing import, so the next drift is loud.
-          uv pip install --system test-install/*.whl pytest pytest-cov requests numpy "psycopg[binary]" redis neo4j pyarrow pandas "adbc-driver-postgresql>=1.12"
+          uv pip install --system test-install/*.whl pytest pytest-cov requests numpy "psycopg[binary]" redis neo4j pyarrow pandas polars "adbc-driver-postgresql>=1.12"
 
       - name: Run pytest on host
         id: pytest

.github/workflows/test-python-examples.yml

@@ -274,11 +274,14 @@ jobs:
           if [[ "$OSTYPE" == "darwin"* ]]; then
             # macOS: Install CPU-only PyTorch (supports MPS acceleration)
             uv pip install --system torch --index-url https://download.pytorch.org/whl/cpu
-            uv pip install --system "numpy<2.0" requests sentence-transformers
+            # numpy is not capped: the old macOS `numpy<2.0` was for Intel Macs, where PyTorch stops at 2.2 (built against
+            # NumPy 1.x). That leg (darwin/amd64) is disabled; re-enabling it needs the cap back for that platform only.
+            # On Apple Silicon the cap breaks pyarrow, which requires NumPy 2 (example 27).
+            uv pip install --system numpy requests sentence-transformers pandas polars pyarrow
           else
             # Linux & Windows: Install CPU-only PyTorch to save space (avoid CUDA)
             uv pip install --system torch --index-url https://download.pytorch.org/whl/cpu
-            uv pip install --system numpy requests sentence-transformers
+            uv pip install --system numpy requests sentence-transformers pandas polars pyarrow
           fi
 
       # Restored before the dataset step, which is what triggers the model

examples/27_dataframe_arrow_ingest.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+"""Example 27: DataFrames and Arrow tables in, Polars out.
+
+A pandas or Polars DataFrame, or an Arrow table, loads straight into a type with
+Database.insert_dataframe / Database.insert_arrow, the way DuckDB scans a DataFrame
+in place: no loop over rows and no CSV in between. A result set comes back out as a
+Polars DataFrame with ResultSet.to_polars().
+
+Workflow covered:
+- a pandas DataFrame into a DOCUMENT type (timestamps, nulls, a list column)
+- a Polars DataFrame into a VERTEX type, then edges between those vertices
+- an Arrow table into a type that declares some of its properties, so the int64 column
+  converts to the declared INTEGER and DATETIME_MICROS properties
+- a column that would lose data in a declared property (a float into an INTEGER)
+  refused before a row is written
+- the same data out again through to_polars()
+
+Requirements:
+- pandas, polars and pyarrow
+  (pip install 'arcadedb-embedded[pandas,polars]')
+"""
+
+from __future__ import annotations
+
+import argparse
+import shutil
+from datetime import datetime
+from pathlib import Path
+
+import pandas as pd
+import polars as pl
+import pyarrow as pa
+
+import arcadedb_embedded as arcadedb
+
+
+def documents_from_pandas(db, rows: int) -> None:
+    print(f"\n=== pandas DataFrame into a document type: {rows:,} rows ===")
+    db.command("sql", "CREATE DOCUMENT TYPE Trip")
+    df = pd.DataFrame(
+        {
+            "id": range(rows),
+            # NaN is pandas' missing value: it loads as a null
+            "fare": [None if i % 10 == 0 else 2.5 + i % 40 for i in range(rows)],
+            "picked_up": pd.Timestamp("2026-01-01")
+            + pd.to_timedelta(range(rows), "min"),
+            "stops": [[i % 7, i % 11] for i in range(rows)],
+        }
+    )
+    written = db.insert_dataframe("Trip", df, batch_size=10_000)
+    stats = db.query(
+        "sql", "SELECT count(*) AS n, count(fare) AS with_fare FROM Trip"
+    ).to_list()[0]
+    print(
+        f"written={written:,} stored={stats['n']:,} with a fare={stats['with_fare']:,}"
+    )
+
+
+def vertices_from_polars(db) -> None:
+    print("\n=== Polars DataFrame into a vertex type, then edges ===")
+    db.command("sql", "CREATE VERTEX TYPE Station")
+    db.command("sql", "CREATE EDGE TYPE Line")
+    stations = pl.DataFrame(
+        {"name": ["Central", "North", "Harbour"], "opened": [1998, 2004, 2011]}
+    )
+    print(f"stations written={db.insert_dataframe('Station', stations)}")
+    rids = {
+        row["name"]: str(row["rid"])
+        for row in db.query("sql", "SELECT name, @rid AS rid FROM Station").to_list()
+    }
+    # An edge needs the RIDs of both ends, so edges go through graph_batch().new_edges()
+    with db.graph_batch() as batch:
+        batch.new_edges(
+            [rids["Central"], rids["North"]], "Line", [rids["North"], rids["Harbour"]]
+        )
+    print("edges:", db.query("sql", "SELECT count(*) AS n FROM Line").to_list()[0]["n"])
+
+
+def arrow_into_declared_types(db) -> None:
+    print("\n=== Arrow table into declared properties ===")
+    db.command("sql", "CREATE DOCUMENT TYPE Reading")
+    db.command("sql", "CREATE PROPERTY Reading.sensor INTEGER")
+    db.command("sql", "CREATE PROPERTY Reading.at DATETIME_MICROS")
+    table = pa.table(
+        {
+            "sensor": pa.array([1, 2, 3], pa.int64()),  # converts to INTEGER
+            "at": pa.array(
+                [datetime(2026, 5, 1, 12, 0, 0, 123456)] * 3, pa.timestamp("us")
+            ),
+            "value": pa.array([0.5, None, 2.5], pa.float32()),  # undeclared: DOUBLE
+        }
+    )
+    print(f"written={db.insert_arrow('Reading', table)}")
+    try:
+        db.insert_arrow("Reading", pa.table({"sensor": [1.5]}))
+    except TypeError as exc:
+        print(f"refused before anything was written: {exc}")
+
+
+def polars_out(db) -> None:
+    print("\n=== to_polars() ===")
+    frame = db.query(
+        "sql", "SELECT sensor, at, value FROM Reading ORDER BY sensor"
+    ).to_polars()
+    print(frame)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--rows", type=int, default=100_000)
+    parser.add_argument("--db-path", default="./my_test_databases/dataframe_ingest_db")
+    args = parser.parse_args()
+
+    path = Path(args.db_path)
+    shutil.rmtree(path, ignore_errors=True)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with arcadedb.create_database(str(path)) as db:
+        documents_from_pandas(db, args.rows)
+        vertices_from_polars(db)
+        arrow_into_declared_types(db)
+        polars_out(db)
+
+
+if __name__ == "__main__":
+    main()

examples/README.md

@@ -391,6 +391,23 @@ half-applied after an interruption
 
 ---
 
+### 🐼 [27_dataframe_arrow_ingest.py](./27_dataframe_arrow_ingest.py)
+**DataFrame And Arrow Ingest | pandas | Polars | Arrow | Declared Types**
+
+DataFrames and Arrow tables loaded straight into a type, and a result back out as Polars:
+- a pandas DataFrame into a document type (timestamps, nulls, a list column), in batches
+- a Polars DataFrame into a vertex type, then edges through `graph_batch().new_edges()`
+- an Arrow table into declared `INTEGER` and `DATETIME_MICROS` properties, and a float
+  column refused before a row is written
+- `to_polars()` on the result
+- synthetic data; flags: `--rows` (default 100,000) and `--db-path`
+- requires `pip install 'arcadedb-embedded[pandas,polars]'`
+
+**Learn:** How a DataFrame becomes documents or vertices without a row loop, and what each
+Arrow type is stored as
+
+---
+
 ### 🧠 [29_hybrid_memory_retrieval.py](./29_hybrid_memory_retrieval.py)
 **Agent Memory | Tenant Pre-filter | Vector + Full-Text + Graph | `vector.fuse`**
 

pyproject.toml

@@ -57,6 +57,7 @@ test = [
     "redis>=5.0",            # Redis wire
     "neo4j>=5.0",            # Bolt
     "pyarrow>=23.0.1",       # ResultSet.to_arrow(); <23.0.1 has PYSEC-2026-113
+    "polars>=1.0",           # ResultSet.to_polars(), insert_dataframe() with a Polars frame
     # ResultSet.to_dataframe(). Absent until 2026-08-04, which meant its two
     # tests called importorskip("pandas") and skipped in every CI run this
     # package has ever had: a public, documented method with no executed
@@ -100,6 +101,15 @@ pandas = [
     "pandas>=2.2.0",
     "numpy>=1.26.4",
 ]
+# Optional: ResultSet.to_polars() and insert_dataframe() with a Polars frame. Both go
+# through Arrow, so pyarrow comes with it. insert_dataframe() with a pandas frame needs
+# `pip install 'arcadedb-embedded[arrow,pandas]'` (the pandas extra alone has no pyarrow,
+# because to_dataframe() does not need it).
+polars = [
+    "polars>=1.0",
+    "pyarrow>=23.0.1",
+    "numpy>=1.26.4",
+]
 examples = [
     "py7zr>=1.1.3",  # Required for Stack Exchange data downloads
 ]