From 56a401c62a3b1085d01128f856b40a1481da38b0 Mon Sep 17 00:00:00 2001
From: Francesc Alted <francesc@blosc.org>
Date: Wed, 2 Sep 2026 19:27:58 +0200
Subject: [PATCH 01/11] Add HTTP/2 transport benchmarks and findings

  Add Caddy and Toxiproxy fixtures for comparing HTTP/1.1 and HTTP/2
  across peer reads and server-side slices. Verify negotiated protocols,
  record local and cat2.cloud results, and document the decision to retain
  HTTP/2 for Client requests without enabling it for peer chunk traffic.
---
 caterva2/tests/test_peers.py                  |  54 +++
 examples/benchmarks/http2/Caddyfile           |   9 +
 examples/benchmarks/http2/README.md           |  94 ++++
 examples/benchmarks/http2/peer_read.py        | 108 +++++
 .../benchmarks/http2/results-2026-09-02.md    | 114 +++++
 .../benchmarks/http2/simulated_latency.py     | 184 ++++++++
 examples/benchmarks/http2/single_fetch.py     | 115 +++++
 plans/http2-optim2.md                         | 436 ++++++++++++++++++
 8 files changed, 1114 insertions(+)
 create mode 100644 examples/benchmarks/http2/Caddyfile
 create mode 100644 examples/benchmarks/http2/README.md
 create mode 100644 examples/benchmarks/http2/peer_read.py
 create mode 100644 examples/benchmarks/http2/results-2026-09-02.md
 create mode 100644 examples/benchmarks/http2/simulated_latency.py
 create mode 100644 examples/benchmarks/http2/single_fetch.py
 create mode 100644 plans/http2-optim2.md

diff --git a/caterva2/tests/test_peers.py b/caterva2/tests/test_peers.py
index 9da03121..26896dfb 100644
--- a/caterva2/tests/test_peers.py
+++ b/caterva2/tests/test_peers.py
@@ -9,9 +9,11 @@
 import asyncio
 import json
 import os
+import pathlib
 import shutil
 import signal
 import socket
+import ssl
 import subprocess
 import sys
 import time
@@ -1020,6 +1022,58 @@ async def afetch(self, slice_, **kwargs):
     assert len(proxy.calls) == 2
 
 
+def test_caddy_fixture_negotiates_http2(tmp_path):
+    """The optional local fixture is real TLS+h2, with direct Uvicorn as h1."""
+    caddy = shutil.which("caddy")
+    if caddy is None:
+        pytest.skip("Caddy is not installed")
+
+    upstream_port, h2_port = _unused_tcp_ports(2)
+    server_dir = tmp_path / "server"
+    (server_dir / "public").mkdir(parents=True)
+    server = _start(server_dir, upstream_port)
+    env = dict(
+        os.environ,
+        CATERVA2_UPSTREAM=f"127.0.0.1:{upstream_port}",
+        CATERVA2_H2_ADDRESS=f"localhost:{h2_port}",
+        XDG_DATA_HOME=str(tmp_path / "caddy-data"),
+        XDG_CONFIG_HOME=str(tmp_path / "caddy-config"),
+    )
+    caddyfile = pathlib.Path(__file__).parents[2] / "examples" / "benchmarks" / "http2" / "Caddyfile"
+    proxy = subprocess.Popen(
+        [caddy, "run", "--config", str(caddyfile)],
+        env=env,
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    )
+    try:
+        root_ca = tmp_path / "caddy-data" / "caddy" / "pki" / "authorities" / "local" / "root.crt"
+        for _ in range(50):
+            if proxy.poll() is not None:
+                raise RuntimeError(f"Caddy exited during startup with status {proxy.returncode}")
+            if root_ca.exists():
+                try:
+                    ssl_context = ssl.create_default_context(cafile=str(root_ca))
+                    with httpx.Client(http2=True, verify=ssl_context, timeout=1) as client:
+                        response = client.get(f"https://localhost:{h2_port}/api/roots")
+                    if response.is_success:
+                        break
+                except httpx.TransportError:
+                    pass
+            time.sleep(0.1)
+        else:
+            raise RuntimeError("Caddy HTTP/2 fixture did not start")
+
+        assert response.http_version == "HTTP/2"
+        direct = httpx.get(f"http://127.0.0.1:{upstream_port}/api/roots", timeout=1)
+        assert direct.http_version == "HTTP/1.1"
+    finally:
+        proxy.send_signal(signal.SIGTERM)
+        proxy.wait(timeout=10)
+        server.send_signal(signal.SIGTERM)
+        server.wait(timeout=10)
+
+
 def test_concurrent_fetches_of_different_datasets_dont_serialize(two_dataset_peers):
     """Interleaved concurrent fetches of two different datasets under a tiny
     shared quota: correctness under load is the regression net (per
diff --git a/examples/benchmarks/http2/Caddyfile b/examples/benchmarks/http2/Caddyfile
new file mode 100644
index 00000000..1f1b8022
--- /dev/null
+++ b/examples/benchmarks/http2/Caddyfile
@@ -0,0 +1,9 @@
+{
+	# The local CA is deliberate: this fixture tests real TLS ALPN, not h2c.
+	local_certs
+}
+
+{$CATERVA2_H2_ADDRESS:localhost:8443} {
+	tls internal
+	reverse_proxy {$CATERVA2_UPSTREAM:127.0.0.1:8000}
+}
diff --git a/examples/benchmarks/http2/README.md b/examples/benchmarks/http2/README.md
new file mode 100644
index 00000000..4905df98
--- /dev/null
+++ b/examples/benchmarks/http2/README.md
@@ -0,0 +1,94 @@
+# Local HTTP/2 peer-read fixture
+
+This fixture puts Caddy in front of an ordinary Uvicorn Caterva2 server:
+
+```text
+benchmark --HTTPS/HTTP2--> Caddy --HTTP/1.1--> Uvicorn
+```
+
+It deliberately keeps direct Uvicorn access available as the HTTP/1.1 control.
+Caddy is optional and is not a Caterva2 dependency.
+
+## Start the fixture
+
+Start a Caterva2 server on `127.0.0.1:8000`, with a multi-chunk dataset in its
+`@public` root. Then, from the repository root, run:
+
+```console
+caddy run --config examples/benchmarks/http2/Caddyfile
+```
+
+Caddy's local CA must be trusted by HTTPX. `caddy trust` installs it in the local
+trust store on supported systems. Alternatively, point `SSL_CERT_FILE` at the
+exported Caddy root certificate. Do not disable certificate verification: doing
+so would make this fixture less representative and could hide configuration
+mistakes.
+
+Check both paths before benchmarking:
+
+```console
+curl -sS -o /dev/null -w '%{http_version}\n' \
+  http://127.0.0.1:8000/api/roots
+curl -sS -o /dev/null -w '%{http_version}\n' \
+  https://localhost:8443/api/roots
+```
+
+The expected outputs are `1.1` and `2`, respectively.
+
+## Run the benchmark
+
+Both URLs must address the same Caterva2 server and dataset:
+
+```console
+python examples/benchmarks/http2/peer_read.py \
+  --http1-url http://127.0.0.1:8000 \
+  --http2-url https://localhost:8443 \
+  --path @public/example.b2nd \
+  --concurrency 4 \
+  --repeat 5
+```
+
+The script verifies the negotiated protocol before collecting timings. Each trial
+uses a new sparse cache, so the timed operation is a cold peer read. HTTP/1.1 and
+HTTP/2 use the same `RemoteSource` and `Proxy.afetch` implementation and the same
+concurrency.
+
+Run several times and compare the distributions. Loopback is useful for correctness
+and protocol verification, but meaningful latency benefits require a controlled WAN
+test or a genuinely remote peer.
+
+## Simulated latency
+
+With Caddy and Toxiproxy installed, the self-contained benchmark creates a temporary
+64-chunk dataset and applies half the requested latency in each TCP direction:
+
+```console
+python examples/benchmarks/http2/simulated_latency.py \
+  --rtt-ms 50 --concurrency 8 --repeat 5
+```
+
+Both protocols traverse the same Toxiproxy listener, TLS connection, Caddy instance,
+and Uvicorn process. The only difference is whether HTTPX offers HTTP/2 during ALPN.
+Temporary servers, proxies, certificates, data, and caches are removed after the run.
+
+The same fixture can reproduce the single-large-response workload with approximately
+the same 26.8 MB materialized slice as `get-slice.py`:
+
+```console
+python examples/benchmarks/http2/simulated_latency.py \
+  --workload single-fetch --rtt-ms 50 \
+  --chunks 10 --items-per-chunk 838860 --slice 5:9 \
+  --repeat 12 --pause 0.25
+```
+
+## Single server-side slice
+
+To compare the one-response workload in `examples/get-slice.py`, while separating
+network transfer from cframe parsing and NumPy materialization:
+
+```console
+python examples/benchmarks/http2/single_fetch.py --repeat 20
+```
+
+Both persistent clients connect to the same endpoint. One forces HTTP/1.1 and the
+other offers HTTP/2; every response is checked against the expected protocol.
diff --git a/examples/benchmarks/http2/peer_read.py b/examples/benchmarks/http2/peer_read.py
new file mode 100644
index 00000000..de58c463
--- /dev/null
+++ b/examples/benchmarks/http2/peer_read.py
@@ -0,0 +1,108 @@
+"""Compare cold peer reads over pooled HTTP/1.1 and multiplexed HTTP/2."""
+
+import argparse
+import asyncio
+import pathlib
+import statistics
+import tempfile
+import time
+
+import blosc2
+import httpx
+
+from caterva2.c2cache.remote import RemoteSource
+
+
+class BenchmarkRemoteSource(RemoteSource):
+    """RemoteSource with a selectable transport, confined to this benchmark."""
+
+    def __init__(self, path: str, urlbase: str, *, http2: bool):
+        super().__init__(path, urlbase=urlbase, use_chunk_api=True)
+        self._benchmark_http2 = http2
+
+    async def aget_chunk(self, nchunk: int) -> bytes:
+        if self._aclient is None:
+            self._aclient = httpx.AsyncClient(http2=self._benchmark_http2, timeout=5)
+        return await super().aget_chunk(nchunk)
+
+
+async def assert_protocol(urlbase: str, expected: str, *, http2: bool) -> None:
+    """Fail rather than silently benchmarking HTTPX's HTTP/1.1 fallback."""
+    async with httpx.AsyncClient(http2=http2, follow_redirects=True, timeout=10) as client:
+        response = await client.get(f"{urlbase.rstrip('/')}/api/roots")
+        response.raise_for_status()
+    if response.http_version != expected:
+        raise RuntimeError(
+            f"{urlbase} negotiated {response.http_version}, expected {expected}; "
+            "benchmark result would be invalid"
+        )
+
+
+async def cold_read(urlbase: str, path: str, concurrency: int, cache: pathlib.Path, *, http2: bool) -> float:
+    source = BenchmarkRemoteSource(path, urlbase=urlbase, http2=http2)
+    proxy = blosc2.Proxy(source, urlpath=str(cache), mode="w")
+    try:
+        started = time.perf_counter()
+        await proxy.afetch(None, max_concurrency=concurrency)
+        return time.perf_counter() - started
+    finally:
+        # Proxy does not own the remote source's HTTP client.
+        await source.aclose()
+
+
+def report(label: str, samples: list[float]) -> None:
+    ordered = sorted(samples)
+    p95_index = min(len(ordered) - 1, int(0.95 * len(ordered)))
+    print(
+        f"{label} n={len(samples)} median={statistics.median(samples):.6f}s "
+        f"p95={ordered[p95_index]:.6f}s min={ordered[0]:.6f}s max={ordered[-1]:.6f}s"
+    )
+
+
+async def main(args: argparse.Namespace) -> None:
+    await assert_protocol(args.http1_url, "HTTP/1.1", http2=False)
+    await assert_protocol(args.http2_url, "HTTP/2", http2=True)
+
+    # Alternate order to reduce systematic server/OS-cache and network drift. Every
+    # operation still gets a new sparse cache and HTTP client.
+    samples = {"http1": [], "http2": []}
+    cases = {
+        "http1": (args.http1_url, False),
+        "http2": (args.http2_url, True),
+    }
+    with tempfile.TemporaryDirectory(prefix="caterva2-http-benchmark-") as tmp:
+        tmpdir = pathlib.Path(tmp)
+        for trial in range(args.repeat):
+            order = ("http1", "http2") if trial % 2 == 0 else ("http2", "http1")
+            for label in order:
+                urlbase, use_http2 = cases[label]
+                sample = await cold_read(
+                    urlbase,
+                    args.path,
+                    args.concurrency,
+                    tmpdir / f"{label}-trial-{trial}.b2nd",
+                    http2=use_http2,
+                )
+                samples[label].append(sample)
+                print(f"{label} trial={trial + 1} seconds={sample:.6f}")
+    http1, http2 = samples["http1"], samples["http2"]
+    report("http1", http1)
+    report("http2", http2)
+    print(f"median_ratio_http1_over_http2={statistics.median(http1) / statistics.median(http2):.3f}")
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--http1-url", required=True)
+    parser.add_argument("--http2-url", required=True)
+    parser.add_argument("--path", required=True, help="remote dataset path, e.g. @public/example.b2nd")
+    parser.add_argument("--concurrency", type=int, default=4)
+    parser.add_argument("--repeat", type=int, default=5)
+    args = parser.parse_args()
+    if args.concurrency < 1 or args.repeat < 1:
+        parser.error("--concurrency and --repeat must be positive")
+    return args
+
+
+if __name__ == "__main__":
+    asyncio.run(main(parse_args()))
diff --git a/examples/benchmarks/http2/results-2026-09-02.md b/examples/benchmarks/http2/results-2026-09-02.md
new file mode 100644
index 00000000..bf56306b
--- /dev/null
+++ b/examples/benchmarks/http2/results-2026-09-02.md
@@ -0,0 +1,114 @@
+# Preliminary `cat2.cloud` peer-read results — 2026-09-02
+
+These are exploratory measurements, not release criteria. They compare the same
+`RemoteSource` + `Proxy.afetch` cold-cache operation against the same deployed
+endpoint, forcing HTTPX to use HTTP/1.1 or offer HTTP/2. The harness asserted that
+the negotiated protocols were `HTTP/1.1` and `HTTP/2`, respectively.
+
+Dataset: `@public/examples/numbers_color.b2nd` at `https://cat2.cloud/demo`
+
+Shape: `(10, 368, 744, 4)`
+
+Chunk shape: `(1, 368, 744, 4)` (10 chunks)
+
+Trials: 5 per protocol and concurrency, alternating protocol order
+
+| Max concurrency | HTTP/1.1 median | HTTP/2 median | HTTP/1.1 / HTTP/2 |
+|---:|---:|---:|---:|
+| 1 | 1.037 s | 0.984 s | 1.055 |
+| 4 | 0.555 s | 0.557 s | 0.996 |
+| 8 | 0.479 s | 0.563 s | 0.850 |
+| 16 | 0.402 s | 0.538 s | 0.747 |
+
+On this workload HTTP/2 did not improve the useful concurrent cases. It was about
+5% faster for serial reads, effectively tied at concurrency 4, and slower at 8 and
+16. Both protocols showed occasional network outliers across exploratory runs.
+
+Likely explanations to investigate include HTTP/2 implementation overhead for only
+10 moderate-sized chunks, server/proxy scheduling, and the ability of several
+HTTP/1.1 TCP connections to transfer large chunks in parallel. These measurements
+do not establish behavior for a higher-RTT peer, more chunks, smaller ranges, or a
+controlled network.
+
+Next useful experiment: run a larger multi-chunk dataset through the local Caddy
+fixture under controlled latency and bandwidth, with more repetitions and connection
+counts. Until then, enabling HTTP/2 remains safe protocol negotiation with HTTP/1.1
+fallback, not a demonstrated peer-read optimization.
+
+## Controlled 50 ms RTT simulation
+
+The self-contained Toxiproxy fixture applied 25 ms in each TCP direction. Both
+protocols used the same TLS endpoint, Toxiproxy listener, Caddy instance, Uvicorn
+server, generated dataset, trial count, and concurrency. The only changed input was
+HTTPX's HTTP/2 offer. Protocol assertions again verified `HTTP/1.1` and `HTTP/2`.
+
+Dataset: 64 random, effectively incompressible chunks
+
+Concurrency: 8
+
+Trials: 7 per protocol and chunk size, alternating protocol order
+
+| Chunk size | HTTP/1.1 median | HTTP/2 median | HTTP/1.1 / HTTP/2 |
+|---:|---:|---:|---:|
+| 64 KiB | 0.543 s | 0.578 s | 0.939 |
+| 512 KiB | 0.557 s | 0.628 s | 0.887 |
+
+HTTP/2 remained slower: approximately 6% for 64 KiB chunks and 13% for 512 KiB
+chunks. A preliminary version of the fixture routed HTTP/1.1 directly to clear-text
+Uvicorn and was discarded as unfair; the table contains only corrected same-path
+measurements.
+
+This simulation models fixed latency but not constrained bandwidth, packet loss, or
+internet jitter. At equal concurrency, HTTP/1.1 establishes several connections in
+parallel, so handshake latency does not necessarily accumulate serially. HTTP/2's
+lower connection count is still an operational benefit, but no elapsed-time benefit
+has been demonstrated for peer chunk reads.
+
+### Local single-response result
+
+The single-response workload was also reproduced locally through the same Caddy and
+Toxiproxy path at 50 ms RTT. A generated 10-chunk array was sliced at `5:9`, yielding
+26,843,520 materialized bytes. Both persistent clients used the same TLS endpoint;
+only the HTTPX protocol offer differed.
+
+| Metric | HTTP/1.1 median | HTTP/2 median |
+|---|---:|---:|
+| Network response | 0.121 s | 0.202 s |
+| NumPy materialization | 0.010 s | 0.010 s |
+| End to end | 0.132 s | 0.213 s |
+
+HTTP/2 was approximately 66% slower locally. Unlike `cat2.cloud`, neither protocol
+showed large timing outliers. This does not reproduce the deployed HTTP/2 advantage;
+instead, it shows that the production result is specific to the deployed nginx,
+network, or connection behavior rather than an intrinsic advantage for one large
+HTTP response. The local HTTP/2 penalty is consistent with single-connection framing
+or flow-control overhead, but this benchmark does not isolate its cause.
+
+## Single `api/fetch` response
+
+This reproduces `examples/get-slice.py` against
+`@public/examples/lung-jpeg2000_10x.b2nd`, slice `5:9`. Two persistent clients
+connected to the same `cat2.cloud` endpoint; one forced HTTP/1.1 and the other
+offered HTTP/2. Protocol order alternated, requests were spaced by 0.5 seconds,
+and each response's negotiated version was asserted.
+
+The response materializes to 26,846,976 bytes of NumPy data. There were 12 trials
+per protocol.
+
+| Metric | HTTP/1.1 median | HTTP/2 median |
+|---|---:|---:|
+| Buffered network response | 1.361 s | 0.318 s |
+| Cframe parsing | 0.000185 s | 0.000169 s |
+| NumPy materialization | 0.108 s | 0.106 s |
+| End to end | 1.468 s | 0.426 s |
+
+HTTP/1.1 network times ranged from 0.309 to 3.127 seconds and worsened markedly
+during the run. HTTP/2 ranged from 0.245 to 0.635 seconds. The HTTP/1.1 median was
+4.28 times the HTTP/2 median in this sample. Materialization was effectively equal,
+so the difference is in receiving the response rather than decoding it.
+
+This supports the historical observation that HTTP/2 improved `get-slice.py`. It
+does not conflict with the peer-chunk results: this workload is one large response,
+so HTTP/1.1 gains nothing from a pool of parallel connections. The cause of the
+large persistent-HTTP/1.1 degradation remains to be isolated; possible contributors
+include nginx connection handling, intermediary shaping, and transport behavior.
diff --git a/examples/benchmarks/http2/simulated_latency.py b/examples/benchmarks/http2/simulated_latency.py
new file mode 100644
index 00000000..bccad68f
--- /dev/null
+++ b/examples/benchmarks/http2/simulated_latency.py
@@ -0,0 +1,184 @@
+"""Self-contained peer-read benchmark with symmetric TCP latency."""
+
+import argparse
+import asyncio
+import os
+import pathlib
+import shutil
+import signal
+import ssl
+import subprocess
+import tempfile
+import time
+
+import blosc2
+import httpx
+import numpy as np
+import peer_read
+import single_fetch
+
+from caterva2.tests.test_peers import _start, _unused_tcp_ports
+
+
+def wait_for_cli(cli: str, api_url: str, process: subprocess.Popen) -> None:
+    for _ in range(50):
+        if process.poll() is not None:
+            raise RuntimeError(f"Toxiproxy exited during startup with status {process.returncode}")
+        result = subprocess.run(
+            [cli, "--host", api_url, "list"], capture_output=True, check=False, text=True
+        )
+        if result.returncode == 0:
+            return
+        time.sleep(0.1)
+    raise RuntimeError("Toxiproxy API did not start")
+
+
+def toxiproxy(cli: str, api_url: str, *args: str) -> None:
+    subprocess.run([cli, "--host", api_url, *args], check=True, capture_output=True, text=True)
+
+
+async def run(args: argparse.Namespace) -> None:
+    caddy = shutil.which("caddy")
+    toxiproxy_server = shutil.which("toxiproxy-server")
+    toxiproxy_cli = shutil.which("toxiproxy-cli")
+    missing = [name for name, path in (("caddy", caddy), ("toxiproxy", toxiproxy_server)) if not path]
+    if missing:
+        raise RuntimeError(f"missing benchmark tools: {', '.join(missing)}")
+
+    with tempfile.TemporaryDirectory(prefix="caterva2-http2-latency-") as tmp:
+        tmpdir = pathlib.Path(tmp)
+        server_port, caddy_port, delayed_port, toxiproxy_port = _unused_tcp_ports(4)
+        server_dir = tmpdir / "server"
+        public = server_dir / "public"
+        public.mkdir(parents=True)
+        data = np.random.default_rng(42).random((args.chunks, args.items_per_chunk))
+        blosc2.asarray(
+            data,
+            chunks=(1, args.items_per_chunk),
+            blocks=(1, args.items_per_chunk),
+            urlpath=str(public / "latency.b2nd"),
+        )
+
+        server = _start(server_dir, server_port)
+        caddy_env = dict(
+            os.environ,
+            CATERVA2_UPSTREAM=f"127.0.0.1:{server_port}",
+            CATERVA2_H2_ADDRESS=f"localhost:{caddy_port}",
+            XDG_DATA_HOME=str(tmpdir / "caddy-data"),
+            XDG_CONFIG_HOME=str(tmpdir / "caddy-config"),
+        )
+        caddyfile = pathlib.Path(__file__).with_name("Caddyfile")
+        caddy_process = subprocess.Popen(
+            [caddy, "run", "--config", str(caddyfile)],
+            env=caddy_env,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+        toxiproxy_process = subprocess.Popen(
+            [toxiproxy_server, "-host", "127.0.0.1", "-port", str(toxiproxy_port)],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+        try:
+            api_url = f"http://127.0.0.1:{toxiproxy_port}"
+            wait_for_cli(toxiproxy_cli, api_url, toxiproxy_process)
+            one_way_ms = args.rtt_ms / 2
+            name = "peer-tls"
+            toxiproxy(
+                toxiproxy_cli,
+                api_url,
+                "create",
+                "--listen",
+                f"127.0.0.1:{delayed_port}",
+                "--upstream",
+                f"127.0.0.1:{caddy_port}",
+                name,
+            )
+            for direction in ("downstream", "upstream"):
+                direction_flag = "--downstream" if direction == "downstream" else "--upstream"
+                toxiproxy(
+                    toxiproxy_cli,
+                    api_url,
+                    "toxic",
+                    "add",
+                    "--type",
+                    "latency",
+                    "--attribute",
+                    f"latency={one_way_ms:g}",
+                    direction_flag,
+                    name,
+                )
+
+            root_ca = tmpdir / "caddy-data" / "caddy" / "pki" / "authorities" / "local" / "root.crt"
+            for _ in range(50):
+                if caddy_process.poll() is not None:
+                    raise RuntimeError(f"Caddy exited during startup with status {caddy_process.returncode}")
+                if root_ca.exists():
+                    try:
+                        context = ssl.create_default_context(cafile=str(root_ca))
+                        with httpx.Client(http2=True, verify=context, timeout=1) as client:
+                            ready = client.get(f"https://localhost:{caddy_port}/api/roots")
+                        if ready.is_success and ready.http_version == "HTTP/2":
+                            break
+                    except httpx.TransportError:
+                        pass
+                time.sleep(0.1)
+            else:
+                raise RuntimeError("Caddy verified HTTP/2 endpoint did not start")
+
+            os.environ["SSL_CERT_FILE"] = str(root_ca)
+            print(
+                f"dataset_chunks={args.chunks} chunk_bytes={args.items_per_chunk * 8} "
+                f"simulated_rtt_ms={args.rtt_ms:g}"
+            )
+            delayed_url = f"https://localhost:{delayed_port}"
+            if args.workload == "peer-read":
+                await peer_read.main(
+                    argparse.Namespace(
+                        # Same TLS endpoint and proxy path. Only HTTPX's protocol
+                        # offer differs between the two benchmark arms.
+                        http1_url=delayed_url,
+                        http2_url=delayed_url,
+                        path="@public/latency.b2nd",
+                        concurrency=args.concurrency,
+                        repeat=args.repeat,
+                    )
+                )
+            else:
+                single_fetch.main(
+                    argparse.Namespace(
+                        urlbase=delayed_url,
+                        path="@public/latency.b2nd",
+                        slice=args.slice,
+                        repeat=args.repeat,
+                        timeout=30,
+                        pause=args.pause,
+                    )
+                )
+        finally:
+            for process in (toxiproxy_process, caddy_process, server):
+                if process.poll() is None:
+                    process.send_signal(signal.SIGTERM)
+                    process.wait(timeout=10)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--workload", choices=("peer-read", "single-fetch"), default="peer-read")
+    parser.add_argument("--rtt-ms", type=float, default=50)
+    parser.add_argument("--chunks", type=int, default=64)
+    parser.add_argument("--items-per-chunk", type=int, default=65536)
+    parser.add_argument("--concurrency", type=int, default=8)
+    parser.add_argument("--repeat", type=int, default=5)
+    parser.add_argument("--slice", default="5:9", help="slice used by the single-fetch workload")
+    parser.add_argument("--pause", type=float, default=0.25)
+    args = parser.parse_args()
+    if min(args.rtt_ms, args.chunks, args.items_per_chunk, args.concurrency, args.repeat) <= 0:
+        parser.error("all numeric arguments must be positive")
+    if args.pause < 0:
+        parser.error("--pause cannot be negative")
+    return args
+
+
+if __name__ == "__main__":
+    asyncio.run(run(parse_args()))
diff --git a/examples/benchmarks/http2/single_fetch.py b/examples/benchmarks/http2/single_fetch.py
new file mode 100644
index 00000000..e9f3272e
--- /dev/null
+++ b/examples/benchmarks/http2/single_fetch.py
@@ -0,0 +1,115 @@
+"""Compare one large api/fetch response over persistent HTTP/1.1 and HTTP/2."""
+
+import argparse
+import statistics
+import time
+
+import blosc2
+import httpx
+
+
+def percentile(samples: list[float], fraction: float) -> float:
+    ordered = sorted(samples)
+    return ordered[min(len(ordered) - 1, int(fraction * len(ordered)))]
+
+
+def fetch_once(
+    client: httpx.Client, url: str, slice_: str, expected: str
+) -> tuple[float, float, float | None, int | None]:
+    started = time.perf_counter()
+    response = client.get(url, params={"slice_": slice_})
+    network = time.perf_counter() - started
+    response.raise_for_status()
+    if response.http_version != expected:
+        raise RuntimeError(f"negotiated {response.http_version}, expected {expected}")
+
+    started = time.perf_counter()
+    array = blosc2.ndarray_from_cframe(response.content)
+    parse = time.perf_counter() - started
+
+    try:
+        started = time.perf_counter()
+        numpy_array = array[:]
+        materialize = time.perf_counter() - started
+    except RuntimeError:
+        # Network and cframe timings remain useful when the local environment
+        # lacks the codec needed to materialize this particular dataset.
+        return network, parse, None, None
+    return network, parse, materialize, numpy_array.nbytes
+
+
+def report(label: str, samples: list[tuple[float, float, float | None, int | None]]) -> None:
+    print(f"{label} trials={len(samples)} numpy_bytes={samples[0][3]}")
+    for index, metric in enumerate(("network", "parse", "numpy")):
+        values = [sample[index] for sample in samples if sample[index] is not None]
+        if not values:
+            print(f"  {metric}: unavailable (local codec could not materialize the cframe)")
+            continue
+        print(
+            f"  {metric}: median={statistics.median(values):.6f}s "
+            f"p95={percentile(values, 0.95):.6f}s min={min(values):.6f}s max={max(values):.6f}s"
+        )
+    totals = [sample[0] + sample[1] + sample[2] for sample in samples if sample[2] is not None]
+    if totals:
+        print(
+            f"  end_to_end: median={statistics.median(totals):.6f}s "
+            f"p95={percentile(totals, 0.95):.6f}s min={min(totals):.6f}s max={max(totals):.6f}s"
+        )
+
+
+def main(args: argparse.Namespace) -> None:
+    url = f"{args.urlbase.rstrip('/')}/api/fetch/{args.path}"
+    samples = {"http1": [], "http2": []}
+    cases = {
+        "http1": (httpx.Client(http1=True, http2=False, timeout=args.timeout), "HTTP/1.1"),
+        "http2": (httpx.Client(http1=True, http2=True, timeout=args.timeout), "HTTP/2"),
+    }
+    try:
+        # Establish both TLS connections and populate server/OS caches outside the
+        # samples. The response body is consumed because Client.get buffers it.
+        for label, (client, expected) in cases.items():
+            warm = client.get(url, params={"slice_": args.slice})
+            warm.raise_for_status()
+            if warm.http_version != expected:
+                raise RuntimeError(f"{label} negotiated {warm.http_version}, expected {expected}")
+
+        for trial in range(args.repeat):
+            order = ("http1", "http2") if trial % 2 == 0 else ("http2", "http1")
+            for label in order:
+                client, expected = cases[label]
+                sample = fetch_once(client, url, args.slice, expected)
+                samples[label].append(sample)
+                print(
+                    f"{label} trial={trial + 1} network={sample[0]:.6f}s "
+                    f"parse={sample[1]:.6f}s "
+                    f"numpy={f'{sample[2]:.6f}s' if sample[2] is not None else 'unavailable'}"
+                )
+                if args.pause:
+                    time.sleep(args.pause)
+    finally:
+        for client, _ in cases.values():
+            client.close()
+
+    report("http1", samples["http1"])
+    report("http2", samples["http2"])
+    h1_network = statistics.median(sample[0] for sample in samples["http1"])
+    h2_network = statistics.median(sample[0] for sample in samples["http2"])
+    print(f"median_network_ratio_http1_over_http2={h1_network / h2_network:.3f}")
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--urlbase", default="https://cat2.cloud/demo")
+    parser.add_argument("--path", default="@public/examples/lung-jpeg2000_10x.b2nd")
+    parser.add_argument("--slice", default="5:9")
+    parser.add_argument("--repeat", type=int, default=20)
+    parser.add_argument("--timeout", type=float, default=30)
+    parser.add_argument("--pause", type=float, default=0.25, help="seconds between measured requests")
+    args = parser.parse_args()
+    if args.repeat < 1 or args.timeout <= 0 or args.pause < 0:
+        parser.error("--repeat and --timeout must be positive; --pause cannot be negative")
+    return args
+
+
+if __name__ == "__main__":
+    main(parse_args())
diff --git a/plans/http2-optim2.md b/plans/http2-optim2.md
new file mode 100644
index 00000000..8b174382
--- /dev/null
+++ b/plans/http2-optim2.md
@@ -0,0 +1,436 @@
+# HTTP/2 Optimization Plan v2: Measured Concurrency for Chunk I/O
+
+**Status:** Proposed for discussion
+**Scope:** `caterva2`, `python-blosc2`, peer-cache reads, direct object-store reads,
+and chunk ingestion
+**Principle:** Concurrency is the optimization; HTTP/2 is a transport that may make
+that concurrency cheaper when it is actually negotiated.
+
+---
+
+## 1. Goals and Non-Goals
+
+### Goals
+
+1. Reduce latency for operations that require many independent chunk or byte-range
+   requests.
+2. Reuse connections and bound concurrency so clients do not overload themselves,
+   Caterva2 peers, reverse proxies, or object stores.
+3. Use HTTP/2 multiplexing where the complete path supports it and measurements show
+   a benefit.
+4. Add enough protocol and performance observability to detect silent HTTP/1.1
+   fallback and regressions.
+5. Preserve cache correctness, retry semantics, duplicate-write behavior, Pyodide
+   compatibility, and existing synchronous APIs.
+
+### Non-Goals
+
+* HTTP/2 is not required for ordinary Caterva2 slicing that already completes in one
+  `api/fetch` request.
+* This plan does not assume that every S3-compatible endpoint supports HTTP/2.
+* It does not promise a particular speedup or prescribe flow-control window sizes
+  before measurement.
+* HTTP/3 is out of scope until the HTTP/1.1 and HTTP/2 work is complete.
+
+---
+
+## 2. Current State
+
+The implementation must be inventoried before changes are made. At the time this
+plan was written:
+
+* `caterva2.Client` already constructs `httpx.Client(http2=True)` outside Emscripten.
+* `httpx[http2]` is already a project dependency.
+* `c2cache` and current `python-blosc2` integrations already contain real asynchronous
+  chunk-fetch paths and bounded/gathered `Proxy.afetch` behavior. These should be
+  extended rather than reimplemented.
+* Some `c2cache` `httpx.AsyncClient` and `httpx.Client` instances do not enable
+  HTTP/2.
+* The bundled Caterva2 command starts Uvicorn, which serves HTTP/1.1. Setting
+  `http2=True` on a client therefore does not make a default local Caterva2 server
+  speak HTTP/2.
+* The deployed `https://cat2.cloud/demo/` endpoint has been verified to negotiate
+  `h2` through TLS ALPN and to return an HTTP/2 response. Its reverse proxy may still
+  communicate with Uvicorn over HTTP/1.1; client-facing multiplexing remains useful.
+
+This inventory must be repeated against the exact `python-blosc2` revision selected
+for implementation, because that repository evolves independently.
+
+---
+
+## 3. Supported Test and Deployment Paths
+
+### 3.1 Local HTTP/1.1 baseline
+
+Run Caterva2 directly under Uvicorn. This is the control configuration and must stay
+fully supported.
+
+### 3.2 Local HTTP/2 configuration
+
+Provide a documented, reproducible TLS configuration using either:
+
+* Caddy or nginx terminating HTTP/2 and proxying to Uvicorn; or
+* an HTTP/2-capable ASGI server such as Hypercorn, if it is shown to run Caterva2
+  correctly.
+
+The reverse-proxy configuration is preferred for production-parity testing. Test
+certificates may be generated locally, but certificate verification must remain on;
+the benchmark client should trust the test CA explicitly.
+
+HTTPX generally negotiates HTTP/2 over TLS using ALPN. Clear-text `h2c` is not the
+primary test path.
+
+### 3.3 Production validation
+
+Run non-destructive protocol and read benchmarks against `cat2.cloud/demo` when
+appropriate. Production tests must use bounded load and must not be the only
+performance evidence.
+
+### 3.4 Mandatory protocol assertion
+
+Every HTTP/2 benchmark must inspect `response.http_version` and fail or mark the case
+invalid unless it equals `HTTP/2`. Also record:
+
+* effective URL and redirects;
+* negotiated protocol;
+* number of TCP connections where measurable;
+* reverse-proxy/server configuration identifier.
+
+This prevents HTTPX's normal HTTP/1.1 fallback from being mistaken for an HTTP/2
+result.
+
+---
+
+## 4. Workloads Where Concurrency May Help
+
+### 4.1 Peer-cache cold reads
+
+A slice that touches many uncached remote chunks currently requires multiple peer
+requests. Fetch missing chunks concurrently while retaining sparse-cache behavior.
+
+Requirements:
+
+* configurable `max_concurrency`, with a conservative default;
+* no unbounded `asyncio.gather()`;
+* already-cached chunks are never fetched again unnecessarily;
+* per-request and overall operation timeouts;
+* retry only transport/transient failures, with a small bounded retry count and
+  backoff/jitter if measurements justify it;
+* preserve successfully cached chunks when another chunk fails;
+* cancellation closes responses and releases stream capacity;
+* peer-offline classification must not turn application deliberate HTTP errors or local
+  programming errors into connectivity failures;
+* client ownership and shutdown must follow the Caterva2 application lifespan.
+
+HTTP/2 clients should be persistent per origin or per suitable lifecycle scope, not
+created for each chunk. Async clients must not be shared unsafely across event loops.
+
+### 4.2 Direct HTTPS object-store range reads
+
+For a Blosc2 source exposed through HTTPS, issue one request per required range and
+schedule them with bounded concurrency. Test each service independently:
+
+* AWS S3 REST endpoints;
+* Cloudflare R2;
+* GCS;
+* MinIO or other S3-compatible services;
+* CDN-fronted object URLs.
+
+Do not infer protocol support from an `https://` URL. Record the negotiated protocol
+for every service and endpoint form. Document whether access is anonymous, uses a
+presigned URL, or requires request signing. Plain HTTPX does not itself provide AWS
+SigV4 signing.
+
+`s3://` through `fsspec`/`s3fs` is a separate transport and must not be described as
+accelerated by changes to an HTTPX client. Any AWS CRT experiment belongs in a
+separate optional work item.
+
+### 4.3 Parallel chunk ingestion
+
+Retain `Client.fill_chunk()` and add an explicit bounded batch/async API rather than
+making callers coordinate arbitrary threads around the synchronous method. A
+provisional shape is:
+
+```python
+results = await client.afill_chunks(
+    remotepath,
+    chunks,
+    max_concurrency=8,
+)
+```
+
+The final API design must specify:
+
+* accepted chunk iterable/mapping format;
+* input order and result association;
+* bounded memory use and whether request bodies are streamed;
+* fail-fast versus collect-errors behavior;
+* partial-success reporting;
+* retry policy;
+* cancellation semantics;
+* behavior for an already-written slot;
+* connection and async-client lifetime.
+
+Multiple processes cannot share one HTTP/2 connection, so process-based ingestion
+must be benchmarked and documented separately.
+
+### 4.4 Interactive request cancellation
+
+HTTP/2 permits cancellation of one stream without closing the whole connection, but
+starting a replacement request does not cancel the old one automatically.
+
+Treat interactive cancellation as a separate feature:
+
+* browser clients use `AbortController`;
+* Python async callers cancel the relevant task;
+* response bodies are explicitly closed/released;
+* cancellation is propagated through intermediate server requests where applicable;
+* tests verify resource release and correctness, without depending on a particular
+  wire frame unless the transport contract guarantees it.
+
+---
+
+## 5. Benchmark Design
+
+Benchmarking precedes broad implementation and all performance claims.
+
+### 5.1 Fair comparison matrix
+
+For each applicable workload, compare:
+
+1. HTTP/1.1 sequential baseline;
+2. HTTP/1.1 persistent pooled client at concurrency `N`;
+3. HTTP/2 persistent client at the same concurrency `N`;
+4. selected concurrency sweep, for example `1, 2, 4, 8, 16, 32`.
+
+The key comparison is pooled HTTP/1.1 versus HTTP/2 at equal concurrency. A
+sequential-only HTTP/1.1 comparison exaggerates HTTP/2's contribution.
+
+### 5.2 Environments
+
+Measure at least:
+
+* loopback/local with the documented TLS proxy;
+* controlled latency and bandwidth, including optional packet loss;
+* `cat2.cloud/demo` under a deliberately low request rate;
+* each supported object-store endpoint.
+
+Keep dataset, slice, compression, concurrency, cache state, client limits, and server
+limits identical across comparable cases.
+
+### 5.3 Workload cases
+
+* cold peer slice touching many chunks;
+* warm peer slice, demonstrating cache behavior;
+* direct object read touching many byte ranges;
+* batch upload of many compressed chunks;
+* mixed-size chunks/ranges, including a slow response to expose head-of-line effects;
+* cancellation during an in-flight operation.
+
+### 5.4 Metrics
+
+Record:
+
+* elapsed time and throughput;
+* p50, p95, and p99 per-request latency;
+* time to first useful chunk/block;
+* negotiated HTTP version;
+* opened/reused TCP connections;
+* transferred payload and header bytes where measurable;
+* client, proxy, and server CPU;
+* peak client/server memory;
+* error, retry, timeout, and cancellation counts;
+* cache hits and misses.
+
+Run warmups and repeated trials and report distributions, not a single best run. Do
+not state expected multipliers in advance.
+
+---
+
+## 6. Implementation Phases
+
+### Phase 0 — Inventory and reproducible protocol fixtures
+
+Deliverables:
+
+* exact inventory of synchronous/asynchronous clients in both repositories;
+* local HTTP/1.1 fixture;
+* local TLS HTTP/2 reverse-proxy fixture;
+* protocol assertion helper;
+* documentation of the production topology relevant to HTTP/2;
+* basic connection/protocol logging available to benchmarks.
+
+Exit criterion: a test proves HTTP/1.1 locally under Uvicorn and HTTP/2 locally
+through the selected fixture, without disabling certificate verification.
+
+### Phase 1 — Baseline benchmark suite
+
+Implement the fair comparison matrix for peer reads and direct HTTPS range reads
+before changing concurrency or transport behavior.
+
+Exit criterion: repeatable results identify whether the bottleneck is RTT,
+connection setup, transfer bandwidth, decompression, server work, cache locking, or
+another component.
+
+### Phase 2 — Bounded peer-cache concurrency
+
+Reuse the existing `C2Array.aget_chunk`/`Proxy.afetch` path. Add or normalize:
+
+* explicit HTTP/2 enablement where absent;
+* bounded concurrency;
+* lifecycle-managed persistent clients;
+* cancellation, retry, and partial-cache behavior;
+* tests under both HTTP/1.1 and HTTP/2.
+
+Exit criterion: correctness tests pass for failures and cancellation, and benchmarks
+show a worthwhile improvement without unacceptable memory or server-load growth.
+
+### Phase 3 — Object-store range concurrency
+
+Implement bounded concurrent HTTPS range reads in `python-blosc2`, with HTTP/1.1
+fallback and per-endpoint protocol reporting.
+
+Exit criterion: functionality is transport-independent, service compatibility is
+documented, and any claimed HTTP/2 benefit is demonstrated for the named endpoint.
+
+### Phase 4 — Batch/async chunk ingestion
+
+Design and implement `afill_chunks` (final name subject to API review), preserving
+the atomic per-slot behavior of `fill_chunk` and reporting partial results clearly.
+
+Exit criterion: duplicate writes, partial failures, cancellation, retries, and
+bounded memory are tested under HTTP/1.1 and HTTP/2.
+
+### Phase 5 — Flow-control investigation, only if needed
+
+Capture stream and connection window behavior for large chunks. Determine whether
+flow control is demonstrably limiting throughput and whether the selected HTTPX,
+HTTP/2 server, and reverse proxy expose supported tuning controls.
+
+The HTTP/2 initial stream window is 65,535 bytes, but this alone does not imply an
+RTT stall every 64 KiB: receivers normally issue window updates as data is consumed.
+Connection-level and stream-level flow control must be analyzed separately.
+
+Exit criterion: tune windows only when traces and benchmarks show a bottleneck.
+Otherwise close this phase with documented evidence and no configuration change.
+
+### Phase 6 — Rollout and operational guidance
+
+* document Uvicorn-only and reverse-proxy deployments;
+* expose conservative concurrency settings;
+* add protocol/concurrency/cache observability;
+* stage rollout with HTTP/1.1 fallback;
+* retain an easy configuration switch to disable HTTP/2 if interoperability issues
+  occur.
+
+---
+
+## 7. Correctness and Safety Test Matrix
+
+Every concurrent implementation must cover:
+
+* out-of-order completion;
+* one transient failure among successful chunks;
+* persistent transport failure;
+* deliberate HTTP 4xx/5xx response;
+* timeout while other streams continue;
+* caller cancellation;
+* server disconnect;
+* duplicate chunk write;
+* incomplete trailing chunk;
+* cold, partially warm, and fully warm sparse cache;
+* client shutdown with requests in flight;
+* HTTP/1.1 fallback;
+* Pyodide/Emscripten behavior where the synchronous Caterva2 client intentionally
+  avoids HTTP/2.
+
+Concurrency settings must have documented upper bounds. Tests should demonstrate
+that peak outstanding requests and buffered response memory stay within them.
+
+---
+
+## 8. Decision Criteria
+
+Adopt HTTP/2 by default for a path only when:
+
+1. it negotiates reliably in the supported deployment;
+2. equal-concurrency benchmarks show a meaningful benefit or a clear reduction in
+   connection cost;
+3. memory, CPU, and tail latency remain acceptable;
+4. cancellation, retries, and HTTP/1.1 fallback are correct;
+5. operational complexity is documented and justified.
+
+If HTTP/1.1 pooling performs equally well, keep the concurrency improvements and do
+not force HTTP/2. If one HTTP/2 connection becomes a bottleneck, test a small bounded
+number of connections rather than assuming that one connection is universally
+optimal.
+
+---
+
+## 9. Open Questions for Discussion
+
+1. Should the reproducible local fixture use Caddy, nginx, Hypercorn, or more than
+   one of these?
+2. What conservative default and maximum should `max_concurrency` use for peer
+   reads, object ranges, and writes?
+3. Should HTTP clients live per peer, per origin, or per application lifespan, and
+   how will shutdown be coordinated?
+4. Which object-store endpoint and authentication modes are officially supported?
+5. Should batch ingestion fail fast or return one result/error per chunk by default?
+6. What measured improvement is sufficient to justify enabling HTTP/2 by default?
+7. Which proxy/server metrics can be collected in CI, and which require a separate
+   performance environment?
+
+---
+
+## 10. Preliminary Findings and Branch Decision (2026-09-02)
+
+### What was measured
+
+The experimental harness under `examples/benchmarks/http2/` verifies the negotiated
+protocol and compares HTTP/1.1 and HTTP/2 while holding application behavior and
+concurrency constant. Detailed commands and results are recorded alongside it.
+
+Three classes of experiment were performed:
+
+1. **Concurrent chunk reads from `cat2.cloud`:** HTTP/2 was approximately tied with
+   HTTP/1.1 at concurrency 4 and slower at concurrency 8 and 16 for the tested
+   10-chunk dataset.
+2. **Local peer reads with 50 ms simulated RTT:** with 64 chunks and concurrency 8,
+   HTTP/2 was about 6% slower for 64 KiB chunks and 13% slower for 512 KiB chunks.
+3. **One large server-side `api/fetch`:** against `cat2.cloud`, HTTP/2 was much more
+   stable and faster than forced HTTP/1.1 for a 26.8 MB slice. The corresponding
+   local Caddy test with 50 ms simulated RTT produced the opposite result, with
+   HTTP/2 about 66% slower.
+
+### Conclusions
+
+* HTTP/2 is not intrinsically faster for Caterva2 traffic. Workload, real network
+  behavior, TLS termination, reverse-proxy configuration, and TCP implementation
+  materially affect the result.
+* For independent chunk reads, several pooled HTTP/1.1 connections can outperform
+  several HTTP/2 streams sharing one TCP connection.
+* The strong single-slice HTTP/2 result on `cat2.cloud` appears deployment-specific.
+  The local latency fixture cannot reproduce it and therefore cannot be used to
+  choose production transport defaults.
+* Local Caddy/Toxiproxy tests remain useful for protocol correctness, fallback,
+  controlled comparisons, and regressions. They are not a substitute for tests
+  between real remote Caterva2 deployments.
+* A decision about peer transport performance requires at least two controlled real
+  remote servers, verified protocols, repeated interleaved trials, and both chunked
+  and single-response workloads.
+
+### Decision for this branch
+
+* Keep the existing `caterva2.Client(http2=True)` behavior. It predates this work,
+  falls back to HTTP/1.1 when necessary, and the deployed single-slice workload
+  provides evidence in its favor.
+* Do **not** change `c2cache` peer clients to `http2=True` by default. Existing peer
+  behavior remains HTTP/1.1 until representative remote-peer measurements justify a
+  change.
+* Keep the benchmark scripts, Caddy fixture, Toxiproxy simulation, protocol
+  assertions, and recorded results from this branch.
+* Stop flow-control tuning and further local performance measurements for now.
+* Resume performance work only when controlled real remote Caterva2 servers are
+  available. At that point HTTP/2 should remain a measured per-path choice rather
+  than a project-wide assumption.

From 211e1bc13646d5e2552c46b1d67abafca4ae753d Mon Sep 17 00:00:00 2001
From: Francesc Alted <francesc@blosc.org>
Date: Thu, 3 Sep 2026 07:33:27 +0200
Subject: [PATCH 02/11] Add --slice to example

---
 examples/benchmarks/http2/peer_read.py | 32 ++++++++++++++++++++++++--
 1 file changed, 30 insertions(+), 2 deletions(-)

diff --git a/examples/benchmarks/http2/peer_read.py b/examples/benchmarks/http2/peer_read.py
index de58c463..5e1f22f5 100644
--- a/examples/benchmarks/http2/peer_read.py
+++ b/examples/benchmarks/http2/peer_read.py
@@ -38,12 +38,35 @@ async def assert_protocol(urlbase: str, expected: str, *, http2: bool) -> None:
         )
 
 
-async def cold_read(urlbase: str, path: str, concurrency: int, cache: pathlib.Path, *, http2: bool) -> float:
+def parse_slice_tuple(slice_str: str | None) -> tuple[slice, ...] | None:
+    if not slice_str:
+        return None
+    parts = []
+    for s in slice_str.split(","):
+        s = s.strip()
+        if ":" in s:
+            start, stop = (int(x.strip()) if x.strip() else None for x in s.split(":", 1))
+            parts.append(slice(start, stop))
+        else:
+            v = int(s)
+            parts.append(slice(v, v + 1))
+    return tuple(parts)
+
+
+async def cold_read(
+    urlbase: str,
+    path: str,
+    concurrency: int,
+    cache: pathlib.Path,
+    *,
+    http2: bool,
+    slice_: tuple[slice, ...] | None = None,
+) -> float:
     source = BenchmarkRemoteSource(path, urlbase=urlbase, http2=http2)
     proxy = blosc2.Proxy(source, urlpath=str(cache), mode="w")
     try:
         started = time.perf_counter()
-        await proxy.afetch(None, max_concurrency=concurrency)
+        await proxy.afetch(slice_, max_concurrency=concurrency)
         return time.perf_counter() - started
     finally:
         # Proxy does not own the remote source's HTTP client.
@@ -70,6 +93,7 @@ async def main(args: argparse.Namespace) -> None:
         "http1": (args.http1_url, False),
         "http2": (args.http2_url, True),
     }
+    slice_tuple = parse_slice_tuple(args.slice)
     with tempfile.TemporaryDirectory(prefix="caterva2-http-benchmark-") as tmp:
         tmpdir = pathlib.Path(tmp)
         for trial in range(args.repeat):
@@ -82,6 +106,7 @@ async def main(args: argparse.Namespace) -> None:
                     args.concurrency,
                     tmpdir / f"{label}-trial-{trial}.b2nd",
                     http2=use_http2,
+                    slice_=slice_tuple,
                 )
                 samples[label].append(sample)
                 print(f"{label} trial={trial + 1} seconds={sample:.6f}")
@@ -96,6 +121,9 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--http1-url", required=True)
     parser.add_argument("--http2-url", required=True)
     parser.add_argument("--path", required=True, help="remote dataset path, e.g. @public/example.b2nd")
+    parser.add_argument(
+        "--slice", default=None, help="optional slice string, e.g. '9500:10500, 9500:10500, 9500:10500'"
+    )
     parser.add_argument("--concurrency", type=int, default=4)
     parser.add_argument("--repeat", type=int, default=5)
     args = parser.parse_args()

From d50ffad072e23aae8534bbae91592a605b217198 Mon Sep 17 00:00:00 2001
From: Francesc Alted <francesc@blosc.org>
Date: Thu, 3 Sep 2026 07:34:31 +0200
Subject: [PATCH 03/11] Update plan with latest measurements

---
 plans/http2-optim2.md | 95 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 95 insertions(+)

diff --git a/plans/http2-optim2.md b/plans/http2-optim2.md
index 8b174382..518b2fed 100644
--- a/plans/http2-optim2.md
+++ b/plans/http2-optim2.md
@@ -434,3 +434,98 @@ Three classes of experiment were performed:
 * Resume performance work only when controlled real remote Caterva2 servers are
   available. At that point HTTP/2 should remain a measured per-path choice rather
   than a project-wide assumption.
+
+---
+
+## 11. Remote Server Audit and Definitive Findings (2026-09-03)
+
+### 11.1 Local Simulation vs. Production Network Ground Truth
+
+Further evaluation confirmed that local loopback simulation (Caddy + Toxiproxy) is not
+suitable for choosing production transport defaults:
+* User-space latency injection (Toxiproxy) buffers and delays bytes in Go user-space,
+  distorting real kernel TCP congestion control (CWND ramp-up, BDP, ACK pacing, packet loss).
+* Loopback bandwidth is effectively infinite, which disproportionately amplifies
+  client-side pure-Python CPU overhead (`httpx` / `h2` frame parsing) and misrepresents
+  WAN network bottlenecks.
+* Production validation against the live remote server (`https://cat2.cloud/demo`) was
+  selected as the authoritative environment for performance decisions.
+
+### 11.2 Root Cause of the 2026-09-02 Single-Slice Anomaly
+
+On 2026-09-02, HTTP/1.1 appeared ~4.3x slower than HTTP/2 on `cat2.cloud` for single
+large `api/fetch` responses (1.361 s vs 0.318 s). An audit of the production Nginx
+reverse proxy revealed two major configuration bottlenecks:
+
+1. **Missing Upstream Keepalives:** Nginx defaulted to HTTP/1.0 with `Connection: close`
+   toward Uvicorn, tearing down and recreating the Unix domain socket on every request.
+2. **Buffer Overflow and Disk Spooling:** Nginx default proxy buffers were only 32 KB
+   total (`proxy_buffers 8 4k`). When Uvicorn returned the 2.68 MB compressed slice,
+   the tiny RAM buffer filled in microseconds and Nginx spooled the response to temporary
+   files on disk (`/var/lib/nginx/proxy/...`), introducing disk I/O latency and locks.
+
+The Nginx deployment was updated with:
+* **Upstream keepalive pool:**
+  ```nginx
+  upstream demo {
+      server unix:/home/demo/caterva2-deploy/_caterva2/state/uvicorn.socket;
+      keepalive 32;
+  }
+  ```
+* **Streaming RAM buffers (2 MB pool, zero disk spooling):**
+  ```nginx
+  proxy_http_version 1.1;
+  proxy_set_header Connection "";
+  proxy_buffering on;
+  proxy_buffer_size 128k;
+  proxy_buffers 16 128k;
+  proxy_busy_buffers_size 256k;
+  ```
+* **TCP stack optimizations:** `tcp_nopush on;` and `tcp_nodelay on;`.
+
+**Result:** Single-slice HTTP/1.1 network response time dropped from **1.361 s to 0.226 s**
+(a 6x improvement), completely eliminating the previous gap and tying with HTTP/2 (0.242 s).
+Both protocols transfer single slices at the physical bandwidth limit of the WAN/Wi-Fi link.
+
+### 11.3 Multi-Chunk Peer-Read Benchmark on `gaia-3d.b2nd`
+
+To test concurrent chunk retrieval on a representative large-scale dataset,
+`examples/benchmarks/http2/peer_read.py` was extended to support arbitrary `--slice`
+ranges. Slicing `@public/large/gaia-3d.b2nd` at `(9500:10500, 9500:10500, 9500:10500)`
+retrieved 64 independent chunks (~20–50 KiB compressed each).
+
+Measurements across concurrency levels against the tuned `cat2.cloud` server (median of
+alternating trials):
+
+| Max Concurrency | HTTP/1.1 Median | HTTP/2 Median | Ratio (H1 / H2) | Winner |
+|---:|---:|---:|---:|---|
+| **1 (serial)** | 4.652 s | 4.647 s | 1.001 | Dead tie |
+| **4** | 1.367 s | 1.433 s | 0.954 | HTTP/1.1 (~5% faster) |
+| **8** | 0.838 s | 0.970 s | 0.863 | **HTTP/1.1 (~15% faster)** |
+| **16** | 0.612 s | 0.818 s | 0.748 | **HTTP/1.1 (~30% faster)** |
+
+#### Physical Mechanisms:
+* **Multiple TCP Congestion Windows:** At concurrency 16, pooled HTTP/1.1 opens 16
+  independent TCP connections, each with its own kernel TCP congestion window,
+  saturating WAN bandwidth in parallel. HTTP/2 forces all 16 streams through a single
+  TCP connection and a single congestion window.
+* **Resilience to Packet Loss:** Packet drops on real WANs cause TCP Head-of-Line
+  blocking across all HTTP/2 multiplexed streams on that connection, whereas pooled
+  HTTP/1.1 connections continue uninterrupted.
+* **Client CPU Overhead:** Demultiplexing binary chunk frames across 16 active streams
+  in pure Python (`h2`) incurs noticeable CPU overhead compared to streaming raw socket
+  bytes in HTTP/1.1.
+
+### 11.4 Final Transport Architecture Decision
+
+1. **`caterva2.Client`:** Retain `http2=True`. For interactive users and notebook
+   sessions issuing single queries/slices, HTTP/2 matches HTTP/1.1 throughput (~0.22 s)
+   while protecting public servers from TCP socket exhaustion and supporting `RST_STREAM`
+   stream cancellation.
+2. **`caterva2.c2cache` & `python-blosc2` (`C2Array.aget_chunk`):** Retain pooled HTTP/1.1
+   (`http2=False`). For bulk concurrent chunk downloads, connection pooling consistently
+   outperforms HTTP/2 multiplexing by 15% to 30%. No code changes are required in either
+   repository.
+3. **Production Reverse-Proxy Profile:** Document the Nginx upstream keepalive (`keepalive 32;`)
+   and in-memory streaming buffer directives (`proxy_buffers 16 128k;`) as standard
+   operational requirements for Caterva2 reverse-proxy deployments.

From 779ff5a2e6c65724c0d9216629003cff69124cb7 Mon Sep 17 00:00:00 2001
From: Francesc Alted <francesc@blosc.org>
Date: Thu, 3 Sep 2026 07:37:53 +0200
Subject: [PATCH 04/11] Add AGENTS.md with repository rules for trailing
 newlines and pre-commit

---
 AGENTS.md | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 AGENTS.md

diff --git a/AGENTS.md b/AGENTS.md
new file mode 100644
index 00000000..1ed207b9
--- /dev/null
+++ b/AGENTS.md
@@ -0,0 +1,5 @@
+# Agent Guidelines for Caterva2
+
+- **Trailing Newline**: Always ensure every created or modified file ends with a single trailing newline (`\n`) to satisfy pre-commit's `end-of-file-fixer`.
+- **Pre-commit Compliance**: Ensure all code changes adhere to repository pre-commit hooks (formatting, linting, and whitespace).
+- **Documentation**: Maintain documentation integrity, preserving existing comments and docstrings unless explicitly directed otherwise.

From 6a2901edff4723f58b3a64c7be251a165b627d4c Mon Sep 17 00:00:00 2001
From: Francesc Alted <francesc@blosc.org>
Date: Thu, 3 Sep 2026 11:46:59 +0200
Subject: [PATCH 05/11] Do not re-create an existing hdf5 file

---
 caterva2/tests/test_hdf5_tree.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/caterva2/tests/test_hdf5_tree.py b/caterva2/tests/test_hdf5_tree.py
index 7ff0fdad..ec1c934c 100644
--- a/caterva2/tests/test_hdf5_tree.py
+++ b/caterva2/tests/test_hdf5_tree.py
@@ -44,7 +44,9 @@ def fill_h5_public(client):
     dest_dir = pathlib.Path(TEST_STATE_DIR) / "server/public"
     dest_dir.mkdir(parents=True, exist_ok=True)
     fname = "test_tree.h5"
-    _make_h5(dest_dir / fname)
+    target = dest_dir / fname
+    if not target.exists():
+        _make_h5(target)
     return fname, client.get(TEST_CATERVA2_ROOT)
 
 

From 02e8bbd285e8ec198bfc1abbe3a365907a97ac0b Mon Sep 17 00:00:00 2001
From: Francesc Alted <francesc@blosc.org>
Date: Thu, 3 Sep 2026 11:55:18 +0200
Subject: [PATCH 06/11] Switch from Ubicloud to GitHub Action for arm64

---
 .github/workflows/python-app-arm64.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/python-app-arm64.yml b/.github/workflows/python-app-arm64.yml
index 46ae448f..a9c1759a 100644
--- a/.github/workflows/python-app-arm64.yml
+++ b/.github/workflows/python-app-arm64.yml
@@ -1,4 +1,4 @@
-# This workflow will run on an external ubicloud arm64 runner on latest ubuntu (ubicloud standard)
+# This workflow will run on a GitHub-hosted arm64 runner on Ubuntu 24.04
 
 name: Python application (ubuntu, arm64)
 
@@ -13,15 +13,15 @@ on:
 permissions:
   contents: read
 
-# One run per branch: pushing again supersedes the run already going, which
-# matters most on the paid arm64 runner.  A run for `main` is never cancelled.
+# One run per branch: pushing again supersedes the run already going.
+# A run for `main` is never cancelled.
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 
 jobs:
   build:
-    runs-on: ubicloud-standard-2-arm
+    runs-on: ubuntu-24.04-arm
 
     steps:
     - uses: actions/checkout@v5

From 1677af3f094aa4f9ae5b44270b56cbd91e7a1dce Mon Sep 17 00:00:00 2001
From: Francesc Alted <francesc@blosc.org>
Date: Wed, 2 Sep 2026 19:27:58 +0200
Subject: [PATCH 07/11] Add HTTP/2 transport benchmarks and findings

  Add Caddy and Toxiproxy fixtures for comparing HTTP/1.1 and HTTP/2
  across peer reads and server-side slices. Verify negotiated protocols,
  record local and cat2.cloud results, and document the decision to retain
  HTTP/2 for Client requests without enabling it for peer chunk traffic.
---
 caterva2/tests/test_peers.py                  |  54 +++
 examples/benchmarks/http2/Caddyfile           |   9 +
 examples/benchmarks/http2/README.md           |  94 ++++
 examples/benchmarks/http2/peer_read.py        | 108 +++++
 .../benchmarks/http2/results-2026-09-02.md    | 114 +++++
 .../benchmarks/http2/simulated_latency.py     | 184 ++++++++
 examples/benchmarks/http2/single_fetch.py     | 115 +++++
 plans/http2-optim2.md                         | 436 ++++++++++++++++++
 8 files changed, 1114 insertions(+)
 create mode 100644 examples/benchmarks/http2/Caddyfile
 create mode 100644 examples/benchmarks/http2/README.md
 create mode 100644 examples/benchmarks/http2/peer_read.py
 create mode 100644 examples/benchmarks/http2/results-2026-09-02.md
 create mode 100644 examples/benchmarks/http2/simulated_latency.py
 create mode 100644 examples/benchmarks/http2/single_fetch.py
 create mode 100644 plans/http2-optim2.md

diff --git a/caterva2/tests/test_peers.py b/caterva2/tests/test_peers.py
index 9da03121..26896dfb 100644
--- a/caterva2/tests/test_peers.py
+++ b/caterva2/tests/test_peers.py
@@ -9,9 +9,11 @@
 import asyncio
 import json
 import os
+import pathlib
 import shutil
 import signal
 import socket
+import ssl
 import subprocess
 import sys
 import time
@@ -1020,6 +1022,58 @@ async def afetch(self, slice_, **kwargs):
     assert len(proxy.calls) == 2
 
 
+def test_caddy_fixture_negotiates_http2(tmp_path):
+    """The optional local fixture is real TLS+h2, with direct Uvicorn as h1."""
+    caddy = shutil.which("caddy")
+    if caddy is None:
+        pytest.skip("Caddy is not installed")
+
+    upstream_port, h2_port = _unused_tcp_ports(2)
+    server_dir = tmp_path / "server"
+    (server_dir / "public").mkdir(parents=True)
+    server = _start(server_dir, upstream_port)
+    env = dict(
+        os.environ,
+        CATERVA2_UPSTREAM=f"127.0.0.1:{upstream_port}",
+        CATERVA2_H2_ADDRESS=f"localhost:{h2_port}",
+        XDG_DATA_HOME=str(tmp_path / "caddy-data"),
+        XDG_CONFIG_HOME=str(tmp_path / "caddy-config"),
+    )
+    caddyfile = pathlib.Path(__file__).parents[2] / "examples" / "benchmarks" / "http2" / "Caddyfile"
+    proxy = subprocess.Popen(
+        [caddy, "run", "--config", str(caddyfile)],
+        env=env,
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    )
+    try:
+        root_ca = tmp_path / "caddy-data" / "caddy" / "pki" / "authorities" / "local" / "root.crt"
+        for _ in range(50):
+            if proxy.poll() is not None:
+                raise RuntimeError(f"Caddy exited during startup with status {proxy.returncode}")
+            if root_ca.exists():
+                try:
+                    ssl_context = ssl.create_default_context(cafile=str(root_ca))
+                    with httpx.Client(http2=True, verify=ssl_context, timeout=1) as client:
+                        response = client.get(f"https://localhost:{h2_port}/api/roots")
+                    if response.is_success:
+                        break
+                except httpx.TransportError:
+                    pass
+            time.sleep(0.1)
+        else:
+            raise RuntimeError("Caddy HTTP/2 fixture did not start")
+
+        assert response.http_version == "HTTP/2"
+        direct = httpx.get(f"http://127.0.0.1:{upstream_port}/api/roots", timeout=1)
+        assert direct.http_version == "HTTP/1.1"
+    finally:
+        proxy.send_signal(signal.SIGTERM)
+        proxy.wait(timeout=10)
+        server.send_signal(signal.SIGTERM)
+        server.wait(timeout=10)
+
+
 def test_concurrent_fetches_of_different_datasets_dont_serialize(two_dataset_peers):
     """Interleaved concurrent fetches of two different datasets under a tiny
     shared quota: correctness under load is the regression net (per
diff --git a/examples/benchmarks/http2/Caddyfile b/examples/benchmarks/http2/Caddyfile
new file mode 100644
index 00000000..1f1b8022
--- /dev/null
+++ b/examples/benchmarks/http2/Caddyfile
@@ -0,0 +1,9 @@
+{
+	# The local CA is deliberate: this fixture tests real TLS ALPN, not h2c.
+	local_certs
+}
+
+{$CATERVA2_H2_ADDRESS:localhost:8443} {
+	tls internal
+	reverse_proxy {$CATERVA2_UPSTREAM:127.0.0.1:8000}
+}
diff --git a/examples/benchmarks/http2/README.md b/examples/benchmarks/http2/README.md
new file mode 100644
index 00000000..4905df98
--- /dev/null
+++ b/examples/benchmarks/http2/README.md
@@ -0,0 +1,94 @@
+# Local HTTP/2 peer-read fixture
+
+This fixture puts Caddy in front of an ordinary Uvicorn Caterva2 server:
+
+```text
+benchmark --HTTPS/HTTP2--> Caddy --HTTP/1.1--> Uvicorn
+```
+
+It deliberately keeps direct Uvicorn access available as the HTTP/1.1 control.
+Caddy is optional and is not a Caterva2 dependency.
+
+## Start the fixture
+
+Start a Caterva2 server on `127.0.0.1:8000`, with a multi-chunk dataset in its
+`@public` root. Then, from the repository root, run:
+
+```console
+caddy run --config examples/benchmarks/http2/Caddyfile
+```
+
+Caddy's local CA must be trusted by HTTPX. `caddy trust` installs it in the local
+trust store on supported systems. Alternatively, point `SSL_CERT_FILE` at the
+exported Caddy root certificate. Do not disable certificate verification: doing
+so would make this fixture less representative and could hide configuration
+mistakes.
+
+Check both paths before benchmarking:
+
+```console
+curl -sS -o /dev/null -w '%{http_version}\n' \
+  http://127.0.0.1:8000/api/roots
+curl -sS -o /dev/null -w '%{http_version}\n' \
+  https://localhost:8443/api/roots
+```
+
+The expected outputs are `1.1` and `2`, respectively.
+
+## Run the benchmark
+
+Both URLs must address the same Caterva2 server and dataset:
+
+```console
+python examples/benchmarks/http2/peer_read.py \
+  --http1-url http://127.0.0.1:8000 \
+  --http2-url https://localhost:8443 \
+  --path @public/example.b2nd \
+  --concurrency 4 \
+  --repeat 5
+```
+
+The script verifies the negotiated protocol before collecting timings. Each trial
+uses a new sparse cache, so the timed operation is a cold peer read. HTTP/1.1 and
+HTTP/2 use the same `RemoteSource` and `Proxy.afetch` implementation and the same
+concurrency.
+
+Run several times and compare the distributions. Loopback is useful for correctness
+and protocol verification, but meaningful latency benefits require a controlled WAN
+test or a genuinely remote peer.
+
+## Simulated latency
+
+With Caddy and Toxiproxy installed, the self-contained benchmark creates a temporary
+64-chunk dataset and applies half the requested latency in each TCP direction:
+
+```console
+python examples/benchmarks/http2/simulated_latency.py \
+  --rtt-ms 50 --concurrency 8 --repeat 5
+```
+
+Both protocols traverse the same Toxiproxy listener, TLS connection, Caddy instance,
+and Uvicorn process. The only difference is whether HTTPX offers HTTP/2 during ALPN.
+Temporary servers, proxies, certificates, data, and caches are removed after the run.
+
+The same fixture can reproduce the single-large-response workload with approximately
+the same 26.8 MB materialized slice as `get-slice.py`:
+
+```console
+python examples/benchmarks/http2/simulated_latency.py \
+  --workload single-fetch --rtt-ms 50 \
+  --chunks 10 --items-per-chunk 838860 --slice 5:9 \
+  --repeat 12 --pause 0.25
+```
+
+## Single server-side slice
+
+To compare the one-response workload in `examples/get-slice.py`, while separating
+network transfer from cframe parsing and NumPy materialization:
+
+```console
+python examples/benchmarks/http2/single_fetch.py --repeat 20
+```
+
+Both persistent clients connect to the same endpoint. One forces HTTP/1.1 and the
+other offers HTTP/2; every response is checked against the expected protocol.
diff --git a/examples/benchmarks/http2/peer_read.py b/examples/benchmarks/http2/peer_read.py
new file mode 100644
index 00000000..de58c463
--- /dev/null
+++ b/examples/benchmarks/http2/peer_read.py
@@ -0,0 +1,108 @@
+"""Compare cold peer reads over pooled HTTP/1.1 and multiplexed HTTP/2."""
+
+import argparse
+import asyncio
+import pathlib
+import statistics
+import tempfile
+import time
+
+import blosc2
+import httpx
+
+from caterva2.c2cache.remote import RemoteSource
+
+
+class BenchmarkRemoteSource(RemoteSource):
+    """RemoteSource with a selectable transport, confined to this benchmark."""
+
+    def __init__(self, path: str, urlbase: str, *, http2: bool):
+        super().__init__(path, urlbase=urlbase, use_chunk_api=True)
+        self._benchmark_http2 = http2
+
+    async def aget_chunk(self, nchunk: int) -> bytes:
+        if self._aclient is None:
+            self._aclient = httpx.AsyncClient(http2=self._benchmark_http2, timeout=5)
+        return await super().aget_chunk(nchunk)
+
+
+async def assert_protocol(urlbase: str, expected: str, *, http2: bool) -> None:
+    """Fail rather than silently benchmarking HTTPX's HTTP/1.1 fallback."""
+    async with httpx.AsyncClient(http2=http2, follow_redirects=True, timeout=10) as client:
+        response = await client.get(f"{urlbase.rstrip('/')}/api/roots")
+        response.raise_for_status()
+    if response.http_version != expected:
+        raise RuntimeError(
+            f"{urlbase} negotiated {response.http_version}, expected {expected}; "
+            "benchmark result would be invalid"
+        )
+
+
+async def cold_read(urlbase: str, path: str, concurrency: int, cache: pathlib.Path, *, http2: bool) -> float:
+    source = BenchmarkRemoteSource(path, urlbase=urlbase, http2=http2)
+    proxy = blosc2.Proxy(source, urlpath=str(cache), mode="w")
+    try:
+        started = time.perf_counter()
+        await proxy.afetch(None, max_concurrency=concurrency)
+        return time.perf_counter() - started
+    finally:
+        # Proxy does not own the remote source's HTTP client.
+        await source.aclose()
+
+
+def report(label: str, samples: list[float]) -> None:
+    ordered = sorted(samples)
+    p95_index = min(len(ordered) - 1, int(0.95 * len(ordered)))
+    print(
+        f"{label} n={len(samples)} median={statistics.median(samples):.6f}s "
+        f"p95={ordered[p95_index]:.6f}s min={ordered[0]:.6f}s max={ordered[-1]:.6f}s"
+    )
+
+
+async def main(args: argparse.Namespace) -> None:
+    await assert_protocol(args.http1_url, "HTTP/1.1", http2=False)
+    await assert_protocol(args.http2_url, "HTTP/2", http2=True)
+
+    # Alternate order to reduce systematic server/OS-cache and network drift. Every
+    # operation still gets a new sparse cache and HTTP client.
+    samples = {"http1": [], "http2": []}
+    cases = {
+        "http1": (args.http1_url, False),
+        "http2": (args.http2_url, True),
+    }
+    with tempfile.TemporaryDirectory(prefix="caterva2-http-benchmark-") as tmp:
+        tmpdir = pathlib.Path(tmp)
+        for trial in range(args.repeat):
+            order = ("http1", "http2") if trial % 2 == 0 else ("http2", "http1")
+            for label in order:
+                urlbase, use_http2 = cases[label]
+                sample = await cold_read(
+                    urlbase,
+                    args.path,
+                    args.concurrency,
+                    tmpdir / f"{label}-trial-{trial}.b2nd",
+                    http2=use_http2,
+                )
+                samples[label].append(sample)
+                print(f"{label} trial={trial + 1} seconds={sample:.6f}")
+    http1, http2 = samples["http1"], samples["http2"]
+    report("http1", http1)
+    report("http2", http2)
+    print(f"median_ratio_http1_over_http2={statistics.median(http1) / statistics.median(http2):.3f}")
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--http1-url", required=True)
+    parser.add_argument("--http2-url", required=True)
+    parser.add_argument("--path", required=True, help="remote dataset path, e.g. @public/example.b2nd")
+    parser.add_argument("--concurrency", type=int, default=4)
+    parser.add_argument("--repeat", type=int, default=5)
+    args = parser.parse_args()
+    if args.concurrency < 1 or args.repeat < 1:
+        parser.error("--concurrency and --repeat must be positive")
+    return args
+
+
+if __name__ == "__main__":
+    asyncio.run(main(parse_args()))
diff --git a/examples/benchmarks/http2/results-2026-09-02.md b/examples/benchmarks/http2/results-2026-09-02.md
new file mode 100644
index 00000000..bf56306b
--- /dev/null
+++ b/examples/benchmarks/http2/results-2026-09-02.md
@@ -0,0 +1,114 @@
+# Preliminary `cat2.cloud` peer-read results — 2026-09-02
+
+These are exploratory measurements, not release criteria. They compare the same
+`RemoteSource` + `Proxy.afetch` cold-cache operation against the same deployed
+endpoint, forcing HTTPX to use HTTP/1.1 or offer HTTP/2. The harness asserted that
+the negotiated protocols were `HTTP/1.1` and `HTTP/2`, respectively.
+
+Dataset: `@public/examples/numbers_color.b2nd` at `https://cat2.cloud/demo`
+
+Shape: `(10, 368, 744, 4)`
+
+Chunk shape: `(1, 368, 744, 4)` (10 chunks)
+
+Trials: 5 per protocol and concurrency, alternating protocol order
+
+| Max concurrency | HTTP/1.1 median | HTTP/2 median | HTTP/1.1 / HTTP/2 |
+|---:|---:|---:|---:|
+| 1 | 1.037 s | 0.984 s | 1.055 |
+| 4 | 0.555 s | 0.557 s | 0.996 |
+| 8 | 0.479 s | 0.563 s | 0.850 |
+| 16 | 0.402 s | 0.538 s | 0.747 |
+
+On this workload HTTP/2 did not improve the useful concurrent cases. It was about
+5% faster for serial reads, effectively tied at concurrency 4, and slower at 8 and
+16. Both protocols showed occasional network outliers across exploratory runs.
+
+Likely explanations to investigate include HTTP/2 implementation overhead for only
+10 moderate-sized chunks, server/proxy scheduling, and the ability of several
+HTTP/1.1 TCP connections to transfer large chunks in parallel. These measurements
+do not establish behavior for a higher-RTT peer, more chunks, smaller ranges, or a
+controlled network.
+
+Next useful experiment: run a larger multi-chunk dataset through the local Caddy
+fixture under controlled latency and bandwidth, with more repetitions and connection
+counts. Until then, enabling HTTP/2 remains safe protocol negotiation with HTTP/1.1
+fallback, not a demonstrated peer-read optimization.
+
+## Controlled 50 ms RTT simulation
+
+The self-contained Toxiproxy fixture applied 25 ms in each TCP direction. Both
+protocols used the same TLS endpoint, Toxiproxy listener, Caddy instance, Uvicorn
+server, generated dataset, trial count, and concurrency. The only changed input was
+HTTPX's HTTP/2 offer. Protocol assertions again verified `HTTP/1.1` and `HTTP/2`.
+
+Dataset: 64 random, effectively incompressible chunks
+
+Concurrency: 8
+
+Trials: 7 per protocol and chunk size, alternating protocol order
+
+| Chunk size | HTTP/1.1 median | HTTP/2 median | HTTP/1.1 / HTTP/2 |
+|---:|---:|---:|---:|
+| 64 KiB | 0.543 s | 0.578 s | 0.939 |
+| 512 KiB | 0.557 s | 0.628 s | 0.887 |
+
+HTTP/2 remained slower: approximately 6% for 64 KiB chunks and 13% for 512 KiB
+chunks. A preliminary version of the fixture routed HTTP/1.1 directly to clear-text
+Uvicorn and was discarded as unfair; the table contains only corrected same-path
+measurements.
+
+This simulation models fixed latency but not constrained bandwidth, packet loss, or
+internet jitter. At equal concurrency, HTTP/1.1 establishes several connections in
+parallel, so handshake latency does not necessarily accumulate serially. HTTP/2's
+lower connection count is still an operational benefit, but no elapsed-time benefit
+has been demonstrated for peer chunk reads.
+
+### Local single-response result
+
+The single-response workload was also reproduced locally through the same Caddy and
+Toxiproxy path at 50 ms RTT. A generated 10-chunk array was sliced at `5:9`, yielding
+26,843,520 materialized bytes. Both persistent clients used the same TLS endpoint;
+only the HTTPX protocol offer differed.
+
+| Metric | HTTP/1.1 median | HTTP/2 median |
+|---|---:|---:|
+| Network response | 0.121 s | 0.202 s |
+| NumPy materialization | 0.010 s | 0.010 s |
+| End to end | 0.132 s | 0.213 s |
+
+HTTP/2 was approximately 66% slower locally. Unlike `cat2.cloud`, neither protocol
+showed large timing outliers. This does not reproduce the deployed HTTP/2 advantage;
+instead, it shows that the production result is specific to the deployed nginx,
+network, or connection behavior rather than an intrinsic advantage for one large
+HTTP response. The local HTTP/2 penalty is consistent with single-connection framing
+or flow-control overhead, but this benchmark does not isolate its cause.
+
+## Single `api/fetch` response
+
+This reproduces `examples/get-slice.py` against
+`@public/examples/lung-jpeg2000_10x.b2nd`, slice `5:9`. Two persistent clients
+connected to the same `cat2.cloud` endpoint; one forced HTTP/1.1 and the other
+offered HTTP/2. Protocol order alternated, requests were spaced by 0.5 seconds,
+and each response's negotiated version was asserted.
+
+The response materializes to 26,846,976 bytes of NumPy data. There were 12 trials
+per protocol.
+
+| Metric | HTTP/1.1 median | HTTP/2 median |
+|---|---:|---:|
+| Buffered network response | 1.361 s | 0.318 s |
+| Cframe parsing | 0.000185 s | 0.000169 s |
+| NumPy materialization | 0.108 s | 0.106 s |
+| End to end | 1.468 s | 0.426 s |
+
+HTTP/1.1 network times ranged from 0.309 to 3.127 seconds and worsened markedly
+during the run. HTTP/2 ranged from 0.245 to 0.635 seconds. The HTTP/1.1 median was
+4.28 times the HTTP/2 median in this sample. Materialization was effectively equal,
+so the difference is in receiving the response rather than decoding it.
+
+This supports the historical observation that HTTP/2 improved `get-slice.py`. It
+does not conflict with the peer-chunk results: this workload is one large response,
+so HTTP/1.1 gains nothing from a pool of parallel connections. The cause of the
+large persistent-HTTP/1.1 degradation remains to be isolated; possible contributors
+include nginx connection handling, intermediary shaping, and transport behavior.
diff --git a/examples/benchmarks/http2/simulated_latency.py b/examples/benchmarks/http2/simulated_latency.py
new file mode 100644
index 00000000..bccad68f
--- /dev/null
+++ b/examples/benchmarks/http2/simulated_latency.py
@@ -0,0 +1,184 @@
+"""Self-contained peer-read benchmark with symmetric TCP latency."""
+
+import argparse
+import asyncio
+import os
+import pathlib
+import shutil
+import signal
+import ssl
+import subprocess
+import tempfile
+import time
+
+import blosc2
+import httpx
+import numpy as np
+import peer_read
+import single_fetch
+
+from caterva2.tests.test_peers import _start, _unused_tcp_ports
+
+
+def wait_for_cli(cli: str, api_url: str, process: subprocess.Popen) -> None:
+    for _ in range(50):
+        if process.poll() is not None:
+            raise RuntimeError(f"Toxiproxy exited during startup with status {process.returncode}")
+        result = subprocess.run(
+            [cli, "--host", api_url, "list"], capture_output=True, check=False, text=True
+        )
+        if result.returncode == 0:
+            return
+        time.sleep(0.1)
+    raise RuntimeError("Toxiproxy API did not start")
+
+
+def toxiproxy(cli: str, api_url: str, *args: str) -> None:
+    subprocess.run([cli, "--host", api_url, *args], check=True, capture_output=True, text=True)
+
+
+async def run(args: argparse.Namespace) -> None:
+    caddy = shutil.which("caddy")
+    toxiproxy_server = shutil.which("toxiproxy-server")
+    toxiproxy_cli = shutil.which("toxiproxy-cli")
+    missing = [name for name, path in (("caddy", caddy), ("toxiproxy", toxiproxy_server)) if not path]
+    if missing:
+        raise RuntimeError(f"missing benchmark tools: {', '.join(missing)}")
+
+    with tempfile.TemporaryDirectory(prefix="caterva2-http2-latency-") as tmp:
+        tmpdir = pathlib.Path(tmp)
+        server_port, caddy_port, delayed_port, toxiproxy_port = _unused_tcp_ports(4)
+        server_dir = tmpdir / "server"
+        public = server_dir / "public"
+        public.mkdir(parents=True)
+        data = np.random.default_rng(42).random((args.chunks, args.items_per_chunk))
+        blosc2.asarray(
+            data,
+            chunks=(1, args.items_per_chunk),
+            blocks=(1, args.items_per_chunk),
+            urlpath=str(public / "latency.b2nd"),
+        )
+
+        server = _start(server_dir, server_port)
+        caddy_env = dict(
+            os.environ,
+            CATERVA2_UPSTREAM=f"127.0.0.1:{server_port}",
+            CATERVA2_H2_ADDRESS=f"localhost:{caddy_port}",
+            XDG_DATA_HOME=str(tmpdir / "caddy-data"),
+            XDG_CONFIG_HOME=str(tmpdir / "caddy-config"),
+        )
+        caddyfile = pathlib.Path(__file__).with_name("Caddyfile")
+        caddy_process = subprocess.Popen(
+            [caddy, "run", "--config", str(caddyfile)],
+            env=caddy_env,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+        toxiproxy_process = subprocess.Popen(
+            [toxiproxy_server, "-host", "127.0.0.1", "-port", str(toxiproxy_port)],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+        try:
+            api_url = f"http://127.0.0.1:{toxiproxy_port}"
+            wait_for_cli(toxiproxy_cli, api_url, toxiproxy_process)
+            one_way_ms = args.rtt_ms / 2
+            name = "peer-tls"
+            toxiproxy(
+                toxiproxy_cli,
+                api_url,
+                "create",
+                "--listen",
+                f"127.0.0.1:{delayed_port}",
+                "--upstream",
+                f"127.0.0.1:{caddy_port}",
+                name,
+            )
+            for direction in ("downstream", "upstream"):
+                direction_flag = "--downstream" if direction == "downstream" else "--upstream"
+                toxiproxy(
+                    toxiproxy_cli,
+                    api_url,
+                    "toxic",
+                    "add",
+                    "--type",
+                    "latency",
+                    "--attribute",
+                    f"latency={one_way_ms:g}",
+                    direction_flag,
+                    name,
+                )
+
+            root_ca = tmpdir / "caddy-data" / "caddy" / "pki" / "authorities" / "local" / "root.crt"
+            for _ in range(50):
+                if caddy_process.poll() is not None:
+                    raise RuntimeError(f"Caddy exited during startup with status {caddy_process.returncode}")
+                if root_ca.exists():
+                    try:
+                        context = ssl.create_default_context(cafile=str(root_ca))
+                        with httpx.Client(http2=True, verify=context, timeout=1) as client:
+                            ready = client.get(f"https://localhost:{caddy_port}/api/roots")
+                        if ready.is_success and ready.http_version == "HTTP/2":
+                            break
+                    except httpx.TransportError:
+                        pass
+                time.sleep(0.1)
+            else:
+                raise RuntimeError("Caddy verified HTTP/2 endpoint did not start")
+
+            os.environ["SSL_CERT_FILE"] = str(root_ca)
+            print(
+                f"dataset_chunks={args.chunks} chunk_bytes={args.items_per_chunk * 8} "
+                f"simulated_rtt_ms={args.rtt_ms:g}"
+            )
+            delayed_url = f"https://localhost:{delayed_port}"
+            if args.workload == "peer-read":
+                await peer_read.main(
+                    argparse.Namespace(
+                        # Same TLS endpoint and proxy path. Only HTTPX's protocol
+                        # offer differs between the two benchmark arms.
+                        http1_url=delayed_url,
+                        http2_url=delayed_url,
+                        path="@public/latency.b2nd",
+                        concurrency=args.concurrency,
+                        repeat=args.repeat,
+                    )
+                )
+            else:
+                single_fetch.main(
+                    argparse.Namespace(
+                        urlbase=delayed_url,
+                        path="@public/latency.b2nd",
+                        slice=args.slice,
+                        repeat=args.repeat,
+                        timeout=30,
+                        pause=args.pause,
+                    )
+                )
+        finally:
+            for process in (toxiproxy_process, caddy_process, server):
+                if process.poll() is None:
+                    process.send_signal(signal.SIGTERM)
+                    process.wait(timeout=10)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--workload", choices=("peer-read", "single-fetch"), default="peer-read")
+    parser.add_argument("--rtt-ms", type=float, default=50)
+    parser.add_argument("--chunks", type=int, default=64)
+    parser.add_argument("--items-per-chunk", type=int, default=65536)
+    parser.add_argument("--concurrency", type=int, default=8)
+    parser.add_argument("--repeat", type=int, default=5)
+    parser.add_argument("--slice", default="5:9", help="slice used by the single-fetch workload")
+    parser.add_argument("--pause", type=float, default=0.25)
+    args = parser.parse_args()
+    if min(args.rtt_ms, args.chunks, args.items_per_chunk, args.concurrency, args.repeat) <= 0:
+        parser.error("all numeric arguments must be positive")
+    if args.pause < 0:
+        parser.error("--pause cannot be negative")
+    return args
+
+
+if __name__ == "__main__":
+    asyncio.run(run(parse_args()))
diff --git a/examples/benchmarks/http2/single_fetch.py b/examples/benchmarks/http2/single_fetch.py
new file mode 100644
index 00000000..e9f3272e
--- /dev/null
+++ b/examples/benchmarks/http2/single_fetch.py
@@ -0,0 +1,115 @@
+"""Compare one large api/fetch response over persistent HTTP/1.1 and HTTP/2."""
+
+import argparse
+import statistics
+import time
+
+import blosc2
+import httpx
+
+
+def percentile(samples: list[float], fraction: float) -> float:
+    ordered = sorted(samples)
+    return ordered[min(len(ordered) - 1, int(fraction * len(ordered)))]
+
+
+def fetch_once(
+    client: httpx.Client, url: str, slice_: str, expected: str
+) -> tuple[float, float, float | None, int | None]:
+    started = time.perf_counter()
+    response = client.get(url, params={"slice_": slice_})
+    network = time.perf_counter() - started
+    response.raise_for_status()
+    if response.http_version != expected:
+        raise RuntimeError(f"negotiated {response.http_version}, expected {expected}")
+
+    started = time.perf_counter()
+    array = blosc2.ndarray_from_cframe(response.content)
+    parse = time.perf_counter() - started
+
+    try:
+        started = time.perf_counter()
+        numpy_array = array[:]
+        materialize = time.perf_counter() - started
+    except RuntimeError:
+        # Network and cframe timings remain useful when the local environment
+        # lacks the codec needed to materialize this particular dataset.
+        return network, parse, None, None
+    return network, parse, materialize, numpy_array.nbytes
+
+
+def report(label: str, samples: list[tuple[float, float, float | None, int | None]]) -> None:
+    print(f"{label} trials={len(samples)} numpy_bytes={samples[0][3]}")
+    for index, metric in enumerate(("network", "parse", "numpy")):
+        values = [sample[index] for sample in samples if sample[index] is not None]
+        if not values:
+            print(f"  {metric}: unavailable (local codec could not materialize the cframe)")
+            continue
+        print(
+            f"  {metric}: median={statistics.median(values):.6f}s "
+            f"p95={percentile(values, 0.95):.6f}s min={min(values):.6f}s max={max(values):.6f}s"
+        )
+    totals = [sample[0] + sample[1] + sample[2] for sample in samples if sample[2] is not None]
+    if totals:
+        print(
+            f"  end_to_end: median={statistics.median(totals):.6f}s "
+            f"p95={percentile(totals, 0.95):.6f}s min={min(totals):.6f}s max={max(totals):.6f}s"
+        )
+
+
+def main(args: argparse.Namespace) -> None:
+    url = f"{args.urlbase.rstrip('/')}/api/fetch/{args.path}"
+    samples = {"http1": [], "http2": []}
+    cases = {
+        "http1": (httpx.Client(http1=True, http2=False, timeout=args.timeout), "HTTP/1.1"),
+        "http2": (httpx.Client(http1=True, http2=True, timeout=args.timeout), "HTTP/2"),
+    }
+    try:
+        # Establish both TLS connections and populate server/OS caches outside the
+        # samples. The response body is consumed because Client.get buffers it.
+        for label, (client, expected) in cases.items():
+            warm = client.get(url, params={"slice_": args.slice})
+            warm.raise_for_status()
+            if warm.http_version != expected:
+                raise RuntimeError(f"{label} negotiated {warm.http_version}, expected {expected}")
+
+        for trial in range(args.repeat):
+            order = ("http1", "http2") if trial % 2 == 0 else ("http2", "http1")
+            for label in order:
+                client, expected = cases[label]
+                sample = fetch_once(client, url, args.slice, expected)
+                samples[label].append(sample)
+                print(
+                    f"{label} trial={trial + 1} network={sample[0]:.6f}s "
+                    f"parse={sample[1]:.6f}s "
+                    f"numpy={f'{sample[2]:.6f}s' if sample[2] is not None else 'unavailable'}"
+                )
+                if args.pause:
+                    time.sleep(args.pause)
+    finally:
+        for client, _ in cases.values():
+            client.close()
+
+    report("http1", samples["http1"])
+    report("http2", samples["http2"])
+    h1_network = statistics.median(sample[0] for sample in samples["http1"])
+    h2_network = statistics.median(sample[0] for sample in samples["http2"])
+    print(f"median_network_ratio_http1_over_http2={h1_network / h2_network:.3f}")
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--urlbase", default="https://cat2.cloud/demo")
+    parser.add_argument("--path", default="@public/examples/lung-jpeg2000_10x.b2nd")
+    parser.add_argument("--slice", default="5:9")
+    parser.add_argument("--repeat", type=int, default=20)
+    parser.add_argument("--timeout", type=float, default=30)
+    parser.add_argument("--pause", type=float, default=0.25, help="seconds between measured requests")
+    args = parser.parse_args()
+    if args.repeat < 1 or args.timeout <= 0 or args.pause < 0:
+        parser.error("--repeat and --timeout must be positive; --pause cannot be negative")
+    return args
+
+
+if __name__ == "__main__":
+    main(parse_args())
diff --git a/plans/http2-optim2.md b/plans/http2-optim2.md
new file mode 100644
index 00000000..8b174382
--- /dev/null
+++ b/plans/http2-optim2.md
@@ -0,0 +1,436 @@
+# HTTP/2 Optimization Plan v2: Measured Concurrency for Chunk I/O
+
+**Status:** Proposed for discussion
+**Scope:** `caterva2`, `python-blosc2`, peer-cache reads, direct object-store reads,
+and chunk ingestion
+**Principle:** Concurrency is the optimization; HTTP/2 is a transport that may make
+that concurrency cheaper when it is actually negotiated.
+
+---
+
+## 1. Goals and Non-Goals
+
+### Goals
+
+1. Reduce latency for operations that require many independent chunk or byte-range
+   requests.
+2. Reuse connections and bound concurrency so clients do not overload themselves,
+   Caterva2 peers, reverse proxies, or object stores.
+3. Use HTTP/2 multiplexing where the complete path supports it and measurements show
+   a benefit.
+4. Add enough protocol and performance observability to detect silent HTTP/1.1
+   fallback and regressions.
+5. Preserve cache correctness, retry semantics, duplicate-write behavior, Pyodide
+   compatibility, and existing synchronous APIs.
+
+### Non-Goals
+
+* HTTP/2 is not required for ordinary Caterva2 slicing that already completes in one
+  `api/fetch` request.
+* This plan does not assume that every S3-compatible endpoint supports HTTP/2.
+* It does not promise a particular speedup or prescribe flow-control window sizes
+  before measurement.
+* HTTP/3 is out of scope until the HTTP/1.1 and HTTP/2 work is complete.
+
+---
+
+## 2. Current State
+
+The implementation must be inventoried before changes are made. At the time this
+plan was written:
+
+* `caterva2.Client` already constructs `httpx.Client(http2=True)` outside Emscripten.
+* `httpx[http2]` is already a project dependency.
+* `c2cache` and current `python-blosc2` integrations already contain real asynchronous
+  chunk-fetch paths and bounded/gathered `Proxy.afetch` behavior. These should be
+  extended rather than reimplemented.
+* Some `c2cache` `httpx.AsyncClient` and `httpx.Client` instances do not enable
+  HTTP/2.
+* The bundled Caterva2 command starts Uvicorn, which serves HTTP/1.1. Setting
+  `http2=True` on a client therefore does not make a default local Caterva2 server
+  speak HTTP/2.
+* The deployed `https://cat2.cloud/demo/` endpoint has been verified to negotiate
+  `h2` through TLS ALPN and to return an HTTP/2 response. Its reverse proxy may still
+  communicate with Uvicorn over HTTP/1.1; client-facing multiplexing remains useful.
+
+This inventory must be repeated against the exact `python-blosc2` revision selected
+for implementation, because that repository evolves independently.
+
+---
+
+## 3. Supported Test and Deployment Paths
+
+### 3.1 Local HTTP/1.1 baseline
+
+Run Caterva2 directly under Uvicorn. This is the control configuration and must stay
+fully supported.
+
+### 3.2 Local HTTP/2 configuration
+
+Provide a documented, reproducible TLS configuration using either:
+
+* Caddy or nginx terminating HTTP/2 and proxying to Uvicorn; or
+* an HTTP/2-capable ASGI server such as Hypercorn, if it is shown to run Caterva2
+  correctly.
+
+The reverse-proxy configuration is preferred for production-parity testing. Test
+certificates may be generated locally, but certificate verification must remain on;
+the benchmark client should trust the test CA explicitly.
+
+HTTPX generally negotiates HTTP/2 over TLS using ALPN. Clear-text `h2c` is not the
+primary test path.
+
+### 3.3 Production validation
+
+Run non-destructive protocol and read benchmarks against `cat2.cloud/demo` when
+appropriate. Production tests must use bounded load and must not be the only
+performance evidence.
+
+### 3.4 Mandatory protocol assertion
+
+Every HTTP/2 benchmark must inspect `response.http_version` and fail or mark the case
+invalid unless it equals `HTTP/2`. Also record:
+
+* effective URL and redirects;
+* negotiated protocol;
+* number of TCP connections where measurable;
+* reverse-proxy/server configuration identifier.
+
+This prevents HTTPX's normal HTTP/1.1 fallback from being mistaken for an HTTP/2
+result.
+
+---
+
+## 4. Workloads Where Concurrency May Help
+
+### 4.1 Peer-cache cold reads
+
+A slice that touches many uncached remote chunks currently requires multiple peer
+requests. Fetch missing chunks concurrently while retaining sparse-cache behavior.
+
+Requirements:
+
+* configurable `max_concurrency`, with a conservative default;
+* no unbounded `asyncio.gather()`;
+* already-cached chunks are never fetched again unnecessarily;
+* per-request and overall operation timeouts;
+* retry only transport/transient failures, with a small bounded retry count and
+  backoff/jitter if measurements justify it;
+* preserve successfully cached chunks when another chunk fails;
+* cancellation closes responses and releases stream capacity;
+* peer-offline classification must not turn application deliberate HTTP errors or local
+  programming errors into connectivity failures;
+* client ownership and shutdown must follow the Caterva2 application lifespan.
+
+HTTP/2 clients should be persistent per origin or per suitable lifecycle scope, not
+created for each chunk. Async clients must not be shared unsafely across event loops.
+
+### 4.2 Direct HTTPS object-store range reads
+
+For a Blosc2 source exposed through HTTPS, issue one request per required range and
+schedule them with bounded concurrency. Test each service independently:
+
+* AWS S3 REST endpoints;
+* Cloudflare R2;
+* GCS;
+* MinIO or other S3-compatible services;
+* CDN-fronted object URLs.
+
+Do not infer protocol support from an `https://` URL. Record the negotiated protocol
+for every service and endpoint form. Document whether access is anonymous, uses a
+presigned URL, or requires request signing. Plain HTTPX does not itself provide AWS
+SigV4 signing.
+
+`s3://` through `fsspec`/`s3fs` is a separate transport and must not be described as
+accelerated by changes to an HTTPX client. Any AWS CRT experiment belongs in a
+separate optional work item.
+
+### 4.3 Parallel chunk ingestion
+
+Retain `Client.fill_chunk()` and add an explicit bounded batch/async API rather than
+making callers coordinate arbitrary threads around the synchronous method. A
+provisional shape is:
+
+```python
+results = await client.afill_chunks(
+    remotepath,
+    chunks,
+    max_concurrency=8,
+)
+```
+
+The final API design must specify:
+
+* accepted chunk iterable/mapping format;
+* input order and result association;
+* bounded memory use and whether request bodies are streamed;
+* fail-fast versus collect-errors behavior;
+* partial-success reporting;
+* retry policy;
+* cancellation semantics;
+* behavior for an already-written slot;
+* connection and async-client lifetime.
+
+Multiple processes cannot share one HTTP/2 connection, so process-based ingestion
+must be benchmarked and documented separately.
+
+### 4.4 Interactive request cancellation
+
+HTTP/2 permits cancellation of one stream without closing the whole connection, but
+starting a replacement request does not cancel the old one automatically.
+
+Treat interactive cancellation as a separate feature:
+
+* browser clients use `AbortController`;
+* Python async callers cancel the relevant task;
+* response bodies are explicitly closed/released;
+* cancellation is propagated through intermediate server requests where applicable;
+* tests verify resource release and correctness, without depending on a particular
+  wire frame unless the transport contract guarantees it.
+
+---
+
+## 5. Benchmark Design
+
+Benchmarking precedes broad implementation and all performance claims.
+
+### 5.1 Fair comparison matrix
+
+For each applicable workload, compare:
+
+1. HTTP/1.1 sequential baseline;
+2. HTTP/1.1 persistent pooled client at concurrency `N`;
+3. HTTP/2 persistent client at the same concurrency `N`;
+4. selected concurrency sweep, for example `1, 2, 4, 8, 16, 32`.
+
+The key comparison is pooled HTTP/1.1 versus HTTP/2 at equal concurrency. A
+sequential-only HTTP/1.1 comparison exaggerates HTTP/2's contribution.
+
+### 5.2 Environments
+
+Measure at least:
+
+* loopback/local with the documented TLS proxy;
+* controlled latency and bandwidth, including optional packet loss;
+* `cat2.cloud/demo` under a deliberately low request rate;
+* each supported object-store endpoint.
+
+Keep dataset, slice, compression, concurrency, cache state, client limits, and server
+limits identical across comparable cases.
+
+### 5.3 Workload cases
+
+* cold peer slice touching many chunks;
+* warm peer slice, demonstrating cache behavior;
+* direct object read touching many byte ranges;
+* batch upload of many compressed chunks;
+* mixed-size chunks/ranges, including a slow response to expose head-of-line effects;
+* cancellation during an in-flight operation.
+
+### 5.4 Metrics
+
+Record:
+
+* elapsed time and throughput;
+* p50, p95, and p99 per-request latency;
+* time to first useful chunk/block;
+* negotiated HTTP version;
+* opened/reused TCP connections;
+* transferred payload and header bytes where measurable;
+* client, proxy, and server CPU;
+* peak client/server memory;
+* error, retry, timeout, and cancellation counts;
+* cache hits and misses.
+
+Run warmups and repeated trials and report distributions, not a single best run. Do
+not state expected multipliers in advance.
+
+---
+
+## 6. Implementation Phases
+
+### Phase 0 — Inventory and reproducible protocol fixtures
+
+Deliverables:
+
+* exact inventory of synchronous/asynchronous clients in both repositories;
+* local HTTP/1.1 fixture;
+* local TLS HTTP/2 reverse-proxy fixture;
+* protocol assertion helper;
+* documentation of the production topology relevant to HTTP/2;
+* basic connection/protocol logging available to benchmarks.
+
+Exit criterion: a test proves HTTP/1.1 locally under Uvicorn and HTTP/2 locally
+through the selected fixture, without disabling certificate verification.
+
+### Phase 1 — Baseline benchmark suite
+
+Implement the fair comparison matrix for peer reads and direct HTTPS range reads
+before changing concurrency or transport behavior.
+
+Exit criterion: repeatable results identify whether the bottleneck is RTT,
+connection setup, transfer bandwidth, decompression, server work, cache locking, or
+another component.
+
+### Phase 2 — Bounded peer-cache concurrency
+
+Reuse the existing `C2Array.aget_chunk`/`Proxy.afetch` path. Add or normalize:
+
+* explicit HTTP/2 enablement where absent;
+* bounded concurrency;
+* lifecycle-managed persistent clients;
+* cancellation, retry, and partial-cache behavior;
+* tests under both HTTP/1.1 and HTTP/2.
+
+Exit criterion: correctness tests pass for failures and cancellation, and benchmarks
+show a worthwhile improvement without unacceptable memory or server-load growth.
+
+### Phase 3 — Object-store range concurrency
+
+Implement bounded concurrent HTTPS range reads in `python-blosc2`, with HTTP/1.1
+fallback and per-endpoint protocol reporting.
+
+Exit criterion: functionality is transport-independent, service compatibility is
+documented, and any claimed HTTP/2 benefit is demonstrated for the named endpoint.
+
+### Phase 4 — Batch/async chunk ingestion
+
+Design and implement `afill_chunks` (final name subject to API review), preserving
+the atomic per-slot behavior of `fill_chunk` and reporting partial results clearly.
+
+Exit criterion: duplicate writes, partial failures, cancellation, retries, and
+bounded memory are tested under HTTP/1.1 and HTTP/2.
+
+### Phase 5 — Flow-control investigation, only if needed
+
+Capture stream and connection window behavior for large chunks. Determine whether
+flow control is demonstrably limiting throughput and whether the selected HTTPX,
+HTTP/2 server, and reverse proxy expose supported tuning controls.
+
+The HTTP/2 initial stream window is 65,535 bytes, but this alone does not imply an
+RTT stall every 64 KiB: receivers normally issue window updates as data is consumed.
+Connection-level and stream-level flow control must be analyzed separately.
+
+Exit criterion: tune windows only when traces and benchmarks show a bottleneck.
+Otherwise close this phase with documented evidence and no configuration change.
+
+### Phase 6 — Rollout and operational guidance
+
+* document Uvicorn-only and reverse-proxy deployments;
+* expose conservative concurrency settings;
+* add protocol/concurrency/cache observability;
+* stage rollout with HTTP/1.1 fallback;
+* retain an easy configuration switch to disable HTTP/2 if interoperability issues
+  occur.
+
+---
+
+## 7. Correctness and Safety Test Matrix
+
+Every concurrent implementation must cover:
+
+* out-of-order completion;
+* one transient failure among successful chunks;
+* persistent transport failure;
+* deliberate HTTP 4xx/5xx response;
+* timeout while other streams continue;
+* caller cancellation;
+* server disconnect;
+* duplicate chunk write;
+* incomplete trailing chunk;
+* cold, partially warm, and fully warm sparse cache;
+* client shutdown with requests in flight;
+* HTTP/1.1 fallback;
+* Pyodide/Emscripten behavior where the synchronous Caterva2 client intentionally
+  avoids HTTP/2.
+
+Concurrency settings must have documented upper bounds. Tests should demonstrate
+that peak outstanding requests and buffered response memory stay within them.
+
+---
+
+## 8. Decision Criteria
+
+Adopt HTTP/2 by default for a path only when:
+
+1. it negotiates reliably in the supported deployment;
+2. equal-concurrency benchmarks show a meaningful benefit or a clear reduction in
+   connection cost;
+3. memory, CPU, and tail latency remain acceptable;
+4. cancellation, retries, and HTTP/1.1 fallback are correct;
+5. operational complexity is documented and justified.
+
+If HTTP/1.1 pooling performs equally well, keep the concurrency improvements and do
+not force HTTP/2. If one HTTP/2 connection becomes a bottleneck, test a small bounded
+number of connections rather than assuming that one connection is universally
+optimal.
+
+---
+
+## 9. Open Questions for Discussion
+
+1. Should the reproducible local fixture use Caddy, nginx, Hypercorn, or more than
+   one of these?
+2. What conservative default and maximum should `max_concurrency` use for peer
+   reads, object ranges, and writes?
+3. Should HTTP clients live per peer, per origin, or per application lifespan, and
+   how will shutdown be coordinated?
+4. Which object-store endpoint and authentication modes are officially supported?
+5. Should batch ingestion fail fast or return one result/error per chunk by default?
+6. What measured improvement is sufficient to justify enabling HTTP/2 by default?
+7. Which proxy/server metrics can be collected in CI, and which require a separate
+   performance environment?
+
+---
+
+## 10. Preliminary Findings and Branch Decision (2026-09-02)
+
+### What was measured
+
+The experimental harness under `examples/benchmarks/http2/` verifies the negotiated
+protocol and compares HTTP/1.1 and HTTP/2 while holding application behavior and
+concurrency constant. Detailed commands and results are recorded alongside it.
+
+Three classes of experiment were performed:
+
+1. **Concurrent chunk reads from `cat2.cloud`:** HTTP/2 was approximately tied with
+   HTTP/1.1 at concurrency 4 and slower at concurrency 8 and 16 for the tested
+   10-chunk dataset.
+2. **Local peer reads with 50 ms simulated RTT:** with 64 chunks and concurrency 8,
+   HTTP/2 was about 6% slower for 64 KiB chunks and 13% slower for 512 KiB chunks.
+3. **One large server-side `api/fetch`:** against `cat2.cloud`, HTTP/2 was much more
+   stable and faster than forced HTTP/1.1 for a 26.8 MB slice. The corresponding
+   local Caddy test with 50 ms simulated RTT produced the opposite result, with
+   HTTP/2 about 66% slower.
+
+### Conclusions
+
+* HTTP/2 is not intrinsically faster for Caterva2 traffic. Workload, real network
+  behavior, TLS termination, reverse-proxy configuration, and TCP implementation
+  materially affect the result.
+* For independent chunk reads, several pooled HTTP/1.1 connections can outperform
+  several HTTP/2 streams sharing one TCP connection.
+* The strong single-slice HTTP/2 result on `cat2.cloud` appears deployment-specific.
+  The local latency fixture cannot reproduce it and therefore cannot be used to
+  choose production transport defaults.
+* Local Caddy/Toxiproxy tests remain useful for protocol correctness, fallback,
+  controlled comparisons, and regressions. They are not a substitute for tests
+  between real remote Caterva2 deployments.
+* A decision about peer transport performance requires at least two controlled real
+  remote servers, verified protocols, repeated interleaved trials, and both chunked
+  and single-response workloads.
+
+### Decision for this branch
+
+* Keep the existing `caterva2.Client(http2=True)` behavior. It predates this work,
+  falls back to HTTP/1.1 when necessary, and the deployed single-slice workload
+  provides evidence in its favor.
+* Do **not** change `c2cache` peer clients to `http2=True` by default. Existing peer
+  behavior remains HTTP/1.1 until representative remote-peer measurements justify a
+  change.
+* Keep the benchmark scripts, Caddy fixture, Toxiproxy simulation, protocol
+  assertions, and recorded results from this branch.
+* Stop flow-control tuning and further local performance measurements for now.
+* Resume performance work only when controlled real remote Caterva2 servers are
+  available. At that point HTTP/2 should remain a measured per-path choice rather
+  than a project-wide assumption.

From 87e33108db7caa1b089b0ab04787aec119763fc9 Mon Sep 17 00:00:00 2001
From: Francesc Alted <francesc@blosc.org>
Date: Thu, 3 Sep 2026 07:33:27 +0200
Subject: [PATCH 08/11] Add --slice to example

---
 examples/benchmarks/http2/peer_read.py | 32 ++++++++++++++++++++++++--
 1 file changed, 30 insertions(+), 2 deletions(-)

diff --git a/examples/benchmarks/http2/peer_read.py b/examples/benchmarks/http2/peer_read.py
index de58c463..5e1f22f5 100644
--- a/examples/benchmarks/http2/peer_read.py
+++ b/examples/benchmarks/http2/peer_read.py
@@ -38,12 +38,35 @@ async def assert_protocol(urlbase: str, expected: str, *, http2: bool) -> None:
         )
 
 
-async def cold_read(urlbase: str, path: str, concurrency: int, cache: pathlib.Path, *, http2: bool) -> float:
+def parse_slice_tuple(slice_str: str | None) -> tuple[slice, ...] | None:
+    if not slice_str:
+        return None
+    parts = []
+    for s in slice_str.split(","):
+        s = s.strip()
+        if ":" in s:
+            start, stop = (int(x.strip()) if x.strip() else None for x in s.split(":", 1))
+            parts.append(slice(start, stop))
+        else:
+            v = int(s)
+            parts.append(slice(v, v + 1))
+    return tuple(parts)
+
+
+async def cold_read(
+    urlbase: str,
+    path: str,
+    concurrency: int,
+    cache: pathlib.Path,
+    *,
+    http2: bool,
+    slice_: tuple[slice, ...] | None = None,
+) -> float:
     source = BenchmarkRemoteSource(path, urlbase=urlbase, http2=http2)
     proxy = blosc2.Proxy(source, urlpath=str(cache), mode="w")
     try:
         started = time.perf_counter()
-        await proxy.afetch(None, max_concurrency=concurrency)
+        await proxy.afetch(slice_, max_concurrency=concurrency)
         return time.perf_counter() - started
     finally:
         # Proxy does not own the remote source's HTTP client.
@@ -70,6 +93,7 @@ async def main(args: argparse.Namespace) -> None:
         "http1": (args.http1_url, False),
         "http2": (args.http2_url, True),
     }
+    slice_tuple = parse_slice_tuple(args.slice)
     with tempfile.TemporaryDirectory(prefix="caterva2-http-benchmark-") as tmp:
         tmpdir = pathlib.Path(tmp)
         for trial in range(args.repeat):
@@ -82,6 +106,7 @@ async def main(args: argparse.Namespace) -> None:
                     args.concurrency,
                     tmpdir / f"{label}-trial-{trial}.b2nd",
                     http2=use_http2,
+                    slice_=slice_tuple,
                 )
                 samples[label].append(sample)
                 print(f"{label} trial={trial + 1} seconds={sample:.6f}")
@@ -96,6 +121,9 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--http1-url", required=True)
     parser.add_argument("--http2-url", required=True)
     parser.add_argument("--path", required=True, help="remote dataset path, e.g. @public/example.b2nd")
+    parser.add_argument(
+        "--slice", default=None, help="optional slice string, e.g. '9500:10500, 9500:10500, 9500:10500'"
+    )
     parser.add_argument("--concurrency", type=int, default=4)
     parser.add_argument("--repeat", type=int, default=5)
     args = parser.parse_args()

From 01ae4bca3b5361d2ce9020b6b631fac8f53ce196 Mon Sep 17 00:00:00 2001
From: Francesc Alted <francesc@blosc.org>
Date: Thu, 3 Sep 2026 07:34:31 +0200
Subject: [PATCH 09/11] Update plan with latest measurements

---
 plans/http2-optim2.md | 95 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 95 insertions(+)

diff --git a/plans/http2-optim2.md b/plans/http2-optim2.md
index 8b174382..518b2fed 100644
--- a/plans/http2-optim2.md
+++ b/plans/http2-optim2.md
@@ -434,3 +434,98 @@ Three classes of experiment were performed:
 * Resume performance work only when controlled real remote Caterva2 servers are
   available. At that point HTTP/2 should remain a measured per-path choice rather
   than a project-wide assumption.
+
+---
+
+## 11. Remote Server Audit and Definitive Findings (2026-09-03)
+
+### 11.1 Local Simulation vs. Production Network Ground Truth
+
+Further evaluation confirmed that local loopback simulation (Caddy + Toxiproxy) is not
+suitable for choosing production transport defaults:
+* User-space latency injection (Toxiproxy) buffers and delays bytes in Go user-space,
+  distorting real kernel TCP congestion control (CWND ramp-up, BDP, ACK pacing, packet loss).
+* Loopback bandwidth is effectively infinite, which disproportionately amplifies
+  client-side pure-Python CPU overhead (`httpx` / `h2` frame parsing) and misrepresents
+  WAN network bottlenecks.
+* Production validation against the live remote server (`https://cat2.cloud/demo`) was
+  selected as the authoritative environment for performance decisions.
+
+### 11.2 Root Cause of the 2026-09-02 Single-Slice Anomaly
+
+On 2026-09-02, HTTP/1.1 appeared ~4.3x slower than HTTP/2 on `cat2.cloud` for single
+large `api/fetch` responses (1.361 s vs 0.318 s). An audit of the production Nginx
+reverse proxy revealed two major configuration bottlenecks:
+
+1. **Missing Upstream Keepalives:** Nginx defaulted to HTTP/1.0 with `Connection: close`
+   toward Uvicorn, tearing down and recreating the Unix domain socket on every request.
+2. **Buffer Overflow and Disk Spooling:** Nginx default proxy buffers were only 32 KB
+   total (`proxy_buffers 8 4k`). When Uvicorn returned the 2.68 MB compressed slice,
+   the tiny RAM buffer filled in microseconds and Nginx spooled the response to temporary
+   files on disk (`/var/lib/nginx/proxy/...`), introducing disk I/O latency and locks.
+
+The Nginx deployment was updated with:
+* **Upstream keepalive pool:**
+  ```nginx
+  upstream demo {
+      server unix:/home/demo/caterva2-deploy/_caterva2/state/uvicorn.socket;
+      keepalive 32;
+  }
+  ```
+* **Streaming RAM buffers (2 MB pool, zero disk spooling):**
+  ```nginx
+  proxy_http_version 1.1;
+  proxy_set_header Connection "";
+  proxy_buffering on;
+  proxy_buffer_size 128k;
+  proxy_buffers 16 128k;
+  proxy_busy_buffers_size 256k;
+  ```
+* **TCP stack optimizations:** `tcp_nopush on;` and `tcp_nodelay on;`.
+
+**Result:** Single-slice HTTP/1.1 network response time dropped from **1.361 s to 0.226 s**
+(a 6x improvement), completely eliminating the previous gap and tying with HTTP/2 (0.242 s).
+Both protocols transfer single slices at the physical bandwidth limit of the WAN/Wi-Fi link.
+
+### 11.3 Multi-Chunk Peer-Read Benchmark on `gaia-3d.b2nd`
+
+To test concurrent chunk retrieval on a representative large-scale dataset,
+`examples/benchmarks/http2/peer_read.py` was extended to support arbitrary `--slice`
+ranges. Slicing `@public/large/gaia-3d.b2nd` at `(9500:10500, 9500:10500, 9500:10500)`
+retrieved 64 independent chunks (~20–50 KiB compressed each).
+
+Measurements across concurrency levels against the tuned `cat2.cloud` server (median of
+alternating trials):
+
+| Max Concurrency | HTTP/1.1 Median | HTTP/2 Median | Ratio (H1 / H2) | Winner |
+|---:|---:|---:|---:|---|
+| **1 (serial)** | 4.652 s | 4.647 s | 1.001 | Dead tie |
+| **4** | 1.367 s | 1.433 s | 0.954 | HTTP/1.1 (~5% faster) |
+| **8** | 0.838 s | 0.970 s | 0.863 | **HTTP/1.1 (~15% faster)** |
+| **16** | 0.612 s | 0.818 s | 0.748 | **HTTP/1.1 (~30% faster)** |
+
+#### Physical Mechanisms:
+* **Multiple TCP Congestion Windows:** At concurrency 16, pooled HTTP/1.1 opens 16
+  independent TCP connections, each with its own kernel TCP congestion window,
+  saturating WAN bandwidth in parallel. HTTP/2 forces all 16 streams through a single
+  TCP connection and a single congestion window.
+* **Resilience to Packet Loss:** Packet drops on real WANs cause TCP Head-of-Line
+  blocking across all HTTP/2 multiplexed streams on that connection, whereas pooled
+  HTTP/1.1 connections continue uninterrupted.
+* **Client CPU Overhead:** Demultiplexing binary chunk frames across 16 active streams
+  in pure Python (`h2`) incurs noticeable CPU overhead compared to streaming raw socket
+  bytes in HTTP/1.1.
+
+### 11.4 Final Transport Architecture Decision
+
+1. **`caterva2.Client`:** Retain `http2=True`. For interactive users and notebook
+   sessions issuing single queries/slices, HTTP/2 matches HTTP/1.1 throughput (~0.22 s)
+   while protecting public servers from TCP socket exhaustion and supporting `RST_STREAM`
+   stream cancellation.
+2. **`caterva2.c2cache` & `python-blosc2` (`C2Array.aget_chunk`):** Retain pooled HTTP/1.1
+   (`http2=False`). For bulk concurrent chunk downloads, connection pooling consistently
+   outperforms HTTP/2 multiplexing by 15% to 30%. No code changes are required in either
+   repository.
+3. **Production Reverse-Proxy Profile:** Document the Nginx upstream keepalive (`keepalive 32;`)
+   and in-memory streaming buffer directives (`proxy_buffers 16 128k;`) as standard
+   operational requirements for Caterva2 reverse-proxy deployments.

From 1e00a8b4f60ad9ad67433605cecf3dc78c0e8281 Mon Sep 17 00:00:00 2001
From: Francesc Alted <francesc@blosc.org>
Date: Thu, 3 Sep 2026 07:37:53 +0200
Subject: [PATCH 10/11] Add AGENTS.md with repository rules for trailing
 newlines and pre-commit

---
 AGENTS.md | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 AGENTS.md

diff --git a/AGENTS.md b/AGENTS.md
new file mode 100644
index 00000000..1ed207b9
--- /dev/null
+++ b/AGENTS.md
@@ -0,0 +1,5 @@
+# Agent Guidelines for Caterva2
+
+- **Trailing Newline**: Always ensure every created or modified file ends with a single trailing newline (`\n`) to satisfy pre-commit's `end-of-file-fixer`.
+- **Pre-commit Compliance**: Ensure all code changes adhere to repository pre-commit hooks (formatting, linting, and whitespace).
+- **Documentation**: Maintain documentation integrity, preserving existing comments and docstrings unless explicitly directed otherwise.

From 901bde6c202b6dbfe8c45e32904458b0da3cc10a Mon Sep 17 00:00:00 2001
From: Francesc Alted <francesc@blosc.org>
Date: Thu, 3 Sep 2026 12:32:03 +0200
Subject: [PATCH 11/11] Better handling of GIL deadlocks during file locking
 across threads

---
 caterva2/services/server.py | 185 +++++++++++++++++++++---------------
 1 file changed, 106 insertions(+), 79 deletions(-)

diff --git a/caterva2/services/server.py b/caterva2/services/server.py
index b492e7f1..715e1dfb 100644
--- a/caterva2/services/server.py
+++ b/caterva2/services/server.py
@@ -23,6 +23,7 @@
 import shutil
 import string
 import tarfile
+import threading
 import time
 import traceback
 import types
@@ -100,6 +101,29 @@ def dataset_lock(abspath) -> asyncio.Lock:
     return lock
 
 
+_thread_locks: weakref.WeakValueDictionary[str, threading.Lock] = weakref.WeakValueDictionary()
+_thread_locks_guard = threading.Lock()
+
+
+def dataset_thread_lock(abspath) -> threading.Lock:
+    """Serialize threadpool writes on one array within this process.
+
+    Python-Blosc2's file locking (`locking=True`) executes in C without releasing
+    the GIL during open/lock. If two threads in the same process attempt to
+    open/lock the same file concurrently, one can block on the OS file lock
+    while holding Python's GIL, deadlocking the process. Guarding operations on
+    the same array with a threading.Lock ensures the waiting thread yields the
+    GIL cleanly.
+    """
+    key = str(abspath)
+    with _thread_locks_guard:
+        lock = _thread_locks.get(key)
+        if lock is None:
+            lock = threading.Lock()
+            _thread_locks[key] = lock
+        return lock
+
+
 mimetypes.add_type("text/markdown", ".md")  # Because in macOS this is not by default
 mimetypes.add_type("application/x-ipynb+json", ".ipynb")
 
@@ -1550,10 +1574,12 @@ def publish_dataset(abspath: pathlib.Path, path: pathlib.Path) -> str:
         with contextlib.suppress(Exception):
             fs.rm(staging)
         raise
-    array = blosc2.open(abspath, mode="a", locking=True)
-    with array.schunk.holding_lock():
-        array.schunk.vlmeta[PUBLISHED_URL] = destination
-        array.schunk.vlmeta[FILL_STATE] = PUBLISHED
+    with dataset_thread_lock(abspath):
+        array = blosc2.open(abspath, mode="a", locking=True)
+        with array.schunk.holding_lock():
+            array.schunk.vlmeta[PUBLISHED_URL] = destination
+            array.schunk.vlmeta[FILL_STATE] = PUBLISHED
+        del array
     return destination
 
 
@@ -1568,82 +1594,83 @@ def store_chunk(abspath: pathlib.Path, nchunk: int, chunk: bytes) -> dict:
     both find the slot free would otherwise both write it, and the second would
     move every chunk that came after the first.
     """
-    try:
-        array = blosc2.open(abspath, mode="a", locking=True)
-    except Exception as exc:
-        srv_utils.raise_bad_request(f"{abspath.name} cannot be opened for writing: {exc}")
-    if not isinstance(array, blosc2.NDArray):
-        srv_utils.raise_bad_request(
-            f"{abspath.name} is not an NDArray, so it has no chunks of a shape to write into"
-        )
-    schunk = array.schunk
-    if not 0 <= nchunk < schunk.nchunks:
-        srv_utils.raise_not_found(f"{abspath.name} has no chunk {nchunk}")
-    try:
-        nbytes, _, blocksize = blosc2.get_cbuffer_sizes(chunk)
-        typesize = chunk_typesize(chunk)
-    except Exception:
-        srv_utils.raise_bad_request("the body is not a Blosc2 chunk")
-    # A chunk of another geometry would be stored and then read as nonsense, so
-    # it is refused here rather than left for whoever reads the array next
-    if nbytes != schunk.chunksize:
-        srv_utils.raise_bad_request(
-            f"the chunk holds {nbytes} bytes where this array's chunks hold {schunk.chunksize}"
-        )
-    if blocksize != schunk.blocksize:
-        srv_utils.raise_bad_request(
-            f"the chunk is split into blocks of {blocksize} bytes where this array's are "
-            f"{schunk.blocksize}; compress it against the array's blocks"
-        )
-    # The one part of the geometry the sizes do not carry, and the one whose
-    # mismatch is silent: the shuffle filters read and write on a stride of it,
-    # so a chunk compressed against another typesize decompresses to the right
-    # number of bytes with every one of them in the wrong place -- no error
-    # anywhere, just an array of scrambled values
-    if typesize != filter_typesize(schunk.typesize):
-        srv_utils.raise_bad_request(
-            f"the chunk was compressed with a typesize of {typesize} where this array's is "
-            f"{schunk.typesize}; compress it against the array's dtype"
-        )
-    complete = False
-    with schunk.holding_lock():
-        if not chunk_is_unwritten(schunk, nchunk):
-            raise fastapi.HTTPException(
-                status_code=409, detail=f"chunk {nchunk} of {abspath.name} was already written"
+    with dataset_thread_lock(abspath):
+        try:
+            array = blosc2.open(abspath, mode="a", locking=True)
+        except Exception as exc:
+            srv_utils.raise_bad_request(f"{abspath.name} cannot be opened for writing: {exc}")
+        if not isinstance(array, blosc2.NDArray):
+            srv_utils.raise_bad_request(
+                f"{abspath.name} is not an NDArray, so it has no chunks of a shape to write into"
             )
-        schunk.update_chunk(nchunk, chunk)
-        if FILL_NONCE not in schunk.vlmeta:
-            # What names *this* array, as against another one that came to sit at
-            # the same path with the same size.  A client caching the array reads
-            # it from api/info and can tell the two apart, which a size and an
-            # mtime cannot always do.  Written once, by whichever writer arrived
-            # first, and never again
-            schunk.vlmeta[FILL_NONCE] = uuid.uuid4().hex
-            # Said out loud rather than left to be inferred from the absence of
-            # it, and free here: the same locked region, the same trailer
-            schunk.vlmeta[FILL_STATE] = FILLING
-        written, nchunks = count_written(abspath)
-        state = schunk.vlmeta.get(FILL_STATE, FILLING)
-        if written == nchunks and state == FILLING:
-            # Exactly once, whichever writer got here: the lock is held, so of two
-            # writers that both see the array complete only one makes this move,
-            # and that one owns the publishing.  Recorded even where there is
-            # nowhere to publish to, because "every slot is claimed" is worth
-            # saying on its own: it is what tells a reader the array can no
-            # longer change under a cache of it
-            complete = bool(settings.publish_root)
-            state = PUBLISHING if complete else COMPLETE
-            schunk.vlmeta[FILL_STATE] = state
-    # Drop the handle before anything reads the file again: a handle left open
-    # over a frame another one writes is the stale-handle hazard, and it is silent
-    del array, schunk
-    return {
-        "nchunk": nchunk,
-        "written": written,
-        "nchunks": nchunks,
-        "state": state,
-        "publish": complete,
-    }
+        schunk = array.schunk
+        if not 0 <= nchunk < schunk.nchunks:
+            srv_utils.raise_not_found(f"{abspath.name} has no chunk {nchunk}")
+        try:
+            nbytes, _, blocksize = blosc2.get_cbuffer_sizes(chunk)
+            typesize = chunk_typesize(chunk)
+        except Exception:
+            srv_utils.raise_bad_request("the body is not a Blosc2 chunk")
+        # A chunk of another geometry would be stored and then read as nonsense, so
+        # it is refused here rather than left for whoever reads the array next
+        if nbytes != schunk.chunksize:
+            srv_utils.raise_bad_request(
+                f"the chunk holds {nbytes} bytes where this array's chunks hold {schunk.chunksize}"
+            )
+        if blocksize != schunk.blocksize:
+            srv_utils.raise_bad_request(
+                f"the chunk is split into blocks of {blocksize} bytes where this array's are "
+                f"{schunk.blocksize}; compress it against the array's blocks"
+            )
+        # The one part of the geometry the sizes do not carry, and the one whose
+        # mismatch is silent: the shuffle filters read and write on a stride of it,
+        # so a chunk compressed against another typesize decompresses to the right
+        # number of bytes with every one of them in the wrong place -- no error
+        # anywhere, just an array of scrambled values
+        if typesize != filter_typesize(schunk.typesize):
+            srv_utils.raise_bad_request(
+                f"the chunk was compressed with a typesize of {typesize} where this array's is "
+                f"{schunk.typesize}; compress it against the array's dtype"
+            )
+        complete = False
+        with schunk.holding_lock():
+            if not chunk_is_unwritten(schunk, nchunk):
+                raise fastapi.HTTPException(
+                    status_code=409, detail=f"chunk {nchunk} of {abspath.name} was already written"
+                )
+            schunk.update_chunk(nchunk, chunk)
+            if FILL_NONCE not in schunk.vlmeta:
+                # What names *this* array, as against another one that came to sit at
+                # the same path with the same size.  A client caching the array reads
+                # it from api/info and can tell the two apart, which a size and an
+                # mtime cannot always do.  Written once, by whichever writer arrived
+                # first, and never again
+                schunk.vlmeta[FILL_NONCE] = uuid.uuid4().hex
+                # Said out loud rather than left to be inferred from the absence of
+                # it, and free here: the same locked region, the same trailer
+                schunk.vlmeta[FILL_STATE] = FILLING
+            written, nchunks = count_written(abspath)
+            state = schunk.vlmeta.get(FILL_STATE, FILLING)
+            if written == nchunks and state == FILLING:
+                # Exactly once, whichever writer got here: the lock is held, so of two
+                # writers that both see the array complete only one makes this move,
+                # and that one owns the publishing.  Recorded even where there is
+                # nowhere to publish to, because "every slot is claimed" is worth
+                # saying on its own: it is what tells a reader the array can no
+                # longer change under a cache of it
+                complete = bool(settings.publish_root)
+                state = PUBLISHING if complete else COMPLETE
+                schunk.vlmeta[FILL_STATE] = state
+        # Drop the handle before anything reads the file again: a handle left open
+        # over a frame another one writes is the stale-handle hazard, and it is silent
+        del array, schunk
+        return {
+            "nchunk": nchunk,
+            "written": written,
+            "nchunks": nchunks,
+            "state": state,
+            "publish": complete,
+        }
 
 
 @app.post("/api/chunk/{path:path}")