From c2d94e337acc8bc4c78c10fa2f64fb89c43b3306 Mon Sep 17 00:00:00 2001 From: philippe Date: Wed, 28 Oct 2026 10:56:45 -0400 Subject: [PATCH 1/3] Make shared storage opt-in, move msgspec to extras Dash() no longer starts shared storage or the multiplexed streaming downlink unless shared_storage= or DASH_SHARED_STORAGE is set. Streaming callbacks still work on a per-callback connection without it. msgspec moves from install requirements to the new dash[shared_storage] extra (persistent LocalSharedStorage modes) and to dash[diskcache] and dash[redis]. Without it, in-memory LocalSharedStorage falls back to JSON on its socket transport. --- .ai/ARCHITECTURE.md | 64 ++++++----- CHANGELOG.md | 3 +- dash/_callback_context.py | 2 +- dash/_shared_storage/__init__.py | 2 +- dash/_shared_storage/_codec.py | 52 ++++++--- dash/_shared_storage/_env.py | 6 +- dash/_shared_storage/_persistence.py | 4 +- dash/_shared_storage/_transport.py | 6 +- dash/_shared_storage/diskcache.py | 3 +- dash/_shared_storage/local.py | 4 + dash/_shared_storage/redis.py | 3 +- dash/backends/_fastapi.py | 2 +- dash/backends/_flask.py | 2 +- dash/backends/_quart.py | 2 +- dash/dash.py | 23 ++-- requirements/ci.txt | 1 + requirements/diskcache.txt | 1 + requirements/install.txt | 1 - requirements/redis.txt | 3 +- requirements/shared_storage.txt | 1 + setup.py | 3 +- tests/shared_storage/test_env_config.py | 8 +- tests/shared_storage/test_no_msgspec.py | 103 ++++++++++++++++++ .../test_stream_callbacks_integration.py | 8 +- tests/streaming/test_stream_transport.py | 34 ++++++ tests/streaming/test_stream_wsgi.py | 22 +++- 26 files changed, 285 insertions(+), 78 deletions(-) create mode 100644 requirements/shared_storage.txt create mode 100644 tests/shared_storage/test_no_msgspec.py diff --git a/.ai/ARCHITECTURE.md b/.ai/ARCHITECTURE.md index 1ad587e2c4..deb28e15d4 100644 --- a/.ai/ARCHITECTURE.md +++ b/.ai/ARCHITECTURE.md @@ -727,7 +727,7 @@ Special handling for Colab: - `prevent_initial_callbacks` - Skip callbacks on load (default: `False`) - `background_callback_manager` - DiskcacheManager or CeleryManager - `on_error` - Global callback error handler -- `shared_storage` - Cross-process state + pub/sub backend (default: `LocalSharedStorage`). Pass `None` to disable, or a `BaseSharedStorage` subclass/instance to swap. See [Shared Storage](#shared-storage). +- `shared_storage` - Cross-process state + pub/sub backend (default: off). Pass `LocalSharedStorage` to enable, or a `BaseSharedStorage` subclass/instance to swap. See [Shared Storage](#shared-storage). **WebSocket Callbacks:** - `websocket_callbacks` - Enable WebSocket for all callbacks (default: `False`). Requires FastAPI backend. @@ -820,7 +820,9 @@ channel. Unlike `dcc.Store` (which lives in the browser), shared storage lives on the server and lets callbacks running in different workers see the same keys and topics without standing up an external service like Redis. -Enabled by default on every app. It starts lazily on first use, so it costs +Opt-in: off unless the app passes `shared_storage=` or `DASH_SHARED_STORAGE` +is set, so existing deployments do not grow a socket owner or a streaming +downlink on upgrade. Once enabled it starts lazily on first use, so it costs nothing until touched. ### Accessing It @@ -836,13 +838,21 @@ def handler(n): return store.get("clicks", 0) # Or off the app directly: -app = dash.Dash() +app = dash.Dash(shared_storage=dash.LocalSharedStorage) app.shared_storage.set("key", {"any": "json-compatible value"}) ``` Values (and published messages) must be **JSON-compatible** — dict / list / str / int / float / bool / None — the same constraint as `dcc.Store` and callback -outputs. Messages are encoded with `msgspec` (msgpack), a hard dependency. +outputs. Messages are encoded with `msgspec` (msgpack) from the +`dash[shared_storage]` extra (also pulled in by `dash[diskcache]` and +`dash[redis]`). Without it, `LocalSharedStorage` in `"memory"` mode still works +and its socket transport falls back to stdlib JSON (both ends run the same +install, so they always agree). JSON turns non-string dict keys into strings and +cannot carry `bytes`, so keep values strictly JSON-shaped. Everything that +writes encoded bytes somewhere lasting (the persistent modes, Diskcache, Redis) +raises an `ImportError` naming the extra that brings it instead, so a stored +format never depends on what happens to be installed. ### API @@ -895,7 +905,7 @@ more than one container/pod (see [Deployment Topology](#deployment-topology-and- | Backend | Backing store | Reaches | Extra | |---------|---------------|---------|-------| -| `LocalSharedStorage` *(default)* | in-memory, owner-elected socket | one **container** (all its workers) | none | +| `LocalSharedStorage` | in-memory, owner-elected socket | one **container** (all its workers) | none (`dash[shared_storage]` for the persistent modes) | | `DiskcacheSharedStorage` | a `diskcache.Cache` | one **host** (all processes sharing the dir) | `dash[diskcache]` | | `RedisSharedStorage` | Redis (Streams for pub/sub) | **any** process/container/pod | `dash[redis]` | @@ -909,14 +919,15 @@ from dash._shared_storage import ( LocalSharedStorage, DiskcacheSharedStorage, RedisSharedStorage, ) -Dash(shared_storage=LocalSharedStorage) # default +Dash() # disabled (default) +Dash(shared_storage=LocalSharedStorage) Dash(shared_storage=DiskcacheSharedStorage(directory="/tmp/ss")) Dash(shared_storage=RedisSharedStorage(url="redis://host:6379")) -Dash(shared_storage=None) # disabled ``` When disabled, accessing `app.shared_storage` / `dash.ctx.shared_storage` -raises `SharedStorageError`. +raises `SharedStorageError`, and streaming callbacks still work, each on its own +connection instead of the multiplexed downlink. **`LocalSharedStorage`** — in-memory, no external service. On first use every worker races to become the single **owner** for the machine by binding a stable @@ -993,7 +1004,7 @@ subscriptions likewise poll without blocking and sleep on the loop. ### Deployment Topology and Shared Storage -The default `LocalSharedStorage` is **per-container**: its socket/loopback +`LocalSharedStorage` is **per-container**: its socket/loopback election only reaches processes in the same network + filesystem namespace. - **Single process / single pod** (e.g. Plotly Cloud apps): fine — one owner, @@ -1009,16 +1020,16 @@ election only reaches processes in the same network + filesystem namespace. A hosting platform can switch the backend without editing the app through `DASH_SHARED_STORAGE` (`_shared_storage/_env.py`), read only when the app did not pass `shared_storage=` (the default is a sentinel, so an explicit argument, -`None` included, always wins): `local`, `none`, `diskcache:///abs/path`, or a -`redis://` / `rediss://` URL. A `key_prefix` query parameter on the Redis URL -(`redis://host/0?key_prefix=myapp`) sets `RedisSharedStorage(key_prefix=)` and -is stripped before the URL reaches redis-py; the diskcache path is -percent-decoded. `cluster://` is reserved and raises; anything else raises -`InvalidConfig` at construction, without echoing the value (it can hold -credentials). The value becomes a zero-argument -factory, so nothing is built or connected until `app.shared_storage` is first -read, but a missing extra (`dash[redis]`, `dash[diskcache]`) fails at -construction with the backend's own `ImportError`. +`None` included, always wins): `local`, `none` (same as unset or empty), +`diskcache:///abs/path`, or a `redis://` / `rediss://` URL. A `key_prefix` query +parameter on the Redis URL (`redis://host/0?key_prefix=myapp`) sets +`RedisSharedStorage(key_prefix=)` and is stripped before the URL reaches +redis-py; the diskcache path is percent-decoded. `cluster://` is reserved and +raises; anything else raises `InvalidConfig` at construction, without echoing +the value (it can hold credentials). The value becomes a zero-argument factory, +so nothing is built or connected until `app.shared_storage` is first read, but a +missing extra (`dash[redis]`, `dash[diskcache]`, `dash[shared_storage]`) fails +at construction with the backend's own `ImportError`. ### Custom and Out-of-Tree Backends @@ -1039,12 +1050,13 @@ and returns a `Subscription` (`__iter__` / `__aiter__` / `close`) that raises top-level `dash`; the poll-loop helpers (`PollResult`, the polling subscription) are private and not part of the contract. -Core only ships backends whose dependency is already a Dash extra (`msgspec` -base; `dash[diskcache]`; `dash[redis]`). Anything needing a heavier dependency -belongs **out of tree** behind this same interface — e.g. a Postgres backend -(`LISTEN`/`NOTIFY` for push pub/sub + a table for KV and replay) lives in its own -package so a `psycopg` connection is never pulled into Dash core. It plugs in the -same way as a built-in: `Dash(shared_storage=PostgresSharedStorage(...))`. +Core only ships backends whose dependency is already a Dash extra +(`dash[shared_storage]`; `dash[diskcache]`; `dash[redis]`). Anything needing a +heavier dependency belongs **out of tree** behind this same interface — e.g. a +Postgres backend (`LISTEN`/`NOTIFY` for push pub/sub + a table for KV and +replay) lives in its own package so a `psycopg` connection is never pulled into +Dash core. It plugs in the same way as a built-in: +`Dash(shared_storage=PostgresSharedStorage(...))`. ### Module Layout @@ -1412,7 +1424,7 @@ with an empty result on the terminal frame. ### Multiplexed downlink and the stream SharedWorker -When the app has a shared-storage backend (the default `LocalSharedStorage`), +When the app has a shared-storage backend (opt-in, see Shared Storage), HTTP streams do not each hold their own response. The callback's POST carries `streamConnection: {requestId}` and returns a fast ack; a *pump* (`dash/_stream_hub.py`) drives the generator as an asyncio task and publishes diff --git a/CHANGELOG.md b/CHANGELOG.md index 62f302b6f9..14ca695428 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,12 +5,13 @@ This project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] ### Added +- [#](https://github.com/plotly/dash/pull/) Make shared storage and the multiplexed streaming downlink opt-in (`shared_storage=` or `DASH_SHARED_STORAGE`), and move `msgspec` to a new `dash[shared_storage]` extra for the persistent `LocalSharedStorage` modes (`dash[diskcache]` and `dash[redis]` include it). - [#4026](https://github.com/plotly/dash/pull/4026) Add `DASH_SECRET_KEY` so a hosting platform can give every worker and pod the same signing secret for page, background and stream tokens without editing the app (`server.secret_key` still wins and is not changed). - [#4026](https://github.com/plotly/dash/pull/4026) Add `DASH_SHARED_STORAGE` to pick the shared-storage backend when the app does not pass `shared_storage=`: `local`, `none`, `diskcache:///abs/path`, or a `redis://` / `rediss://` URL. - [#4024](https://github.com/plotly/dash/pull/4024) Add support for social media meta tags in regular Dash apps using `app.title`, `app.description`, and `app.image`. - [#3976](https://github.com/plotly/dash/pull/3976) Add a new `scrollToTop` prop to `dcc.Link` to control whether the page scrolls to the top after client-side navigation. It defaults to `True` to preserve the existing behavior. Fixes [#3974](https://github.com/plotly/dash/issues/3974). - [#3947](https://github.com/plotly/dash/pull/3947) Make `plotly-cloud` a default install dependency of Dash instead of an optional extra, so the `plotly` CLI and Dash's cloud integration work out of the box. The `dash[cloud]` extra is kept for backward compatibility. -- [#3930](https://github.com/plotly/dash/pull/3930) Add shared storage: a backend-agnostic cross-process state manager (key/value with optional TTL, plus ordered replayable pub/sub) on every app via `dash.ctx.shared_storage`, started lazily and disabled with `shared_storage=None`. Ships `LocalSharedStorage` (default, in-memory with optional disk persistence), `DiskcacheSharedStorage`, and `RedisSharedStorage` for horizontally-scaled deployments; see `.ai/ARCHITECTURE.md`. +- [#3930](https://github.com/plotly/dash/pull/3930) Add shared storage: a backend-agnostic cross-process state manager (key/value with optional TTL, plus ordered replayable pub/sub) via `dash.ctx.shared_storage`, started lazily. Ships `LocalSharedStorage` (in-memory with optional disk persistence), `DiskcacheSharedStorage`, and `RedisSharedStorage` for horizontally-scaled deployments; see `.ai/ARCHITECTURE.md`. - [#3931](https://github.com/plotly/dash/pull/3931) Add streaming callbacks: an `async def` generator callback streams its yields to the browser as they are produced (`dash.Patch` yields apply incrementally). A browser's streams share one SharedWorker-hosted connection so they do not count against the per-host connection limit; closing a tab cancels its streams. - [#3977](https://github.com/plotly/dash/pull/3977) Add partial WebSocket prop reads with `get_prop(..., path=...)`. Closes [#3975](https://github.com/plotly/dash/issues/3975). - [#3765](https://github.com/plotly/dash/pull/3765) Add opt-in partial pattern matching for callback `Input`, `Output`, and `State` via `partial_pattern=True`, so dictionary ID patterns can match component IDs with extra keys and combine with `ALL`/`MATCH` wildcards. Fixes [#3764](https://github.com/plotly/dash/issues/3764). diff --git a/dash/_callback_context.py b/dash/_callback_context.py index 430d04591a..2245ddad76 100644 --- a/dash/_callback_context.py +++ b/dash/_callback_context.py @@ -216,7 +216,7 @@ def shared_storage(self): A backend-agnostic key/value + publish/subscribe store shared across worker processes -- for cross-callback or session state without an - external service. Requires ``Dash(shared_storage=...)`` (on by default). + external service. Requires ``Dash(shared_storage=...)`` (off by default). """ return get_app().shared_storage diff --git a/dash/_shared_storage/__init__.py b/dash/_shared_storage/__init__.py index 2dba0ac90c..a26079595c 100644 --- a/dash/_shared_storage/__init__.py +++ b/dash/_shared_storage/__init__.py @@ -2,7 +2,7 @@ Exposes the ``BaseSharedStorage`` interface and its backends: -- ``LocalSharedStorage`` (default): in-memory, owner-elected -- one container. +- ``LocalSharedStorage``: in-memory, owner-elected -- one container. - ``DiskcacheSharedStorage``: on a ``diskcache.Cache`` -- one machine, not pods. - ``RedisSharedStorage``: on Redis -- across processes and containers. """ diff --git a/dash/_shared_storage/_codec.py b/dash/_shared_storage/_codec.py index 4884c1625f..f2191f0280 100644 --- a/dash/_shared_storage/_codec.py +++ b/dash/_shared_storage/_codec.py @@ -1,27 +1,53 @@ -"""Codec for the shared-storage socket transport. +"""Codec for the shared-storage transport and the persistent backends. -Uses ``msgspec`` (msgpack: fast, compact), a hard dependency of Dash. The -format is data-only, so bytes read off the socket cannot execute code. +Uses ``msgspec`` (msgpack: fast, compact) from the ``dash[shared_storage]`` +extra, also part of ``dash[diskcache]`` and ``dash[redis]``. Without it, only the in-memory transport works and it falls back to +stdlib JSON; the backends that write encoded bytes somewhere lasting (disk +persistence, diskcache, Redis) call :func:`require_msgspec` so the stored +format never depends on what happens to be installed. Both formats are +data-only, so bytes read off the socket cannot execute code. Payloads must be JSON-compatible (dict / list / str / int / float / bool / None) -- the same constraint as ``dcc.Store`` and callback outputs. """ +import json from typing import Any -import msgspec +try: + import msgspec +except ImportError: + msgspec = None # type: ignore[assignment] -# msgspec raises this on malformed input. It subclasses ``ValueError`` in -# current msgspec but not in every version, so callers should catch it by name. -DecodeError = msgspec.DecodeError +HAS_MSGSPEC = msgspec is not None -_encoder = msgspec.msgpack.Encoder() -_decoder = msgspec.msgpack.Decoder() +if msgspec is not None: + # msgspec raises this on malformed input. It subclasses ``ValueError`` in + # current msgspec but not in every version, so callers should catch it by + # name. + DecodeError: Any = msgspec.DecodeError + _encoder = msgspec.msgpack.Encoder() + _decoder = msgspec.msgpack.Decoder() + def encode(obj: Any) -> bytes: + return _encoder.encode(obj) -def encode(obj: Any) -> bytes: - return _encoder.encode(obj) + def decode(data: bytes) -> Any: + return _decoder.decode(data) +else: + DecodeError = json.JSONDecodeError -def decode(data: bytes) -> Any: - return _decoder.decode(data) + def encode(obj: Any) -> bytes: + return json.dumps(obj, separators=(",", ":")).encode("utf-8") + + def decode(data: bytes) -> Any: + return json.loads(data) + + +def require_msgspec(what: str, extra: str = "shared_storage") -> None: + if not HAS_MSGSPEC: + raise ImportError( + f"{what} requires the {extra} extra:\n\n" + f' $ pip install "dash[{extra}]"\n' + ) diff --git a/dash/_shared_storage/_env.py b/dash/_shared_storage/_env.py index 9c539452af..e8d80d1b83 100644 --- a/dash/_shared_storage/_env.py +++ b/dash/_shared_storage/_env.py @@ -49,10 +49,10 @@ def storage_from_env(value: Optional[str]) -> Any: """ raw = (value or "").strip() lowered = raw.lower() - if lowered in ("", "local"): - return LocalSharedStorage - if lowered == "none": + if lowered in ("", "none"): return None + if lowered == "local": + return LocalSharedStorage try: scheme = urlparse(raw).scheme.lower() diff --git a/dash/_shared_storage/_persistence.py b/dash/_shared_storage/_persistence.py index 1912a9bd08..1032a20bc4 100644 --- a/dash/_shared_storage/_persistence.py +++ b/dash/_shared_storage/_persistence.py @@ -4,8 +4,8 @@ keys are sharded across fixed-size chunk files via a key->chunk index, so a single mutation rewrites only the affected chunk rather than the whole store (the design is adapted from raposa's ``BinaryStorage``). Serialization is -msgspec msgpack via :mod:`._codec`; each chunk write is atomic (temp file + -``os.replace``). +msgspec msgpack via :mod:`._codec` (the ``dash[shared_storage]`` extra); each +chunk write is atomic (temp file + ``os.replace``). Two modes drive *when* the store is written: diff --git a/dash/_shared_storage/_transport.py b/dash/_shared_storage/_transport.py index c8be5a7863..97eb2f3f94 100644 --- a/dash/_shared_storage/_transport.py +++ b/dash/_shared_storage/_transport.py @@ -3,9 +3,9 @@ A single owner process serves one :class:`StoreEngine` over a stream socket (AF_UNIX where available, TCP loopback otherwise). Messages are length-prefixed: a 4-byte big-endian length followed by a body encoded by ``_codec`` (msgspec -msgpack). The codec is data-only -- the socket is -localhost-only, but deserializing untrusted bytes off a socket is a -remote-code-execution hazard, so the transport carries only data. The channel is +msgpack, or JSON without the ``dash[shared_storage]`` extra). The codec is +data-only -- the socket is localhost-only, but deserializing untrusted bytes off +a socket is a remote-code-execution hazard, so the transport carries only data. The channel is additionally gated by a per-owner random token; only same-host clients that read the owner's advertisement file (written 0600) can attach. diff --git a/dash/_shared_storage/diskcache.py b/dash/_shared_storage/diskcache.py index a232dc37ca..e5b2416c34 100644 --- a/dash/_shared_storage/diskcache.py +++ b/dash/_shared_storage/diskcache.py @@ -20,7 +20,7 @@ import time from typing import Any, Optional -from ._codec import decode, encode +from ._codec import decode, encode, require_msgspec from ._engine import DEFAULT_BUFFER, PollResult from ._polling import PollingSubscription from .base import BaseSharedStorage, Subscription, check_topic_ttl @@ -37,6 +37,7 @@ def _require_diskcache(): + require_msgspec("DiskcacheSharedStorage", "diskcache") try: import diskcache # type: ignore[import-not-found,import-untyped] # pylint: disable=import-outside-toplevel diff --git a/dash/_shared_storage/local.py b/dash/_shared_storage/local.py index c81ddfe299..9c857f9a48 100644 --- a/dash/_shared_storage/local.py +++ b/dash/_shared_storage/local.py @@ -25,6 +25,7 @@ import time from typing import Any, Optional, Tuple +from ._codec import require_msgspec from ._engine import DEFAULT_BUFFER, PollResult, StoreEngine from ._persistence import _Persistence, default_store_dir from ._transport import ( @@ -477,6 +478,7 @@ class LocalSharedStorage(BaseSharedStorage): ``flush_interval`` seconds and on clean exit, and recovered on start; up to one interval of writes may be lost on an unclean crash. + The persistent modes need the ``dash[shared_storage]`` extra (msgspec). ``path`` is the on-disk store directory for the persistent modes; it defaults to a per-namespace folder under the user cache directory. ``flush_interval`` (seconds) only applies to ``"persist-reset"``. @@ -501,6 +503,8 @@ def __init__( raise SharedStorageError( f"flush_interval must be positive, got {flush_interval!r}" ) + if mode != "memory": + require_msgspec(f"LocalSharedStorage(mode={mode!r})") ns = namespace or _default_namespace() if mode != "memory" and path is None: path = default_store_dir(ns) diff --git a/dash/_shared_storage/redis.py b/dash/_shared_storage/redis.py index 1fdcbefb05..6c3743ef47 100644 --- a/dash/_shared_storage/redis.py +++ b/dash/_shared_storage/redis.py @@ -27,7 +27,7 @@ import time from typing import Any, Dict, List, Optional, Set, Tuple -from ._codec import decode, encode +from ._codec import decode, encode, require_msgspec from ._engine import DEFAULT_BUFFER, PollResult from ._polling import PollingSubscription from .base import ( @@ -93,6 +93,7 @@ def _require_redis(): + require_msgspec("RedisSharedStorage", "redis") try: import redis # type: ignore[import-not-found,import-untyped] # pylint: disable=import-outside-toplevel diff --git a/dash/backends/_fastapi.py b/dash/backends/_fastapi.py index ec1514a405..ee59f91084 100644 --- a/dash/backends/_fastapi.py +++ b/dash/backends/_fastapi.py @@ -718,7 +718,7 @@ async def _dispatch( downlink = body.get("streamDownlink") if downlink is not None: connection_id = get_stream_connection_id() - if connection_id is None: + if connection_id is None or not dash_app.shared_storage_enabled: return Response(status_code=403) return _ndjson_response( async_downlink_marker( diff --git a/dash/backends/_flask.py b/dash/backends/_flask.py index 8054a25ac7..03ffedb2b7 100644 --- a/dash/backends/_flask.py +++ b/dash/backends/_flask.py @@ -310,7 +310,7 @@ def _serve_downlink(downlink): # The connection id is derived from the signed end_id, never taken # from the client, so a page can only ever read its own topic. connection_id = get_stream_connection_id() - if connection_id is None: + if connection_id is None or not dash_app.shared_storage_enabled: return Response(status=403) envelopes = poll_downlink( dash_app.shared_storage, connection_id, downlink.get("from") diff --git a/dash/backends/_quart.py b/dash/backends/_quart.py index ac58e1acf5..408463343e 100644 --- a/dash/backends/_quart.py +++ b/dash/backends/_quart.py @@ -447,7 +447,7 @@ async def _dispatch(): # pylint: disable=too-many-return-statements downlink = body.get("streamDownlink") if downlink is not None: connection_id = get_stream_connection_id() - if connection_id is None: + if connection_id is None or not dash_app.shared_storage_enabled: return Response(status=403) # type: ignore[return-value] marker = async_downlink_marker( dash_app.shared_storage, diff --git a/dash/dash.py b/dash/dash.py index 72a1d7b604..601ef964af 100644 --- a/dash/dash.py +++ b/dash/dash.py @@ -479,11 +479,15 @@ class Dash(ObsoleteChecker): :param shared_storage: Backend for ``dash.ctx.shared_storage`` and the streaming-callback transport: a ``BaseSharedStorage`` subclass or - instance, or ``None`` to disable. Default ``LocalSharedStorage`` (one - machine or pod). When not passed, the ``DASH_SHARED_STORAGE`` - environment variable can choose it: ``local``, ``none``, - ``diskcache:///absolute/path``, or a ``redis://`` / ``rediss://`` URL. - An explicit argument, including ``None``, always wins. + instance, or ``None`` to disable. Off by default: pass + ``LocalSharedStorage`` for one machine or pod. Without it, streaming + callbacks still work, each on its own connection. When not passed, the + ``DASH_SHARED_STORAGE`` environment variable can choose it: ``local``, + ``none``, ``diskcache:///absolute/path``, or a ``redis://`` / + ``rediss://`` URL. An explicit argument, including ``None``, always + wins. The persistent ``LocalSharedStorage`` modes need + ``pip install "dash[shared_storage]"``; the diskcache and Redis backends + need their own ``dash[diskcache]`` / ``dash[redis]`` extra. Stream requests are signed per page load, so with several workers or pods they all need the same signing secret: set ``server.secret_key`` @@ -1036,7 +1040,7 @@ def layout(self, value: Any): @property def shared_storage_enabled(self) -> bool: - """Whether this app has a shared-storage backend (not ``None``). + """Whether this app has a shared-storage backend (off by default). Cheap to read and does not start the backend, unlike ``shared_storage``. """ @@ -1047,13 +1051,14 @@ def shared_storage(self) -> BaseSharedStorage: """The app's shared storage (state manager + pub/sub), backend-agnostic. Started on first access in the worker that uses it. Access it from a - callback via ``dash.ctx.shared_storage``. Raises if the app was created - with ``shared_storage=None``. + callback via ``dash.ctx.shared_storage``. Raises if the app has no + shared storage (pass ``shared_storage=`` or set ``DASH_SHARED_STORAGE``). """ if self._shared_storage_arg is None: raise SharedStorageError( "Shared storage is disabled for this app; construct it with " - "Dash(shared_storage=...) to use dash.ctx.shared_storage." + "Dash(shared_storage=...) or set DASH_SHARED_STORAGE to use " + "dash.ctx.shared_storage." ) if self._shared_storage_instance is None: with self._shared_storage_lock: diff --git a/requirements/ci.txt b/requirements/ci.txt index af242627b2..d502cd9f01 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -8,6 +8,7 @@ ipython<9.0.0 mimesis<=11.1.0; python_version < "3.10" mimesis<=22.2.0; python_version >= "3.10" mock==5.2.0 +msgspec # dash[shared_storage] numpy<=2.5.4 orjson>=3.10.11 openpyxl diff --git a/requirements/diskcache.txt b/requirements/diskcache.txt index 59be4d2b79..47526ef01b 100644 --- a/requirements/diskcache.txt +++ b/requirements/diskcache.txt @@ -2,3 +2,4 @@ diskcache>=5.2.1 multiprocess>=0.70.12 psutil>=5.8.0 +msgspec # DiskcacheSharedStorage diff --git a/requirements/install.txt b/requirements/install.txt index 795f0c85c8..176d9c8f5e 100644 --- a/requirements/install.txt +++ b/requirements/install.txt @@ -10,4 +10,3 @@ setuptools janus>=1.0.0 pydantic>=2.10 comm -msgspec diff --git a/requirements/redis.txt b/requirements/redis.txt index c4903b4540..01c0b4f7ba 100644 --- a/requirements/redis.txt +++ b/requirements/redis.txt @@ -1,4 +1,5 @@ -# Dependency used by RedisSharedStorage. +# Dependencies used by RedisSharedStorage. # Kept in sync with the redis pin in celery.txt; in Dash 5.0 redis will be # removed from the celery extra and live here only. redis>=3.5.3,<=8.1.0 +msgspec diff --git a/requirements/shared_storage.txt b/requirements/shared_storage.txt new file mode 100644 index 0000000000..bdc9e5a1e2 --- /dev/null +++ b/requirements/shared_storage.txt @@ -0,0 +1 @@ +msgspec diff --git a/setup.py b/setup.py index ac7163e384..d657d96916 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,8 @@ def read_req_file(req_type): "fastapi": read_req_file("fastapi"), "quart": read_req_file("quart"), "cloud": read_req_file("cloud"), - "ag-grid": read_req_file("ag-grid") + "ag-grid": read_req_file("ag-grid"), + "shared_storage": read_req_file("shared_storage") }, entry_points={ "console_scripts": [ diff --git a/tests/shared_storage/test_env_config.py b/tests/shared_storage/test_env_config.py index 68535f1065..aca7dc66fc 100644 --- a/tests/shared_storage/test_env_config.py +++ b/tests/shared_storage/test_env_config.py @@ -24,19 +24,19 @@ def _built(app): return storage if storage is None else storage() -def test_unset_defaults_to_local(monkeypatch): +def test_unset_disables(monkeypatch): monkeypatch.delenv("DASH_SHARED_STORAGE", raising=False) app = Dash(__name__) - assert app._shared_storage_arg is LocalSharedStorage + assert not app.shared_storage_enabled -@pytest.mark.parametrize("value", ["local", "LOCAL", " local ", ""]) +@pytest.mark.parametrize("value", ["local", "LOCAL", " local "]) def test_local(monkeypatch, value): app = _app(monkeypatch, value) assert app._shared_storage_arg is LocalSharedStorage -@pytest.mark.parametrize("value", ["none", "None"]) +@pytest.mark.parametrize("value", ["none", "None", ""]) def test_none_disables(monkeypatch, value): app = _app(monkeypatch, value) assert not app.shared_storage_enabled diff --git a/tests/shared_storage/test_no_msgspec.py b/tests/shared_storage/test_no_msgspec.py new file mode 100644 index 0000000000..aaae43413a --- /dev/null +++ b/tests/shared_storage/test_no_msgspec.py @@ -0,0 +1,103 @@ +"""Without msgspec, the in-memory backend still works over its socket +transport, and every backend that writes encoded bytes somewhere lasting +refuses to start with a pointer to the extra that brings it. + +Each case runs in a fresh interpreter with msgspec blocked, so the codec picks +its fallback at import time just as it would in a real install. +""" +import subprocess +import sys +import textwrap +import uuid + +import pytest + +PRELUDE = "import sys\nsys.modules['msgspec'] = None\n" + + +def _run(code): + proc = subprocess.run( + [sys.executable, "-c", PRELUDE + textwrap.dedent(code)], + capture_output=True, + text=True, + timeout=60, + check=False, + ) + assert proc.returncode == 0, proc.stderr + return proc.stdout + + +def test_memory_mode_crosses_processes_as_json(): + out = _run( + f""" + from dash._shared_storage import LocalSharedStorage, _codec + + assert not _codec.HAS_MSGSPEC + owner = LocalSharedStorage(namespace="nomsgspec-{uuid.uuid4().hex[:12]}") + owner.start() + client = LocalSharedStorage(namespace=owner._coord._namespace) + client.start() + assert owner._coord.is_owner() and not client._coord.is_owner() + + client.set("k", {{"a": [1, 2.5, None, True, "x"]}}) + assert owner.get("k") == {{"a": [1, 2.5, None, True, "x"]}} + sub = client.subscribe("t") + owner.publish("t", {{"n": 1}}) + print(sub.poll(timeout=5)) + sub.close() + client.close() + owner.close() + """ + ) + assert out.strip() == "[(1, {'n': 1})]" + + +@pytest.mark.parametrize( + "setup, what, extra", + [ + ( + "from dash._shared_storage import LocalSharedStorage\n" + "LocalSharedStorage(mode='persist', path='{tmp}')", + "LocalSharedStorage(mode='persist')", + "shared_storage", + ), + ( + "from dash._shared_storage import LocalSharedStorage\n" + "LocalSharedStorage(mode='persist-reset', path='{tmp}')", + "LocalSharedStorage(mode='persist-reset')", + "shared_storage", + ), + ( + "from dash._shared_storage import DiskcacheSharedStorage\n" + "DiskcacheSharedStorage(directory='{tmp}')", + "DiskcacheSharedStorage", + "diskcache", + ), + ( + "from dash._shared_storage import RedisSharedStorage\n" + "RedisSharedStorage(url='redis://localhost:6399/0')", + "RedisSharedStorage", + "redis", + ), + ( + "import os\nos.environ['DASH_SHARED_STORAGE'] = 'redis://localhost:6399/0'\n" + "from dash import Dash\nDash(__name__)", + "RedisSharedStorage", + "redis", + ), + ( + "import os\nos.environ['DASH_SHARED_STORAGE'] = 'diskcache://{tmp}'\n" + "from dash import Dash\nDash(__name__)", + "DiskcacheSharedStorage", + "diskcache", + ), + ], +) +def test_lasting_backends_need_the_extra(tmp_path, setup, what, extra): + out = _run( + "try:\n" + + textwrap.indent(setup.format(tmp=tmp_path), " ") + + "\nexcept ImportError as err:\n print(err)\n" + ) + assert f"{what} requires the {extra} extra" in out + assert f'pip install "dash[{extra}]"' in out diff --git a/tests/streaming/test_stream_callbacks_integration.py b/tests/streaming/test_stream_callbacks_integration.py index 1643c1e40c..da4d4effcc 100644 --- a/tests/streaming/test_stream_callbacks_integration.py +++ b/tests/streaming/test_stream_callbacks_integration.py @@ -11,6 +11,7 @@ no_update, set_props, ) +from dash._shared_storage import LocalSharedStorage from dash.testing.wait import until @@ -216,8 +217,6 @@ def test_stst020_multiplexed_transport_over_shared_storage(dash_duo): stream client rejects) and the downlink would 403, so nothing would render. Two callbacks share the one downlink, so this also covers multiplexing. """ - from dash._shared_storage import LocalSharedStorage - app = Dash(__name__, shared_storage=LocalSharedStorage()) app.layout = html.Div( [ @@ -279,7 +278,7 @@ def _open_tab(dash_duo): def test_stst010_many_tabs_share_one_downlink(dash_duo): """Browsers cap connections per host (~6); with the downlink hosted in a SharedWorker, eight streaming tabs share one and all keep streaming.""" - app = Dash(__name__) + app = Dash(__name__, shared_storage=LocalSharedStorage) app.layout = html.Div([html.Div(id="out", children="idle")]) @app.callback(Output("out", "children"), Input("out", "id")) @@ -303,7 +302,7 @@ async def stream_cb(_): def test_stst011_closing_a_tab_cancels_its_stream(dash_duo): """A tab closing while others keep the shared downlink open cancels that tab's callback server-side instead of running it to completion.""" - app = Dash(__name__) + app = Dash(__name__, shared_storage=LocalSharedStorage) app.layout = html.Div([html.Div(id="out", children="idle")]) events = [] @@ -338,7 +337,6 @@ def test_stst012_idle_stream_topics_are_released(dash_duo, monkeypatch): behind for the life of the process. A page that streams again after its topic was dropped still gets every frame of the new run.""" from dash import _stream_hub - from dash._shared_storage import LocalSharedStorage ttl = 1.0 monkeypatch.setattr(_stream_hub, "STREAM_TOPIC_TTL", ttl) diff --git a/tests/streaming/test_stream_transport.py b/tests/streaming/test_stream_transport.py index b3d04fb468..e61c7ef67d 100644 --- a/tests/streaming/test_stream_transport.py +++ b/tests/streaming/test_stream_transport.py @@ -161,6 +161,40 @@ def test_flask_downlink_rejects_missing_end_id(): storage.close() +@pytest.mark.parametrize("flavor", ["flask", "fastapi", "quart"]) +def test_downlink_without_shared_storage_is_rejected(flavor): + # The renderer never opens a downlink when shared storage is off, so a + # request for one gets 403 instead of a SharedStorageError 500. + body = {"streamDownlink": {"from": 0}} + if flavor == "flask": + app = Dash(__name__) + app.layout = html.Div() + resp = app.server.test_client().post(_uplink_url(app), json=body) + assert resp.status_code == 403 + return + if flavor == "fastapi": + pytest.importorskip("httpx", reason="fastapi.testclient requires httpx") + from fastapi import FastAPI + from fastapi.testclient import TestClient + + app = Dash(__name__, server=FastAPI()) + app.layout = html.Div() + app._setup_server() # pylint: disable=protected-access + with TestClient(app.server) as client: + assert client.post(_uplink_url(app), json=body).status_code == 403 + return + quart = pytest.importorskip("quart") + app = Dash(__name__, server=quart.Quart(__name__)) + app.layout = html.Div() + app._setup_server() # pylint: disable=protected-access + + async def run(): + resp = await app.server.test_client().post(_uplink_url(app), json=body) + return resp.status_code + + assert asyncio.run(run()) == 403 + + def test_flask_forged_stream_tokens_warn_once(monkeypatch, caplog): # A token that fails verification is almost always a secret mismatch # between workers, so say so, but once per process, not per poll. diff --git a/tests/streaming/test_stream_wsgi.py b/tests/streaming/test_stream_wsgi.py index 1d51f0c630..e0548c1970 100644 --- a/tests/streaming/test_stream_wsgi.py +++ b/tests/streaming/test_stream_wsgi.py @@ -33,13 +33,15 @@ import asyncio import os from dash import Dash, Input, Output, html + from dash._shared_storage import LocalSharedStorage, _codec - app = Dash(__name__) + app = Dash(__name__, shared_storage=LocalSharedStorage) server = app.server @server.after_request def tag_worker(response): response.headers["X-Worker-Pid"] = str(os.getpid()) + response.headers["X-Has-Msgspec"] = str(_codec.HAS_MSGSPEC) return response app.layout = html.Div([html.Button("go", id="btn", n_clicks=0), html.Div(id="out")]) @@ -62,8 +64,9 @@ def _free_port(): @contextlib.contextmanager -def _gunicorn(tmp_path, workers, env=None): +def _gunicorn(tmp_path, workers, env=None, config=""): (tmp_path / "wsgi_app.py").write_text(APP) + (tmp_path / "gunicorn_conf.py").write_text(config) port = _free_port() proc_env = {k: v for k, v in os.environ.items() if k.lower() != "dash_secret_key"} proc_env.update(env or {}) @@ -79,6 +82,8 @@ def _gunicorn(tmp_path, workers, env=None): str(workers), "--timeout", "30", + "--config", + "gunicorn_conf.py", ], cwd=tmp_path, env=proc_env, @@ -170,3 +175,16 @@ def test_stwg003_workers_without_shared_secret_refuse_streams(tmp_path): page_pid, polls = _downlink_polls(url) for pid, status in polls: assert status == (200 if pid == page_pid else 403) + + +def test_stwg004_workers_stream_without_msgspec(dash_br, tmp_path): + # Without the shared_storage extra, the in-memory backend still brokers + # streams across workers, encoding its socket messages as JSON. + no_msgspec = "import sys\nsys.modules['msgspec'] = None\n" + with _gunicorn( + tmp_path, workers=4, env={"DASH_SECRET_KEY": "shared"}, config=no_msgspec + ) as url: + assert requests.get(url, timeout=5).headers["X-Has-Msgspec"] == "False" + _, polls = _downlink_polls(url) + assert {status for _, status in polls} == {200} + _stream_completes(dash_br, url) From 7b397aef5e2161d5619451ac86ca6f858fbb3931 Mon Sep 17 00:00:00 2001 From: philippe Date: Wed, 28 Oct 2026 10:57:14 -0400 Subject: [PATCH 2/3] Add PR number to CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14ca695428..4029c03251 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] ### Added -- [#](https://github.com/plotly/dash/pull/) Make shared storage and the multiplexed streaming downlink opt-in (`shared_storage=` or `DASH_SHARED_STORAGE`), and move `msgspec` to a new `dash[shared_storage]` extra for the persistent `LocalSharedStorage` modes (`dash[diskcache]` and `dash[redis]` include it). +- [#4095](https://github.com/plotly/dash/pull/4095) Make shared storage and the multiplexed streaming downlink opt-in (`shared_storage=` or `DASH_SHARED_STORAGE`), and move `msgspec` to a new `dash[shared_storage]` extra for the persistent `LocalSharedStorage` modes (`dash[diskcache]` and `dash[redis]` include it). - [#4026](https://github.com/plotly/dash/pull/4026) Add `DASH_SECRET_KEY` so a hosting platform can give every worker and pod the same signing secret for page, background and stream tokens without editing the app (`server.secret_key` still wins and is not changed). - [#4026](https://github.com/plotly/dash/pull/4026) Add `DASH_SHARED_STORAGE` to pick the shared-storage backend when the app does not pass `shared_storage=`: `local`, `none`, `diskcache:///abs/path`, or a `redis://` / `rediss://` URL. - [#4024](https://github.com/plotly/dash/pull/4024) Add support for social media meta tags in regular Dash apps using `app.title`, `app.description`, and `app.image`. From 945c00471346034244a53e415d25784f5aec6f46 Mon Sep 17 00:00:00 2001 From: philippe Date: Wed, 28 Oct 2026 11:41:41 -0400 Subject: [PATCH 3/3] Enable shared storage in the Windows stream worker URL test --- tests/unit/test_resources.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_resources.py b/tests/unit/test_resources.py index cbbf52964b..7ca8509389 100644 --- a/tests/unit/test_resources.py +++ b/tests/unit/test_resources.py @@ -272,7 +272,11 @@ def test_index_with_windows_extended_length_paths(tmp_path, monkeypatch): icons.mkdir(parents=True) (icons / "favicon.ico").write_bytes(b"") - app = dash.Dash(__name__, assets_folder="\\\\?\\" + str(tmp_path / "assets")) + app = dash.Dash( + __name__, + assets_folder="\\\\?\\" + str(tmp_path / "assets"), + shared_storage=dash.LocalSharedStorage, + ) app.layout = html.Div() response = app.server.test_client().get("/")