From 1ee9e2be8904a718076d155b7f97e3a65a48c0e5 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 27 Jul 2026 14:57:13 -0500 Subject: [PATCH 1/8] PYTHON-5945 Add OpenTelemetry Simple Command Support --- .evergreen/generated_configs/variants.yml | 12 + .evergreen/scripts/generate_config.py | 15 + .evergreen/scripts/setup_tests.py | 11 +- .evergreen/scripts/utils.py | 1 + doc/changelog.rst | 6 + justfile | 2 +- pymongo/_otel.py | 268 +++++++++++++++ pymongo/_telemetry.py | 38 ++- pymongo/asynchronous/command_runner.py | 28 +- pymongo/asynchronous/mongo_client.py | 19 ++ pymongo/client_options.py | 13 + pymongo/common.py | 22 ++ pymongo/pool_shared.py | 2 +- pymongo/synchronous/command_runner.py | 28 +- pymongo/synchronous/mongo_client.py | 19 ++ pyproject.toml | 2 + requirements/opentelemetry.txt | 1 + test/asynchronous/test_otel.py | 388 ++++++++++++++++++++++ test/test_otel.py | 382 +++++++++++++++++++++ uv.lock | 340 ++++++++++--------- 20 files changed, 1421 insertions(+), 176 deletions(-) create mode 100644 pymongo/_otel.py create mode 100644 requirements/opentelemetry.txt create mode 100644 test/asynchronous/test_otel.py create mode 100644 test/test_otel.py diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index 26027888fe..b9b054bda0 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -468,6 +468,18 @@ buildvariants: - windows-2022-latest-small batchtime: 1440 + # Otel tests + - name: otel-rhel8 + tasks: + - name: .test-non-standard .standalone-noauth-nossl + display_name: OTel RHEL8 + run_on: + - rhel8.10-small + expansions: + TEST_NAME: otel + COVERAGE: "1" + tags: [pr] + # Perf tests - name: performance-benchmarks tasks: diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 97fd9b0176..a3979472e4 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -513,6 +513,21 @@ def create_doctests_variants(): ] +def create_otel_variants(): + host = DEFAULT_HOST + # Merge otel's coverage into the combined report; see setup_tests.py's COVERAGE handling. + expansions = dict(TEST_NAME="otel", COVERAGE="1") + return [ + create_variant( + [".test-non-standard .standalone-noauth-nossl"], + get_variant_name("OTel", host), + host=host, + tags=["pr"], + expansions=expansions, + ) + ] + + def create_atlas_connect_variants(): host = DEFAULT_HOST return [ diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 5a3f441f58..4f6833e289 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -42,6 +42,7 @@ "enterprise_auth": "gssapi", "kms": "encryption", "ocsp": "ocsp", + "otel": "opentelemetry", "pyopenssl": "ocsp", } @@ -502,6 +503,11 @@ def handle_test_env() -> None: if test_name == "numpy": UV_ARGS.append("--with numpy") + if test_name == "otel": + # The SDK is test-only tooling (for the in-memory span exporter); the driver + # itself must not depend on it, only on opentelemetry-api (the "opentelemetry" extra). + UV_ARGS.append("--with opentelemetry-sdk") + if test_name == "perf": data_dir = ROOT / "specifications/source/benchmarking/data" if not data_dir.exists(): @@ -523,9 +529,10 @@ def handle_test_env() -> None: else: TEST_ARGS = f"test/performance/async_perf_test.py {TEST_ARGS}" - # Add coverage if requested. + # Add coverage if requested, either via --cov or a pre-set COVERAGE expansion + # (e.g. a buildvariant that wants its coverage merged into the combined report). # Only cover CPython. PyPy reports suspiciously low coverage. - if opts.cov and platform.python_implementation() == "CPython": + if (opts.cov or is_set("COVERAGE")) and platform.python_implementation() == "CPython": # Keep in sync with combine-coverage.sh. # coverage >=5 is needed for relative_files=true. UV_ARGS.append("--group coverage") diff --git a/.evergreen/scripts/utils.py b/.evergreen/scripts/utils.py index a7decdfaa8..880e28cec1 100644 --- a/.evergreen/scripts/utils.py +++ b/.evergreen/scripts/utils.py @@ -43,6 +43,7 @@ class Distro: "load_balancer": "load_balancer", "mockupdb": "mockupdb", "ocsp": "ocsp", + "otel": "otel", "perf": "perf", "sfp": "sfp", "numpy": "", diff --git a/doc/changelog.rst b/doc/changelog.rst index 08d3908e62..dde6388937 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -152,6 +152,12 @@ PyMongo 4.18 brings a number of changes including: :meth:`~pymongo.synchronous.database.Database.aggregate`, and :meth:`~pymongo.asynchronous.collection.AsyncCollection.list_search_indexes` and :meth:`~pymongo.synchronous.collection.Collection.list_search_indexes`. +- Added optional OpenTelemetry command-span support, conforming to the + `OpenTelemetry driver specification `_. + Enable it with the ``tracing`` :class:`~pymongo.mongo_client.MongoClient` + option or the ``OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED`` environment + variable. Install the ``opentelemetry-api`` package, or use the + ``pymongo[opentelemetry]`` extra, to enable this feature. Changes in Version 4.17.0 (2026/04/20) -------------------------------------- diff --git a/justfile b/justfile index df678cdbac..b6156fe72c 100644 --- a/justfile +++ b/justfile @@ -2,7 +2,7 @@ set shell := ["bash", "-c"] # Commonly used command segments. -typing_run := "uv run --group typing --extra aws --extra encryption --with numpy --extra ocsp --extra snappy --extra test --extra zstd" +typing_run := "uv run --group typing --extra aws --extra encryption --with numpy --extra ocsp --extra opentelemetry --with opentelemetry-sdk --extra snappy --extra test --extra zstd" docs_run := "uv run --extra docs" doc_build := "./doc/_build" mypy_args := "--install-types --non-interactive" diff --git a/pymongo/_otel.py b/pymongo/_otel.py new file mode 100644 index 0000000000..713f963bbc --- /dev/null +++ b/pymongo/_otel.py @@ -0,0 +1,268 @@ +# Copyright 2026-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Optional OpenTelemetry command-span support. + +Kept separate from :mod:`pymongo._telemetry` so that module stays free of +``opentelemetry`` import guards. Every function here is a no-op when +``opentelemetry`` isn't installed or tracing isn't enabled. +""" + +from __future__ import annotations + +import os +from collections.abc import Mapping, MutableMapping +from typing import TYPE_CHECKING, Any, Optional, TypedDict + +from bson import json_util +from bson.json_util import _truncate_documents +from pymongo._version import __version__ +from pymongo.logger import _HELLO_COMMANDS, _JSON_OPTIONS, _SENSITIVE_COMMANDS + +try: + from opentelemetry import trace + from opentelemetry.trace import SpanKind, Status, StatusCode + + _HAS_OPENTELEMETRY = True + # Safe to cache at import time: opentelemetry.trace.get_tracer() returns a + # ProxyTracer when no real TracerProvider is registered yet, and that proxy + # transparently starts delegating to the real tracer once the application + # calls trace.set_tracer_provider() later, so this doesn't bind us to a + # permanently-inert no-op tracer. + _TRACER: Optional[Tracer] = trace.get_tracer("PyMongo", __version__) +except ImportError: + _HAS_OPENTELEMETRY = False + _TRACER = None + +if TYPE_CHECKING: + from opentelemetry.trace import Span, Tracer + + from pymongo.pool_shared import _ConnectionTelemetryInfo + from pymongo.typings import _DocumentOut + + +class TracingOptions(TypedDict): + """The shape of the ``MongoClient`` ``tracing`` option. + + ``query_text_max_length`` is None when the client didn't configure it, so + the environment variable can be consulted; any explicit value (including + 0, to force ``db.query.text`` off) overrides the environment variable. + """ + + enabled: bool + query_text_max_length: Optional[int] + + +_OTEL_ENABLED_ENV = "OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED" +_OTEL_QUERY_TEXT_MAX_LENGTH_ENV = "OTEL_PYTHON_INSTRUMENTATION_MONGODB_QUERY_TEXT_MAX_LENGTH" +_TRUTHY = frozenset({"1", "true", "yes"}) + +# Fields redacted from the db.query.text attribute, mirroring the fields excluded +# from the equivalent CommandStartedEvent.command per the OpenTelemetry spec. +_QUERY_TEXT_EXCLUDED_FIELDS = frozenset({"lsid", "$db", "$clusterTime", "signature"}) + +# getMore's own command value is the cursor id, not the collection name; the +# collection lives under a separate "collection" key instead. +# See _gen_get_more_command in pymongo/message.py. +_GET_MORE = "getMore" + +# explain wraps the real command (e.g. find/aggregate) rather than naming a +# collection directly: {"explain": {"find": "coll", ...}}. See _Query.as_command +# in pymongo/message.py. +_EXPLAIN = "explain" + +# Commands against this database (e.g. user/role management, renameCollection) +# never have a real collection name, even when their command value is a string. +_ADMIN_DB = "admin" + + +def _env_truthy(name: str) -> bool: + """Return True if the environment variable ``name`` is set to "1", "true", or "yes".""" + return os.getenv(name, "").strip().lower() in _TRUTHY + + +def _is_tracing_enabled(tracing_options: Optional[TracingOptions]) -> bool: + """Return True if OTel command spans should be created for this client. + + The ``MongoClient`` ``tracing.enabled`` option and the + ``OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED`` environment variable both + gate enablement; either one being truthy is sufficient. + """ + if not _HAS_OPENTELEMETRY: + return False + if tracing_options and tracing_options.get("enabled"): + return True + return _env_truthy(_OTEL_ENABLED_ENV) + + +def _get_query_text_max_length(tracing_options: Optional[TracingOptions]) -> int: + """Return the configured db.query.text truncation length, or 0 to omit the attribute. + + An explicit client value (including 0) always wins; the environment + variable is only consulted when the client didn't configure it at all. + """ + client_value = tracing_options.get("query_text_max_length") if tracing_options else None + if client_value is not None: + return max(0, client_value) + try: + return max(0, int(os.getenv(_OTEL_QUERY_TEXT_MAX_LENGTH_ENV, "0"))) + except ValueError: + return 0 + + +def _build_query_text(cmd: Mapping[str, Any], max_length: int) -> str: + """Serialize ``cmd`` to extended JSON, redacted and truncated to ``max_length``. + + Mirrors the truncation approach used for log messages: truncate field + values first, which usually keeps the result well-formed JSON (unlike a + blind cut of the fully-serialized string), then fall back to a hard + string cut as a safety net for whatever the field truncation's size + estimate still leaves over ``max_length``. The "..." marker is carved out + of the budget (not appended on top of it) so the result never exceeds + ``max_length``. + """ + filtered = {k: v for k, v in cmd.items() if k not in _QUERY_TEXT_EXCLUDED_FIELDS} + truncated_cmd = _truncate_documents(filtered, max_length)[0] + # default=repr mirrors the structured logger: tracing is best-effort and must + # not raise for commands containing custom/codec-managed Python types. + text = json_util.dumps(truncated_cmd, json_options=_JSON_OPTIONS, default=repr) + if len(text) > max_length: + suffix = "..." + text = text[: max(0, max_length - len(suffix))] + suffix + return text + + +def _extract_collection_name( + command_name: str, dbname: str, cmd: Mapping[str, Any] +) -> Optional[str]: + """Return the collection name targeted by ``cmd``, or None if it doesn't target one. + + Always None for commands against the admin database: several (e.g. dropUser, + renameCollection) carry a string command value that names a user, role, or + namespace rather than a collection. + """ + if dbname == _ADMIN_DB: + return None + if command_name == _EXPLAIN: + inner = cmd.get(_EXPLAIN) + if not isinstance(inner, Mapping) or not inner: + return None + inner_name = next(iter(inner)) + return _extract_collection_name(inner_name, dbname, inner) + key = "collection" if command_name == _GET_MORE else command_name + value = cmd.get(key) + return value if isinstance(value, str) else None + + +def _build_query_summary(command_name: str, dbname: str, collection: Optional[str]) -> str: + """Build the ``db.query.summary`` attribute value for a command.""" + if collection: + return f"{command_name} {dbname}.{collection}" + return f"{command_name} {dbname}" + + +def _is_sensitive_command(command_name: str, speculative_hello: bool) -> bool: + """Mirror the redaction rules in ``pymongo.logger.LogMessage._is_sensitive``.""" + if command_name in _SENSITIVE_COMMANDS: + return True + return command_name in _HELLO_COMMANDS and speculative_hello + + +def _format_lsid(lsid: Mapping[str, Any]) -> Optional[str]: + """Return the ``db.mongodb.lsid`` attribute value for a session id document.""" + id_value = lsid.get("id") + if id_value is None: + return None + try: + return str(id_value.as_uuid()) + except (AttributeError, ValueError): + return str(id_value) + + +def start_command_span( + tracing_options: Optional[TracingOptions], + conn: _ConnectionTelemetryInfo, + cmd: MutableMapping[str, Any], + dbname: str, + command_name: str, + speculative_hello: bool, +) -> Optional[Span]: + """Start and return a CLIENT-kind span for a server command, or None. + + Returns None when tracing is disabled/unavailable or the command is + sensitive (mirroring the redaction applied to logs). + """ + if not _is_tracing_enabled(tracing_options): + return None + if _is_sensitive_command(command_name, speculative_hello): + return None + + collection = _extract_collection_name(command_name, dbname, cmd) + address = conn.address + transport = "unix" if address[1] is None else "tcp" + attributes: dict[str, Any] = { + "db.system.name": "mongodb", + "db.namespace": dbname, + "db.command.name": command_name, + "db.query.summary": _build_query_summary(command_name, dbname, collection), + "server.address": address[0], + "network.transport": transport, + "db.mongodb.driver_connection_id": conn.id, + } + if address[1] is not None: + attributes["server.port"] = address[1] + if collection: + attributes["db.collection.name"] = collection + if conn.server_connection_id is not None: + attributes["db.mongodb.server_connection_id"] = conn.server_connection_id + lsid = cmd.get("lsid") + if isinstance(lsid, Mapping): + formatted_lsid = _format_lsid(lsid) + if formatted_lsid is not None: + attributes["db.mongodb.lsid"] = formatted_lsid + txn_number = cmd.get("txnNumber") + if txn_number is not None: + attributes["db.mongodb.txn_number"] = txn_number + max_query_text_length = _get_query_text_max_length(tracing_options) + if max_query_text_length > 0: + attributes["db.query.text"] = _build_query_text(cmd, max_query_text_length) + + assert _TRACER is not None # _is_tracing_enabled already checked _HAS_OPENTELEMETRY + return _TRACER.start_span(command_name, kind=SpanKind.CLIENT, attributes=attributes) + + +def end_command_span_success(span: Optional[Span], reply: _DocumentOut) -> None: + """Set the cursor id (if any) and end the span.""" + if span is None: + return + cursor = reply.get("cursor") + if isinstance(cursor, Mapping) and "id" in cursor: + span.set_attribute("db.mongodb.cursor_id", cursor["id"]) + span.end() + + +def end_command_span_failure( + span: Optional[Span], + failure: _DocumentOut, + exc: BaseException, +) -> None: + """Record the exception, set the error status, and end the span.""" + if span is None: + return + span.record_exception(exc) + code = failure.get("code") + if code is not None: + span.set_attribute("db.response.status_code", str(code)) + span.set_status(Status(StatusCode.ERROR, description=failure.get("errmsg"))) + span.end() diff --git a/pymongo/_telemetry.py b/pymongo/_telemetry.py index 499309b336..84b11a7408 100644 --- a/pymongo/_telemetry.py +++ b/pymongo/_telemetry.py @@ -22,7 +22,8 @@ from collections.abc import MutableMapping from typing import TYPE_CHECKING, Any, Optional -from pymongo import _op_id +from pymongo import _op_id, _otel +from pymongo.errors import OperationFailure from pymongo.logger import ( _COMMAND_LOGGER, _CONNECTION_LOGGER, @@ -94,8 +95,12 @@ class _CommandTelemetry: "_publish", "_request_id", "_should_log", + "_span", + "_speculative_hello", "_start", "_topology_id", + "_tracing_enabled", + "_tracing_options", ) def __init__( @@ -107,13 +112,18 @@ def __init__( dbname: str, request_id: int, op_id: Optional[int], + tracing_options: Optional[_otel.TracingOptions] = None, + speculative_hello: bool = False, name: Optional[str] = None, ) -> None: # NOTE: the _run_command fast path in command_runner.py inline this gate for performance # They must be kept in sync with any gating changes self._should_log = topology_id is not None and _is_debug_enabled(_COMMAND_LOGGER) self._publish = listeners is not None and listeners.enabled_for_commands - self._active = self._should_log or self._publish + self._tracing_options = tracing_options + self._tracing_enabled = _otel._is_tracing_enabled(tracing_options) + self._span: Optional[Any] = None + self._active = self._should_log or self._publish or self._tracing_enabled self._start = 0.0 self._duration_s = 0.0 if not self._active: @@ -126,6 +136,7 @@ def __init__( self._dbname = dbname self._request_id = request_id self._op_id = op_id if op_id is not None else _op_id.OP_ID.get() + self._speculative_hello = speculative_hello def _emit_log(self, message: _CommandStatusMessage, **extra: Any) -> None: _debug_log( @@ -145,7 +156,7 @@ def _emit_log(self, message: _CommandStatusMessage, **extra: Any) -> None: ) def started(self, orig: MutableMapping[str, Any], ensure_db: bool) -> None: - """Emit the STARTED log entry and APM event, and start the duration clock.""" + """Emit the STARTED log entry and APM event, start the span, and start the duration clock.""" self._start = time.monotonic() if not self._active: return @@ -164,6 +175,15 @@ def started(self, orig: MutableMapping[str, Any], ensure_db: bool) -> None: self._op_id, service_id=self._conn.service_id, ) + if self._tracing_enabled: + self._span = _otel.start_command_span( + self._tracing_options, + self._conn, + self._cmd, + self._dbname, + self._name, + self._speculative_hello, + ) @property def duration_s(self) -> float: @@ -176,7 +196,7 @@ def succeeded( command_name: str, speculative_hello: bool, ) -> None: - """Emit the SUCCEEDED log entry and APM event.""" + """Emit the SUCCEEDED log entry and APM event, and end the span.""" self._duration_s = _monotonic_duration(self._start) if not self._active: return @@ -202,14 +222,16 @@ def succeeded( speculative_hello=speculative_hello, database_name=self._dbname, ) + if self._span is not None: + _otel.end_command_span_success(self._span, reply) def failed( self, failure: _DocumentOut, command_name: str, - is_server_side_error: bool, + exc: BaseException, ) -> None: - """Emit the FAILED log entry and APM event.""" + """Emit the FAILED log entry and APM event, and end the span.""" self._duration_s = _monotonic_duration(self._start) if not self._active: return @@ -219,7 +241,7 @@ def failed( _CommandStatusMessage.FAILED, durationMS=duration, failure=failure, - isServerSideError=is_server_side_error, + isServerSideError=isinstance(exc, OperationFailure), ) if self._publish: assert self._listeners is not None @@ -234,6 +256,8 @@ def failed( service_id=self._conn.service_id, database_name=self._dbname, ) + if self._span is not None: + _otel.end_command_span_failure(self._span, failure, exc) class _CmapTelemetry: diff --git a/pymongo/asynchronous/command_runner.py b/pymongo/asynchronous/command_runner.py index aa0839fa4a..f13d893dbf 100644 --- a/pymongo/asynchronous/command_runner.py +++ b/pymongo/asynchronous/command_runner.py @@ -48,7 +48,7 @@ ) from bson import _decode_all_selective -from pymongo import _csot, helpers_shared, message +from pymongo import _csot, _otel, helpers_shared, message from pymongo._telemetry import _CommandTelemetry from pymongo.compression_support import _NO_COMPRESSION from pymongo.errors import NotPrimaryError, OperationFailure @@ -163,14 +163,28 @@ async def _run_command( if orig is None: orig = cmd - # Fast path: skip telemetry construction when logging and APM are disabled - # Inline enabled check here for performance + # Fast path: skip telemetry construction when logging, APM, and tracing are all + # disabled. Inline enabled check here for performance; it must stay in sync with + # _CommandTelemetry.__init__'s own gate, including its tracing term, or a + # tracing-only client would build no telemetry and so emit no command spans. + tracing_options = client.options.tracing if client is not None else None telemetry: Optional[_CommandTelemetry] = None - if (topology_id is not None and _is_debug_enabled(_COMMAND_LOGGER)) or ( - listeners is not None and listeners.enabled_for_commands + if ( + (topology_id is not None and _is_debug_enabled(_COMMAND_LOGGER)) + or (listeners is not None and listeners.enabled_for_commands) + or _otel._is_tracing_enabled(tracing_options) ): telemetry = _CommandTelemetry( - topology_id, conn, listeners, cmd, dbname, request_id, op_id, name=name + topology_id, + conn, + listeners, + cmd, + dbname, + request_id, + op_id, + tracing_options=tracing_options, + speculative_hello=speculative_hello, + name=name, ) telemetry.started(orig, ensure_db) start = 0.0 @@ -224,7 +238,7 @@ async def _run_command( else: failure = _convert_exception(exc) if telemetry is not None: - telemetry.failed(failure, command_name, isinstance(exc, OperationFailure)) + telemetry.failed(failure, command_name, exc) raise if telemetry is not None: diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index a0f1397a4d..af21740385 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -635,8 +635,27 @@ def __init__( If enabled, server overload errors will cause retry attempts to select a server that has not yet returned an overload error, if possible. Defaults to ``False``. + | **OpenTelemetry options:** + | (Requires the ``opentelemetry-api`` package; install with the ``pymongo[opentelemetry]`` extra.) + + - `tracing`: (dict) Configuration for OpenTelemetry command spans, with keys: + + - ``enabled``: (boolean) Whether to create spans for server commands issued by + this client. Defaults to ``False``. Also controlled by the + ``OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED`` environment variable; either + being enabled is sufficient. + - ``query_text_max_length``: (int) The maximum length of the ``db.query.text`` + span attribute. Unset by default, which defers to the + ``OTEL_PYTHON_INSTRUMENTATION_MONGODB_QUERY_TEXT_MAX_LENGTH`` environment + variable (itself defaulting to ``0``, which omits the attribute). Setting + this explicitly, including to ``0``, always overrides the environment + variable. + .. seealso:: The MongoDB documentation on `connections `_. + .. versionchanged:: 4.18 + Added the ``tracing`` keyword argument. + .. versionchanged:: 4.17 Added the ``max_adaptive_retries`` and ``enable_overload_retargeting`` URI and keyword arguments. diff --git a/pymongo/client_options.py b/pymongo/client_options.py index 99a2bcb4e4..d2d33f6805 100644 --- a/pymongo/client_options.py +++ b/pymongo/client_options.py @@ -40,6 +40,7 @@ if TYPE_CHECKING: from bson.codec_options import CodecOptions + from pymongo import _otel from pymongo.auth_shared import MongoCredential from pymongo.encryption_options import AutoEncryptionOpts from pymongo.pyopenssl_context import SSLContext @@ -247,6 +248,10 @@ def __init__( if "enable_overload_retargeting" in options else options.get("enableoverloadretargeting", common.ENABLE_OVERLOAD_RETARGETING) ) + self.__tracing = cast( + "_otel.TracingOptions", + options.get("tracing") or {"enabled": False, "query_text_max_length": None}, + ) @property def _options(self) -> Mapping[str, Any]: @@ -374,3 +379,11 @@ def enable_overload_retargeting(self) -> bool: .. versionadded:: 4.17 """ return self.__enable_overload_retargeting + + @property + def tracing(self) -> _otel.TracingOptions: + """The configured ``tracing`` option for OpenTelemetry command spans. + + .. versionadded:: 4.18 + """ + return self.__tracing diff --git a/pymongo/common.py b/pymongo/common.py index dc18b937e5..2b8367429a 100644 --- a/pymongo/common.py +++ b/pymongo/common.py @@ -51,6 +51,7 @@ from pymongo.write_concern import DEFAULT_WRITE_CONCERN, WriteConcern, validate_boolean if TYPE_CHECKING: + from pymongo import _otel from pymongo.typings import _AgnosticClientSession @@ -610,6 +611,26 @@ def validate_server_api_or_none(option: Any, value: Any) -> Optional[ServerApi]: return value +def validate_tracing_or_none(option: str, value: Any) -> Optional[_otel.TracingOptions]: + """Validate the tracing keyword arg.""" + if value is None: + return value + validate_is_mapping(option, value) + unknown = set(value) - {"enabled", "query_text_max_length"} + if unknown: + raise ConfigurationError(f"Unknown tracing option(s): {sorted(unknown)}") + enabled = value.get("enabled", False) + validate_boolean("tracing.enabled", enabled) + query_text_max_length = value.get("query_text_max_length") + if query_text_max_length is not None: + # bool is a subclass of int; reject it explicitly rather than silently + # treating True/False as 1/0. + if isinstance(query_text_max_length, bool): + raise TypeError("tracing.query_text_max_length must be an integer, not a boolean") + validate_non_negative_integer("tracing.query_text_max_length", query_text_max_length) + return {"enabled": enabled, "query_text_max_length": query_text_max_length} + + def validate_is_callable_or_none(option: Any, value: Any) -> Optional[Callable[..., Any]]: """Validates that 'value' is a callable.""" if value is None: @@ -788,6 +809,7 @@ def validate_server_monitoring_mode(option: str, value: str) -> str: "authoidcallowedhosts": validate_list, "max_adaptive_retries": validate_non_negative_integer, "enable_overload_retargeting": validate_boolean_or_string, + "tracing": validate_tracing_or_none, } # Dictionary where keys are any URI option name, and values are the diff --git a/pymongo/pool_shared.py b/pymongo/pool_shared.py index 8cd546bda6..63a71f3b79 100644 --- a/pymongo/pool_shared.py +++ b/pymongo/pool_shared.py @@ -57,7 +57,7 @@ class _ConnectionTelemetryInfo(Protocol): id: int server_connection_id: Optional[int] - address: tuple[str, int] + address: _Address service_id: Optional[ObjectId] diff --git a/pymongo/synchronous/command_runner.py b/pymongo/synchronous/command_runner.py index 21da51e2fc..27380eb068 100644 --- a/pymongo/synchronous/command_runner.py +++ b/pymongo/synchronous/command_runner.py @@ -48,7 +48,7 @@ ) from bson import _decode_all_selective -from pymongo import _csot, helpers_shared, message +from pymongo import _csot, _otel, helpers_shared, message from pymongo._telemetry import _CommandTelemetry from pymongo.compression_support import _NO_COMPRESSION from pymongo.errors import NotPrimaryError, OperationFailure @@ -163,14 +163,28 @@ def _run_command( if orig is None: orig = cmd - # Fast path: skip telemetry construction when logging and APM are disabled - # Inline enabled check here for performance + # Fast path: skip telemetry construction when logging, APM, and tracing are all + # disabled. Inline enabled check here for performance; it must stay in sync with + # _CommandTelemetry.__init__'s own gate, including its tracing term, or a + # tracing-only client would build no telemetry and so emit no command spans. + tracing_options = client.options.tracing if client is not None else None telemetry: Optional[_CommandTelemetry] = None - if (topology_id is not None and _is_debug_enabled(_COMMAND_LOGGER)) or ( - listeners is not None and listeners.enabled_for_commands + if ( + (topology_id is not None and _is_debug_enabled(_COMMAND_LOGGER)) + or (listeners is not None and listeners.enabled_for_commands) + or _otel._is_tracing_enabled(tracing_options) ): telemetry = _CommandTelemetry( - topology_id, conn, listeners, cmd, dbname, request_id, op_id, name=name + topology_id, + conn, + listeners, + cmd, + dbname, + request_id, + op_id, + tracing_options=tracing_options, + speculative_hello=speculative_hello, + name=name, ) telemetry.started(orig, ensure_db) start = 0.0 @@ -224,7 +238,7 @@ def _run_command( else: failure = _convert_exception(exc) if telemetry is not None: - telemetry.failed(failure, command_name, isinstance(exc, OperationFailure)) + telemetry.failed(failure, command_name, exc) raise if telemetry is not None: diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index 574fbc00de..1321e2139f 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -636,8 +636,27 @@ def __init__( If enabled, server overload errors will cause retry attempts to select a server that has not yet returned an overload error, if possible. Defaults to ``False``. + | **OpenTelemetry options:** + | (Requires the ``opentelemetry-api`` package; install with the ``pymongo[opentelemetry]`` extra.) + + - `tracing`: (dict) Configuration for OpenTelemetry command spans, with keys: + + - ``enabled``: (boolean) Whether to create spans for server commands issued by + this client. Defaults to ``False``. Also controlled by the + ``OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED`` environment variable; either + being enabled is sufficient. + - ``query_text_max_length``: (int) The maximum length of the ``db.query.text`` + span attribute. Unset by default, which defers to the + ``OTEL_PYTHON_INSTRUMENTATION_MONGODB_QUERY_TEXT_MAX_LENGTH`` environment + variable (itself defaulting to ``0``, which omits the attribute). Setting + this explicitly, including to ``0``, always overrides the environment + variable. + .. seealso:: The MongoDB documentation on `connections `_. + .. versionchanged:: 4.18 + Added the ``tracing`` keyword argument. + .. versionchanged:: 4.17 Added the ``max_adaptive_retries`` and ``enable_overload_retargeting`` URI and keyword arguments. diff --git a/pyproject.toml b/pyproject.toml index 2f7a68c837..fce3ad3445 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -103,6 +103,7 @@ docs = ["requirements/docs.txt"] encryption = ["requirements/encryption.txt"] gssapi = ["requirements/gssapi.txt"] ocsp = ["requirements/ocsp.txt"] +opentelemetry = ["requirements/opentelemetry.txt"] snappy = ["requirements/snappy.txt"] test = ["requirements/test.txt"] zstd = ["requirements/zstd.txt"] @@ -151,6 +152,7 @@ markers = [ "encryption: encryption tests", "load_balancer: load balancer tests", "mockupdb: tests that rely on mockupdb", + "otel: tests that rely on opentelemetry", "default: default test suite", "default_async: default async test suite", ] diff --git a/requirements/opentelemetry.txt b/requirements/opentelemetry.txt new file mode 100644 index 0000000000..d20388d07f --- /dev/null +++ b/requirements/opentelemetry.txt @@ -0,0 +1 @@ +opentelemetry-api>=1.20.0 diff --git a/test/asynchronous/test_otel.py b/test/asynchronous/test_otel.py new file mode 100644 index 0000000000..7eaeafe734 --- /dev/null +++ b/test/asynchronous/test_otel.py @@ -0,0 +1,388 @@ +# Copyright 2026-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test OpenTelemetry command-span support.""" + +from __future__ import annotations + +import os +import sys +from typing import Optional +from unittest.mock import patch + +sys.path[0:0] = [""] + +import pytest + +import pymongo._otel as _otel +from pymongo import common +from pymongo.errors import ConfigurationError, OperationFailure +from pymongo.typings import _Address +from test.asynchronous import AsyncIntegrationTest, unittest + +_HAS_OTEL_TEST_DEPS = False +if _otel._HAS_OPENTELEMETRY: + try: + from opentelemetry import trace + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + _HAS_OTEL_TEST_DEPS = True + except ImportError: + pass + +_IS_SYNC = False + +pytestmark = pytest.mark.otel + + +def _shared_test_provider() -> TracerProvider: + """Return a process-wide SDK TracerProvider for tests to attach exporters to. + + ``trace.set_tracer_provider`` only takes effect once per process (later calls + are silently ignored), so tests must share one provider and each register + their own span processor rather than trying to install a fresh provider. + """ + current = trace.get_tracer_provider() + if isinstance(current, TracerProvider): + return current + provider = TracerProvider() + trace.set_tracer_provider(provider) + return provider + + +@unittest.skipUnless(_HAS_OTEL_TEST_DEPS, "opentelemetry-sdk is not installed") +class TestOTelSpans(AsyncIntegrationTest): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.exporter = InMemorySpanExporter() + _shared_test_provider().add_span_processor(SimpleSpanProcessor(cls.exporter)) + + async def asyncSetUp(self): + await super().asyncSetUp() + self.exporter.clear() + + def spans(self, name: str | None = None): + finished = self.exporter.get_finished_spans() + if name is None: + return list(finished) + return [s for s in finished if s.name == name] + + # TODO(PYTHON-5947): once the unified test format runner supports + # expectTracingMessages/operation spans, this is superseded by the spec's + # find_without_query_text.yml and insert.yml. + async def test_span_created_for_insert_and_find(self): + client = await self.async_rs_or_single_client(tracing={"enabled": True}) + coll = client[self.db.name].test_otel + await coll.drop() + self.exporter.clear() + await coll.insert_one({"x": 1}) + + insert_spans = self.spans("insert") + self.assertEqual(len(insert_spans), 1) + attrs = insert_spans[0].attributes + self.assertEqual(attrs["db.system.name"], "mongodb") + self.assertEqual(attrs["db.namespace"], self.db.name) + self.assertEqual(attrs["db.collection.name"], "test_otel") + self.assertEqual(attrs["db.command.name"], "insert") + self.assertEqual(attrs["db.query.summary"], f"insert {self.db.name}.test_otel") + self.assertIn("server.address", attrs) + self.assertIn("server.port", attrs) + self.assertIn(attrs["network.transport"], ("tcp", "unix")) + self.assertIn("db.mongodb.driver_connection_id", attrs) + self.assertNotIn("db.query.text", attrs) + + self.exporter.clear() + docs = await coll.find({}).to_list() + self.assertEqual(len(docs), 1) + find_spans = self.spans("find") + self.assertEqual(len(find_spans), 1) + self.assertEqual(find_spans[0].attributes["db.command.name"], "find") + + async def test_span_created_for_get_more(self): + client = await self.async_rs_or_single_client(tracing={"enabled": True}) + coll = client[self.db.name].test_otel_getmore + await coll.drop() + await coll.insert_many([{"x": i} for i in range(5)]) + self.exporter.clear() + + docs = await coll.find({}, batch_size=2).to_list() + self.assertEqual(len(docs), 5) + + get_more_spans = self.spans("getMore") + self.assertGreater(len(get_more_spans), 0) + for span in get_more_spans: + self.assertEqual(span.attributes["db.collection.name"], "test_otel_getmore") + self.assertEqual(span.attributes["db.command.name"], "getMore") + + async def test_explain_retains_collection_name(self): + # explain wraps the real command ({"explain": {"find": "coll", ...}}), the + # same shape as getMore's indirection, so it needs the same handling. + client = await self.async_rs_or_single_client(tracing={"enabled": True}) + self.exporter.clear() + await client[self.db.name].command("explain", {"find": "test_otel", "filter": {}}) + + spans = self.spans("explain") + self.assertEqual(len(spans), 1) + attrs = spans[0].attributes + self.assertEqual(attrs["db.collection.name"], "test_otel") + self.assertEqual(attrs["db.query.summary"], f"explain {self.db.name}.test_otel") + + async def test_server_port_omitted_for_unix_socket(self): + class _FakeUnixConn: + id = 1 + server_connection_id: Optional[int] = None + address: _Address = ("/tmp/fake-otel-test.sock", None) + service_id = None + + self.exporter.clear() + span = _otel.start_command_span( + {"enabled": True, "query_text_max_length": None}, + _FakeUnixConn(), + {"ping": 1}, + "admin", + "ping", + False, + ) + _otel.end_command_span_success(span, {"ok": 1}) + + spans = self.spans("ping") + self.assertEqual(len(spans), 1) + attrs = spans[0].attributes + self.assertNotIn("server.port", attrs) + self.assertEqual(attrs["network.transport"], "unix") + + async def test_sensitive_command_produces_no_span(self): + client = await self.async_rs_or_single_client(tracing={"enabled": True}) + self.exporter.clear() + with self.assertRaises(OperationFailure): + await client.admin.command("saslStart", mechanism="SCRAM-SHA-256", payload=b"") + + names = [s.name for s in self.spans()] + self.assertNotIn("saslStart", names) + + async def test_admin_command_omits_collection_name(self): + # usersInfo's command value is a username string, not a collection, and + # it always runs against admin; querying a nonexistent user is a no-op. + client = await self.async_rs_or_single_client(tracing={"enabled": True}) + self.exporter.clear() + await client.admin.command("usersInfo", "pymongo_otel_nonexistent_user") + + spans = self.spans("usersInfo") + self.assertEqual(len(spans), 1) + attrs = spans[0].attributes + self.assertEqual(attrs["db.namespace"], "admin") + self.assertNotIn("db.collection.name", attrs) + self.assertEqual(attrs["db.query.summary"], "usersInfo admin") + + async def test_failure_records_exception_and_status_code(self): + client = await self.async_rs_or_single_client(tracing={"enabled": True}) + self.exporter.clear() + with self.assertRaises(OperationFailure): + await client[self.db.name].command("thisCommandDoesNotExist") + + spans = self.spans() + self.assertEqual(len(spans), 1) + span = spans[0] + self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) + self.assertIn("db.response.status_code", span.attributes) + self.assertTrue(any(event.name == "exception" for event in span.events)) + + async def test_tracing_disabled_by_default(self): + client = await self.async_rs_or_single_client() + self.exporter.clear() + await client.admin.command("ping") + self.assertEqual(self.spans(), []) + + # TODO(PYTHON-5947): once operation spans exist, also assert that the + # "ping" *operation* span (not just the command span) is absent/present + # here, and that self.spans() counts both. + async def test_prose_1_tracing_enable_disable_via_env_var(self): + """Prose Test 1: Tracing Enable/Disable via Environment Variable.""" + with patch.dict(os.environ, {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED": "false"}): + client = await self.async_rs_or_single_client() + self.exporter.clear() + await client.admin.command("ping") + self.assertEqual(self.spans(), []) + + with patch.dict(os.environ, {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED": "true"}): + client = await self.async_rs_or_single_client() + self.exporter.clear() + await client.admin.command("ping") + self.assertIn("ping", [s.name for s in self.spans()]) + + # TODO(PYTHON-5947): once operation spans exist, self.spans("find") will + # also match the outer find *operation* span; disambiguate (e.g. by + # db.command.name vs db.operation.name) so this only asserts on the + # command span's db.query.text attribute. + async def test_prose_2_command_payload_emission_via_env_var(self): + """Prose Test 2: Command Payload Emission via Environment Variable.""" + env = { + "OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED": "true", + "OTEL_PYTHON_INSTRUMENTATION_MONGODB_QUERY_TEXT_MAX_LENGTH": "1024", + } + with patch.dict(os.environ, env): + client = await self.async_rs_or_single_client() + self.exporter.clear() + await client[self.db.name].test_otel.find({}).to_list() + spans = self.spans("find") + self.assertEqual(len(spans), 1) + self.assertIn("db.query.text", spans[0].attributes) + + with patch.dict(os.environ, {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED": "true"}): + client = await self.async_rs_or_single_client() + self.exporter.clear() + await client[self.db.name].test_otel.find({}).to_list() + spans = self.spans("find") + self.assertEqual(len(spans), 1) + self.assertNotIn("db.query.text", spans[0].attributes) + + # TODO(PYTHON-5947): once the unified test format runner supports + # expectTracingMessages/operation spans, this is superseded by the spec's + # find.yml (db.query.text assertion). + async def test_query_text_included_when_configured(self): + client = await self.async_rs_or_single_client( + tracing={"enabled": True, "query_text_max_length": 1000} + ) + coll = client[self.db.name].test_otel + await coll.drop() + self.exporter.clear() + await coll.insert_one({"x": 1}) + + spans = self.spans("insert") + self.assertEqual(len(spans), 1) + self.assertIn("db.query.text", spans[0].attributes) + self.assertNotIn("lsid", spans[0].attributes["db.query.text"]) + + async def test_explicit_query_text_max_length_zero_overrides_env_var(self): + # An explicit client-side 0 must win over the environment variable, unlike + # unset (which defers to it) - otherwise an app can't reliably opt out. + env = {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_QUERY_TEXT_MAX_LENGTH": "1024"} + with patch.dict(os.environ, env): + client = await self.async_rs_or_single_client( + tracing={"enabled": True, "query_text_max_length": 0} + ) + self.exporter.clear() + await client.admin.command("ping") + + spans = self.spans("ping") + self.assertEqual(len(spans), 1) + self.assertNotIn("db.query.text", spans[0].attributes) + + async def test_query_text_truncation_shrinks_oversized_field_values(self): + client = await self.async_rs_or_single_client( + tracing={"enabled": True, "query_text_max_length": 200} + ) + coll = client[self.db.name].test_otel + await coll.drop() + self.exporter.clear() + await coll.insert_one({"x": "a" * 500}) + + spans = self.spans("insert") + self.assertEqual(len(spans), 1) + query_text = spans[0].attributes["db.query.text"] + # The oversized field value must be truncated at the field level (not + # just a blind cut of the fully-serialized string), and the result must + # never exceed the configured bound, even when a "..." marker is added. + self.assertLessEqual(len(query_text), 200) + self.assertNotIn("a" * 500, query_text) + + +# TODO(PYTHON-5947): superseded once the unified test format's +# expectTracingMessages/observeTracingMessages tests exercise this validator +# indirectly through real client construction; remove this class then. +class TestValidateTracingOrNone(unittest.TestCase): + def test_none(self): + self.assertIsNone(common.validate_tracing_or_none("tracing", None)) + + def test_defaults(self): + self.assertEqual( + common.validate_tracing_or_none("tracing", {}), + {"enabled": False, "query_text_max_length": None}, + ) + + def test_enabled_and_query_text_max_length(self): + self.assertEqual( + common.validate_tracing_or_none( + "tracing", {"enabled": True, "query_text_max_length": 500} + ), + {"enabled": True, "query_text_max_length": 500}, + ) + + def test_explicit_zero_query_text_max_length_preserved(self): + # 0 must stay distinct from "unset" (None) so it can override the + # environment variable instead of being treated as not configured. + result = common.validate_tracing_or_none( + "tracing", {"enabled": True, "query_text_max_length": 0} + ) + self.assertEqual(result["query_text_max_length"], 0) + + def test_rejects_non_mapping(self): + with self.assertRaises(TypeError): + common.validate_tracing_or_none("tracing", "enabled") + + def test_rejects_unknown_option(self): + with self.assertRaisesRegex(ConfigurationError, "Unknown tracing option"): + common.validate_tracing_or_none("tracing", {"bogus": True}) + + def test_rejects_non_boolean_enabled(self): + with self.assertRaises(TypeError): + common.validate_tracing_or_none("tracing", {"enabled": "yes"}) + + def test_rejects_non_integer_query_text_max_length(self): + with self.assertRaises(TypeError): + common.validate_tracing_or_none("tracing", {"query_text_max_length": [1]}) + + def test_rejects_negative_query_text_max_length(self): + with self.assertRaises(ValueError): + common.validate_tracing_or_none("tracing", {"query_text_max_length": -1}) + + +class TestOTelTracerCaching(unittest.TestCase): + """Regression test for the tracer-caching implementation in ``pymongo/_otel.py``. + + ``opentelemetry.trace.get_tracer()`` must only be called once, at import + time (cached as module-level ``_otel._TRACER``). Calling it per command + allocates two objects, takes a process-wide lock, and mutates the global + ``warnings`` filter list on every call, even on a cache hit. + """ + + @unittest.skipUnless(_otel._HAS_OPENTELEMETRY, "opentelemetry is not installed") + def test_start_command_span_does_not_call_get_tracer(self): + class _FakeConn: + id = 1 + server_connection_id: Optional[int] = None + address: _Address = ("localhost", 27017) + service_id = None + + with patch.object(_otel, "trace") as mock_trace: + for _ in range(3): + span = _otel.start_command_span( + {"enabled": True, "query_text_max_length": None}, + _FakeConn(), + {"ping": 1}, + "admin", + "ping", + False, + ) + self.assertIsNotNone(span) + span.end() + + mock_trace.get_tracer.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_otel.py b/test/test_otel.py new file mode 100644 index 0000000000..d0e3a55fe5 --- /dev/null +++ b/test/test_otel.py @@ -0,0 +1,382 @@ +# Copyright 2026-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test OpenTelemetry command-span support.""" + +from __future__ import annotations + +import os +import sys +from typing import Optional +from unittest.mock import patch + +sys.path[0:0] = [""] + +import pytest + +import pymongo._otel as _otel +from pymongo import common +from pymongo.errors import ConfigurationError, OperationFailure +from pymongo.typings import _Address +from test import IntegrationTest, unittest + +_HAS_OTEL_TEST_DEPS = False +if _otel._HAS_OPENTELEMETRY: + try: + from opentelemetry import trace + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + _HAS_OTEL_TEST_DEPS = True + except ImportError: + pass + +_IS_SYNC = True + +pytestmark = pytest.mark.otel + + +def _shared_test_provider() -> TracerProvider: + """Return a process-wide SDK TracerProvider for tests to attach exporters to. + + ``trace.set_tracer_provider`` only takes effect once per process (later calls + are silently ignored), so tests must share one provider and each register + their own span processor rather than trying to install a fresh provider. + """ + current = trace.get_tracer_provider() + if isinstance(current, TracerProvider): + return current + provider = TracerProvider() + trace.set_tracer_provider(provider) + return provider + + +@unittest.skipUnless(_HAS_OTEL_TEST_DEPS, "opentelemetry-sdk is not installed") +class TestOTelSpans(IntegrationTest): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.exporter = InMemorySpanExporter() + _shared_test_provider().add_span_processor(SimpleSpanProcessor(cls.exporter)) + + def setUp(self): + super().setUp() + self.exporter.clear() + + def spans(self, name: str | None = None): + finished = self.exporter.get_finished_spans() + if name is None: + return list(finished) + return [s for s in finished if s.name == name] + + # TODO(PYTHON-5947): once the unified test format runner supports + # expectTracingMessages/operation spans, this is superseded by the spec's + # find_without_query_text.yml and insert.yml. + def test_span_created_for_insert_and_find(self): + client = self.rs_or_single_client(tracing={"enabled": True}) + coll = client[self.db.name].test_otel + coll.drop() + self.exporter.clear() + coll.insert_one({"x": 1}) + + insert_spans = self.spans("insert") + self.assertEqual(len(insert_spans), 1) + attrs = insert_spans[0].attributes + self.assertEqual(attrs["db.system.name"], "mongodb") + self.assertEqual(attrs["db.namespace"], self.db.name) + self.assertEqual(attrs["db.collection.name"], "test_otel") + self.assertEqual(attrs["db.command.name"], "insert") + self.assertEqual(attrs["db.query.summary"], f"insert {self.db.name}.test_otel") + self.assertIn("server.address", attrs) + self.assertIn("server.port", attrs) + self.assertIn(attrs["network.transport"], ("tcp", "unix")) + self.assertIn("db.mongodb.driver_connection_id", attrs) + self.assertNotIn("db.query.text", attrs) + + self.exporter.clear() + docs = coll.find({}).to_list() + self.assertEqual(len(docs), 1) + find_spans = self.spans("find") + self.assertEqual(len(find_spans), 1) + self.assertEqual(find_spans[0].attributes["db.command.name"], "find") + + def test_span_created_for_get_more(self): + client = self.rs_or_single_client(tracing={"enabled": True}) + coll = client[self.db.name].test_otel_getmore + coll.drop() + coll.insert_many([{"x": i} for i in range(5)]) + self.exporter.clear() + + docs = coll.find({}, batch_size=2).to_list() + self.assertEqual(len(docs), 5) + + get_more_spans = self.spans("getMore") + self.assertGreater(len(get_more_spans), 0) + for span in get_more_spans: + self.assertEqual(span.attributes["db.collection.name"], "test_otel_getmore") + self.assertEqual(span.attributes["db.command.name"], "getMore") + + def test_explain_retains_collection_name(self): + # explain wraps the real command ({"explain": {"find": "coll", ...}}), the + # same shape as getMore's indirection, so it needs the same handling. + client = self.rs_or_single_client(tracing={"enabled": True}) + self.exporter.clear() + client[self.db.name].command("explain", {"find": "test_otel", "filter": {}}) + + spans = self.spans("explain") + self.assertEqual(len(spans), 1) + attrs = spans[0].attributes + self.assertEqual(attrs["db.collection.name"], "test_otel") + self.assertEqual(attrs["db.query.summary"], f"explain {self.db.name}.test_otel") + + def test_server_port_omitted_for_unix_socket(self): + class _FakeUnixConn: + id = 1 + server_connection_id: Optional[int] = None + address: _Address = ("/tmp/fake-otel-test.sock", None) + service_id = None + + self.exporter.clear() + span = _otel.start_command_span( + {"enabled": True, "query_text_max_length": None}, + _FakeUnixConn(), + {"ping": 1}, + "admin", + "ping", + False, + ) + _otel.end_command_span_success(span, {"ok": 1}) + + spans = self.spans("ping") + self.assertEqual(len(spans), 1) + attrs = spans[0].attributes + self.assertNotIn("server.port", attrs) + self.assertEqual(attrs["network.transport"], "unix") + + def test_sensitive_command_produces_no_span(self): + client = self.rs_or_single_client(tracing={"enabled": True}) + self.exporter.clear() + with self.assertRaises(OperationFailure): + client.admin.command("saslStart", mechanism="SCRAM-SHA-256", payload=b"") + + names = [s.name for s in self.spans()] + self.assertNotIn("saslStart", names) + + def test_admin_command_omits_collection_name(self): + # usersInfo's command value is a username string, not a collection, and + # it always runs against admin; querying a nonexistent user is a no-op. + client = self.rs_or_single_client(tracing={"enabled": True}) + self.exporter.clear() + client.admin.command("usersInfo", "pymongo_otel_nonexistent_user") + + spans = self.spans("usersInfo") + self.assertEqual(len(spans), 1) + attrs = spans[0].attributes + self.assertEqual(attrs["db.namespace"], "admin") + self.assertNotIn("db.collection.name", attrs) + self.assertEqual(attrs["db.query.summary"], "usersInfo admin") + + def test_failure_records_exception_and_status_code(self): + client = self.rs_or_single_client(tracing={"enabled": True}) + self.exporter.clear() + with self.assertRaises(OperationFailure): + client[self.db.name].command("thisCommandDoesNotExist") + + spans = self.spans() + self.assertEqual(len(spans), 1) + span = spans[0] + self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) + self.assertIn("db.response.status_code", span.attributes) + self.assertTrue(any(event.name == "exception" for event in span.events)) + + def test_tracing_disabled_by_default(self): + client = self.rs_or_single_client() + self.exporter.clear() + client.admin.command("ping") + self.assertEqual(self.spans(), []) + + # TODO(PYTHON-5947): once operation spans exist, also assert that the + # "ping" *operation* span (not just the command span) is absent/present + # here, and that self.spans() counts both. + def test_prose_1_tracing_enable_disable_via_env_var(self): + """Prose Test 1: Tracing Enable/Disable via Environment Variable.""" + with patch.dict(os.environ, {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED": "false"}): + client = self.rs_or_single_client() + self.exporter.clear() + client.admin.command("ping") + self.assertEqual(self.spans(), []) + + with patch.dict(os.environ, {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED": "true"}): + client = self.rs_or_single_client() + self.exporter.clear() + client.admin.command("ping") + self.assertIn("ping", [s.name for s in self.spans()]) + + # TODO(PYTHON-5947): once operation spans exist, self.spans("find") will + # also match the outer find *operation* span; disambiguate (e.g. by + # db.command.name vs db.operation.name) so this only asserts on the + # command span's db.query.text attribute. + def test_prose_2_command_payload_emission_via_env_var(self): + """Prose Test 2: Command Payload Emission via Environment Variable.""" + env = { + "OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED": "true", + "OTEL_PYTHON_INSTRUMENTATION_MONGODB_QUERY_TEXT_MAX_LENGTH": "1024", + } + with patch.dict(os.environ, env): + client = self.rs_or_single_client() + self.exporter.clear() + client[self.db.name].test_otel.find({}).to_list() + spans = self.spans("find") + self.assertEqual(len(spans), 1) + self.assertIn("db.query.text", spans[0].attributes) + + with patch.dict(os.environ, {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED": "true"}): + client = self.rs_or_single_client() + self.exporter.clear() + client[self.db.name].test_otel.find({}).to_list() + spans = self.spans("find") + self.assertEqual(len(spans), 1) + self.assertNotIn("db.query.text", spans[0].attributes) + + # TODO(PYTHON-5947): once the unified test format runner supports + # expectTracingMessages/operation spans, this is superseded by the spec's + # find.yml (db.query.text assertion). + def test_query_text_included_when_configured(self): + client = self.rs_or_single_client(tracing={"enabled": True, "query_text_max_length": 1000}) + coll = client[self.db.name].test_otel + coll.drop() + self.exporter.clear() + coll.insert_one({"x": 1}) + + spans = self.spans("insert") + self.assertEqual(len(spans), 1) + self.assertIn("db.query.text", spans[0].attributes) + self.assertNotIn("lsid", spans[0].attributes["db.query.text"]) + + def test_explicit_query_text_max_length_zero_overrides_env_var(self): + # An explicit client-side 0 must win over the environment variable, unlike + # unset (which defers to it) - otherwise an app can't reliably opt out. + env = {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_QUERY_TEXT_MAX_LENGTH": "1024"} + with patch.dict(os.environ, env): + client = self.rs_or_single_client(tracing={"enabled": True, "query_text_max_length": 0}) + self.exporter.clear() + client.admin.command("ping") + + spans = self.spans("ping") + self.assertEqual(len(spans), 1) + self.assertNotIn("db.query.text", spans[0].attributes) + + def test_query_text_truncation_shrinks_oversized_field_values(self): + client = self.rs_or_single_client(tracing={"enabled": True, "query_text_max_length": 200}) + coll = client[self.db.name].test_otel + coll.drop() + self.exporter.clear() + coll.insert_one({"x": "a" * 500}) + + spans = self.spans("insert") + self.assertEqual(len(spans), 1) + query_text = spans[0].attributes["db.query.text"] + # The oversized field value must be truncated at the field level (not + # just a blind cut of the fully-serialized string), and the result must + # never exceed the configured bound, even when a "..." marker is added. + self.assertLessEqual(len(query_text), 200) + self.assertNotIn("a" * 500, query_text) + + +# TODO(PYTHON-5947): superseded once the unified test format's +# expectTracingMessages/observeTracingMessages tests exercise this validator +# indirectly through real client construction; remove this class then. +class TestValidateTracingOrNone(unittest.TestCase): + def test_none(self): + self.assertIsNone(common.validate_tracing_or_none("tracing", None)) + + def test_defaults(self): + self.assertEqual( + common.validate_tracing_or_none("tracing", {}), + {"enabled": False, "query_text_max_length": None}, + ) + + def test_enabled_and_query_text_max_length(self): + self.assertEqual( + common.validate_tracing_or_none( + "tracing", {"enabled": True, "query_text_max_length": 500} + ), + {"enabled": True, "query_text_max_length": 500}, + ) + + def test_explicit_zero_query_text_max_length_preserved(self): + # 0 must stay distinct from "unset" (None) so it can override the + # environment variable instead of being treated as not configured. + result = common.validate_tracing_or_none( + "tracing", {"enabled": True, "query_text_max_length": 0} + ) + self.assertEqual(result["query_text_max_length"], 0) + + def test_rejects_non_mapping(self): + with self.assertRaises(TypeError): + common.validate_tracing_or_none("tracing", "enabled") + + def test_rejects_unknown_option(self): + with self.assertRaisesRegex(ConfigurationError, "Unknown tracing option"): + common.validate_tracing_or_none("tracing", {"bogus": True}) + + def test_rejects_non_boolean_enabled(self): + with self.assertRaises(TypeError): + common.validate_tracing_or_none("tracing", {"enabled": "yes"}) + + def test_rejects_non_integer_query_text_max_length(self): + with self.assertRaises(TypeError): + common.validate_tracing_or_none("tracing", {"query_text_max_length": [1]}) + + def test_rejects_negative_query_text_max_length(self): + with self.assertRaises(ValueError): + common.validate_tracing_or_none("tracing", {"query_text_max_length": -1}) + + +class TestOTelTracerCaching(unittest.TestCase): + """Regression test for the tracer-caching implementation in ``pymongo/_otel.py``. + + ``opentelemetry.trace.get_tracer()`` must only be called once, at import + time (cached as module-level ``_otel._TRACER``). Calling it per command + allocates two objects, takes a process-wide lock, and mutates the global + ``warnings`` filter list on every call, even on a cache hit. + """ + + @unittest.skipUnless(_otel._HAS_OPENTELEMETRY, "opentelemetry is not installed") + def test_start_command_span_does_not_call_get_tracer(self): + class _FakeConn: + id = 1 + server_connection_id: Optional[int] = None + address: _Address = ("localhost", 27017) + service_id = None + + with patch.object(_otel, "trace") as mock_trace: + for _ in range(3): + span = _otel.start_command_span( + {"enabled": True, "query_text_max_length": None}, + _FakeConn(), + {"ping": 1}, + "admin", + "ping", + False, + ) + self.assertIsNotNone(span) + span.end() + + mock_trace.get_tracer.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/uv.lock b/uv.lock index f2ae10151d..add1dd9fdf 100644 --- a/uv.lock +++ b/uv.lock @@ -58,9 +58,9 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "exceptiongroup", marker = "python_full_version < '3.10'" }, - { name = "idna", marker = "python_full_version < '3.10'" }, - { name = "typing-extensions", marker = "python_full_version < '3.10'" }, + { name = "exceptiongroup" }, + { name = "idna" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/96/f0/5eb65b2bb0d09ac6776f2eb54adee6abe8228ea05b20a5ad0e4945de8aac/anyio-4.12.1.tar.gz", hash = "sha256:41cfcc3a4c85d3f05c932da7c26d0201ac36f72abd4435ba90d0464a3ffed703", size = 228685, upload-time = "2026-01-06T11:45:21.246Z" } wheels = [ @@ -76,9 +76,9 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "exceptiongroup", marker = "python_full_version == '3.10.*'" }, - { name = "idna", marker = "python_full_version >= '3.10'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.10' and python_full_version < '3.15'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a9/d2/f4d173e22df740bc37b1db102b386ba719b66e95b0f0d751f556b387e6d2/anyio-4.15.1.tar.gz", hash = "sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94", size = 276966, upload-time = "2026-09-05T10:42:39.44Z" } wheels = [ @@ -358,9 +358,9 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "botocore", version = "1.42.97", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "jmespath", marker = "python_full_version < '3.10'" }, - { name = "s3transfer", version = "0.16.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "botocore", version = "1.42.97", source = { registry = "https://pypi.org/simple" } }, + { name = "jmespath" }, + { name = "s3transfer", version = "0.16.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/55/7d/5c6fa0bb9fd5caf865b9356411793900304328bcd0bc1eda96a32a1368a6/boto3-1.42.97.tar.gz", hash = "sha256:2833dbeda3670ea610ad48dff7d27cdc829dbbfcdfbc6b750b673948e949b6f0", size = 113217, upload-time = "2026-04-27T20:39:17.646Z" } wheels = [ @@ -376,9 +376,9 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "botocore", version = "1.43.94", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "jmespath", marker = "python_full_version >= '3.10'" }, - { name = "s3transfer", version = "0.19.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "botocore", version = "1.43.94", source = { registry = "https://pypi.org/simple" } }, + { name = "jmespath" }, + { name = "s3transfer", version = "0.19.2", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/d5/0f/a1d691e9c51ae5dce5a7394e550d7e36a995494e1c2795ce210d3520b257/boto3-1.43.94.tar.gz", hash = "sha256:aa832d88e85035cebb83f04700c65623328b88fe828ff7ed9539483bd9d608c5", size = 112663, upload-time = "2026-09-14T20:10:05.902Z" } wheels = [ @@ -394,9 +394,9 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "jmespath", marker = "python_full_version < '3.10'" }, - { name = "python-dateutil", marker = "python_full_version < '3.10'" }, - { name = "urllib3", version = "1.26.20", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3", version = "1.26.20", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/c6/95/c37edb602948fad2253ffd1bb3dba5b938645bd1845ee4160350136a0f41/botocore-1.42.97.tar.gz", hash = "sha256:5c0bb00e32d16ff6d278cc8c9e10dc3672d9c1d569031635ac3c908a60de8310", size = 15269348, upload-time = "2026-04-27T20:39:05.625Z" } wheels = [ @@ -412,9 +412,9 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "jmespath", marker = "python_full_version >= '3.10'" }, - { name = "python-dateutil", marker = "python_full_version >= '3.10'" }, - { name = "urllib3", version = "2.7.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3", version = "2.7.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/fa/5b/5a7cc096db8ddb5f6c221112d5a4a46888d9e38c1f5d5a06ae7e3e0755f2/botocore-1.43.94.tar.gz", hash = "sha256:cb97cedec5fceee3261f7fff75d0b0aa2cf68b32eeb2e8c2649e4657419773a6", size = 16104907, upload-time = "2026-09-14T20:10:00.315Z" } wheels = [ @@ -439,7 +439,7 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "pycparser", version = "2.23", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10' and implementation_name != 'PyPy'" }, + { name = "pycparser", version = "2.23", source = { registry = "https://pypi.org/simple" }, marker = "implementation_name != 'PyPy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } wheels = [ @@ -537,7 +537,7 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "pycparser", version = "3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10' and implementation_name != 'PyPy'" }, + { name = "pycparser", version = "3.0", source = { registry = "https://pypi.org/simple" }, marker = "implementation_name != 'PyPy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } wheels = [ @@ -830,7 +830,7 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593, upload-time = "2024-12-21T18:38:44.339Z" } wheels = [ @@ -976,7 +976,7 @@ wheels = [ [package.optional-dependencies] toml = [ - { name = "tomli", marker = "python_full_version < '3.10'" }, + { name = "tomli" }, ] [[package]] @@ -1083,7 +1083,7 @@ wheels = [ [package.optional-dependencies] toml = [ - { name = "tomli", marker = "python_full_version >= '3.10' and python_full_version <= '3.11'" }, + { name = "tomli", marker = "python_full_version <= '3.11'" }, ] [[package]] @@ -1353,8 +1353,8 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version <= '3.9' and platform_python_implementation != 'PyPy'" }, - { name = "typing-extensions", marker = "python_full_version <= '3.9'" }, + { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_python_implementation != 'PyPy'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/b2/7ffa7fe8207a8c42147ffe70c3e360b228160c1d85dc3faff16aaa3244c0/cryptography-47.0.0.tar.gz", hash = "sha256:9f8e55fe4e63613a5e1cc5819030f27b97742d720203a087802ce4ce9ceb52bb", size = 830863, upload-time = "2026-04-24T19:54:57.056Z" } wheels = [ @@ -1418,9 +1418,9 @@ resolution-markers = [ "python_full_version > '3.9' and python_full_version < '3.10'", ] dependencies = [ - { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version > '3.9' and python_full_version < '3.10' and platform_python_implementation != 'PyPy'" }, + { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10' and platform_python_implementation != 'PyPy'" }, { name = "cffi", version = "2.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10' and platform_python_implementation != 'PyPy'" }, - { name = "typing-extensions", marker = "python_full_version > '3.9' and python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/bb/ad/5d6702db60b1e40b41ef513b6967ff5848f307d50f8449baf1634f5908f1/cryptography-50.0.1.tar.gz", hash = "sha256:5dd9bda1c12b4162f6ff568eeb5e0ff956c28d14406e875cfe8a63a2d414ff20", size = 880381, upload-time = "2026-08-25T19:45:45.499Z" } wheels = [ @@ -1520,7 +1520,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -1554,10 +1554,10 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10' and platform_python_implementation == 'CPython' and sys_platform == 'win32'" }, - { name = "greenlet", version = "3.2.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10' and platform_python_implementation == 'CPython'" }, - { name = "zope-event", version = "6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "zope-interface", version = "8.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_python_implementation == 'CPython' and sys_platform == 'win32'" }, + { name = "greenlet", version = "3.2.5", source = { registry = "https://pypi.org/simple" }, marker = "platform_python_implementation == 'CPython'" }, + { name = "zope-event", version = "6.0", source = { registry = "https://pypi.org/simple" } }, + { name = "zope-interface", version = "8.0.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/9f/5c/92002455a57cb3634383e2b822e3bccf409f43cde34528e46428971475cf/gevent-26.7.0.tar.gz", hash = "sha256:5b333a556e38a302b1b8c80525bef16d437e16f1e7767947789406841856a102", size = 6729213, upload-time = "2026-07-22T20:16:04.713Z" } wheels = [ @@ -1617,10 +1617,10 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "cffi", version = "2.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10' and platform_python_implementation == 'CPython' and sys_platform == 'win32'" }, - { name = "greenlet", version = "3.5.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10' and platform_python_implementation == 'CPython'" }, - { name = "zope-event", version = "6.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "zope-interface", version = "8.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "cffi", version = "2.1.1", source = { registry = "https://pypi.org/simple" }, marker = "platform_python_implementation == 'CPython' and sys_platform == 'win32'" }, + { name = "greenlet", version = "3.5.6", source = { registry = "https://pypi.org/simple" }, marker = "platform_python_implementation == 'CPython'" }, + { name = "zope-event", version = "6.2", source = { registry = "https://pypi.org/simple" } }, + { name = "zope-interface", version = "8.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/b8/eb/5f2db8013f1a4a6df2c23201f384a066f13ff5764a9f62a608c8a50ac8cc/gevent-26.8.0.tar.gz", hash = "sha256:96039f41bbde6dcd72559e5ffbd408a04f46774b47d991d4cf032da8fa79e5a0", size = 6625998, upload-time = "2026-08-10T18:02:28.038Z" } wheels = [ @@ -1903,7 +1903,7 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "zipp", version = "3.23.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "zipp", version = "3.23.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/f3/49/3b30cad09e7771a4982d9975a8cbf64f00d4a1ececb53297f1d9a7be1b10/importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb", size = 57107, upload-time = "2025-12-21T10:00:19.278Z" } wheels = [ @@ -1919,7 +1919,7 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "zipp", version = "4.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "zipp", version = "4.1.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/6f/7e/1e7e8dc30634b93ebb3d58a3dea569ad146e656218d3960ab04f62047b29/importlib_metadata-9.0.1.tar.gz", hash = "sha256:ab830580bc0ef3db61ce8fae716389e5462b67e033018bab6d8f80ef17172f99", size = 59124, upload-time = "2026-08-28T15:30:34.646Z" } wheels = [ @@ -2295,6 +2295,39 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, ] +[[package]] +name = "opentelemetry-api" +version = "1.41.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version > '3.9' and python_full_version < '3.10'", + "python_full_version <= '3.9'", +] +dependencies = [ + { name = "importlib-metadata", version = "8.7.1", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fa/fc/b7564cbef36601aef0d6c9bc01f7badb64be8e862c2e1c3c5c3b43b53e4f/opentelemetry_api-1.41.1.tar.gz", hash = "sha256:0ad1814d73b875f84494387dae86ce0b12c68556331ce6ce8fe789197c949621", size = 71416, upload-time = "2026-04-24T13:15:38.262Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/59/3e7118ed140f76b0982ba4321bdaed1997a0473f9720de2d10788a577033/opentelemetry_api-1.41.1-py3-none-any.whl", hash = "sha256:a22df900e75c76dc08440710e51f52f1aa6b451b429298896023e60db5b3139f", size = 69007, upload-time = "2026-04-24T13:15:15.662Z" }, +] + +[[package]] +name = "opentelemetry-api" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.11'", + "python_full_version == '3.10.*'", +] +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ee/8b/aa9e2d8b8dfa7c946f7dec5d1f8f6ba8eca062f43509a06bdb5ce93d26c0/opentelemetry_api-1.44.0.tar.gz", hash = "sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a", size = 72406, upload-time = "2026-07-16T15:25:32.678Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/6f/a04e900f465ff3221ccc395522503e2d10e79fa21f2723c8e177aae1e0d1/opentelemetry_api-1.44.0-py3-none-any.whl", hash = "sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef", size = 60018, upload-time = "2026-07-16T15:25:11.657Z" }, +] + [[package]] name = "packaging" version = "26.3" @@ -2432,6 +2465,10 @@ ocsp = [ { name = "requests", version = "2.34.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "service-identity" }, ] +opentelemetry = [ + { name = "opentelemetry-api", version = "1.41.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "opentelemetry-api", version = "1.44.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, +] snappy = [ { name = "python-snappy" }, ] @@ -2491,6 +2528,7 @@ requires-dist = [ { name = "dnspython", specifier = ">=2.7.0,<3.0.0" }, { name = "furo", marker = "extra == 'docs'", specifier = "==2025.12.19" }, { name = "importlib-metadata", marker = "python_full_version < '3.13' and extra == 'test'", specifier = ">=7.0" }, + { name = "opentelemetry-api", marker = "extra == 'opentelemetry'", specifier = ">=1.20.0" }, { name = "pykerberos", marker = "os_name != 'nt' and extra == 'gssapi'", specifier = ">=1.2.4" }, { name = "pymongo-auth-aws", marker = "extra == 'aws'", specifier = ">=1.3.0,<2.0.0" }, { name = "pymongo-auth-aws", marker = "extra == 'encryption'", specifier = ">=1.3.0,<2.0.0" }, @@ -2508,7 +2546,7 @@ requires-dist = [ { name = "sphinxcontrib-shellcheck", marker = "extra == 'docs'", specifier = ">=1.1.2,<2" }, { name = "winkerberos", marker = "os_name == 'nt' and extra == 'gssapi'", specifier = ">=0.12.2" }, ] -provides-extras = ["aws", "docs", "encryption", "gssapi", "ocsp", "snappy", "test", "zstd"] +provides-extras = ["aws", "docs", "encryption", "gssapi", "ocsp", "opentelemetry", "snappy", "test", "zstd"] [package.metadata.requires-dev] coverage = [{ name = "coverage", extras = ["toml"], specifier = ">=5,<=7.15.0" }] @@ -2572,8 +2610,8 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "cryptography", version = "47.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version <= '3.9'" }, - { name = "typing-extensions", marker = "python_full_version <= '3.9'" }, + { name = "cryptography", version = "47.0.0", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1a/51/27a5ad5f939d08f690a326ef9582cda7140555180db71695f6fb747d6a36/pyopenssl-26.2.0.tar.gz", hash = "sha256:8c6fcecd1183a7fc897548dfe388b0cdb7f37e018200d8409cf33959dbe35387", size = 182195, upload-time = "2026-05-04T23:06:09.72Z" } wheels = [ @@ -2590,8 +2628,8 @@ resolution-markers = [ "python_full_version > '3.9' and python_full_version < '3.10'", ] dependencies = [ - { name = "cryptography", version = "50.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version > '3.9'" }, - { name = "typing-extensions", marker = "python_full_version > '3.9' and python_full_version < '3.13'" }, + { name = "cryptography", version = "50.0.1", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3f/e8/7325d258199b159eb2c03fe32107533e2832e70e63f4fb88a6aa00023201/pyopenssl-26.4.0.tar.gz", hash = "sha256:28dfcce0162b9211413e26dfbfdf1d24317fbeba18fc93c12400a1856b2a0bc7", size = 182046, upload-time = "2026-08-01T19:50:50.512Z" } wheels = [ @@ -2620,13 +2658,13 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.10'" }, - { name = "iniconfig", version = "2.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "packaging", marker = "python_full_version < '3.10'" }, - { name = "pluggy", marker = "python_full_version < '3.10'" }, - { name = "pygments", marker = "python_full_version < '3.10'" }, - { name = "tomli", marker = "python_full_version < '3.10'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup" }, + { name = "iniconfig", version = "2.1.0", source = { registry = "https://pypi.org/simple" } }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" } wheels = [ @@ -2642,13 +2680,13 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, - { name = "exceptiongroup", marker = "python_full_version == '3.10.*'" }, - { name = "iniconfig", version = "2.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "packaging", marker = "python_full_version >= '3.10'" }, - { name = "pluggy", marker = "python_full_version >= '3.10'" }, - { name = "pygments", marker = "python_full_version >= '3.10'" }, - { name = "tomli", marker = "python_full_version == '3.10.*'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig", version = "2.3.0", source = { registry = "https://pypi.org/simple" } }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } wheels = [ @@ -2664,9 +2702,9 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "backports-asyncio-runner", marker = "python_full_version < '3.10'" }, - { name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "typing-extensions", marker = "python_full_version < '3.10'" }, + { name = "backports-asyncio-runner" }, + { name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/86/9e3c5f48f7b7b638b216e4b9e645f54d199d7abbbab7a64a13b4e12ba10f/pytest_asyncio-1.2.0.tar.gz", hash = "sha256:c609a64a2a8768462d0c99811ddb8bd2583c33fd33cf7f21af1c142e824ffb57", size = 50119, upload-time = "2025-09-12T07:33:53.816Z" } wheels = [ @@ -2682,9 +2720,9 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "backports-asyncio-runner", marker = "python_full_version == '3.10.*'" }, - { name = "pytest", version = "9.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.10' and python_full_version < '3.13'" }, + { name = "backports-asyncio-runner", marker = "python_full_version < '3.11'" }, + { name = "pytest", version = "9.1.1", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/43/7c/d36d04db312ecf4298932ef77e6e4a9e8ad017906e24e34f0b0c361a2473/pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42", size = 58514, upload-time = "2026-05-26T09:56:04.083Z" } wheels = [ @@ -2734,10 +2772,10 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "certifi", marker = "python_full_version < '3.10'" }, - { name = "charset-normalizer", marker = "python_full_version < '3.10'" }, - { name = "idna", marker = "python_full_version < '3.10'" }, - { name = "urllib3", version = "1.26.20", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3", version = "1.26.20", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" } wheels = [ @@ -2753,10 +2791,10 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "certifi", marker = "python_full_version >= '3.10'" }, - { name = "charset-normalizer", marker = "python_full_version >= '3.10'" }, - { name = "idna", marker = "python_full_version >= '3.10'" }, - { name = "urllib3", version = "2.7.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3", version = "2.7.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } wheels = [ @@ -2777,7 +2815,7 @@ name = "roman-numerals-py" version = "4.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "roman-numerals", marker = "python_full_version >= '3.11'" }, + { name = "roman-numerals" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cb/b5/de96fca640f4f656eb79bbee0e79aeec52e3e0e359f8a3e6a0d366378b64/roman_numerals_py-4.1.0.tar.gz", hash = "sha256:f5d7b2b4ca52dd855ef7ab8eb3590f428c0b1ea480736ce32b01fef2a5f8daf9", size = 4274, upload-time = "2025-12-17T18:25:41.153Z" } wheels = [ @@ -2818,7 +2856,7 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "botocore", version = "1.42.97", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "botocore", version = "1.42.97", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/46/29/af14f4ef3c11a50435308660e2cc68761c9a7742475e0585cd4396b91777/s3transfer-0.16.1.tar.gz", hash = "sha256:8e424355754b9ccb32467bdc568edf55be82692ef2002d934b1311dbb3b9e524", size = 154801, upload-time = "2026-04-22T20:36:06.475Z" } wheels = [ @@ -2834,7 +2872,7 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "botocore", version = "1.43.94", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "botocore", version = "1.43.94", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/76/43/35e4d8aa320bffe8287fe8f65f578fa2d2db0a64212f0e710dce58267854/s3transfer-0.19.2.tar.gz", hash = "sha256:ba0309fd86be3c27dbf78cdd813c13c5e1df16e5874b99d2535ebbdfb9892993", size = 165592, upload-time = "2026-07-22T19:30:44.432Z" } wheels = [ @@ -3020,24 +3058,24 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "alabaster", version = "0.7.16", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "babel", marker = "python_full_version < '3.10'" }, - { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, - { name = "docutils", marker = "python_full_version < '3.10'" }, - { name = "imagesize", version = "1.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "importlib-metadata", version = "8.7.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "jinja2", marker = "python_full_version < '3.10'" }, - { name = "packaging", marker = "python_full_version < '3.10'" }, - { name = "pygments", marker = "python_full_version < '3.10'" }, - { name = "requests", version = "2.32.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "snowballstemmer", marker = "python_full_version < '3.10'" }, - { name = "sphinxcontrib-applehelp", marker = "python_full_version < '3.10'" }, - { name = "sphinxcontrib-devhelp", marker = "python_full_version < '3.10'" }, - { name = "sphinxcontrib-htmlhelp", marker = "python_full_version < '3.10'" }, - { name = "sphinxcontrib-jsmath", marker = "python_full_version < '3.10'" }, - { name = "sphinxcontrib-qthelp", marker = "python_full_version < '3.10'" }, - { name = "sphinxcontrib-serializinghtml", marker = "python_full_version < '3.10'" }, - { name = "tomli", marker = "python_full_version < '3.10'" }, + { name = "alabaster", version = "0.7.16", source = { registry = "https://pypi.org/simple" } }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils" }, + { name = "imagesize", version = "1.5.0", source = { registry = "https://pypi.org/simple" } }, + { name = "importlib-metadata", version = "8.7.1", source = { registry = "https://pypi.org/simple" } }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests", version = "2.32.5", source = { registry = "https://pypi.org/simple" } }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, + { name = "tomli" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5b/be/50e50cb4f2eff47df05673d361095cafd95521d2a22521b920c67a372dcb/sphinx-7.4.7.tar.gz", hash = "sha256:242f92a7ea7e6c5b406fdc2615413890ba9f699114a9c09192d7dfead2ee9cfe", size = 8067911, upload-time = "2024-07-20T14:46:56.059Z" } wheels = [ @@ -3052,23 +3090,23 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "alabaster", version = "1.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "babel", marker = "python_full_version == '3.10.*'" }, - { name = "colorama", marker = "python_full_version == '3.10.*' and sys_platform == 'win32'" }, - { name = "docutils", marker = "python_full_version == '3.10.*'" }, - { name = "imagesize", version = "2.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "jinja2", marker = "python_full_version == '3.10.*'" }, - { name = "packaging", marker = "python_full_version == '3.10.*'" }, - { name = "pygments", marker = "python_full_version == '3.10.*'" }, - { name = "requests", version = "2.34.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "snowballstemmer", marker = "python_full_version == '3.10.*'" }, - { name = "sphinxcontrib-applehelp", marker = "python_full_version == '3.10.*'" }, - { name = "sphinxcontrib-devhelp", marker = "python_full_version == '3.10.*'" }, - { name = "sphinxcontrib-htmlhelp", marker = "python_full_version == '3.10.*'" }, - { name = "sphinxcontrib-jsmath", marker = "python_full_version == '3.10.*'" }, - { name = "sphinxcontrib-qthelp", marker = "python_full_version == '3.10.*'" }, - { name = "sphinxcontrib-serializinghtml", marker = "python_full_version == '3.10.*'" }, - { name = "tomli", marker = "python_full_version == '3.10.*'" }, + { name = "alabaster", version = "1.0.0", source = { registry = "https://pypi.org/simple" } }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils" }, + { name = "imagesize", version = "2.0.1", source = { registry = "https://pypi.org/simple" } }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests", version = "2.34.2", source = { registry = "https://pypi.org/simple" } }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, + { name = "tomli" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/be0b61178fe2cdcb67e2a92fc9ebb488e3c51c4f74a36a7824c0adf23425/sphinx-8.1.3.tar.gz", hash = "sha256:43c1911eecb0d3e161ad78611bc905d1ad0e523e4ddc202a58a821773dc4c927", size = 8184611, upload-time = "2024-10-13T20:27:13.93Z" } wheels = [ @@ -3083,23 +3121,23 @@ resolution-markers = [ "python_full_version >= '3.11'", ] dependencies = [ - { name = "alabaster", version = "1.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "babel", marker = "python_full_version >= '3.11'" }, - { name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" }, - { name = "docutils", marker = "python_full_version >= '3.11'" }, - { name = "imagesize", version = "2.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "jinja2", marker = "python_full_version >= '3.11'" }, - { name = "packaging", marker = "python_full_version >= '3.11'" }, - { name = "pygments", marker = "python_full_version >= '3.11'" }, - { name = "requests", version = "2.34.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "roman-numerals-py", marker = "python_full_version >= '3.11'" }, - { name = "snowballstemmer", marker = "python_full_version >= '3.11'" }, - { name = "sphinxcontrib-applehelp", marker = "python_full_version >= '3.11'" }, - { name = "sphinxcontrib-devhelp", marker = "python_full_version >= '3.11'" }, - { name = "sphinxcontrib-htmlhelp", marker = "python_full_version >= '3.11'" }, - { name = "sphinxcontrib-jsmath", marker = "python_full_version >= '3.11'" }, - { name = "sphinxcontrib-qthelp", marker = "python_full_version >= '3.11'" }, - { name = "sphinxcontrib-serializinghtml", marker = "python_full_version >= '3.11'" }, + { name = "alabaster", version = "1.0.0", source = { registry = "https://pypi.org/simple" } }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils" }, + { name = "imagesize", version = "2.0.1", source = { registry = "https://pypi.org/simple" } }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests", version = "2.34.2", source = { registry = "https://pypi.org/simple" } }, + { name = "roman-numerals-py" }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/38/ad/4360e50ed56cb483667b8e6dadf2d3fda62359593faabbe749a27c4eaca6/sphinx-8.2.3.tar.gz", hash = "sha256:398ad29dee7f63a75888314e9424d40f52ce5a6a87ae88e7071e80af296ec348", size = 8321876, upload-time = "2025-03-02T22:31:59.658Z" } wheels = [ @@ -3116,16 +3154,16 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.11'" }, - { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "colorama" }, + { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.10.*'" }, { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "starlette", version = "0.49.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "starlette", version = "0.49.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.10.*'" }, { name = "starlette", version = "1.6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "uvicorn", version = "0.39.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "uvicorn", version = "0.39.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.10.*'" }, { name = "uvicorn", version = "0.53.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "watchfiles", version = "1.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "watchfiles", version = "1.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.10.*'" }, { name = "watchfiles", version = "1.2.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "websockets", version = "15.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "websockets", version = "15.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.10.*'" }, { name = "websockets", version = "16.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a5/2c/155e1de2c1ba96a72e5dba152c509a8b41e047ee5c2def9e9f0d812f8be7/sphinx_autobuild-2024.10.3.tar.gz", hash = "sha256:248150f8f333e825107b6d4b86113ab28fa51750e5f9ae63b59dc339be951fb1", size = 14023, upload-time = "2024-10-02T23:15:30.172Z" } @@ -3141,12 +3179,12 @@ resolution-markers = [ "python_full_version >= '3.11'", ] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.11'" }, - { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "starlette", version = "1.6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "uvicorn", version = "0.53.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "watchfiles", version = "1.2.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "websockets", version = "17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "colorama" }, + { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" } }, + { name = "starlette", version = "1.6.0", source = { registry = "https://pypi.org/simple" } }, + { name = "uvicorn", version = "0.53.0", source = { registry = "https://pypi.org/simple" } }, + { name = "watchfiles", version = "1.2.0", source = { registry = "https://pypi.org/simple" } }, + { name = "websockets", version = "17.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/e0/3c/a59a3a453d4133777f7ed2e83c80b7dc817d43c74b74298ca0af869662ad/sphinx_autobuild-2025.8.25.tar.gz", hash = "sha256:9cf5aab32853c8c31af572e4fecdc09c997e2b8be5a07daf2a389e270e85b213", size = 15200, upload-time = "2025-08-25T18:44:55.436Z" } wheels = [ @@ -3280,8 +3318,8 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "anyio", version = "4.12.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "typing-extensions", marker = "python_full_version < '3.10'" }, + { name = "anyio", version = "4.12.1", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/de/1a/608df0b10b53b0beb96a37854ee05864d182ddd4b1156a22f1ad3860425a/starlette-0.49.3.tar.gz", hash = "sha256:1c14546f299b5901a1ea0e34410575bc33bbd741377a10484a54445588d00284", size = 2655031, upload-time = "2025-11-01T15:12:26.13Z" } wheels = [ @@ -3297,8 +3335,8 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "anyio", version = "4.15.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.10' and python_full_version < '3.13'" }, + { name = "anyio", version = "4.15.1", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b5/b4/205b0d5241d934e8add0c38aa924c4f9fb7330834ff11e5444db964ec3f9/starlette-1.6.0.tar.gz", hash = "sha256:d4e3ac5e546444960c710297a3c9fc3f7ebae1b7e963f3d36173b49da535be9b", size = 2716969, upload-time = "2026-08-08T18:27:57.512Z" } wheels = [ @@ -3426,9 +3464,9 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "h11", marker = "python_full_version < '3.10'" }, - { name = "typing-extensions", marker = "python_full_version < '3.10'" }, + { name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" } }, + { name = "h11" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ae/4f/f9fdac7cf6dd79790eb165639b5c452ceeabc7bbabbba4569155470a287d/uvicorn-0.39.0.tar.gz", hash = "sha256:610512b19baa93423d2892d7823741f6d27717b642c8964000d7194dded19302", size = 82001, upload-time = "2025-12-21T13:05:17.973Z" } wheels = [ @@ -3444,9 +3482,9 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "click", version = "8.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "h11", marker = "python_full_version >= '3.10'" }, - { name = "typing-extensions", marker = "python_full_version == '3.10.*'" }, + { name = "click", version = "8.5.0", source = { registry = "https://pypi.org/simple" } }, + { name = "h11" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5d/ad/04bbb797c84fc1f26cb171f7394716f4865ffb8d8c5e1eef42565c2dfa6b/uvicorn-0.53.0.tar.gz", hash = "sha256:a9356f0cb89b3b8621529c5d5eebd69bfe154f4c3f68b4cf2de47e45fa855c2e", size = 110881, upload-time = "2026-09-14T07:44:23.815Z" } wheels = [ @@ -3462,7 +3500,7 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "anyio", version = "4.12.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "anyio", version = "4.12.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/c2/c9/8869df9b2a2d6c59d79220a4db37679e74f807c559ffe5265e08b227a210/watchfiles-1.1.1.tar.gz", hash = "sha256:a173cb5c16c4f40ab19cecf48a534c409f7ea983ab8fed0741304a1c0a31b3f2", size = 94440, upload-time = "2025-10-14T15:06:21.08Z" } wheels = [ @@ -3585,7 +3623,7 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "anyio", version = "4.15.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "anyio", version = "4.15.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/cd/41/5e1a4bb12aac5f1493fa1bdc11154eca3b258ca4eba65d39c473fe19d8e9/watchfiles-1.2.0.tar.gz", hash = "sha256:c995fba777f1ea992f090f9236e9284cf7a5d1a0130dd5a3d82c598cacd76838", size = 108252, upload-time = "2026-05-18T04:32:04.251Z" } wheels = [ @@ -4135,7 +4173,7 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "setuptools", version = "82.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "setuptools", version = "82.0.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/c2/d8/9c8b0c6bb1db09725395618f68d3b8a08089fca0aed28437500caaf713ee/zope_event-6.0.tar.gz", hash = "sha256:0ebac894fa7c5f8b7a89141c272133d8c1de6ddc75ea4b1f327f00d1f890df92", size = 18731, upload-time = "2025-09-12T07:10:13.551Z" } wheels = [ From 61da46892a205d4a7b8853d03f0fd6bad0f52595 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 25 Sep 2026 06:19:09 -0500 Subject: [PATCH 2/8] PYTHON-5945 Address review feedback on tracing options and span attributes Fix tracing.enabled so an explicit client value overrides the OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED environment variable, fix db.query.text truncation so budgets smaller than the "..." marker still honor the bound, and fix collection-name extraction so user and role management commands do not expose usernames as db.collection.name. --- pymongo/_otel.py | 70 +++++++++++--- pymongo/_telemetry.py | 2 +- pymongo/asynchronous/command_runner.py | 5 +- pymongo/asynchronous/mongo_client.py | 7 +- pymongo/client_options.py | 2 +- pymongo/common.py | 11 ++- pymongo/message.py | 8 +- pymongo/synchronous/command_runner.py | 5 +- pymongo/synchronous/mongo_client.py | 7 +- test/asynchronous/test_otel.py | 129 ++++++++++++++++++++++++- test/test_otel.py | 129 ++++++++++++++++++++++++- 11 files changed, 343 insertions(+), 32 deletions(-) diff --git a/pymongo/_otel.py b/pymongo/_otel.py index 713f963bbc..d3719050f1 100644 --- a/pymongo/_otel.py +++ b/pymongo/_otel.py @@ -55,12 +55,13 @@ class TracingOptions(TypedDict): """The shape of the ``MongoClient`` ``tracing`` option. - ``query_text_max_length`` is None when the client didn't configure it, so - the environment variable can be consulted; any explicit value (including - 0, to force ``db.query.text`` off) overrides the environment variable. + ``enabled`` and ``query_text_max_length`` are None when the client didn't + configure them, so the environment variables can be consulted; any explicit + value (including ``False``, to force tracing off, or 0, to force + ``db.query.text`` off) overrides the environment variable. """ - enabled: bool + enabled: Optional[bool] query_text_max_length: Optional[int] @@ -68,6 +69,12 @@ class TracingOptions(TypedDict): _OTEL_QUERY_TEXT_MAX_LENGTH_ENV = "OTEL_PYTHON_INSTRUMENTATION_MONGODB_QUERY_TEXT_MAX_LENGTH" _TRUTHY = frozenset({"1", "true", "yes"}) +# Redaction compares command names case-insensitively, mirroring the +# normalization in command monitoring, so a differently-cased sensitive +# command still gets no span. +_SENSITIVE_COMMANDS_LOWER = frozenset(name.lower() for name in _SENSITIVE_COMMANDS) +_HELLO_COMMANDS_LOWER = frozenset(name.lower() for name in _HELLO_COMMANDS) + # Fields redacted from the db.query.text attribute, mirroring the fields excluded # from the equivalent CommandStartedEvent.command per the OpenTelemetry spec. _QUERY_TEXT_EXCLUDED_FIELDS = frozenset({"lsid", "$db", "$clusterTime", "signature"}) @@ -86,6 +93,33 @@ class TracingOptions(TypedDict): # never have a real collection name, even when their command value is a string. _ADMIN_DB = "admin" +# Commands whose string command value names a user or role, not a collection +# (against any database): db.collection.name must be omitted rather than +# expose the username or role name as a collection. +_NOT_COLLECTION_COMMANDS = frozenset( + { + "createUser", + "dropAllRolesFromDatabase", + "dropAllUsersFromDatabase", + "dropRole", + "dropUser", + "grantPrivilegesToRole", + "grantPrivilegesToUser", + "grantRolesToRole", + "grantRolesToUser", + "invalidateUserCache", + "revokePrivilegesFromRole", + "revokePrivilegesFromUser", + "revokeRolesFromRole", + "revokeRolesFromUser", + "rolesInfo", + "createRole", + "updateRole", + "updateUser", + "usersInfo", + } +) + def _env_truthy(name: str) -> bool: """Return True if the environment variable ``name`` is set to "1", "true", or "yes".""" @@ -95,14 +129,19 @@ def _env_truthy(name: str) -> bool: def _is_tracing_enabled(tracing_options: Optional[TracingOptions]) -> bool: """Return True if OTel command spans should be created for this client. - The ``MongoClient`` ``tracing.enabled`` option and the - ``OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED`` environment variable both - gate enablement; either one being truthy is sufficient. + An explicit ``MongoClient`` ``tracing.enabled`` value overrides the + ``OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED`` environment variable; + when it isn't configured, the environment variable decides. ``None`` + means there is no client context (connection handshakes, server + monitoring), which is never traced. """ if not _HAS_OPENTELEMETRY: return False - if tracing_options and tracing_options.get("enabled"): - return True + if tracing_options is None: + return False + enabled = tracing_options.get("enabled") + if enabled is not None: + return enabled return _env_truthy(_OTEL_ENABLED_ENV) @@ -138,8 +177,10 @@ def _build_query_text(cmd: Mapping[str, Any], max_length: int) -> str: # not raise for commands containing custom/codec-managed Python types. text = json_util.dumps(truncated_cmd, json_options=_JSON_OPTIONS, default=repr) if len(text) > max_length: - suffix = "..." - text = text[: max(0, max_length - len(suffix))] + suffix + # A budget smaller than the marker truncates without it so the result + # still never exceeds max_length. + suffix = "..." if max_length >= 3 else "" + text = text[: max_length - len(suffix)] + suffix return text @@ -154,6 +195,8 @@ def _extract_collection_name( """ if dbname == _ADMIN_DB: return None + if command_name in _NOT_COLLECTION_COMMANDS: + return None if command_name == _EXPLAIN: inner = cmd.get(_EXPLAIN) if not isinstance(inner, Mapping) or not inner: @@ -174,9 +217,10 @@ def _build_query_summary(command_name: str, dbname: str, collection: Optional[st def _is_sensitive_command(command_name: str, speculative_hello: bool) -> bool: """Mirror the redaction rules in ``pymongo.logger.LogMessage._is_sensitive``.""" - if command_name in _SENSITIVE_COMMANDS: + name = command_name.lower() + if name in _SENSITIVE_COMMANDS_LOWER: return True - return command_name in _HELLO_COMMANDS and speculative_hello + return name in _HELLO_COMMANDS_LOWER and speculative_hello def _format_lsid(lsid: Mapping[str, Any]) -> Optional[str]: diff --git a/pymongo/_telemetry.py b/pymongo/_telemetry.py index 84b11a7408..8df71ef56f 100644 --- a/pymongo/_telemetry.py +++ b/pymongo/_telemetry.py @@ -179,7 +179,7 @@ def started(self, orig: MutableMapping[str, Any], ensure_db: bool) -> None: self._span = _otel.start_command_span( self._tracing_options, self._conn, - self._cmd, + orig, self._dbname, self._name, self._speculative_hello, diff --git a/pymongo/asynchronous/command_runner.py b/pymongo/asynchronous/command_runner.py index f13d893dbf..9e94f9b0b7 100644 --- a/pymongo/asynchronous/command_runner.py +++ b/pymongo/asynchronous/command_runner.py @@ -232,7 +232,10 @@ async def _run_command( parse_write_concern_error=parse_write_concern_error, pool_opts=pool_opts, ) - except Exception as exc: + except BaseException as exc: + # CancelledError (a BaseException) must also end the span: task + # cancellation lands here mid-command, and the span is ended with an + # error status before the cancellation propagates unmasked. if isinstance(exc, (NotPrimaryError, OperationFailure)): failure: _DocumentOut = exc.details # type: ignore[assignment] else: diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index af21740385..5f8f2578f3 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -641,9 +641,10 @@ def __init__( - `tracing`: (dict) Configuration for OpenTelemetry command spans, with keys: - ``enabled``: (boolean) Whether to create spans for server commands issued by - this client. Defaults to ``False``. Also controlled by the - ``OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED`` environment variable; either - being enabled is sufficient. + this client. Unset by default, which defers to the + ``OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED`` environment variable. When set + explicitly, including to ``False``, it overrides the environment variable, so + an application can reliably opt out. - ``query_text_max_length``: (int) The maximum length of the ``db.query.text`` span attribute. Unset by default, which defers to the ``OTEL_PYTHON_INSTRUMENTATION_MONGODB_QUERY_TEXT_MAX_LENGTH`` environment diff --git a/pymongo/client_options.py b/pymongo/client_options.py index d2d33f6805..e2d53b50cc 100644 --- a/pymongo/client_options.py +++ b/pymongo/client_options.py @@ -250,7 +250,7 @@ def __init__( ) self.__tracing = cast( "_otel.TracingOptions", - options.get("tracing") or {"enabled": False, "query_text_max_length": None}, + options.get("tracing") or {"enabled": None, "query_text_max_length": None}, ) @property diff --git a/pymongo/common.py b/pymongo/common.py index 2b8367429a..c068758b04 100644 --- a/pymongo/common.py +++ b/pymongo/common.py @@ -619,15 +619,20 @@ def validate_tracing_or_none(option: str, value: Any) -> Optional[_otel.TracingO unknown = set(value) - {"enabled", "query_text_max_length"} if unknown: raise ConfigurationError(f"Unknown tracing option(s): {sorted(unknown)}") - enabled = value.get("enabled", False) - validate_boolean("tracing.enabled", enabled) + enabled = value.get("enabled") + if enabled is not None: + validate_boolean("tracing.enabled", enabled) query_text_max_length = value.get("query_text_max_length") if query_text_max_length is not None: # bool is a subclass of int; reject it explicitly rather than silently # treating True/False as 1/0. if isinstance(query_text_max_length, bool): raise TypeError("tracing.query_text_max_length must be an integer, not a boolean") - validate_non_negative_integer("tracing.query_text_max_length", query_text_max_length) + # Keep the validator's converted value: it accepts numeric strings and + # the stored option must be an int. + query_text_max_length = validate_non_negative_integer( + "tracing.query_text_max_length", query_text_max_length + ) return {"enabled": enabled, "query_text_max_length": query_text_max_length} diff --git a/pymongo/message.py b/pymongo/message.py index df7d940532..0a7ee2a934 100644 --- a/pymongo/message.py +++ b/pymongo/message.py @@ -127,8 +127,12 @@ def _maybe_add_read_preference( return spec -def _convert_exception(exception: Exception) -> dict[str, Any]: - """Convert an Exception into a failure document for publishing.""" +def _convert_exception(exception: BaseException) -> dict[str, Any]: + """Convert an Exception into a failure document for publishing. + + Accepts BaseException so task cancellation (CancelledError) can be + recorded on a command span's failure path. + """ return {"errmsg": str(exception), "errtype": exception.__class__.__name__} diff --git a/pymongo/synchronous/command_runner.py b/pymongo/synchronous/command_runner.py index 27380eb068..88d66fa78a 100644 --- a/pymongo/synchronous/command_runner.py +++ b/pymongo/synchronous/command_runner.py @@ -232,7 +232,10 @@ def _run_command( parse_write_concern_error=parse_write_concern_error, pool_opts=pool_opts, ) - except Exception as exc: + except BaseException as exc: + # CancelledError (a BaseException) must also end the span: task + # cancellation lands here mid-command, and the span is ended with an + # error status before the cancellation propagates unmasked. if isinstance(exc, (NotPrimaryError, OperationFailure)): failure: _DocumentOut = exc.details # type: ignore[assignment] else: diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index 1321e2139f..391eeeb26b 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -642,9 +642,10 @@ def __init__( - `tracing`: (dict) Configuration for OpenTelemetry command spans, with keys: - ``enabled``: (boolean) Whether to create spans for server commands issued by - this client. Defaults to ``False``. Also controlled by the - ``OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED`` environment variable; either - being enabled is sufficient. + this client. Unset by default, which defers to the + ``OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED`` environment variable. When set + explicitly, including to ``False``, it overrides the environment variable, so + an application can reliably opt out. - ``query_text_max_length``: (int) The maximum length of the ``db.query.text`` span attribute. Unset by default, which defers to the ``OTEL_PYTHON_INSTRUMENTATION_MONGODB_QUERY_TEXT_MAX_LENGTH`` environment diff --git a/test/asynchronous/test_otel.py b/test/asynchronous/test_otel.py index 7eaeafe734..7ae37bf198 100644 --- a/test/asynchronous/test_otel.py +++ b/test/asynchronous/test_otel.py @@ -16,8 +16,10 @@ from __future__ import annotations +import asyncio import os import sys +import time from typing import Optional from unittest.mock import patch @@ -28,8 +30,9 @@ import pymongo._otel as _otel from pymongo import common from pymongo.errors import ConfigurationError, OperationFailure +from pymongo.operations import InsertOne from pymongo.typings import _Address -from test.asynchronous import AsyncIntegrationTest, unittest +from test.asynchronous import AsyncIntegrationTest, async_client_context, unittest _HAS_OTEL_TEST_DEPS = False if _otel._HAS_OPENTELEMETRY: @@ -141,6 +144,20 @@ async def test_explain_retains_collection_name(self): self.assertEqual(attrs["db.collection.name"], "test_otel") self.assertEqual(attrs["db.query.summary"], f"explain {self.db.name}.test_otel") + async def test_user_management_commands_omit_collection_name(self): + # usersInfo's string command value names a user, not a collection: + # db.collection.name must be omitted so usernames aren't mislabeled + # (and exposed) as collections. + client = await self.async_rs_or_single_client(tracing={"enabled": True}) + self.exporter.clear() + await client[self.db.name].command("usersInfo", "someuser") + + spans = self.spans("usersInfo") + self.assertEqual(len(spans), 1) + attrs = spans[0].attributes + self.assertNotIn("db.collection.name", attrs) + self.assertEqual(attrs["db.query.summary"], f"usersInfo {self.db.name}") + async def test_server_port_omitted_for_unix_socket(self): class _FakeUnixConn: id = 1 @@ -282,6 +299,27 @@ async def test_explicit_query_text_max_length_zero_overrides_env_var(self): self.assertEqual(len(spans), 1) self.assertNotIn("db.query.text", spans[0].attributes) + async def test_explicit_enabled_false_overrides_env_var(self): + # An explicit client-side disable must win over the environment + # variable, unlike unset (which defers to it) - otherwise an app + # can't reliably opt out. + with patch.dict(os.environ, {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED": "true"}): + client = await self.async_rs_or_single_client(tracing={"enabled": False}) + self.exporter.clear() + await client.admin.command("ping") + # Only the client's own commands are gated by the client option; + # connection handshakes consult the environment variable directly. + self.assertNotIn("ping", [s.name for s in self.spans()]) + + async def test_unset_enabled_defers_to_env_var(self): + # ``tracing={}`` leaves enabled unconfigured, so the environment + # variable decides. + with patch.dict(os.environ, {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED": "true"}): + client = await self.async_rs_or_single_client(tracing={}) + self.exporter.clear() + await client.admin.command("ping") + self.assertIn("ping", [s.name for s in self.spans()]) + async def test_query_text_truncation_shrinks_oversized_field_values(self): client = await self.async_rs_or_single_client( tracing={"enabled": True, "query_text_max_length": 200} @@ -300,6 +338,81 @@ async def test_query_text_truncation_shrinks_oversized_field_values(self): self.assertLessEqual(len(query_text), 200) self.assertNotIn("a" * 500, query_text) + def test_query_text_tiny_max_length_truncates_without_suffix(self): + # Budgets smaller than the "..." marker must truncate without it so + # the result still honors the configured bound. + for max_length in (1, 2): + text = _otel._build_query_text({"ping": 1}, max_length) + self.assertEqual(len(text), max_length) + + def test_is_sensitive_command_case_insensitive(self): + # Redaction normalizes the command name, mirroring the comparison in + # command monitoring: a differently-cased sensitive command gets no + # span, so its payload cannot leak through db.query.text. + for name in ("saslStart", "SASLSTART", "saslstart", "CreateUser", "createuser"): + self.assertTrue(_otel._is_sensitive_command(name, False)) + self.assertFalse(_otel._is_sensitive_command("find", False)) + + def test_no_client_options_is_never_traced(self): + # ``None`` tracing options mean no client context (connection + # handshakes, server monitoring): never traced, even when the + # environment variable enables driver-level tracing. + with patch.dict(os.environ, {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED": "true"}): + self.assertFalse(_otel._is_tracing_enabled(None)) + + # Client-level bulk_write requires MongoDB 8.0+ (wire version 25). + @async_client_context.require_version_min(8, 0, 0, -24) + async def test_unacknowledged_bulk_write_query_text_includes_documents(self): + # db.query.text is built from the document published in + # CommandStartedEvent, which carries the write documents that the + # wire command omits for unacknowledged bulk writes. + client = await self.async_rs_or_single_client( + tracing={"enabled": True, "query_text_max_length": 1000}, w=0 + ) + self.exporter.clear() + await client.bulk_write( + [InsertOne(namespace=f"{self.db.name}.test_otel", document={"x": 1})], ordered=False + ) + (span,) = self.spans("bulkWrite") + self.assertIn('"x": 1', span.attributes["db.query.text"]) + + @async_client_context.require_failCommand_blockConnection + @async_client_context.require_async + async def test_span_ended_on_task_cancellation(self): + # Task cancellation raises CancelledError (a BaseException), which the + # command runner's cleanup must handle: the span ends with an error + # status and the cancellation still propagates. + client = await self.async_rs_or_single_client(tracing={"enabled": True}) + coll = client[self.db.name].test_otel + await coll.drop() + await coll.insert_many([{"x": i} for i in range(5)]) + self.exporter.clear() + + fail_command = { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": {"failCommands": ["getMore"], "blockConnection": True, "blockTimeMS": 5000}, + } + + async def task(): + cursor = coll.find({}, batch_size=1) + await cursor.next() + async with self.fail_point(fail_command): + await cursor.next() + + running = asyncio.create_task(task()) + await asyncio.sleep(0.1) + start = time.monotonic() + running.cancel() + with self.assertRaises(asyncio.CancelledError): + await running + # The cancellation surfaces once the failPoint's block on the + # connection releases; it must not hang indefinitely. + self.assertLess(time.monotonic() - start, 7) + + (span,) = self.spans("getMore") + self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) + # TODO(PYTHON-5947): superseded once the unified test format's # expectTracingMessages/observeTracingMessages tests exercise this validator @@ -311,9 +424,15 @@ def test_none(self): def test_defaults(self): self.assertEqual( common.validate_tracing_or_none("tracing", {}), - {"enabled": False, "query_text_max_length": None}, + {"enabled": None, "query_text_max_length": None}, ) + def test_explicit_enabled_false_preserved(self): + # False must stay distinct from "unset" (None) so it can override the + # environment variable instead of deferring to it. + result = common.validate_tracing_or_none("tracing", {"enabled": False}) + self.assertIs(result["enabled"], False) + def test_enabled_and_query_text_max_length(self): self.assertEqual( common.validate_tracing_or_none( @@ -350,6 +469,12 @@ def test_rejects_negative_query_text_max_length(self): with self.assertRaises(ValueError): common.validate_tracing_or_none("tracing", {"query_text_max_length": -1}) + def test_numeric_string_query_text_max_length_converted(self): + # The validator's converted value must be kept: a numeric string that + # passes validation cannot flow into max(0, value) at command time. + result = common.validate_tracing_or_none("tracing", {"query_text_max_length": "100"}) + self.assertEqual(result["query_text_max_length"], 100) + class TestOTelTracerCaching(unittest.TestCase): """Regression test for the tracer-caching implementation in ``pymongo/_otel.py``. diff --git a/test/test_otel.py b/test/test_otel.py index d0e3a55fe5..0186588991 100644 --- a/test/test_otel.py +++ b/test/test_otel.py @@ -16,8 +16,10 @@ from __future__ import annotations +import asyncio import os import sys +import time from typing import Optional from unittest.mock import patch @@ -28,8 +30,9 @@ import pymongo._otel as _otel from pymongo import common from pymongo.errors import ConfigurationError, OperationFailure +from pymongo.operations import InsertOne from pymongo.typings import _Address -from test import IntegrationTest, unittest +from test import IntegrationTest, client_context, unittest _HAS_OTEL_TEST_DEPS = False if _otel._HAS_OPENTELEMETRY: @@ -141,6 +144,20 @@ def test_explain_retains_collection_name(self): self.assertEqual(attrs["db.collection.name"], "test_otel") self.assertEqual(attrs["db.query.summary"], f"explain {self.db.name}.test_otel") + def test_user_management_commands_omit_collection_name(self): + # usersInfo's string command value names a user, not a collection: + # db.collection.name must be omitted so usernames aren't mislabeled + # (and exposed) as collections. + client = self.rs_or_single_client(tracing={"enabled": True}) + self.exporter.clear() + client[self.db.name].command("usersInfo", "someuser") + + spans = self.spans("usersInfo") + self.assertEqual(len(spans), 1) + attrs = spans[0].attributes + self.assertNotIn("db.collection.name", attrs) + self.assertEqual(attrs["db.query.summary"], f"usersInfo {self.db.name}") + def test_server_port_omitted_for_unix_socket(self): class _FakeUnixConn: id = 1 @@ -278,6 +295,27 @@ def test_explicit_query_text_max_length_zero_overrides_env_var(self): self.assertEqual(len(spans), 1) self.assertNotIn("db.query.text", spans[0].attributes) + def test_explicit_enabled_false_overrides_env_var(self): + # An explicit client-side disable must win over the environment + # variable, unlike unset (which defers to it) - otherwise an app + # can't reliably opt out. + with patch.dict(os.environ, {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED": "true"}): + client = self.rs_or_single_client(tracing={"enabled": False}) + self.exporter.clear() + client.admin.command("ping") + # Only the client's own commands are gated by the client option; + # connection handshakes consult the environment variable directly. + self.assertNotIn("ping", [s.name for s in self.spans()]) + + def test_unset_enabled_defers_to_env_var(self): + # ``tracing={}`` leaves enabled unconfigured, so the environment + # variable decides. + with patch.dict(os.environ, {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED": "true"}): + client = self.rs_or_single_client(tracing={}) + self.exporter.clear() + client.admin.command("ping") + self.assertIn("ping", [s.name for s in self.spans()]) + def test_query_text_truncation_shrinks_oversized_field_values(self): client = self.rs_or_single_client(tracing={"enabled": True, "query_text_max_length": 200}) coll = client[self.db.name].test_otel @@ -294,6 +332,81 @@ def test_query_text_truncation_shrinks_oversized_field_values(self): self.assertLessEqual(len(query_text), 200) self.assertNotIn("a" * 500, query_text) + def test_query_text_tiny_max_length_truncates_without_suffix(self): + # Budgets smaller than the "..." marker must truncate without it so + # the result still honors the configured bound. + for max_length in (1, 2): + text = _otel._build_query_text({"ping": 1}, max_length) + self.assertEqual(len(text), max_length) + + def test_is_sensitive_command_case_insensitive(self): + # Redaction normalizes the command name, mirroring the comparison in + # command monitoring: a differently-cased sensitive command gets no + # span, so its payload cannot leak through db.query.text. + for name in ("saslStart", "SASLSTART", "saslstart", "CreateUser", "createuser"): + self.assertTrue(_otel._is_sensitive_command(name, False)) + self.assertFalse(_otel._is_sensitive_command("find", False)) + + def test_no_client_options_is_never_traced(self): + # ``None`` tracing options mean no client context (connection + # handshakes, server monitoring): never traced, even when the + # environment variable enables driver-level tracing. + with patch.dict(os.environ, {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED": "true"}): + self.assertFalse(_otel._is_tracing_enabled(None)) + + # Client-level bulk_write requires MongoDB 8.0+ (wire version 25). + @client_context.require_version_min(8, 0, 0, -24) + def test_unacknowledged_bulk_write_query_text_includes_documents(self): + # db.query.text is built from the document published in + # CommandStartedEvent, which carries the write documents that the + # wire command omits for unacknowledged bulk writes. + client = self.rs_or_single_client( + tracing={"enabled": True, "query_text_max_length": 1000}, w=0 + ) + self.exporter.clear() + client.bulk_write( + [InsertOne(namespace=f"{self.db.name}.test_otel", document={"x": 1})], ordered=False + ) + (span,) = self.spans("bulkWrite") + self.assertIn('"x": 1', span.attributes["db.query.text"]) + + @client_context.require_failCommand_blockConnection + @client_context.require_async + def test_span_ended_on_task_cancellation(self): + # Task cancellation raises CancelledError (a BaseException), which the + # command runner's cleanup must handle: the span ends with an error + # status and the cancellation still propagates. + client = self.rs_or_single_client(tracing={"enabled": True}) + coll = client[self.db.name].test_otel + coll.drop() + coll.insert_many([{"x": i} for i in range(5)]) + self.exporter.clear() + + fail_command = { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": {"failCommands": ["getMore"], "blockConnection": True, "blockTimeMS": 5000}, + } + + def task(): + cursor = coll.find({}, batch_size=1) + cursor.next() + with self.fail_point(fail_command): + cursor.next() + + running = asyncio.create_task(task()) + time.sleep(0.1) + start = time.monotonic() + running.cancel() + with self.assertRaises(asyncio.CancelledError): + running + # The cancellation surfaces once the failPoint's block on the + # connection releases; it must not hang indefinitely. + self.assertLess(time.monotonic() - start, 7) + + (span,) = self.spans("getMore") + self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) + # TODO(PYTHON-5947): superseded once the unified test format's # expectTracingMessages/observeTracingMessages tests exercise this validator @@ -305,9 +418,15 @@ def test_none(self): def test_defaults(self): self.assertEqual( common.validate_tracing_or_none("tracing", {}), - {"enabled": False, "query_text_max_length": None}, + {"enabled": None, "query_text_max_length": None}, ) + def test_explicit_enabled_false_preserved(self): + # False must stay distinct from "unset" (None) so it can override the + # environment variable instead of deferring to it. + result = common.validate_tracing_or_none("tracing", {"enabled": False}) + self.assertIs(result["enabled"], False) + def test_enabled_and_query_text_max_length(self): self.assertEqual( common.validate_tracing_or_none( @@ -344,6 +463,12 @@ def test_rejects_negative_query_text_max_length(self): with self.assertRaises(ValueError): common.validate_tracing_or_none("tracing", {"query_text_max_length": -1}) + def test_numeric_string_query_text_max_length_converted(self): + # The validator's converted value must be kept: a numeric string that + # passes validation cannot flow into max(0, value) at command time. + result = common.validate_tracing_or_none("tracing", {"query_text_max_length": "100"}) + self.assertEqual(result["query_text_max_length"], 100) + class TestOTelTracerCaching(unittest.TestCase): """Regression test for the tracer-caching implementation in ``pymongo/_otel.py``. From 139494c2b79003f171dc5e5e96c6dc648435b951 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 7 Oct 2026 05:46:41 -0500 Subject: [PATCH 3/8] PYTHON-5945 Optimize the OpenTelemetry command span hot path Benchmark tracing overhead per the OpenTelemetry spec's performance requirements with tools/otel_bench.py, and optimize the per-command hot path based on the results: - Defer expensive span attributes (db.query.summary, db.mongodb.lsid, db.mongodb.txn_number, db.query.text) until after the sampler's decision, so unsampled and no-op spans skip building them entirely. db.query.text is the big win: it serialized the command to extended JSON on every single command. - Cache connection-static span attributes on the connection (keyed by server connection id) instead of rebuilding them per command. - Resolve the client's tracing option against the environment once, at MongoClient construction, so no command consults the environment. - Skip building the duration timedelta when neither command logging nor APM events are enabled; a tracing-only client doesn't need it. Add PERF_CPU_TIME to the DriverBench performance tests to record per-operation CPU time, which otel_bench.py uses to report the fixed CPU cost of each tracing configuration (tracing off, api-only, SDK with TraceIdRatioBased sampling, SDK always-on). --- .gitignore | 1 + pymongo/_otel.py | 134 +++++++++++++++++++--------- pymongo/_telemetry.py | 92 ++++++++++--------- pymongo/asynchronous/pool.py | 3 + pymongo/client_options.py | 16 ++-- pymongo/pool_shared.py | 4 + pymongo/synchronous/pool.py | 3 + test/asynchronous/test_otel.py | 83 ++++++++++++++++- test/performance/async_perf_test.py | 53 ++++++++--- test/performance/perf_test.py | 53 ++++++++--- test/test_otel.py | 83 ++++++++++++++++- 11 files changed, 403 insertions(+), 122 deletions(-) diff --git a/.gitignore b/.gitignore index 8546aeabb7..0c29bdbd48 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ expansion.yml .evergreen/scripts/test-env.sh specifications/ results.json +otel-bench-results/ .synchro-modified .evergreen/atlas_x509_dev_client_certificate.pem .evergreen/atlas_x509_sfp_client_certificate.pem diff --git a/pymongo/_otel.py b/pymongo/_otel.py index d3719050f1..0476f3b993 100644 --- a/pymongo/_otel.py +++ b/pymongo/_otel.py @@ -41,6 +41,9 @@ # calls trace.set_tracer_provider() later, so this doesn't bind us to a # permanently-inert no-op tracer. _TRACER: Optional[Tracer] = trace.get_tracer("PyMongo", __version__) + # Command spans are always CLIENT kind; hoisted to avoid an attribute + # lookup on every span creation. + _SPAN_KIND_CLIENT = SpanKind.CLIENT except ImportError: _HAS_OPENTELEMETRY = False _TRACER = None @@ -104,8 +107,8 @@ class TracingOptions(TypedDict): "dropRole", "dropUser", "grantPrivilegesToRole", - "grantPrivilegesToUser", "grantRolesToRole", + "grantPrivilegesToUser", "grantRolesToUser", "invalidateUserCache", "revokePrivilegesFromRole", @@ -126,38 +129,80 @@ def _env_truthy(name: str) -> bool: return os.getenv(name, "").strip().lower() in _TRUTHY +def _resolve_tracing_options(tracing: Optional[TracingOptions]) -> TracingOptions: + """Resolve a client's raw ``tracing`` option against the environment, once. + + Called once per ``MongoClient`` construction: an explicit value (including + ``False``, to force tracing off, or ``0``, to force ``db.query.text`` off) + wins over the environment variable; otherwise the environment variable + decides. When opentelemetry isn't installed, tracing is always disabled. + The resolved values are then consulted on every command without touching + the environment again. + """ + if not _HAS_OPENTELEMETRY: + return {"enabled": False, "query_text_max_length": 0} + enabled = tracing.get("enabled") if tracing is not None else None + if enabled is None: + enabled = _env_truthy(_OTEL_ENABLED_ENV) + max_length = tracing.get("query_text_max_length") if tracing is not None else None + if max_length is None: + try: + max_length = int(os.getenv(_OTEL_QUERY_TEXT_MAX_LENGTH_ENV, "0")) + except ValueError: + max_length = 0 + return {"enabled": bool(enabled), "query_text_max_length": max(0, max_length)} + + def _is_tracing_enabled(tracing_options: Optional[TracingOptions]) -> bool: """Return True if OTel command spans should be created for this client. - An explicit ``MongoClient`` ``tracing.enabled`` value overrides the - ``OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED`` environment variable; - when it isn't configured, the environment variable decides. ``None`` - means there is no client context (connection handshakes, server - monitoring), which is never traced. + ``tracing_options`` is the client's resolved ``tracing`` option (see + :func:`_resolve_tracing_options`); ``None`` means there is no client + context (connection handshakes, server monitoring), which is never traced. + This runs on every command and must be cheap: no environment lookups. """ - if not _HAS_OPENTELEMETRY: - return False if tracing_options is None: return False - enabled = tracing_options.get("enabled") - if enabled is not None: - return enabled - return _env_truthy(_OTEL_ENABLED_ENV) + return bool(tracing_options.get("enabled")) def _get_query_text_max_length(tracing_options: Optional[TracingOptions]) -> int: - """Return the configured db.query.text truncation length, or 0 to omit the attribute. + """Return the resolved ``db.query.text`` truncation length, or 0 to omit the attribute.""" + if tracing_options is None: + return 0 + return tracing_options.get("query_text_max_length") or 0 + - An explicit client value (including 0) always wins; the environment - variable is only consulted when the client didn't configure it at all. +def _connection_attributes(conn: _ConnectionTelemetryInfo) -> dict[str, Any]: + """Return the connection-static span attributes as a fresh dict. + + The values are derived from the connection's address and ids, which do not + change during the life of a connection, so they are computed once and + cached on the connection (keyed by the server connection id, which changes + when the underlying socket is re-established, invalidating the cache). A + checked-out connection is owned by one caller at a time, so reading and + updating the cache needs no lock. """ - client_value = tracing_options.get("query_text_max_length") if tracing_options else None - if client_value is not None: - return max(0, client_value) - try: - return max(0, int(os.getenv(_OTEL_QUERY_TEXT_MAX_LENGTH_ENV, "0"))) - except ValueError: - return 0 + server_connection_id = conn.server_connection_id + cached = getattr(conn, "_otel_connection_attributes", None) + if cached is not None and cached[0] == server_connection_id and cached[1]: + return cached[1].copy() + address = conn.address + attrs: dict[str, Any] = { + "db.system.name": "mongodb", + "server.address": address[0], + "network.transport": "unix" if address[1] is None else "tcp", + "db.mongodb.driver_connection_id": conn.id, + } + if address[1] is not None: + attrs["server.port"] = address[1] + if server_connection_id is not None: + attrs["db.mongodb.server_connection_id"] = server_connection_id + if cached is not None: + # Duck-typed fakes without the attribute (test doubles for the + # Protocol) skip caching rather than growing a new attribute. + conn._otel_connection_attributes = (server_connection_id, attrs) + return attrs.copy() def _build_query_text(cmd: Mapping[str, Any], max_length: int) -> str: @@ -246,6 +291,12 @@ def start_command_span( Returns None when tracing is disabled/unavailable or the command is sensitive (mirroring the redaction applied to logs). + + Only cheap attributes (constants and values already held by the + connection) are provided at span creation, so they are visible to the + sampler. The expensive ones are added only when the span is being + recorded, per the OpenTelemetry spec's performance guidelines, so + unsampled configurations pay little more than the span creation itself. """ if not _is_tracing_enabled(tracing_options): return None @@ -253,37 +304,34 @@ def start_command_span( return None collection = _extract_collection_name(command_name, dbname, cmd) - address = conn.address - transport = "unix" if address[1] is None else "tcp" - attributes: dict[str, Any] = { - "db.system.name": "mongodb", - "db.namespace": dbname, - "db.command.name": command_name, - "db.query.summary": _build_query_summary(command_name, dbname, collection), - "server.address": address[0], - "network.transport": transport, - "db.mongodb.driver_connection_id": conn.id, - } - if address[1] is not None: - attributes["server.port"] = address[1] + attributes = _connection_attributes(conn) + attributes["db.namespace"] = dbname + attributes["db.command.name"] = command_name if collection: attributes["db.collection.name"] = collection - if conn.server_connection_id is not None: - attributes["db.mongodb.server_connection_id"] = conn.server_connection_id + + assert _TRACER is not None # _is_tracing_enabled already checked _HAS_OPENTELEMETRY + span = _TRACER.start_span(command_name, kind=_SPAN_KIND_CLIENT, attributes=attributes) + if not span.is_recording(): + # The span was dropped by the sampler (or tracing is a no-op): the + # spec says no further attributes are added to it. + return span + + # Expensive attributes, added after the sampling decision; they are not + # visible to samplers. + span.set_attribute("db.query.summary", _build_query_summary(command_name, dbname, collection)) lsid = cmd.get("lsid") if isinstance(lsid, Mapping): formatted_lsid = _format_lsid(lsid) if formatted_lsid is not None: - attributes["db.mongodb.lsid"] = formatted_lsid + span.set_attribute("db.mongodb.lsid", formatted_lsid) txn_number = cmd.get("txnNumber") if txn_number is not None: - attributes["db.mongodb.txn_number"] = txn_number + span.set_attribute("db.mongodb.txn_number", txn_number) max_query_text_length = _get_query_text_max_length(tracing_options) if max_query_text_length > 0: - attributes["db.query.text"] = _build_query_text(cmd, max_query_text_length) - - assert _TRACER is not None # _is_tracing_enabled already checked _HAS_OPENTELEMETRY - return _TRACER.start_span(command_name, kind=SpanKind.CLIENT, attributes=attributes) + span.set_attribute("db.query.text", _build_query_text(cmd, max_query_text_length)) + return span def end_command_span_success(span: Optional[Span], reply: _DocumentOut) -> None: diff --git a/pymongo/_telemetry.py b/pymongo/_telemetry.py index 8df71ef56f..e6821ef6a9 100644 --- a/pymongo/_telemetry.py +++ b/pymongo/_telemetry.py @@ -200,28 +200,31 @@ def succeeded( self._duration_s = _monotonic_duration(self._start) if not self._active: return - duration = datetime.timedelta(seconds=self._duration_s) - if self._should_log: - self._emit_log( - _CommandStatusMessage.SUCCEEDED, - durationMS=duration, - reply=reply, - speculative_authenticate=speculative_hello, - ) - if self._publish: - assert self._listeners is not None - self._listeners.publish_command_success( - duration, - reply, - command_name, - self._request_id, - self._conn.address, - self._conn.server_connection_id, - self._op_id, - service_id=self._conn.service_id, - speculative_hello=speculative_hello, - database_name=self._dbname, - ) + if self._should_log or self._publish: + # Only logging and APM events format the duration; a tracing-only + # client doesn't pay for the timedelta. + duration = datetime.timedelta(seconds=self._duration_s) + if self._should_log: + self._emit_log( + _CommandStatusMessage.SUCCEEDED, + durationMS=duration, + reply=reply, + speculative_authenticate=speculative_hello, + ) + if self._publish: + assert self._listeners is not None + self._listeners.publish_command_success( + duration, + reply, + command_name, + self._request_id, + self._conn.address, + self._conn.server_connection_id, + self._op_id, + service_id=self._conn.service_id, + speculative_hello=speculative_hello, + database_name=self._dbname, + ) if self._span is not None: _otel.end_command_span_success(self._span, reply) @@ -235,27 +238,30 @@ def failed( self._duration_s = _monotonic_duration(self._start) if not self._active: return - duration = datetime.timedelta(seconds=self._duration_s) - if self._should_log: - self._emit_log( - _CommandStatusMessage.FAILED, - durationMS=duration, - failure=failure, - isServerSideError=isinstance(exc, OperationFailure), - ) - if self._publish: - assert self._listeners is not None - self._listeners.publish_command_failure( - duration, - failure, - command_name, - self._request_id, - self._conn.address, - self._conn.server_connection_id, - self._op_id, - service_id=self._conn.service_id, - database_name=self._dbname, - ) + if self._should_log or self._publish: + # Only logging and APM events format the duration; a tracing-only + # client doesn't pay for the timedelta. + duration = datetime.timedelta(seconds=self._duration_s) + if self._should_log: + self._emit_log( + _CommandStatusMessage.FAILED, + durationMS=duration, + failure=failure, + isServerSideError=isinstance(exc, OperationFailure), + ) + if self._publish: + assert self._listeners is not None + self._listeners.publish_command_failure( + duration, + failure, + command_name, + self._request_id, + self._conn.address, + self._conn.server_connection_id, + self._op_id, + service_id=self._conn.service_id, + database_name=self._dbname, + ) if self._span is not None: _otel.end_command_span_failure(self._span, failure, exc) diff --git a/pymongo/asynchronous/pool.py b/pymongo/asynchronous/pool.py index 69ac14e7e4..2898cd5ced 100644 --- a/pymongo/asynchronous/pool.py +++ b/pymongo/asynchronous/pool.py @@ -163,6 +163,9 @@ def __init__( # For load balancer support. self.service_id: Optional[ObjectId] = None self.server_connection_id: Optional[int] = None + # Cache of connection-static span attributes for OpenTelemetry + # command spans; see pymongo._otel._connection_attributes. + self._otel_connection_attributes: tuple[Optional[int], dict[str, Any]] = (None, {}) # When executing a transaction in load balancing mode, this flag is # set to true to indicate that the session now owns the connection. self.pinned_txn = False diff --git a/pymongo/client_options.py b/pymongo/client_options.py index e2d53b50cc..1e505a58b4 100644 --- a/pymongo/client_options.py +++ b/pymongo/client_options.py @@ -23,7 +23,7 @@ from typing import TYPE_CHECKING, Any, Optional, cast from bson.codec_options import _parse_codec_options -from pymongo import common +from pymongo import _otel, common from pymongo.compression_support import CompressionSettings from pymongo.errors import ConfigurationError from pymongo.monitoring import _EventListener, _EventListeners @@ -40,7 +40,6 @@ if TYPE_CHECKING: from bson.codec_options import CodecOptions - from pymongo import _otel from pymongo.auth_shared import MongoCredential from pymongo.encryption_options import AutoEncryptionOpts from pymongo.pyopenssl_context import SSLContext @@ -248,10 +247,10 @@ def __init__( if "enable_overload_retargeting" in options else options.get("enableoverloadretargeting", common.ENABLE_OVERLOAD_RETARGETING) ) - self.__tracing = cast( - "_otel.TracingOptions", - options.get("tracing") or {"enabled": None, "query_text_max_length": None}, - ) + # Resolve the tracing option against the environment variables once: + # tracing cannot be changed after the client is constructed, so no + # command needs to consult the environment again. + self.__tracing = _otel._resolve_tracing_options(options.get("tracing")) @property def _options(self) -> Mapping[str, Any]: @@ -384,6 +383,11 @@ def enable_overload_retargeting(self) -> bool: def tracing(self) -> _otel.TracingOptions: """The configured ``tracing`` option for OpenTelemetry command spans. + The values are resolved when the client is constructed: an explicit + value wins over the ``OTEL_PYTHON_INSTRUMENTATION_*`` environment + variables, and the environment variables decide when the client + didn't configure the value. + .. versionadded:: 4.18 """ return self.__tracing diff --git a/pymongo/pool_shared.py b/pymongo/pool_shared.py index 63a71f3b79..f3df87f9af 100644 --- a/pymongo/pool_shared.py +++ b/pymongo/pool_shared.py @@ -59,6 +59,10 @@ class _ConnectionTelemetryInfo(Protocol): server_connection_id: Optional[int] address: _Address service_id: Optional[ObjectId] + # Cache of (server_connection_id, connection-static span attributes) + # consulted by pymongo._otel._connection_attributes; (None, {}) until the + # first span is created. + _otel_connection_attributes: tuple[Optional[int], dict[str, Any]] def _get_ssl_session(ssl_sock: Any) -> Optional[Any]: diff --git a/pymongo/synchronous/pool.py b/pymongo/synchronous/pool.py index c977729ba8..9d47a4f031 100644 --- a/pymongo/synchronous/pool.py +++ b/pymongo/synchronous/pool.py @@ -163,6 +163,9 @@ def __init__( # For load balancer support. self.service_id: Optional[ObjectId] = None self.server_connection_id: Optional[int] = None + # Cache of connection-static span attributes for OpenTelemetry + # command spans; see pymongo._otel._connection_attributes. + self._otel_connection_attributes: tuple[Optional[int], dict[str, Any]] = (None, {}) # When executing a transaction in load balancing mode, this flag is # set to true to indicate that the session now owns the connection. self.pinned_txn = False diff --git a/test/asynchronous/test_otel.py b/test/asynchronous/test_otel.py index 7ae37bf198..6977bf19cb 100644 --- a/test/asynchronous/test_otel.py +++ b/test/asynchronous/test_otel.py @@ -20,7 +20,7 @@ import os import sys import time -from typing import Optional +from typing import Any, Optional from unittest.mock import patch sys.path[0:0] = [""] @@ -106,6 +106,8 @@ async def test_span_created_for_insert_and_find(self): self.assertIn("server.port", attrs) self.assertIn(attrs["network.transport"], ("tcp", "unix")) self.assertIn("db.mongodb.driver_connection_id", attrs) + self.assertIn("db.mongodb.server_connection_id", attrs) + self.assertIn("db.mongodb.lsid", attrs) self.assertNotIn("db.query.text", attrs) self.exporter.clear() @@ -164,6 +166,7 @@ class _FakeUnixConn: server_connection_id: Optional[int] = None address: _Address = ("/tmp/fake-otel-test.sock", None) service_id = None + _otel_connection_attributes: tuple[Optional[int], dict[str, Any]] = (None, {}) self.exporter.clear() span = _otel.start_command_span( @@ -476,6 +479,83 @@ def test_numeric_string_query_text_max_length_converted(self): self.assertEqual(result["query_text_max_length"], 100) +class TestResolveTracingOptions(unittest.TestCase): + """The client's ``tracing`` option is resolved against the environment once, at construction.""" + + def _without_otel_env(self): + env = os.environ.copy() + env.pop("OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED", None) + env.pop("OTEL_PYTHON_INSTRUMENTATION_MONGODB_QUERY_TEXT_MAX_LENGTH", None) + return env + + def test_none_without_env(self): + with patch.dict(os.environ, self._without_otel_env(), clear=True): + self.assertEqual( + _otel._resolve_tracing_options(None), + {"enabled": False, "query_text_max_length": 0}, + ) + + def test_unset_defers_to_env(self): + with patch.dict(os.environ, {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED": "true"}): + self.assertTrue( + _otel._resolve_tracing_options({"enabled": None, "query_text_max_length": None})[ + "enabled" + ] + ) + + def test_env_false(self): + with patch.dict(os.environ, {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED": "false"}): + self.assertFalse( + _otel._resolve_tracing_options({"enabled": None, "query_text_max_length": None})[ + "enabled" + ] + ) + + def test_explicit_enabled_overrides_env(self): + env = {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED": "true"} + with patch.dict(os.environ, env): + resolved = _otel._resolve_tracing_options( + {"enabled": False, "query_text_max_length": None} + ) + self.assertIs(resolved["enabled"], False) + resolved = _otel._resolve_tracing_options( + {"enabled": True, "query_text_max_length": None} + ) + self.assertIs(resolved["enabled"], True) + + def test_query_text_max_length_from_env(self): + with patch.dict( + os.environ, {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_QUERY_TEXT_MAX_LENGTH": "1024"} + ): + self.assertEqual( + _otel._resolve_tracing_options({"enabled": None, "query_text_max_length": None})[ + "query_text_max_length" + ], + 1024, + ) + + def test_explicit_zero_query_text_overrides_env(self): + env = {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_QUERY_TEXT_MAX_LENGTH": "1024"} + with patch.dict(os.environ, env): + resolved = _otel._resolve_tracing_options({"enabled": None, "query_text_max_length": 0}) + self.assertEqual(resolved["query_text_max_length"], 0) + + def test_invalid_env_query_text_is_zero(self): + with patch.dict( + os.environ, {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_QUERY_TEXT_MAX_LENGTH": "bogus"} + ): + self.assertEqual( + _otel._resolve_tracing_options({"enabled": None, "query_text_max_length": None})[ + "query_text_max_length" + ], + 0, + ) + + def test_negative_query_text_clamped(self): + resolved = _otel._resolve_tracing_options({"enabled": None, "query_text_max_length": -5}) + self.assertEqual(resolved["query_text_max_length"], 0) + + class TestOTelTracerCaching(unittest.TestCase): """Regression test for the tracer-caching implementation in ``pymongo/_otel.py``. @@ -492,6 +572,7 @@ class _FakeConn: server_connection_id: Optional[int] = None address: _Address = ("localhost", 27017) service_id = None + _otel_connection_attributes: tuple[Optional[int], dict[str, Any]] = (None, {}) with patch.object(_otel, "trace") as mock_trace: for _ in range(3): diff --git a/test/performance/async_perf_test.py b/test/performance/async_perf_test.py index e4a68e228e..5aa5e8d4ba 100644 --- a/test/performance/async_perf_test.py +++ b/test/performance/async_perf_test.py @@ -87,6 +87,11 @@ OUTPUT_FILE = os.environ.get("OUTPUT_FILE") +# When set to a truthy value, also record the median process CPU time per +# iteration in the results. Off by default so the standard results.json +# schema is unchanged. +RECORD_CPU_TIME = os.environ.get("PERF_CPU_TIME", "").lower() in ("1", "true", "yes") + result_data: list = [] @@ -102,11 +107,13 @@ def tearDownModule(): class Timer: def __enter__(self): self.start = time.monotonic() + self.cpu_start = time.process_time() return self def __exit__(self, *args): self.end = time.monotonic() self.interval = self.end - self.start + self.cpu_interval = time.process_time() - self.cpu_start async def concurrent(n_tasks, func): @@ -135,6 +142,29 @@ async def asyncTearDown(self): f"Completed {self.__class__.__name__} {megabytes_per_sec:.3f} MB/s, MEDIAN={self.percentile(50):.3f}s, " f"total time={duration:.3f}s, iterations={len(self.results)}" ) + metrics = [ + { + "name": "megabytes_per_sec", + "type": "MEDIAN", + "value": megabytes_per_sec, + "metadata": { + "improvement_direction": "up", + "measurement_unit": "megabytes_per_second", + }, + } + ] + if RECORD_CPU_TIME: + metrics.append( + { + "name": "cpu_time_median", + "type": "MEDIAN", + "value": self.percentile(50, self.cpu_results), + "metadata": { + "improvement_direction": "down", + "measurement_unit": "cpu_seconds_per_iteration", + }, + } + ) result_data.append( { "info": { @@ -143,17 +173,7 @@ async def asyncTearDown(self): "tasks": self.n_tasks, }, }, - "metrics": [ - { - "name": "megabytes_per_sec", - "type": "MEDIAN", - "value": megabytes_per_sec, - "metadata": { - "improvement_direction": "up", - "measurement_unit": "megabytes_per_second", - }, - }, - ], + "metrics": metrics, } ) @@ -166,9 +186,11 @@ async def do_task(self): async def after(self): pass - def percentile(self, percentile): - if hasattr(self, "results"): - sorted_results = sorted(self.results) + def percentile(self, percentile, values=None): + if values is None: + values = getattr(self, "results", None) + if values: + sorted_results = sorted(values) percentile_index = int(len(sorted_results) * percentile / 100) - 1 return sorted_results[percentile_index] else: @@ -177,6 +199,7 @@ def percentile(self, percentile): async def runTest(self): results = [] + cpu_results = [] start = time.monotonic() i = 0 while True: @@ -189,6 +212,7 @@ async def runTest(self): await concurrent(self.n_tasks, self.do_task) await self.after() results.append(timer.interval) + cpu_results.append(timer.cpu_interval) duration = time.monotonic() - start if duration > MIN_ITERATION_TIME and i >= NUM_ITERATIONS: break @@ -204,6 +228,7 @@ async def runTest(self): break self.results = results + self.cpu_results = cpu_results # SINGLE-DOC BENCHMARKS diff --git a/test/performance/perf_test.py b/test/performance/perf_test.py index 5ec13a2b56..7464e21b13 100644 --- a/test/performance/perf_test.py +++ b/test/performance/perf_test.py @@ -89,6 +89,11 @@ OUTPUT_FILE = os.environ.get("OUTPUT_FILE") +# When set to a truthy value, also record the median process CPU time per +# iteration in the results. Off by default so the standard results.json +# schema is unchanged. +RECORD_CPU_TIME = os.environ.get("PERF_CPU_TIME", "").lower() in ("1", "true", "yes") + result_data: list = [] @@ -104,11 +109,13 @@ def tearDownModule(): class Timer: def __enter__(self): self.start = time.monotonic() + self.cpu_start = time.process_time() return self def __exit__(self, *args): self.end = time.monotonic() self.interval = self.end - self.start + self.cpu_interval = time.process_time() - self.cpu_start def threaded(n_threads, func): @@ -142,6 +149,29 @@ def tearDown(self): f"Completed {self.__class__.__name__} {megabytes_per_sec:.3f} MB/s, MEDIAN={self.percentile(50):.3f}s, " f"total time={duration:.3f}s, iterations={len(self.results)}" ) + metrics = [ + { + "name": "megabytes_per_sec", + "type": "MEDIAN", + "value": megabytes_per_sec, + "metadata": { + "improvement_direction": "up", + "measurement_unit": "megabytes_per_second", + }, + } + ] + if RECORD_CPU_TIME: + metrics.append( + { + "name": "cpu_time_median", + "type": "MEDIAN", + "value": self.percentile(50, self.cpu_results), + "metadata": { + "improvement_direction": "down", + "measurement_unit": "cpu_seconds_per_iteration", + }, + } + ) result_data.append( { "info": { @@ -150,17 +180,7 @@ def tearDown(self): "threads": self.n_threads, }, }, - "metrics": [ - { - "name": "megabytes_per_sec", - "type": "MEDIAN", - "value": megabytes_per_sec, - "metadata": { - "improvement_direction": "up", - "measurement_unit": "megabytes_per_second", - }, - }, - ], + "metrics": metrics, } ) @@ -173,9 +193,11 @@ def do_task(self): def after(self): pass - def percentile(self, percentile): - if hasattr(self, "results"): - sorted_results = sorted(self.results) + def percentile(self, percentile, values=None): + if values is None: + values = getattr(self, "results", None) + if values: + sorted_results = sorted(values) percentile_index = int(len(sorted_results) * percentile / 100) - 1 return sorted_results[percentile_index] else: @@ -184,6 +206,7 @@ def percentile(self, percentile): def runTest(self): results = [] + cpu_results = [] start = time.monotonic() i = 0 while True: @@ -196,6 +219,7 @@ def runTest(self): threaded(self.n_threads, self.do_task) self.after() results.append(timer.interval) + cpu_results.append(timer.cpu_interval) duration = time.monotonic() - start if duration > MIN_ITERATION_TIME and i >= NUM_ITERATIONS: break @@ -209,6 +233,7 @@ def runTest(self): break self.results = results + self.cpu_results = cpu_results def mp_map(self, map_func, files): with mp.Pool(initializer=proc_init, initargs=(client_context.client_options,)) as pool: diff --git a/test/test_otel.py b/test/test_otel.py index 0186588991..76adc91e9d 100644 --- a/test/test_otel.py +++ b/test/test_otel.py @@ -20,7 +20,7 @@ import os import sys import time -from typing import Optional +from typing import Any, Optional from unittest.mock import patch sys.path[0:0] = [""] @@ -106,6 +106,8 @@ def test_span_created_for_insert_and_find(self): self.assertIn("server.port", attrs) self.assertIn(attrs["network.transport"], ("tcp", "unix")) self.assertIn("db.mongodb.driver_connection_id", attrs) + self.assertIn("db.mongodb.server_connection_id", attrs) + self.assertIn("db.mongodb.lsid", attrs) self.assertNotIn("db.query.text", attrs) self.exporter.clear() @@ -164,6 +166,7 @@ class _FakeUnixConn: server_connection_id: Optional[int] = None address: _Address = ("/tmp/fake-otel-test.sock", None) service_id = None + _otel_connection_attributes: tuple[Optional[int], dict[str, Any]] = (None, {}) self.exporter.clear() span = _otel.start_command_span( @@ -470,6 +473,83 @@ def test_numeric_string_query_text_max_length_converted(self): self.assertEqual(result["query_text_max_length"], 100) +class TestResolveTracingOptions(unittest.TestCase): + """The client's ``tracing`` option is resolved against the environment once, at construction.""" + + def _without_otel_env(self): + env = os.environ.copy() + env.pop("OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED", None) + env.pop("OTEL_PYTHON_INSTRUMENTATION_MONGODB_QUERY_TEXT_MAX_LENGTH", None) + return env + + def test_none_without_env(self): + with patch.dict(os.environ, self._without_otel_env(), clear=True): + self.assertEqual( + _otel._resolve_tracing_options(None), + {"enabled": False, "query_text_max_length": 0}, + ) + + def test_unset_defers_to_env(self): + with patch.dict(os.environ, {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED": "true"}): + self.assertTrue( + _otel._resolve_tracing_options({"enabled": None, "query_text_max_length": None})[ + "enabled" + ] + ) + + def test_env_false(self): + with patch.dict(os.environ, {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED": "false"}): + self.assertFalse( + _otel._resolve_tracing_options({"enabled": None, "query_text_max_length": None})[ + "enabled" + ] + ) + + def test_explicit_enabled_overrides_env(self): + env = {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED": "true"} + with patch.dict(os.environ, env): + resolved = _otel._resolve_tracing_options( + {"enabled": False, "query_text_max_length": None} + ) + self.assertIs(resolved["enabled"], False) + resolved = _otel._resolve_tracing_options( + {"enabled": True, "query_text_max_length": None} + ) + self.assertIs(resolved["enabled"], True) + + def test_query_text_max_length_from_env(self): + with patch.dict( + os.environ, {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_QUERY_TEXT_MAX_LENGTH": "1024"} + ): + self.assertEqual( + _otel._resolve_tracing_options({"enabled": None, "query_text_max_length": None})[ + "query_text_max_length" + ], + 1024, + ) + + def test_explicit_zero_query_text_overrides_env(self): + env = {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_QUERY_TEXT_MAX_LENGTH": "1024"} + with patch.dict(os.environ, env): + resolved = _otel._resolve_tracing_options({"enabled": None, "query_text_max_length": 0}) + self.assertEqual(resolved["query_text_max_length"], 0) + + def test_invalid_env_query_text_is_zero(self): + with patch.dict( + os.environ, {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_QUERY_TEXT_MAX_LENGTH": "bogus"} + ): + self.assertEqual( + _otel._resolve_tracing_options({"enabled": None, "query_text_max_length": None})[ + "query_text_max_length" + ], + 0, + ) + + def test_negative_query_text_clamped(self): + resolved = _otel._resolve_tracing_options({"enabled": None, "query_text_max_length": -5}) + self.assertEqual(resolved["query_text_max_length"], 0) + + class TestOTelTracerCaching(unittest.TestCase): """Regression test for the tracer-caching implementation in ``pymongo/_otel.py``. @@ -486,6 +566,7 @@ class _FakeConn: server_connection_id: Optional[int] = None address: _Address = ("localhost", 27017) service_id = None + _otel_connection_attributes: tuple[Optional[int], dict[str, Any]] = (None, {}) with patch.object(_otel, "trace") as mock_trace: for _ in range(3): From 927fe036879f601ce09b425036bce0312148494b Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 7 Oct 2026 05:48:16 -0500 Subject: [PATCH 4/8] add the otel bench --- tools/otel_bench.py | 398 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 398 insertions(+) create mode 100755 tools/otel_bench.py diff --git a/tools/otel_bench.py b/tools/otel_bench.py new file mode 100755 index 0000000000..5b0d44eca6 --- /dev/null +++ b/tools/otel_bench.py @@ -0,0 +1,398 @@ +#!/usr/bin/env python +"""Benchmark the overhead of the driver's OpenTelemetry instrumentation. + +Runs the DriverBench "Small doc insertOne" and "Find one by ID" tasks from +``test/performance`` (both the synchronous and the asynchronous API) under +each tracing configuration, interleaved with a rotated configuration order +across repetitions on a single host, and reports the overhead of each +configuration relative to the untraced baseline. + +Implements the benchmarking requirements of the OpenTelemetry +specification's Performance Implications section (DRIVERS-3620): +https://github.com/mongodb/specifications/pull/1986 + +Configurations (per the spec's table; the harness, not the driver, installs +and configures the SDK and its sampler, and the SDK configurations install +no exporter or span processor. No configuration has an active parent span): + +- ``off`` tracing disabled; the baseline. +- ``api-only`` tracing enabled; no SDK, so every tracing call is a no-op. +- ``sdk-ratio`` tracing enabled; SDK installed; ``TraceIdRatioBased(1%)`` sampler. +- ``sdk-always`` tracing enabled; SDK installed; every span sampled. +- ``main`` the commit before OpenTelemetry support was added, tracing + disabled (the spec's one-time pre-OTel comparison). Requires + ``--worktree-main`` pointing at a worktree of that commit. + +Requires a MongoDB server on localhost:27017 (or via DB_IP/DB_PORT), the +DriverBench datasets (``--data-dir``), and ``opentelemetry-sdk`` installed +in the current environment (the SDK is benchmark tooling only, the driver +itself depends only on ``opentelemetry-api``). + +Examples:: + + python tools/otel_bench.py --verify + python tools/otel_bench.py --fast --reps 1 + python tools/otel_bench.py --worktree-main /path/to/main-worktree +""" + +from __future__ import annotations + +import argparse +import json +import os +import statistics +import subprocess +import sys +import time +from collections import deque +from pathlib import Path +from typing import Any, Optional + +ROOT = Path(__file__).resolve().parent.parent + +# The two micro-benchmarks the spec requires, defined in both perf_test.py +# and async_perf_test.py. +TASKS = ("SmallDocInsertOne", "FindOneByID") + +OFF = "off" +API_ONLY = "api-only" +SDK_RATIO = "sdk-ratio" +SDK_ALWAYS = "sdk-always" +MAIN = "main" + +SDK_CONFIGS = frozenset({SDK_RATIO, SDK_ALWAYS}) +ENABLED_CONFIGS = frozenset({API_ONLY, SDK_RATIO, SDK_ALWAYS}) + +BASE_CONFIGS = (OFF, API_ONLY, SDK_RATIO, SDK_ALWAYS, MAIN) + +# Child pytest invocation; `-m perf` overrides the repo config's +# `-m default or default_async` addopts (the last -m wins). +PYTEST_ARGS = ["-m", "perf", "-v", "--durations=5"] + +# opentelemetry.sdk is imported only for the SDK configurations. The provider +# is installed before pytest runs; the driver's cached ProxyTracer delegates +# to it regardless of import order. No span processor is added. +BOOTSTRAP = """import sys +config = sys.argv[1] +del sys.argv[1] +if config in ("sdk-ratio", "sdk-always"): + from opentelemetry import trace + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.sampling import ALWAYS_ON, TraceIdRatioBased + + sampler = TraceIdRatioBased(0.01) if config == "sdk-ratio" else ALWAYS_ON + trace.set_tracer_provider(TracerProvider(sampler=sampler)) +import pytest + +sys.exit(pytest.main(sys.argv[1:])) +""" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--reps", type=int, default=5, help="Number of interleaved repetitions") + parser.add_argument( + "--fast", + action="store_true", + help="Set FASTBENCH=1 in the benchmark children (quick, unrepresentative)", + ) + parser.add_argument( + "--data-dir", + type=Path, + default=ROOT / "specifications" / "source" / "benchmarking" / "data", + help="Directory containing the DriverBench datasets", + ) + parser.add_argument( + "--worktree-main", + type=Path, + help="Worktree of the commit before OpenTelemetry support was added, " + "for the spec's one-time pre-OTel comparison (the 'main' configuration)", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=ROOT / "otel-bench-results", + help="Directory for results and logs", + ) + parser.add_argument("--configs", help="Comma-separated subset of: " + ",".join(BASE_CONFIGS)) + parser.add_argument( + "--python", default=sys.executable, help="Python interpreter for the benchmark children" + ) + parser.add_argument( + "--verify", + action="store_true", + help="Only check that tracing produces spans end-to-end (with an in-memory exporter) and exit", + ) + return parser.parse_args() + + +def resolve_configs(args: argparse.Namespace) -> tuple[str, ...]: + configs = list(BASE_CONFIGS) if args.worktree_main else [c for c in BASE_CONFIGS if c != MAIN] + if args.configs: + selected = args.configs.split(",") + unknown = set(selected) - set(configs) + if unknown: + raise SystemExit(f"Unknown or unavailable configurations: {sorted(unknown)}") + configs = selected + if OFF not in configs: + raise SystemExit("The 'off' configuration is required as the baseline") + return tuple(configs) + + +def build_command(config: str, api: str, python: str) -> list[str]: + perf_file = ( + "test/performance/perf_test.py" if api == "sync" else "test/performance/async_perf_test.py" + ) + nodes = [f"{perf_file}::Test{task}" for task in TASKS] + if config in SDK_CONFIGS: + return [python, "-c", BOOTSTRAP, config, *PYTEST_ARGS, *nodes] + return [python, "-m", "pytest", *PYTEST_ARGS, *nodes] + + +def child_env(config: str, output_file: Path, args: argparse.Namespace) -> dict[str, str]: + # The harness owns the tracing configuration: strip any ambient OTEL_ + # variables, then set exactly what the configuration requires. + env = {k: v for k, v in os.environ.items() if not k.startswith("OTEL_")} + env["TEST_PATH"] = str(args.data_dir) + env["OUTPUT_FILE"] = str(output_file) + env["PERF_CPU_TIME"] = "1" + if args.fast: + env["FASTBENCH"] = "1" + if config in ENABLED_CONFIGS: + env["OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED"] = "1" + return env + + +def run_one(config: str, api: str, rep: int, args: argparse.Namespace) -> tuple[Path, float]: + output_file = args.output_dir / f"{api}-{config}-rep{rep:02d}.json" + if output_file.exists(): + output_file.unlink() + cmd = build_command(config, api, args.python) + cwd = args.worktree_main if config == MAIN else ROOT + log_file = output_file.with_suffix(".log") + start = time.monotonic() + with open(log_file, "w") as log: + proc = subprocess.run( # noqa: S603 + cmd, + cwd=cwd, + env=child_env(config, output_file, args), + stdout=log, + stderr=subprocess.STDOUT, + check=False, + ) + elapsed = time.monotonic() - start + if proc.returncode != 0: + print(f"FAILED (exit {proc.returncode}): {' '.join(cmd)}\nLog tail:") + print("\n".join(log_file.read_text().splitlines()[-40:])) + raise SystemExit(1) + return output_file, elapsed + + +def parse_results(output_file: Path) -> dict[str, dict[str, float]]: + entries = json.loads(output_file.read_text()) + parsed: dict[str, dict[str, float]] = {} + for entry in entries: + name = entry["info"]["test_name"] + parsed[name] = {metric["name"]: metric["value"] for metric in entry["metrics"]} + return parsed + + +def verify() -> None: + """Check end-to-end that tracing produces spans, using an in-memory exporter. + + This is a wiring check only; measured runs never install an exporter. + """ + os.environ["OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED"] = "1" + from opentelemetry import trace + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + + try: + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + except ImportError: + # The module name changed across SDK versions. + from opentelemetry.sdk.trace.export.in_memory_exporter import ( # type: ignore[import-not-found,no-redef] + InMemorySpanExporter, + ) + + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + trace.set_tracer_provider(provider) + + sys.path.insert(0, str(ROOT)) + from pymongo import MongoClient + + client: MongoClient[dict[str, Any]] = MongoClient() + client.perftest_otel_verify.coll.insert_one({"x": 1}) + client.drop_database("perftest_otel_verify") + client.close() + + spans = exporter.get_finished_spans() + names = [span.name for span in spans] + assert "insert" in names, f"Expected an 'insert' span, got: {names}" + span = next(s for s in spans if s.name == "insert") + attrs = dict(span.attributes or {}) + assert attrs.get("db.system.name") == "mongodb", attrs + assert attrs.get("db.command.name") == "insert", attrs + print(f"OK: {len(names)} span(s) recorded: {names}") + + +def report( + configs: tuple[str, ...], + scores: dict[tuple[str, str, str], list[float]], + cpu_scores: dict[tuple[str, str, str], list[float]], + args: argparse.Namespace, +) -> dict[str, Any]: + num_docs = 1000 if args.fast else 10000 + summary: dict[str, Any] = { + "reps": args.reps, + "fast": args.fast, + "configs": {}, + "pre_otel_compare": None, + } + + for api in ("sync", "async"): + print(f"\n## {api} (median MB/s across repetitions, overhead vs 'off')") + print("| config | " + " | ".join(TASKS) + " |") + print("|" + "---|" * (len(TASKS) + 1)) + off_medians = { + task: statistics.median(scores[(OFF, api, task)]) + for task in TASKS + if scores.get((OFF, api, task)) + } + for config in configs: + if config == MAIN: + continue + cells = [] + for task in TASKS: + values = scores.get((config, api, task), []) + if not values: + cells.append("n/a") + continue + median = statistics.median(values) + if config == OFF: + cells.append(f"{median:.2f} MB/s (baseline)") + else: + overhead = (1 - median / off_medians[task]) * 100 + cells.append(f"{median:.2f} MB/s ({overhead:+.1f}%)") + print(f"| {config} | " + " | ".join(cells) + " |") + + if MAIN in configs and all(scores.get((MAIN, api, task)) for task in TASKS): + row: dict[str, Any] = {} + for task in TASKS: + main_median = statistics.median(scores[(MAIN, api, task)]) + off_median = statistics.median(scores[(OFF, api, task)]) + row[task] = { + "main_median_mb_s": main_median, + "off_median_mb_s": off_median, + "overhead_pct": (1 - off_median / main_median) * 100, + } + summary["pre_otel_compare"] = {"api": api, **row} + overheads = ", ".join(f"{task}: {row[task]['overhead_pct']:+.1f}%" for task in TASKS) + print(f"\nPre-OTel comparison (off on this branch vs main, {api}): {overheads}") + + print("\n## CPU time per operation (us/op median, delta vs 'off' in parens)") + print( + "| config | " + + " | ".join(f"{api}/{task}" for api in ("sync", "async") for task in TASKS) + + " |" + ) + print("|" + "---|" * (len(TASKS) * 2 + 1)) + for config in configs: + if config == MAIN: + continue + cells = [] + for api in ("sync", "async"): + for task in TASKS: + values = cpu_scores.get((config, api, task), []) + base = cpu_scores.get((OFF, api, task), []) + if not values or not base: + cells.append("n/a") + continue + per_op = statistics.median(values) / num_docs * 1e6 + per_op_off = statistics.median(base) / num_docs * 1e6 + delta = "" if config == OFF else f" ({per_op - per_op_off:+.0f})" + cells.append(f"{per_op:.0f}{delta}") + print(f"| {config} | " + " | ".join(cells) + " |") + + for config in configs: + summary["configs"][config] = {} + for api in ("sync", "async"): + for task in TASKS: + values = scores.get((config, api, task), []) + if values: + entry: dict[str, Any] = { + "mb_per_sec_reps": values, + "mb_per_sec_median": statistics.median(values), + } + cpu = cpu_scores.get((config, api, task), []) + if cpu: + entry["cpu_seconds_per_iteration_reps"] = cpu + entry["cpu_us_per_op_median"] = statistics.median(cpu) / num_docs * 1e6 + summary["configs"][config][f"{api}/{task}"] = entry + return summary + + +def main() -> None: + args = parse_args() + if args.verify: + verify() + return + + if args.reps < 1: + raise SystemExit("--reps must be at least 1") + if not args.data_dir.exists(): + raise SystemExit( + f"Dataset directory not found: {args.data_dir}\n" + "Clone the specifications repo and extract the data tarballs, or pass --data-dir." + ) + configs = resolve_configs(args) + args.output_dir.mkdir(parents=True, exist_ok=True) + + scores: dict[tuple[str, str, str], list[float]] = {} + cpu_scores: dict[tuple[str, str, str], list[float]] = {} + started = time.monotonic() + total_runs = args.reps * len(configs) * 2 + done_runs = 0 + summary: Optional[dict[str, Any]] = None + + for rep in range(1, args.reps + 1): + # Rotate the configuration order each repetition so host drift affects + # all configurations equally (the spec's interleaving requirement). + order: deque[str] = deque(configs) + order.rotate(-(rep - 1)) + for config in order: + for api in ("sync", "async"): + output_file, elapsed = run_one(config, api, rep, args) + parsed = parse_results(output_file) + for task in TASKS: + metrics = parsed.get(task, {}) + mbps = metrics.get("megabytes_per_sec") + if mbps is not None: + scores.setdefault((config, api, task), []).append(mbps) + if "cpu_time_median" in metrics: + cpu_scores.setdefault((config, api, task), []).append( + metrics["cpu_time_median"] + ) + done_runs += 1 + results_mb = ", ".join( + f"{task}={parsed[task]['megabytes_per_sec']:.2f}MB/s" for task in TASKS + ) + print( + f"[{done_runs}/{total_runs}] rep {rep}/{args.reps} {config:>9} {api:<5} " + f"({elapsed:.0f}s): {results_mb}", + flush=True, + ) + # Crash-safe: persist intermediate results after every repetition. + summary = report(configs, scores, cpu_scores, args) + (args.output_dir / "otel_bench_results.json").write_text(json.dumps(summary, indent=2)) + + assert summary is not None + print(f"\nTotal: {time.monotonic() - started:.0f}s. Results in {args.output_dir}/") + if summary["pre_otel_compare"]: + print(json.dumps(summary["pre_otel_compare"], indent=2)) + + +if __name__ == "__main__": + main() From 1db0b46d0220a235c94a58bf3a2b11a1418c17fe Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 7 Oct 2026 07:32:56 -0500 Subject: [PATCH 5/8] Merge remote-tracking branch 'upstream/main' into PYTHON-6065 --- .evergreen/generated_configs/functions.yml | 9 +- .evergreen/generated_configs/tasks.yml | 718 +++++++++--------- .evergreen/resync-specs.sh | 69 +- .evergreen/run-tests.sh | 7 +- .evergreen/scripts/cleanup.sh | 7 +- .evergreen/scripts/configure-env.sh | 41 +- .evergreen/scripts/create-spec-pr.sh | 84 +- .evergreen/scripts/dispatch.sh | 46 ++ .evergreen/scripts/generate_config.py | 43 +- .evergreen/scripts/generate_config_utils.py | 2 +- .evergreen/scripts/install-dependencies.sh | 12 +- .evergreen/scripts/mod_wsgi.sh | 263 +++++++ .evergreen/scripts/mod_wsgi_matrix.py | 29 + .evergreen/scripts/mod_wsgi_tester.py | 312 ++++++++ .evergreen/scripts/resync-all-specs.py | 9 +- .evergreen/scripts/resync-all-specs.sh | 7 +- .evergreen/scripts/run-getdata.sh | 7 + .evergreen/scripts/run_server.py | 19 +- .evergreen/scripts/setup-dev-env.sh | 23 + .evergreen/scripts/setup-uv-python.sh | 163 ++-- .evergreen/scripts/setup_tests.py | 28 +- .evergreen/scripts/stop-server.sh | 4 + .evergreen/scripts/utils.py | 25 +- .evergreen/spec-patch/PYTHON-5993.patch | 94 +++ .github/actions/mod-wsgi-versions/action.yml | 23 + .github/dependabot.yml | 26 + .github/workflows/dist.yml | 2 +- .github/workflows/sbom.yml | 1 + .github/workflows/test-mod-wsgi.yml | 179 +++++ .github/workflows/test-python.yml | 19 +- .github/zizmor.yml | 2 +- .gitignore | 8 +- .gitmodules | 3 + CONTRIBUTING.md | 95 ++- doc/changelog.rst | 30 + drivers-evergreen-tools | 1 + justfile | 36 +- pymongo/_client_bulk_shared.py | 155 +++- pymongo/_psl.py | 22 +- pymongo/_telemetry.py | 11 + pymongo/asynchronous/bulk.py | 168 +--- pymongo/asynchronous/client_bulk.py | 151 +--- pymongo/asynchronous/mongo_client.py | 57 +- pymongo/asynchronous/monitor.py | 1 + pymongo/asynchronous/pool.py | 128 +++- pymongo/asynchronous/settings.py | 4 +- pymongo/asynchronous/srv_resolver.py | 105 ++- pymongo/asynchronous/uri_parser.py | 12 +- pymongo/bulk_shared.py | 171 ++++- pymongo/common.py | 1 + pymongo/driver_info.py | 12 +- pymongo/pool_options.py | 196 +++-- pymongo/public_suffix_list.dat | 15 +- pymongo/settings_shared.py | 9 +- pymongo/synchronous/bulk.py | 168 +--- pymongo/synchronous/client_bulk.py | 151 +--- pymongo/synchronous/mongo_client.py | 57 +- pymongo/synchronous/monitor.py | 1 + pymongo/synchronous/pool.py | 128 +++- pymongo/synchronous/settings.py | 4 +- pymongo/synchronous/srv_resolver.py | 105 ++- pymongo/synchronous/uri_parser.py | 12 +- pymongo/typings.py | 1 - pyproject.toml | 5 + requirements/mod_wsgi.txt | 3 + test/asynchronous/test_client.py | 202 ++++- test/asynchronous/test_client_metadata.py | 177 ++++- test/asynchronous/test_dns.py | 150 +++- test/asynchronous/test_pooling.py | 95 +++ test/asynchronous/test_srv_polling.py | 111 ++- test/asynchronous/unified_format.py | 6 + test/bson_corpus/dbref.json | 15 + test/mockupdb/test_handshake.py | 4 +- test/mod_wsgi_test/README.rst | 78 +- test/mod_wsgi_test/apache22amazon.conf | 34 - test/mod_wsgi_test/apache22ubuntu1204.conf | 29 - test/mod_wsgi_test/apache24ubuntu.conf | 47 ++ test/mod_wsgi_test/apache24ubuntu161404.conf | 28 - test/mod_wsgi_test/httpd24fedora.conf | 58 ++ test/mod_wsgi_test/mod_wsgi_test.conf | 2 - .../mod_wsgi_test/mod_wsgi_test_embedded.conf | 2 - test/test_bson.py | 35 +- test/test_client.py | 202 ++++- test/test_client_metadata.py | 177 ++++- test/test_dns.py | 148 +++- test/test_pooling.py | 95 +++ test/test_srv_polling.py | 111 ++- test/unified_format.py | 6 + test/utils_shared.py | 36 + tools/generate_sbom_requirements.py | 4 +- 90 files changed, 4508 insertions(+), 1643 deletions(-) create mode 100755 .evergreen/scripts/dispatch.sh create mode 100755 .evergreen/scripts/mod_wsgi.sh create mode 100644 .evergreen/scripts/mod_wsgi_matrix.py create mode 100644 .evergreen/scripts/mod_wsgi_tester.py create mode 100644 .evergreen/spec-patch/PYTHON-5993.patch create mode 100644 .github/actions/mod-wsgi-versions/action.yml create mode 100644 .github/workflows/test-mod-wsgi.yml create mode 100644 .gitmodules create mode 160000 drivers-evergreen-tools create mode 100644 requirements/mod_wsgi.txt delete mode 100644 test/mod_wsgi_test/apache22amazon.conf delete mode 100644 test/mod_wsgi_test/apache22ubuntu1204.conf create mode 100644 test/mod_wsgi_test/apache24ubuntu.conf delete mode 100644 test/mod_wsgi_test/apache24ubuntu161404.conf create mode 100644 test/mod_wsgi_test/httpd24fedora.conf diff --git a/.evergreen/generated_configs/functions.yml b/.evergreen/generated_configs/functions.yml index 9fbc1f8500..df7393de4c 100644 --- a/.evergreen/generated_configs/functions.yml +++ b/.evergreen/generated_configs/functions.yml @@ -102,7 +102,7 @@ functions: - SSL - ORCHESTRATION_FILE - UV_PYTHON - - TOOLCHAIN_VERSION + - UV_PYTHON_PREFERENCE - STORAGE_ENGINE - REQUIRE_API_VERSION - DRIVERS_TOOLS @@ -111,6 +111,9 @@ functions: - LOAD_BALANCER - LOCAL_ATLAS - NO_EXT + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN type: test - command: expansions.update params: @@ -135,9 +138,9 @@ functions: - AWS_SESSION_TOKEN - COVERAGE - UV_PYTHON + - UV_PYTHON_PREFERENCE - LIBMONGOCRYPT_URL - MONGODB_URI - - TOOLCHAIN_VERSION - DISABLE_TEST_COMMANDS - GREEN_FRAMEWORK - NO_EXT @@ -249,7 +252,7 @@ functions: - test-numpy working_dir: src include_expansions_in_env: - - TOOLCHAIN_VERSION + - UV_PYTHON - COVERAGE type: test diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index e2677e7399..169f8ce85f 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -20,7 +20,7 @@ tasks: vars: TEST_NAME: auth_aws SUB_TEST_NAME: regular - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_MIN_DEPS: "1" tags: [auth-aws, auth-aws-regular] - name: test-auth-aws-5.0-assume-role-python3.11 @@ -34,7 +34,7 @@ tasks: vars: TEST_NAME: auth_aws SUB_TEST_NAME: assume-role - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" tags: [auth-aws, auth-aws-assume-role] - name: test-auth-aws-6.0-ec2-python3.12 commands: @@ -47,7 +47,7 @@ tasks: vars: TEST_NAME: auth_aws SUB_TEST_NAME: ec2 - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" tags: [auth-aws, auth-aws-ec2] - name: test-auth-aws-7.0-env-creds-python3.13 commands: @@ -60,7 +60,7 @@ tasks: vars: TEST_NAME: auth_aws SUB_TEST_NAME: env-creds - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" tags: [auth-aws, auth-aws-env-creds] - name: test-auth-aws-8.0-session-creds-python3.14t commands: @@ -73,7 +73,7 @@ tasks: vars: TEST_NAME: auth_aws SUB_TEST_NAME: session-creds - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t tags: [auth-aws, auth-aws-session-creds, free-threaded] - name: test-auth-aws-9.0-web-identity-python3.14 commands: @@ -86,7 +86,7 @@ tasks: vars: TEST_NAME: auth_aws SUB_TEST_NAME: web-identity - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" tags: [auth-aws, auth-aws-web-identity] - name: test-auth-aws-9.0-web-identity-session-name-python3.14 commands: @@ -100,7 +100,7 @@ tasks: TEST_NAME: auth_aws SUB_TEST_NAME: web-identity AWS_ROLE_SESSION_NAME: test - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" tags: [auth-aws, auth-aws-web-identity] - name: test-auth-aws-rapid-regular-python3.15t commands: @@ -113,7 +113,7 @@ tasks: vars: TEST_NAME: auth_aws SUB_TEST_NAME: regular - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t tags: [auth-aws, auth-aws-regular, free-threaded] - name: test-auth-aws-latest-assume-role-python3.15-cov commands: @@ -126,7 +126,7 @@ tasks: vars: TEST_NAME: auth_aws SUB_TEST_NAME: assume-role - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" COVERAGE: "1" tags: [auth-aws, auth-aws-assume-role, pr] - name: test-auth-aws-ecs @@ -322,7 +322,7 @@ tasks: - func: assume ec2 role - func: run tests vars: - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_MIN_DEPS: "1" tags: [test-no-orchestration, python-3.10] - name: test-no-orchestration-python3.15 @@ -330,14 +330,14 @@ tasks: - func: assume ec2 role - func: run tests vars: - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" tags: [test-no-orchestration, python-3.15] - name: test-no-orchestration-pypy3.11 commands: - func: assume ec2 role - func: run tests vars: - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: [test-no-orchestration, python-pypy3.11] # No toolchain tests @@ -392,7 +392,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "4.4"] @@ -403,7 +403,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "5.0"] @@ -414,7 +414,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "6.0"] @@ -425,7 +425,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "7.0"] @@ -436,7 +436,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] @@ -447,7 +447,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] @@ -458,7 +458,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, rapid] @@ -469,7 +469,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-ecdsa, latest] - name: test-ocsp-ecdsa-invalid-cert-server-does-not-staple-v4.4-python3.10-min-deps @@ -479,7 +479,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "4.4"] @@ -490,7 +490,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "5.0"] @@ -501,7 +501,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "6.0"] @@ -512,7 +512,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "7.0"] @@ -523,7 +523,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] @@ -534,7 +534,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] @@ -545,7 +545,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, rapid] @@ -556,7 +556,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-ecdsa, latest] - name: test-ocsp-ecdsa-delegate-valid-cert-server-does-not-staple-v4.4-python3.10-min-deps @@ -566,7 +566,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "4.4"] @@ -577,7 +577,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "5.0"] @@ -588,7 +588,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "6.0"] @@ -599,7 +599,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "7.0"] @@ -610,7 +610,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] @@ -621,7 +621,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] @@ -632,7 +632,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, rapid] @@ -643,7 +643,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-ecdsa, latest] - name: test-ocsp-ecdsa-delegate-invalid-cert-server-does-not-staple-v4.4-python3.10-min-deps @@ -653,7 +653,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "4.4"] @@ -664,7 +664,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "5.0"] @@ -675,7 +675,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "6.0"] @@ -686,7 +686,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "7.0"] @@ -697,7 +697,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] @@ -708,7 +708,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] @@ -719,7 +719,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, rapid] @@ -730,7 +730,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-ecdsa, latest] - name: test-ocsp-ecdsa-soft-fail-v4.4-python3.10-min-deps @@ -740,7 +740,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "4.4"] @@ -751,7 +751,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "5.0"] @@ -762,7 +762,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "6.0"] @@ -773,7 +773,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "7.0"] @@ -784,7 +784,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] @@ -795,7 +795,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] @@ -806,7 +806,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, rapid] @@ -817,7 +817,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-ecdsa, latest] - name: test-ocsp-ecdsa-valid-cert-server-staples-v4.4-python3.10-min-deps @@ -827,7 +827,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: @@ -843,7 +843,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: @@ -858,7 +858,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: @@ -873,7 +873,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: @@ -888,7 +888,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: @@ -903,7 +903,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: @@ -918,7 +918,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: @@ -933,7 +933,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest COVERAGE: "1" tags: @@ -949,7 +949,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: @@ -964,7 +964,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: @@ -979,7 +979,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: @@ -994,7 +994,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: @@ -1009,7 +1009,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: @@ -1024,7 +1024,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: @@ -1039,7 +1039,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: @@ -1054,7 +1054,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: - ocsp @@ -1068,7 +1068,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: @@ -1083,7 +1083,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: @@ -1098,7 +1098,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: @@ -1113,7 +1113,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: @@ -1128,7 +1128,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: @@ -1143,7 +1143,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: @@ -1158,7 +1158,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: @@ -1173,7 +1173,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: - ocsp @@ -1187,7 +1187,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: @@ -1202,7 +1202,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: @@ -1217,7 +1217,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: @@ -1232,7 +1232,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: @@ -1247,7 +1247,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: @@ -1262,7 +1262,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: @@ -1277,7 +1277,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: @@ -1292,7 +1292,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: - ocsp @@ -1306,7 +1306,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "4.4"] @@ -1317,7 +1317,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "5.0"] @@ -1328,7 +1328,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "6.0"] @@ -1339,7 +1339,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "7.0"] @@ -1350,7 +1350,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] @@ -1361,7 +1361,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] @@ -1372,7 +1372,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, rapid] @@ -1383,7 +1383,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-ecdsa, latest] - name: test-ocsp-ecdsa-delegate-malicious-invalid-cert-muststaple-server-does-not-staple-v4.4-python3.10-min-deps @@ -1393,7 +1393,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "4.4"] @@ -1404,7 +1404,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "5.0"] @@ -1415,7 +1415,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "6.0"] @@ -1426,7 +1426,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "7.0"] @@ -1437,7 +1437,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] @@ -1448,7 +1448,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] @@ -1459,7 +1459,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, rapid] @@ -1470,7 +1470,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-ecdsa, latest] - name: test-ocsp-ecdsa-malicious-no-responder-muststaple-server-does-not-staple-v4.4-python3.10-min-deps @@ -1480,7 +1480,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "4.4"] @@ -1491,7 +1491,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "5.0"] @@ -1502,7 +1502,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "6.0"] @@ -1513,7 +1513,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "7.0"] @@ -1524,7 +1524,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] @@ -1535,7 +1535,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] @@ -1546,7 +1546,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, rapid] @@ -1557,7 +1557,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-ecdsa, latest] - name: test-ocsp-rsa-valid-cert-server-does-not-staple-v4.4-python3.10-min-deps @@ -1567,7 +1567,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "4.4"] @@ -1578,7 +1578,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "5.0"] @@ -1589,7 +1589,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "6.0"] @@ -1600,7 +1600,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "7.0"] @@ -1611,7 +1611,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] @@ -1622,7 +1622,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] @@ -1633,7 +1633,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, rapid] @@ -1644,7 +1644,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-rsa, latest] - name: test-ocsp-rsa-invalid-cert-server-does-not-staple-v4.4-python3.10-min-deps @@ -1654,7 +1654,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "4.4"] @@ -1665,7 +1665,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "5.0"] @@ -1676,7 +1676,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "6.0"] @@ -1687,7 +1687,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "7.0"] @@ -1698,7 +1698,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] @@ -1709,7 +1709,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] @@ -1720,7 +1720,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, rapid] @@ -1731,7 +1731,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-rsa, latest] - name: test-ocsp-rsa-delegate-valid-cert-server-does-not-staple-v4.4-python3.10-min-deps @@ -1741,7 +1741,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "4.4"] @@ -1752,7 +1752,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "5.0"] @@ -1763,7 +1763,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "6.0"] @@ -1774,7 +1774,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "7.0"] @@ -1785,7 +1785,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] @@ -1796,7 +1796,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] @@ -1807,7 +1807,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, rapid] @@ -1818,7 +1818,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-rsa, latest] - name: test-ocsp-rsa-delegate-invalid-cert-server-does-not-staple-v4.4-python3.10-min-deps @@ -1828,7 +1828,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "4.4"] @@ -1839,7 +1839,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "5.0"] @@ -1850,7 +1850,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "6.0"] @@ -1861,7 +1861,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "7.0"] @@ -1872,7 +1872,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] @@ -1883,7 +1883,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] @@ -1894,7 +1894,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, rapid] @@ -1905,7 +1905,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-rsa, latest] - name: test-ocsp-rsa-soft-fail-v4.4-python3.10-min-deps @@ -1915,7 +1915,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "4.4"] @@ -1926,7 +1926,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "5.0"] @@ -1937,7 +1937,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "6.0"] @@ -1948,7 +1948,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "7.0"] @@ -1959,7 +1959,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] @@ -1970,7 +1970,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] @@ -1981,7 +1981,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, rapid] @@ -1992,7 +1992,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-rsa, latest] - name: test-ocsp-rsa-valid-cert-server-staples-v4.4-python3.10-min-deps @@ -2002,7 +2002,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: @@ -2017,7 +2017,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: @@ -2032,7 +2032,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: @@ -2047,7 +2047,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: @@ -2062,7 +2062,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: @@ -2077,7 +2077,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: @@ -2092,7 +2092,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: @@ -2107,7 +2107,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: - ocsp @@ -2121,7 +2121,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: @@ -2136,7 +2136,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: @@ -2151,7 +2151,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: @@ -2166,7 +2166,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: @@ -2181,7 +2181,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: @@ -2196,7 +2196,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: @@ -2211,7 +2211,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: @@ -2226,7 +2226,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: - ocsp @@ -2240,7 +2240,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: @@ -2255,7 +2255,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: @@ -2270,7 +2270,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: @@ -2285,7 +2285,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: @@ -2300,7 +2300,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: @@ -2315,7 +2315,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: @@ -2330,7 +2330,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: @@ -2345,7 +2345,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: - ocsp @@ -2359,7 +2359,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: @@ -2374,7 +2374,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: @@ -2389,7 +2389,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: @@ -2404,7 +2404,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: @@ -2419,7 +2419,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: @@ -2434,7 +2434,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: @@ -2449,7 +2449,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: @@ -2464,7 +2464,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: - ocsp @@ -2478,7 +2478,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "4.4"] @@ -2489,7 +2489,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "5.0"] @@ -2500,7 +2500,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "6.0"] @@ -2511,7 +2511,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "7.0"] @@ -2522,7 +2522,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] @@ -2533,7 +2533,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] @@ -2544,7 +2544,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, rapid] @@ -2555,7 +2555,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-rsa, latest] - name: test-ocsp-rsa-delegate-malicious-invalid-cert-muststaple-server-does-not-staple-v4.4-python3.10-min-deps @@ -2565,7 +2565,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "4.4"] @@ -2576,7 +2576,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "5.0"] @@ -2587,7 +2587,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "6.0"] @@ -2598,7 +2598,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "7.0"] @@ -2609,7 +2609,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] @@ -2620,7 +2620,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] @@ -2631,7 +2631,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, rapid] @@ -2642,7 +2642,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-rsa, latest] - name: test-ocsp-rsa-malicious-no-responder-muststaple-server-does-not-staple-v4.4-python3.10-min-deps @@ -2652,7 +2652,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "4.4"] @@ -2663,7 +2663,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "5.0"] @@ -2674,7 +2674,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "6.0"] @@ -2685,7 +2685,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "7.0"] @@ -2696,7 +2696,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] @@ -2707,7 +2707,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] @@ -2718,7 +2718,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, rapid] @@ -2729,7 +2729,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-rsa, latest] @@ -2784,10 +2784,14 @@ tasks: vars: VERSION: v8.0-perf SSL: ssl + UV_PYTHON: 3.10.11 + UV_PYTHON_PREFERENCE: only-managed - func: run tests vars: TEST_NAME: perf SUB_TEST_NAME: sync + UV_PYTHON: 3.10.11 + UV_PYTHON_PREFERENCE: only-managed - func: attach benchmark test results - func: send dashboard data tags: [perf] @@ -2797,10 +2801,14 @@ tasks: vars: VERSION: v8.0-perf SSL: ssl + UV_PYTHON: 3.10.11 + UV_PYTHON_PREFERENCE: only-managed - func: run tests vars: TEST_NAME: perf SUB_TEST_NAME: async + UV_PYTHON: 3.10.11 + UV_PYTHON_PREFERENCE: only-managed - func: attach benchmark test results - func: send dashboard data tags: [perf] @@ -2810,10 +2818,14 @@ tasks: vars: VERSION: v8.0-perf SSL: nossl + UV_PYTHON: 3.10.11 + UV_PYTHON_PREFERENCE: only-managed - func: run tests vars: TEST_NAME: perf SUB_TEST_NAME: sync + UV_PYTHON: 3.10.11 + UV_PYTHON_PREFERENCE: only-managed - func: attach benchmark test results - func: send dashboard data tags: [perf] @@ -2823,10 +2835,14 @@ tasks: vars: VERSION: v8.0-perf SSL: nossl + UV_PYTHON: 3.10.11 + UV_PYTHON_PREFERENCE: only-managed - func: run tests vars: TEST_NAME: perf SUB_TEST_NAME: async + UV_PYTHON: 3.10.11 + UV_PYTHON_PREFERENCE: only-managed - func: attach benchmark test results - func: send dashboard data tags: [perf] @@ -2841,7 +2857,7 @@ tasks: - func: run tests vars: TEST_NAME: search_index - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" tags: [search_index] # Server version tests @@ -2857,7 +2873,7 @@ tasks: AUTH: auth SSL: nossl TOPOLOGY: replica_set - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_async tags: - server-version @@ -2876,7 +2892,7 @@ tasks: AUTH: auth SSL: nossl TOPOLOGY: replica_set - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_sync tags: - server-version @@ -2897,7 +2913,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_async tags: - server-version @@ -2916,7 +2932,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: replica_set - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_sync tags: - server-version @@ -2937,7 +2953,7 @@ tasks: SSL: nossl TOPOLOGY: replica_set COVERAGE: "1" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_async tags: - server-version @@ -2960,7 +2976,7 @@ tasks: SSL: nossl TOPOLOGY: replica_set COVERAGE: "1" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_sync tags: - server-version @@ -2980,7 +2996,7 @@ tasks: AUTH: noauth SSL: ssl TOPOLOGY: replica_set - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_async tags: - server-version @@ -3000,7 +3016,7 @@ tasks: AUTH: noauth SSL: ssl TOPOLOGY: replica_set - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_sync tags: - server-version @@ -3019,7 +3035,7 @@ tasks: AUTH: auth SSL: nossl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_async tags: - server-version @@ -3040,7 +3056,7 @@ tasks: SSL: nossl TOPOLOGY: sharded_cluster TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_sync tags: - server-version @@ -3063,7 +3079,7 @@ tasks: TOPOLOGY: sharded_cluster TEST_MIN_DEPS: "1" COVERAGE: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_async tags: - server-version @@ -3083,7 +3099,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_async tags: - server-version @@ -3102,7 +3118,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_async tags: - server-version @@ -3121,7 +3137,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_async tags: - server-version @@ -3140,7 +3156,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_async tags: - server-version @@ -3159,7 +3175,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_async tags: - server-version @@ -3179,7 +3195,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_async tags: - server-version @@ -3198,7 +3214,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_async tags: - server-version @@ -3218,7 +3234,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_async tags: - server-version @@ -3241,7 +3257,7 @@ tasks: TOPOLOGY: sharded_cluster TEST_MIN_DEPS: "1" COVERAGE: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_sync tags: - server-version @@ -3261,7 +3277,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_sync tags: - server-version @@ -3280,7 +3296,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_sync tags: - server-version @@ -3299,7 +3315,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_sync tags: - server-version @@ -3318,7 +3334,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_sync tags: - server-version @@ -3337,7 +3353,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_sync tags: - server-version @@ -3357,7 +3373,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_sync tags: - server-version @@ -3376,7 +3392,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_sync tags: - server-version @@ -3396,7 +3412,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_sync tags: - server-version @@ -3415,7 +3431,7 @@ tasks: AUTH: noauth SSL: nossl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_async tags: - server-version @@ -3434,7 +3450,7 @@ tasks: AUTH: noauth SSL: nossl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_sync tags: - server-version @@ -3454,7 +3470,7 @@ tasks: AUTH: noauth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_async tags: - server-version @@ -3473,7 +3489,7 @@ tasks: AUTH: noauth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_sync tags: - server-version @@ -3492,7 +3508,7 @@ tasks: AUTH: auth SSL: nossl TOPOLOGY: standalone - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_async tags: - server-version @@ -3511,7 +3527,7 @@ tasks: AUTH: auth SSL: nossl TOPOLOGY: standalone - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_sync tags: - server-version @@ -3530,7 +3546,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: standalone - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_async tags: - server-version @@ -3551,7 +3567,7 @@ tasks: SSL: ssl TOPOLOGY: standalone TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_sync tags: - server-version @@ -3572,7 +3588,7 @@ tasks: SSL: nossl TOPOLOGY: standalone COVERAGE: "1" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_async tags: - server-version @@ -3594,7 +3610,7 @@ tasks: SSL: nossl TOPOLOGY: standalone COVERAGE: "1" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_sync tags: - server-version @@ -3615,7 +3631,7 @@ tasks: AUTH: noauth SSL: ssl TOPOLOGY: standalone - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_async tags: - server-version @@ -3634,7 +3650,7 @@ tasks: AUTH: noauth SSL: ssl TOPOLOGY: standalone - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_sync tags: - server-version @@ -3658,7 +3674,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "4.4" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_sync tags: - test-standard @@ -3680,7 +3696,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "4.4" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_sync tags: - test-standard @@ -3702,7 +3718,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "4.4" - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_sync tags: - test-standard @@ -3724,7 +3740,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "4.4" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_sync tags: - test-standard @@ -3749,7 +3765,7 @@ tasks: TOPOLOGY: standalone VERSION: "4.4" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_sync tags: - test-standard @@ -3771,7 +3787,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "4.4" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_sync tags: - test-standard @@ -3794,7 +3810,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "4.4" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_sync tags: - test-standard @@ -3817,7 +3833,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "5.0" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_async tags: - test-standard @@ -3839,7 +3855,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "5.0" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_async tags: - test-standard @@ -3861,7 +3877,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "5.0" - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_async tags: - test-standard @@ -3883,7 +3899,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "5.0" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_async tags: - test-standard @@ -3908,7 +3924,7 @@ tasks: TOPOLOGY: standalone VERSION: "5.0" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_async tags: - test-standard @@ -3930,7 +3946,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "5.0" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_async tags: - test-standard @@ -3953,7 +3969,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "5.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_async tags: - test-standard @@ -3978,7 +3994,7 @@ tasks: TOPOLOGY: replica_set VERSION: "6.0" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_sync tags: - test-standard @@ -4000,7 +4016,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "6.0" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_sync tags: - test-standard @@ -4023,7 +4039,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "6.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_sync tags: - test-standard @@ -4046,7 +4062,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "6.0" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_sync tags: - test-standard @@ -4068,7 +4084,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "6.0" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_sync tags: - test-standard @@ -4090,7 +4106,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "6.0" - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_sync tags: - test-standard @@ -4112,7 +4128,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "6.0" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_sync tags: - test-standard @@ -4136,7 +4152,7 @@ tasks: TOPOLOGY: replica_set VERSION: "7.0" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_async tags: - test-standard @@ -4158,7 +4174,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "7.0" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_async tags: - test-standard @@ -4181,7 +4197,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "7.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_async tags: - test-standard @@ -4204,7 +4220,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "7.0" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_async tags: - test-standard @@ -4226,7 +4242,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "7.0" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_async tags: - test-standard @@ -4248,7 +4264,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "7.0" - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_async tags: - test-standard @@ -4270,7 +4286,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "7.0" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_async tags: - test-standard @@ -4292,7 +4308,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "8.0" - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_sync tags: - test-standard @@ -4314,7 +4330,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "8.0" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_sync tags: - test-standard @@ -4338,7 +4354,7 @@ tasks: TOPOLOGY: sharded_cluster VERSION: "8.0" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_sync tags: - test-standard @@ -4360,7 +4376,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "8.0" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_sync tags: - test-standard @@ -4383,7 +4399,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "8.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_sync tags: - test-standard @@ -4406,7 +4422,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "8.0" - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_sync tags: - test-standard @@ -4428,7 +4444,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "8.0" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_sync tags: - test-standard @@ -4451,7 +4467,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "9.0" - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_async tags: - test-standard @@ -4473,7 +4489,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "9.0" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_async tags: - test-standard @@ -4497,7 +4513,7 @@ tasks: TOPOLOGY: sharded_cluster VERSION: "9.0" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_async tags: - test-standard @@ -4519,7 +4535,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "9.0" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_async tags: - test-standard @@ -4542,7 +4558,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "9.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_async tags: - test-standard @@ -4565,7 +4581,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "9.0" - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_async tags: - test-standard @@ -4587,7 +4603,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "9.0" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_async tags: - test-standard @@ -4612,7 +4628,7 @@ tasks: TOPOLOGY: replica_set VERSION: latest COVERAGE: "1" - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_async tags: - test-standard @@ -4637,7 +4653,7 @@ tasks: TOPOLOGY: replica_set VERSION: latest COVERAGE: "1" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_async tags: - test-standard @@ -4663,7 +4679,7 @@ tasks: TOPOLOGY: sharded_cluster VERSION: latest COVERAGE: "1" - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_async tags: - test-standard @@ -4688,7 +4704,7 @@ tasks: TOPOLOGY: sharded_cluster VERSION: latest COVERAGE: "1" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_async tags: - test-standard @@ -4713,7 +4729,7 @@ tasks: TOPOLOGY: standalone VERSION: latest COVERAGE: "1" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_async tags: - test-standard @@ -4738,7 +4754,7 @@ tasks: TOPOLOGY: standalone VERSION: latest COVERAGE: "1" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_async tags: - test-standard @@ -4761,7 +4777,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: rapid - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_sync tags: - test-standard @@ -4783,7 +4799,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: rapid - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_sync tags: - test-standard @@ -4806,7 +4822,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: rapid - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_sync tags: - test-standard @@ -4828,7 +4844,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: rapid - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_sync tags: - test-standard @@ -4850,7 +4866,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: rapid - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_sync tags: - test-standard @@ -4872,7 +4888,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: rapid - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_sync tags: - test-standard @@ -4896,7 +4912,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "8.2" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" tags: - test-string-query-preview - server-8.2 @@ -4919,7 +4935,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "4.4" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" tags: - test-non-standard - server-4.4 @@ -4940,7 +4956,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "4.4" - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" tags: - test-non-standard - server-4.4 @@ -4963,7 +4979,7 @@ tasks: TOPOLOGY: standalone VERSION: "4.4" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" tags: - test-non-standard - server-4.4 @@ -4984,7 +5000,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "4.4" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-non-standard - server-4.4 @@ -5006,7 +5022,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "5.0" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t tags: - test-non-standard - server-5.0 @@ -5028,7 +5044,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "5.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-non-standard - server-5.0 @@ -5050,7 +5066,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "5.0" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" tags: - test-non-standard - server-5.0 @@ -5071,7 +5087,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "5.0" - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" tags: - test-non-standard - server-5.0 @@ -5092,7 +5108,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "6.0" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" tags: - test-non-standard - server-6.0 @@ -5115,7 +5131,7 @@ tasks: TOPOLOGY: sharded_cluster VERSION: "6.0" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" tags: - test-non-standard - server-6.0 @@ -5136,7 +5152,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "6.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-non-standard - server-6.0 @@ -5158,7 +5174,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "6.0" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t tags: - test-non-standard - server-6.0 @@ -5180,7 +5196,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "7.0" - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" tags: - test-non-standard - server-7.0 @@ -5201,7 +5217,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "7.0" - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" tags: - test-non-standard - server-7.0 @@ -5222,7 +5238,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "7.0" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" tags: - test-non-standard - server-7.0 @@ -5243,7 +5259,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "7.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-non-standard - server-7.0 @@ -5265,7 +5281,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "8.0" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" tags: - test-non-standard - server-8.0 @@ -5286,7 +5302,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "8.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-non-standard - server-8.0 @@ -5308,7 +5324,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "8.0" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t tags: - test-non-standard - server-8.0 @@ -5330,7 +5346,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "8.0" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t tags: - test-non-standard - server-8.0 @@ -5354,7 +5370,7 @@ tasks: TOPOLOGY: replica_set VERSION: "9.0" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" tags: - test-non-standard - server-9.0 @@ -5375,7 +5391,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "9.0" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" tags: - test-non-standard - server-9.0 @@ -5396,7 +5412,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "9.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-non-standard - server-9.0 @@ -5418,7 +5434,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "9.0" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" tags: - test-non-standard - server-9.0 @@ -5439,7 +5455,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: latest - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t tags: - test-non-standard - server-latest @@ -5461,7 +5477,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: latest - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-non-standard - server-latest @@ -5485,7 +5501,7 @@ tasks: TOPOLOGY: sharded_cluster VERSION: latest COVERAGE: "1" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" tags: - test-non-standard - server-latest @@ -5508,7 +5524,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" tags: - test-non-standard-no-cov - server-latest @@ -5530,7 +5546,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: latest - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" tags: - test-non-standard - server-latest @@ -5551,7 +5567,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: rapid - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" tags: - test-non-standard - server-rapid @@ -5572,7 +5588,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: rapid - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t tags: - test-non-standard - server-rapid @@ -5594,7 +5610,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: rapid - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" tags: - test-non-standard - server-rapid @@ -5615,7 +5631,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: rapid - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-non-standard - server-rapid @@ -5629,7 +5645,7 @@ tasks: commands: - func: test numpy vars: - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" tags: - binary - vector @@ -5639,7 +5655,7 @@ tasks: commands: - func: test numpy vars: - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" COVERAGE: "1" tags: - binary @@ -5665,7 +5681,7 @@ tasks: TOPOLOGY: sharded_cluster VERSION: "4.4" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" tags: - test-standard-auth - server-4.4 @@ -5686,7 +5702,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "4.4" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-standard-auth - server-4.4 @@ -5708,7 +5724,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "5.0" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" tags: - test-standard-auth - server-5.0 @@ -5729,7 +5745,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "5.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-standard-auth - server-5.0 @@ -5751,7 +5767,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "6.0" - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" tags: - test-standard-auth - server-6.0 @@ -5772,7 +5788,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "6.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-standard-auth - server-6.0 @@ -5794,7 +5810,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "7.0" - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" tags: - test-standard-auth - server-7.0 @@ -5815,7 +5831,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "7.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-standard-auth - server-7.0 @@ -5837,7 +5853,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "8.0" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t tags: - test-standard-auth - server-8.0 @@ -5859,7 +5875,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "8.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-standard-auth - server-8.0 @@ -5881,7 +5897,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "9.0" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" tags: - test-standard-auth - server-9.0 @@ -5902,7 +5918,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "9.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-standard-auth - server-9.0 @@ -5926,7 +5942,7 @@ tasks: TOPOLOGY: sharded_cluster VERSION: latest COVERAGE: "1" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" tags: - test-standard-auth - server-latest @@ -5948,7 +5964,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-standard-auth - server-latest @@ -5970,7 +5986,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: rapid - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t tags: - test-standard-auth - server-rapid @@ -5992,7 +6008,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: rapid - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-standard-auth - server-rapid diff --git a/.evergreen/resync-specs.sh b/.evergreen/resync-specs.sh index 4feca0aa5b..840d05bf1e 100755 --- a/.evergreen/resync-specs.sh +++ b/.evergreen/resync-specs.sh @@ -85,28 +85,59 @@ cp_psl () { echo "Could not find the public suffix list at $src" >&2 return 1 fi - python - "$src" "$PYMONGO"/pymongo/public_suffix_list.dat <<'EOF' + # Use the newest mongodbtoolchain python on CI hosts; python3 fallback locally. + PY=$(ls -1d /opt/mongodbtoolchain/v*/bin/python3 2>/dev/null | sort -V | tail -1) + [ -n "$PY" ] || PY=python3 + [ "$("$PY" -c 'import sys; print(sys.version_info >= (3, 7))' 2>/dev/null)" = True ] \ + || { echo "Error: $PY is not Python 3.7+, which this conversion requires." >&2; return 1; } + "$PY" - "$src" "$PYMONGO"/pymongo/public_suffix_list.dat <<'EOF' +import os import sys src, dst = sys.argv[1], sys.argv[2] -with open(src, encoding="utf-8") as f, open(dst, "w", encoding="utf-8") as out: - for line in f: - rule = line.strip() - if rule and not rule.startswith("//") and not rule.isascii(): - # "!" and "*." prefixes are not valid IDNA, so encode only the - # domain part and re-attach the prefix afterwards. - prefix = "" - for p in ("!", "*."): - if rule.startswith(p): - prefix, rule = p, rule[len(p) :] - break - try: - rule = rule.encode("idna").decode("ascii") - except UnicodeError: - print(f"Could not convert rule to Punycode: {prefix}{rule}", file=sys.stderr) - sys.exit(1) - line = f"{prefix}{rule.lower()}\n" - out.write(line) + +# Convert in memory and write atomically so failures can't truncate the list. +with open(src, encoding="utf-8") as f: + lines = f.readlines() +if not lines: + print(f"Source public suffix list is empty: {src}", file=sys.stderr) + sys.exit(1) + +converted = [] +for line in lines: + rule = line.strip() + if rule and not rule.startswith("//") and not rule.isascii(): + # "!" and "*." prefixes are not valid IDNA, so encode only the + # domain part and re-attach the prefix afterwards. + prefix = "" + for p in ("!", "*."): + if rule.startswith(p): + prefix, rule = p, rule[len(p) :] + break + try: + rule = rule.encode("idna").decode("ascii") + except UnicodeError: + print(f"Could not convert rule to Punycode: {prefix}{rule}", file=sys.stderr) + sys.exit(1) + line = f"{prefix}{rule.lower()}\n" + converted.append(line) + +if len(converted) != len(lines): + print( + f"Conversion lost lines: {len(lines)} in, {len(converted)} out", + file=sys.stderr, + ) + sys.exit(1) + +# A blank or comment-only source must not replace the real list. +if not any(ln.strip() and not ln.strip().startswith("//") for ln in converted): + print(f"Source public suffix list has no rules: {src}", file=sys.stderr) + sys.exit(1) + +tmp = dst + ".tmp" +with open(tmp, "w", encoding="utf-8") as out: + out.writelines(converted) +os.replace(tmp, dst) EOF } diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 4ec527c941..aa1fe97610 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -26,10 +26,13 @@ else exit 1 fi +# No-op on Evergreen (just.sh already ran this); needed for direct/local runs (PYTHON-6135). +. $SCRIPT_DIR/scripts/setup-uv-python.sh + # Start the test runner. -echo "Running tests with UV_PYTHON=${UV_PYTHON:-}..." +echo "Running tests with UV_PYTHON=${UV_PYTHON:-} UV_PYTHON_SEARCH_PATH=${UV_PYTHON_SEARCH_PATH:-}..." echo "UV_ARGS=${UV_ARGS}" uv run ${UV_ARGS} --reinstall-package pymongo .evergreen/scripts/run_tests.py "$@" -echo "Running tests with UV_PYTHON=${UV_PYTHON:-}... done." +echo "Running tests with UV_PYTHON=${UV_PYTHON:-} UV_PYTHON_SEARCH_PATH=${UV_PYTHON_SEARCH_PATH:-}... done." cd $PREV_DIR diff --git a/.evergreen/scripts/cleanup.sh b/.evergreen/scripts/cleanup.sh index f04a936fd2..44b36d3569 100755 --- a/.evergreen/scripts/cleanup.sh +++ b/.evergreen/scripts/cleanup.sh @@ -10,5 +10,10 @@ if [ -f $HERE/env.sh ]; then source $HERE/env.sh fi -rm -rf "${DRIVERS_TOOLS}" || true +# Don't delete the in-tree submodule; clean the ignored credential and state +# files (secrets-export.sh, AWS creds, token files) that `git submodule update` +# leaves behind, so they can't carry into later tasks on a reused host. A +# caller-provided DRIVERS_TOOLS wins, so the checkout actually used is cleaned. +: "${DRIVERS_TOOLS:=$HERE/../../drivers-evergreen-tools}" rm -f $HERE/../../secrets-export.sh || true +git -C "$DRIVERS_TOOLS" clean -fdx 2>/dev/null || true diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index fec3cdadae..5e36125838 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -12,7 +12,8 @@ else fi PROJECT_DIRECTORY="$(pwd)" -DRIVERS_TOOLS="$(dirname $PROJECT_DIRECTORY)/drivers-tools" +# Default to the submodule; an env var override wins. +DRIVERS_TOOLS="${DRIVERS_TOOLS:-$PROJECT_DIRECTORY/drivers-evergreen-tools}" CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo} DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS/.bin" MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" @@ -93,12 +94,40 @@ export PROJECT="${project:-mongo-python-driver}" export PIP_QUIET=1 EOT -# Write the .env file for drivers-tools. -rm -rf $DRIVERS_TOOLS -BRANCH=master -ORG=mongodb-labs -git clone --branch $BRANCH https://github.com/$ORG/drivers-evergreen-tools.git $DRIVERS_TOOLS +# Only touch the in-tree submodule when it is the checkout actually in use; +# an overridden DRIVERS_TOOLS is a checkout we do not own. +if [ "$DRIVERS_TOOLS" = "$PROJECT_DIRECTORY/drivers-evergreen-tools" ]; then + # Initialize the submodule (Evergreen's git.get_project does not); tolerate + # non-git hosts with a warning. + if ! git -C "$PROJECT_DIRECTORY" submodule update --init --recursive; then + echo "WARNING: could not initialize the drivers-evergreen-tools submodule;" \ + "using the existing checkout contents instead." + fi + # Write a uv config boundary into the submodule: it is vendored inside this + # project, so uv run by the tools' own scripts would otherwise walk up to + # pyproject.toml and enforce our required-version pin. Write-if-absent, so an + # upstream uv.toml fails the submodule update loudly instead of being clobbered. + if [ -d "${DRIVERS_TOOLS}" ] && [ ! -f "${DRIVERS_TOOLS}/uv.toml" ]; then + cat < "${DRIVERS_TOOLS}/uv.toml" +# Written by mongo-python-driver to stop uv's config discovery here; see +# .evergreen/scripts/configure-env.sh. +EOT + fi + + # Keep the boundary out of git status via the submodule's local exclude; + # no-op without git. + if _git_dir=$(git -C "${DRIVERS_TOOLS}" rev-parse --absolute-git-dir 2>/dev/null); then + mkdir -p "${_git_dir}/info" + grep -qxF "uv.toml" "${_git_dir}/info/exclude" 2>/dev/null || + printf "uv.toml\n" >> "${_git_dir}/info/exclude" + fi +fi + +# Write the .env file for drivers-tools. Create the checkout if it is missing so +# a failed submodule init does not stop this script before setup-tests.py can +# reach check_drivers_tools() and report the problem actionably. +mkdir -p "${DRIVERS_TOOLS}" cat < ${DRIVERS_TOOLS}/.env SKIP_LEGACY_SHELL=1 DRIVERS_TOOLS="$DRIVERS_TOOLS" diff --git a/.evergreen/scripts/create-spec-pr.sh b/.evergreen/scripts/create-spec-pr.sh index 1de1a7cae1..daa68d0db6 100755 --- a/.evergreen/scripts/create-spec-pr.sh +++ b/.evergreen/scripts/create-spec-pr.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash +set -eu -tools="$(realpath -s "../drivers-tools")" +# Default to the drivers-evergreen-tools submodule when unset; a +# caller-provided DRIVERS_TOOLS wins, as in the other consumers. +tools="$(realpath -s "${DRIVERS_TOOLS:-./drivers-evergreen-tools}")" pushd $tools/.evergreen/github_app || exit owner="mongodb" @@ -31,20 +34,87 @@ echo "Creating the git checkout..." branch="spec-resync-"$(date '+%m-%d-%Y') git remote set-url origin https://x-access-token:${token}@github.com/$owner/$repo.git -git checkout -b $branch "origin/main" +# -B resets the branch if a reused checkout already has it, keeping reruns idempotent. +git checkout -B "$branch" "origin/main" + +# Attribute the commit to the bot user; fall back to the host identity. +bot_login="mongodb-drivers-pr-bot[bot]" +bot_id=$(curl -gsm 10 "https://api.github.com/users/${bot_login}" | jq -r '.id // empty' || true) +if [ -n "$bot_id" ]; then + git config user.name "$bot_login" + git config user.email "${bot_id}+${bot_login}@users.noreply.github.com" +fi + git add ./test git commit -am "resyncing specs $(date '+%m-%d-%Y')" echo "Creating the git checkout... done." -git push origin $branch +# Date-stamped branch: force-push makes same-day reruns idempotent; retry +# absorbs GitHub's intermittent 403s on the receive-pack endpoint. +push_ok=false +for _attempt in 1 2 3; do + if git push --force origin $branch; then + push_ok=true + break + fi + if [ "$_attempt" -lt 3 ]; then + echo "Push attempt ${_attempt} failed; retrying in 30s..." + sleep 30 + fi +done +if [ "$push_ok" != true ]; then + echo "Failed to push $branch after 3 attempts!" >&2 + exit 1 +fi + +# Build the payload as a file so the body content is always properly +# JSON-escaped, rather than interpolated into the command line as text. +payload_file=$(mktemp) +trap 'rm -f "$payload_file"' EXIT +jq -n \ + --arg title "[Spec Resync] $(date '+%m-%d-%Y')" \ + --arg head "${branch}" \ + --rawfile body "$1" \ + '{title: $title, body: $body, head: $head, base: "main"}' > "$payload_file" + +# Reuse an open PR for this branch (same-day rerun) instead of duplicating it. +existing_pr_json=$(curl -gs \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $token" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + --url "https://api.github.com/repos/$owner/$repo/pulls?head=$owner:$branch&state=open") +existing_pr_url=$(echo "$existing_pr_json" | jq -r '.[0].html_url // empty') +if [ -n "$existing_pr_url" ]; then + existing_pr_number=$(echo "$existing_pr_json" | jq -r '.[0].number // empty') + if ! curl -sfX PATCH \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $token" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -d "{\"title\": \"[Spec Resync] $(date '+%m-%d-%Y')\", \"body\": $(jq -Rs . < "$1")}" \ + --url "https://api.github.com/repos/$owner/$repo/pulls/${existing_pr_number}" > /dev/null + then + echo "Failed to update PR summary!" >&2 + exit 1 + fi + echo "$existing_pr_url" + echo "Creating the PR... done. (PR already existed; branch and summary were updated)" + rm -rf $tools + exit 0 +fi + resp=$(curl -L \ -X POST \ -H "Accept: application/vnd.github+json" \ -H "Authorization: Bearer $token" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - -d "{\"title\":\"[Spec Resync] $(date '+%m-%d-%Y')\",\"body\":\"$(cat "$1")\",\"head\":\"${branch}\",\"base\":\"main\"}" \ + -d "@${payload_file}" \ --url https://api.github.com/repos/$owner/$repo/pulls) -echo $resp | jq '.html_url' -echo "Creating the PR... done." -rm -rf $tools +pr_url=$(echo "$resp" | jq -r '.html_url // empty') +if [ -z "$pr_url" ]; then + echo "Failed to create PR! API response:" + echo "$resp" | jq . + exit 1 +fi +echo "$pr_url" +echo "Creating the PR... done." diff --git a/.evergreen/scripts/dispatch.sh b/.evergreen/scripts/dispatch.sh new file mode 100755 index 0000000000..372cf6535c --- /dev/null +++ b/.evergreen/scripts/dispatch.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Route the just test recipes to the mod_wsgi tests or the Evergreen test +# runner. The mod_wsgi setup writes TEST_NAME to the test env file. +set -eu + +SCRIPT_DIR=$(dirname "${BASH_SOURCE:-$0}") +RECIPE="${1:-}" +if [ "$#" -gt 0 ]; then + shift +fi + +mod_wsgi_env() { + [ -f "$SCRIPT_DIR/test-env.sh" ] \ + && grep -q 'TEST_NAME="mod_wsgi"' "$SCRIPT_DIR/test-env.sh" \ + && bash "$SCRIPT_DIR/mod_wsgi.sh" active +} + +case "$RECIPE" in + setup-tests) + if [ "${1:-}" = "mod_wsgi" ]; then + shift + exec bash "$SCRIPT_DIR/mod_wsgi.sh" setup "${1:-}" + fi + exec bash "$SCRIPT_DIR/setup-tests.sh" "$@" + ;; + run-tests) + if mod_wsgi_env; then + if [ "$#" -gt 0 ]; then + echo "Ignoring arguments for the mod_wsgi tests: $*" >&2 + fi + exec bash "$SCRIPT_DIR/mod_wsgi.sh" test + fi + exec bash "$(dirname "$SCRIPT_DIR")/run-tests.sh" "$@" + ;; + teardown-tests) + if [ -f "$SCRIPT_DIR/test-env.sh" ] \ + && grep -q 'TEST_NAME="mod_wsgi"' "$SCRIPT_DIR/test-env.sh"; then + exec bash "$SCRIPT_DIR/mod_wsgi.sh" teardown + fi + exec bash "$SCRIPT_DIR/teardown-tests.sh" "$@" + ;; + *) + echo "Unknown recipe: $RECIPE" + exit 1 + ;; +esac diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index a3979472e4..2ef5a34b9e 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -43,6 +43,7 @@ git_get_project, ) from shrub.v3.evg_task import EvgTask, EvgTaskDependency, EvgTaskRef +from utils import PERF_PYTHON_VERSION ############## # Variants @@ -412,7 +413,7 @@ def create_test_numpy_tasks(): tasks = [] for python in MIN_MAX_PYTHON: tags = ["binary", "vector", f"python-{python}", "test-numpy"] - vars = dict(TOOLCHAIN_VERSION=python) + vars = dict(UV_PYTHON=python) if python == MIN_MAX_PYTHON[-1]: tags.append("pr") vars["COVERAGE"] = "1" @@ -707,7 +708,7 @@ def create_server_version_tasks(): ) server_func = FunctionCall(func="run server", vars=expansions) test_vars = expansions.copy() - test_vars["TOOLCHAIN_VERSION"] = python + test_vars["UV_PYTHON"] = python test_vars["TEST_NAME"] = f"default_{sync}" test_func = FunctionCall(func="run tests", vars=test_vars) tasks.append(EvgTask(name=name, tags=tags, commands=[server_func, test_func])) @@ -769,7 +770,7 @@ def create_test_non_standard_tasks(): name = get_task_name("test-non-standard", python=python, **expansions) server_func = FunctionCall(func="run server", vars=expansions) test_vars = expansions.copy() - test_vars["TOOLCHAIN_VERSION"] = python + test_vars["UV_PYTHON"] = python test_func = FunctionCall(func="run tests", vars=test_vars) tasks.append(EvgTask(name=name, tags=tags, commands=[server_func, test_func])) # For each coverage task, also emit a non-coverage companion so that @@ -787,7 +788,7 @@ def create_test_non_standard_tasks(): nc_name = get_task_name("test-non-standard", python=python, **nc_expansions) nc_server_func = FunctionCall(func="run server", vars=nc_expansions) nc_test_vars = nc_expansions.copy() - nc_test_vars["TOOLCHAIN_VERSION"] = python + nc_test_vars["UV_PYTHON"] = python nc_test_func = FunctionCall(func="run tests", vars=nc_test_vars) tasks.append( EvgTask(name=nc_name, tags=nc_tags, commands=[nc_server_func, nc_test_func]) @@ -818,7 +819,7 @@ def create_string_query_preview_tasks(): name = get_task_name("test-string-query-preview", python=python, **expansions) server_func = FunctionCall(func="run server", vars=expansions) test_vars = expansions.copy() - test_vars["TOOLCHAIN_VERSION"] = python + test_vars["UV_PYTHON"] = python test_func = FunctionCall(func="run tests", vars=test_vars) return [EvgTask(name=name, tags=tags, commands=[server_func, test_func])] @@ -861,7 +862,7 @@ def create_test_standard_auth_tasks(): name = get_task_name("test-standard-auth", python=python, **expansions) server_func = FunctionCall(func="run server", vars=expansions) test_vars = expansions.copy() - test_vars["TOOLCHAIN_VERSION"] = python + test_vars["UV_PYTHON"] = python test_func = FunctionCall(func="run tests", vars=test_vars) tasks.append(EvgTask(name=name, tags=tags, commands=[server_func, test_func])) return tasks @@ -901,7 +902,7 @@ def create_standard_tasks(): name = get_task_name("test-standard", python=python, sync=sync, **expansions) server_func = FunctionCall(func="run server", vars=expansions) test_vars = expansions.copy() - test_vars["TOOLCHAIN_VERSION"] = python + test_vars["UV_PYTHON"] = python test_vars["TEST_NAME"] = f"default_{sync}" test_func = FunctionCall(func="run tests", vars=test_vars) tasks.append(EvgTask(name=name, tags=tags, commands=[server_func, test_func])) @@ -916,7 +917,7 @@ def create_no_orchestration_tasks(): f"python-{python}", ] assume_func = FunctionCall(func="assume ec2 role") - test_vars = dict(TOOLCHAIN_VERSION=python) + test_vars = dict(UV_PYTHON=python) if python == ALL_PYTHONS[0]: test_vars["TEST_MIN_DEPS"] = "1" name = get_task_name("test-no-orchestration", **test_vars) @@ -966,7 +967,7 @@ def create_aws_tasks(): tags = [*base_tags, f"auth-aws-{test_type}"] if "t" in python: tags.append("free-threaded") - test_vars = dict(TEST_NAME="auth_aws", SUB_TEST_NAME=test_type, TOOLCHAIN_VERSION=python) + test_vars = dict(TEST_NAME="auth_aws", SUB_TEST_NAME=test_type, UV_PYTHON=python) if python == MIN_MAX_PYTHON[0]: test_vars["TEST_MIN_DEPS"] = "1" elif python == MIN_MAX_PYTHON[-1]: @@ -984,7 +985,7 @@ def create_aws_tasks(): TEST_NAME="auth_aws", SUB_TEST_NAME="web-identity", AWS_ROLE_SESSION_NAME="test", - TOOLCHAIN_VERSION=python, + UV_PYTHON=python, ) if "t" in python: tags.append("free-threaded") @@ -1033,7 +1034,7 @@ def _create_ocsp_tasks(algo, variant, server_type, base_task_name): ORCHESTRATION_FILE=file_name, OCSP_SERVER_TYPE=server_type, TEST_NAME="ocsp", - TOOLCHAIN_VERSION=python, + UV_PYTHON=python, VERSION=version, ) if python == ALL_PYTHONS[0]: @@ -1096,7 +1097,7 @@ def create_aws_lambda_tasks(): def create_search_index_tasks(): assume_func = FunctionCall(func="assume ec2 role") server_func = FunctionCall(func="run server", vars=dict(TEST_NAME="search_index")) - vars = dict(TEST_NAME="search_index", TOOLCHAIN_VERSION=CPYTHONS[0]) + vars = dict(TEST_NAME="search_index", UV_PYTHON=CPYTHONS[0]) test_func = FunctionCall(func="run tests", vars=vars) task_name = "test-search-index-helpers" tags = ["search_index"] @@ -1107,9 +1108,12 @@ def create_search_index_tasks(): def create_perf_tasks(): tasks = [] for version, ssl, sync in product(["8.0"], ["ssl", "nossl"], ["sync", "async"]): - vars = dict(VERSION=f"v{version}-perf", SSL=ssl) + # The toolchain build may be this same patch version but is not + # optimized, which regressed CPU-bound benchmarks (PYTHON-6135). + perf_vars = dict(UV_PYTHON=PERF_PYTHON_VERSION, UV_PYTHON_PREFERENCE="only-managed") + vars = dict(VERSION=f"v{version}-perf", SSL=ssl, **perf_vars) server_func = FunctionCall(func="run server", vars=vars) - vars = dict(TEST_NAME="perf", SUB_TEST_NAME=sync) + vars = dict(TEST_NAME="perf", SUB_TEST_NAME=sync, **perf_vars) test_func = FunctionCall(func="run tests", vars=vars) attach_func = FunctionCall(func="attach benchmark test results") send_func = FunctionCall(func="send dashboard data") @@ -1328,7 +1332,7 @@ def create_run_server_func(): "SSL", "ORCHESTRATION_FILE", "UV_PYTHON", - "TOOLCHAIN_VERSION", + "UV_PYTHON_PREFERENCE", "STORAGE_ENGINE", "REQUIRE_API_VERSION", "DRIVERS_TOOLS", @@ -1337,6 +1341,11 @@ def create_run_server_func(): "LOAD_BALANCER", "LOCAL_ATLAS", "NO_EXT", + # The AWS credentials are needed to download the "latest" nightly + # build from the private S3 bucket, see DRIVERS-3628. + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", ] args = [".evergreen/just.sh", "run-server", "${TEST_NAME}"] sub_cmd = get_subprocess_exec(include_expansions_in_env=includes, args=args) @@ -1353,9 +1362,9 @@ def create_run_tests_func(): "AWS_SESSION_TOKEN", "COVERAGE", "UV_PYTHON", + "UV_PYTHON_PREFERENCE", "LIBMONGOCRYPT_URL", "MONGODB_URI", - "TOOLCHAIN_VERSION", "DISABLE_TEST_COMMANDS", "GREEN_FRAMEWORK", "NO_EXT", @@ -1378,7 +1387,7 @@ def create_run_tests_func(): def create_test_numpy_func(): - includes = ["TOOLCHAIN_VERSION", "COVERAGE"] + includes = ["UV_PYTHON", "COVERAGE"] test_cmd = get_subprocess_exec( include_expansions_in_env=includes, args=[".evergreen/just.sh", "test-numpy"] ) diff --git a/.evergreen/scripts/generate_config_utils.py b/.evergreen/scripts/generate_config_utils.py index efb8d6a34b..67be0dbec6 100644 --- a/.evergreen/scripts/generate_config_utils.py +++ b/.evergreen/scripts/generate_config_utils.py @@ -178,7 +178,7 @@ def get_common_name(base: str, sep: str, **kwargs) -> str: display_name = f"{display_name}{sep}{version}" for key, value in kwargs.items(): name = value - if key.lower() in ["python", "toolchain_version"]: + if key.lower() in ["python", "uv_python"]: if not value.startswith("pypy"): name = f"Python{value}" else: diff --git a/.evergreen/scripts/install-dependencies.sh b/.evergreen/scripts/install-dependencies.sh index 36e7866735..93b62ce6b6 100755 --- a/.evergreen/scripts/install-dependencies.sh +++ b/.evergreen/scripts/install-dependencies.sh @@ -2,6 +2,13 @@ # Install the necessary dependencies. set -euo pipefail +# The uv tool installs below (pinning uv, rust-just) just need a working +# interpreter, so drop the task's Python selection: it may not exist on this +# host (e.g. 3.15 with a 3.14 toolchain), and uv would fail on a too-old +# interpreter found in the search path (e.g. /usr/bin/python3 on RHEL) rather +# than skip it. Later steps re-source env.sh, restoring the selection. +unset UV_PYTHON UV_PYTHON_SEARCH_PATH UV_PYTHON_PREFERENCE + HERE=$(dirname ${BASH_SOURCE:-$0}) HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )" pushd "$(dirname "$(dirname $HERE)")" > /dev/null @@ -60,8 +67,9 @@ fi # Set up uv if needed. if [ "$_need_setup" = "1" ]; then - # ensure-uv.sh (drivers-evergreen-tools) finds or installs uv and scopes its env. - if [ -n "${DRIVERS_TOOLS:-}" ] && [ -f "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh" ]; then + # Default DRIVERS_TOOLS to the submodule; an env var override wins. + : "${DRIVERS_TOOLS:=$(dirname "$(dirname "$HERE")")/drivers-evergreen-tools}" + if [ -f "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh" ]; then . "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh" ensure_uv || exit 1 fi diff --git a/.evergreen/scripts/mod_wsgi.sh b/.evergreen/scripts/mod_wsgi.sh new file mode 100755 index 0000000000..78233a4ff2 --- /dev/null +++ b/.evergreen/scripts/mod_wsgi.sh @@ -0,0 +1,263 @@ +#!/bin/bash +# Set up, run, or tear down the mod_wsgi tests. On Linux the test runs against +# the host's Apache; on other hosts it runs in an ubuntu:24.04 container with +# its own MongoDB replica set. +# +# Usage: mod_wsgi.sh setup | test | teardown | active +set -euo pipefail + +SCRIPT_DIR=$(dirname "${BASH_SOURCE:-$0}") +ROOT=$(dirname "$(dirname "$SCRIPT_DIR")") +CMD="${1:-}" +MODE="${2:-}" +CONTAINER=pymongo-mod-wsgi +HOME_SRC=/home/smoke/src +# The venv, MongoDB install, and the recorded version selection live outside +# the source copy, so refreshing the checkout never reinstalls them. +VENV=/home/smoke/venv +VERSIONS=/home/smoke/work/versions + +cd "$ROOT" + +case "$CMD" in + setup) + if [ "$MODE" != "standalone" ] && [ "$MODE" != "embedded" ]; then + echo "Usage: mod_wsgi.sh setup " + exit 1 + fi + # The harness recipes (run-tests, teardown-tests) dispatch on this. + echo "export TEST_NAME=\"mod_wsgi\"" > .evergreen/scripts/test-env.sh + echo "export SUB_TEST_NAME=\"$MODE\"" >> .evergreen/scripts/test-env.sh + # A failed setup leaves a partially started session behind. While the + # marker still routes to this session, tear it down; then remove the + # marker, so later just recipes do not route to the half-set-up state. + # The teardown is what stops the server or container setup may have + # started, and the next setup attempt stops any leftovers as well. + cleanup_marker() { + status=$? + if [ "$status" -ne 0 ]; then + bash "$SCRIPT_DIR/mod_wsgi.sh" teardown || true + rm -f .evergreen/scripts/test-env.sh + fi + } + trap cleanup_marker EXIT + ;; + test|teardown|active) ;; + *) + echo "Usage: mod_wsgi.sh setup | test | teardown | active" + exit 1 + ;; +esac + +if [ "$CMD" = "active" ]; then + # Exit 0 when a mod_wsgi session is active in this environment, so the + # dispatch script can route to us without a stale marker. Uses the system + # python and the tester's own STATE_FILE constant, so a missing or + # mismatched uv cannot break the dispatch and the path cannot drift. + if [ "$(uname -s)" = "Linux" ] && [ -z "${MOD_WSGI_DOCKER:-}" ]; then + if python3 -c " +import sys +sys.path.insert(0, '.evergreen/scripts') +from mod_wsgi_tester import STATE_FILE +sys.exit(0 if STATE_FILE.exists() else 1) +"; then + exit 0 + fi + exit 1 + fi + # The docker branch below handles the container case. +fi + +if [ "$(uname -s)" = "Linux" ] && [ -z "${MOD_WSGI_DOCKER:-}" ]; then + # Native: Apache runs on the host; MongoDB is already running, started by + # "just run-server" or the CI workflow. Set MOD_WSGI_DOCKER to force the + # container path on Linux. + if [ "$CMD" != "teardown" ]; then + # Other recipes' exact uv syncs prune the on-demand mod_wsgi install; + # restore it when missing. Checking first avoids reconciling a caller's + # differently-resolved dependencies. + if ! uv run --no-sync python -c "import mod_wsgi" 2>/dev/null; then + uv pip install -r requirements/mod_wsgi.txt --quiet + fi + fi + if [ "$CMD" = "setup" ]; then + uv run --no-sync python .evergreen/scripts/mod_wsgi_tester.py setup "$MODE" + else + uv run --no-sync python .evergreen/scripts/mod_wsgi_tester.py "$CMD" + fi + # Clear the dispatch marker so later just recipes use the harness. + if [ "$CMD" = "teardown" ]; then + rm -f .evergreen/scripts/test-env.sh + fi + exit 0 +fi + +# Docker path for hosts that cannot run Apache and mongod natively. +container_exists() { docker container inspect "$CONTAINER" >/dev/null 2>&1; } +container_running() { + [ "$(docker container inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null)" = "true" ] +} + +if [ "$CMD" = "active" ]; then + if container_exists; then + exit 0 + fi + exit 1 +fi + +# Copy the checkout into the container's home directory, so root-owned test +# artifacts never pollute it. +sync_source() { + docker exec "$CONTAINER" rm -rf "$HOME_SRC" + docker exec -u smoke "$CONTAINER" mkdir -p "$HOME_SRC" + tar -C "$ROOT" -cf - --exclude=.git --exclude=.venv \ + --exclude=access_log --exclude=error_log . | docker exec -i -u smoke "$CONTAINER" tar -C "$HOME_SRC" -xf - +} + +# Start a single-node replica set, if it is not already running. +ensure_mongod() { + docker exec --user smoke -w "$HOME_SRC" "$CONTAINER" bash -c 'set -euo pipefail + export PATH="$HOME/.local/bin:$PATH" + mkdir -p "$HOME/work/db" + if ! pgrep -x mongod >/dev/null; then + "$HOME/work/bin/mongod" --replSet rs0 --bind_ip 127.0.0.1 --port 27017 --dbpath "$HOME/work/db" --fork --logpath "$HOME/work/mongod.log" + fi + ' +} + +# Create the container and install the environment (idempotent). +bootstrap() { + if ! container_exists; then + docker run --name "$CONTAINER" -d ubuntu:24.04 sleep infinity + # A transient failure during initialization (for example in apt-get) + # would leave the half-created container behind, and later bootstraps + # would skip initialization and fail when sync_source expects the + # missing smoke user. Remove the container on failure, so the next + # attempt initializes a fresh one. + initialize() { + docker exec "$CONTAINER" apt-get update -qq + docker exec "$CONTAINER" apt-get install -y -qq apache2 apache2-dev build-essential curl jq git tar gzip ca-certificates + docker exec "$CONTAINER" useradd -m smoke + } + if ! initialize; then + docker rm -f "$CONTAINER" >/dev/null 2>&1 || true + exit 1 + fi + fi + if ! container_running; then + docker start "$CONTAINER" + fi + # Always refresh the checkout copy, so reused containers test the current + # source. The venv and versions file live outside it and survive the + # refresh. + sync_source + # Re-evaluate the newest Python and MongoDB on every bootstrap, so a reused + # container tracks matrix and server releases, but only reinstall when the + # recorded selection differs, since the downloads are slow. + docker exec --user smoke -w "$HOME_SRC" \ + -e VERSIONS="$VERSIONS" -e VENV="$VENV" "$CONTAINER" bash -c 'set -euo pipefail + export PATH="$HOME/.local/bin:$PATH" + export UV_PYTHON_INSTALL_DIR="$HOME/uv-python" + export UV_PROJECT_ENVIRONMENT="$VENV" + if ! command -v uv >/dev/null; then + curl -LsSf https://astral.sh/uv/install.sh | sh >/dev/null + fi + # Mirror the CI jobs and test the newest supported CPython. + LATEST_PYTHON=$(uv run --no-project --with "shrub.py>=3.10.0" python .evergreen/scripts/mod_wsgi_matrix.py | jq -r ".[-1].\"python-version\"") + case "$(uname -m)" in + aarch64|arm64) MARCH=aarch64 ;; + x86_64|amd64) MARCH=x86_64 ;; + *) echo "Unsupported architecture: $(uname -m)"; exit 1 ;; + esac + # MongoDB builds are per-distro; the container is ubuntu:24.04. + MONGODB_URL=$(curl -fsSL https://downloads.mongodb.org/current.json | jq -r " + .versions[] | select(.current and .production_release) | .downloads[] | + select(.target==\"ubuntu2404\" and .arch==\"$MARCH\") | .archive.url" | grep -v enterprise | head -1) + DESIRED="python=$LATEST_PYTHON mongodb=$MONGODB_URL" + if [ "$(cat "$VERSIONS" 2>/dev/null || true)" != "$DESIRED" ]; then + uv python install "$LATEST_PYTHON" >/dev/null + mkdir -p "$HOME/work/bin" + curl -fsSL "$MONGODB_URL" -o "$HOME/work/mongo.tgz" + MEMBER=$(tar -tzf "$HOME/work/mongo.tgz" | grep "bin/mongod$") + # Strip the two leading path components, so ensure_mongod finds the + # binary at "$HOME/work/bin/mongod". + tar -xz -C "$HOME/work/bin" --strip-components=2 -f "$HOME/work/mongo.tgz" "$MEMBER" + # A new interpreter needs a fresh venv; the sync below repopulates it. + rm -rf "$VENV" + printf "%s\n" "$DESIRED" > "$VERSIONS" + fi + # Rebuild the in-place C extensions on every bootstrap; the source + # refresh may carry artifacts built by a different interpreter. + export PYMONGO_C_EXT_MUST_BUILD=1 + uv sync --refresh-package pymongo + # The sync above prunes mod_wsgi, so install it after. + VIRTUAL_ENV="$VENV" uv pip install -r requirements/mod_wsgi.txt --quiet + ' + ensure_mongod + docker exec --user smoke -w "$HOME_SRC" "$CONTAINER" bash -c 'set -euo pipefail + export PATH="$HOME/.local/bin:$PATH" + export UV_PROJECT_ENVIRONMENT="'"$VENV"'" + # Probe replSetGetStatus over a direct connection: it returns immediately + # for an uninitialized replica set, while the default hello-based server + # selection would block for the full server selection timeout first. + if ! uv run --no-sync python -c " +from pymongo import MongoClient +client = MongoClient(\"127.0.0.1:27017\", directConnection=True) +try: + client.admin.command(\"replSetGetStatus\") +except Exception: + # NotYetInitialized (or a transient failure): initiation is still needed. + raise SystemExit(1) +" 2>/dev/null; then + uv run --no-sync python -c " +from pymongo import MongoClient +client = MongoClient(\"127.0.0.1:27017\", directConnection=True) +client.admin.command(\"replSetInitiate\", {\"_id\": \"rs0\", \"members\": [{\"_id\": 0, \"host\": \"127.0.0.1:27017\"}]}) +" + fi + ' +} + +# Bring a running session to readiness for a test run without refreshing the +# checkout copy: Apache may be live in it, and sync_source would unlink its +# pid file and error log. Only a missing, stopped, or never-bootstrapped +# container needs the full bootstrap, since then nothing is live to disturb. +ensure_ready() { + if container_exists && container_running \ + && docker exec "$CONTAINER" test -e "$VERSIONS"; then + ensure_mongod + else + bootstrap + fi +} + +case "$CMD" in + setup) + bootstrap + docker exec --user smoke -w "$HOME_SRC" "$CONTAINER" bash -c 'set -euo pipefail + export PATH="$HOME/.local/bin:$PATH" + export UV_PYTHON_INSTALL_DIR="$HOME/uv-python" + export UV_PROJECT_ENVIRONMENT="'"$VENV"'" + uv run --no-sync python .evergreen/scripts/mod_wsgi_tester.py setup '"$MODE"' + ' + ;; + test) + ensure_ready + docker exec --user smoke -w "$HOME_SRC" "$CONTAINER" bash -c 'set -euo pipefail + export PATH="$HOME/.local/bin:$PATH" + export UV_PROJECT_ENVIRONMENT="'"$VENV"'" + uv run --no-sync python .evergreen/scripts/mod_wsgi_tester.py test + ' + ;; + teardown) + if container_running; then + docker exec --user smoke -w "$HOME_SRC" "$CONTAINER" bash -c 'set -euo pipefail + export PATH="$HOME/.local/bin:$PATH" + export UV_PROJECT_ENVIRONMENT="'"$VENV"'" + uv run --no-sync python .evergreen/scripts/mod_wsgi_tester.py teardown + ' || true + docker stop -t 1 "$CONTAINER" + fi + rm -f .evergreen/scripts/test-env.sh + ;; +esac diff --git a/.evergreen/scripts/mod_wsgi_matrix.py b/.evergreen/scripts/mod_wsgi_matrix.py new file mode 100644 index 0000000000..2b8bb6611b --- /dev/null +++ b/.evergreen/scripts/mod_wsgi_matrix.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import json +import sys + +from generate_config_utils import ALL_VERSIONS, CPYTHONS + +# MongoDB 6.0 with the oldest supported CPython and minimum dependencies, and +# the latest MongoDB with the newest supported CPython. MongoDB 6.0 is the +# oldest version that installs on the GitHub runners. +VERSIONS = [ + { + "python-version": CPYTHONS[0], + "mongodb-version": "6.0", + }, + { + "python-version": CPYTHONS[-1], + "mongodb-version": ALL_VERSIONS[-1], + }, +] + +if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "--index": + # Emit one version pair in GITHUB_OUTPUT format. + entry = VERSIONS[int(sys.argv[2])] + print(f"python-version={entry['python-version']}") + print(f"mongodb-version={entry['mongodb-version']}") + else: + print(json.dumps(VERSIONS)) diff --git a/.evergreen/scripts/mod_wsgi_tester.py b/.evergreen/scripts/mod_wsgi_tester.py new file mode 100644 index 0000000000..c326f6b476 --- /dev/null +++ b/.evergreen/scripts/mod_wsgi_tester.py @@ -0,0 +1,312 @@ +"""Setup, run, and tear down the mod_wsgi tests. + +Usage: + python .evergreen/scripts/mod_wsgi_tester.py setup + python .evergreen/scripts/mod_wsgi_tester.py test + python .evergreen/scripts/mod_wsgi_tester.py teardown + +The commands run as separate workflow steps, so state is passed between them +in a 0600 file in a user-owned cache directory. +""" + +from __future__ import annotations + +import json +import logging +import os +import signal +import socket +import subprocess +import sys +import time +import urllib.request +from pathlib import Path + +logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") +LOGGER = logging.getLogger("mod_wsgi_tester") + +# Resolve __file__, so ROOT is correct when the script is invoked with a +# relative path, as mod_wsgi.sh does. +ROOT = Path(__file__).resolve().parents[2] +# State lives in a user-owned cache directory rather than shared /tmp: another +# local user must not be able to read or plant it, since teardown feeds its +# values into the commands that stop Apache. +STATE_DIR = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "pymongo-mod-wsgi" +STATE_DIR.mkdir(mode=0o700, parents=True, exist_ok=True) +STATE_FILE = STATE_DIR / "state.json" +PORT = 8080 + + +def run_command( + cmd: str | list[str], env: dict[str, str] | None = None, check: bool = True +) -> None: + LOGGER.info(f"Running command: {cmd}") + try: + result = subprocess.run( # noqa: S603 + cmd, + shell=isinstance(cmd, str), + env={**os.environ, **(env or {})}, + cwd=str(ROOT), + capture_output=True, + text=True, + check=False, + ) + except FileNotFoundError: + # A missing binary is a failed command like any other: raised when + # required, logged and swallowed for best-effort calls. The shell + # path used to behave the same way via the exit code, and hosts like + # the Fedora job's container do not ship pkill. + if check: + raise + LOGGER.warning(f"Command not found: {cmd if isinstance(cmd, str) else cmd[0]}") + return + if result.stdout: + LOGGER.info(result.stdout) + if result.returncode != 0: + LOGGER.error(result.stderr) + if check: + raise RuntimeError(f"Command failed with code {result.returncode}: {cmd}") + + +def make_request(url: str, timeout: int = 10) -> None: + for _ in range(timeout): + try: + # Per-request timeout, so an Apache that accepts the connection + # but never answers fails after roughly timeout seconds instead + # of hanging forever. + urllib.request.urlopen(url, timeout=1) # noqa: S310 + return + except OSError: + # URLError (and its HTTPError subclass) plus a socket timeout. + pass + time.sleep(1) + raise TimeoutError(f"Failed to access {url}") + + +def find_mod_wsgi_so() -> str: + # The .so is built against the installing interpreter and lives in the + # installed package. + import mod_wsgi + + so_files = list((Path(mod_wsgi.__file__).parent / "server").glob("mod_wsgi*.so")) + if len(so_files) != 1: + raise ValueError(f"Expected one mod_wsgi.so in {mod_wsgi.__file__}, found {so_files}") + return str(so_files[0]) + + +def find_apache() -> tuple[str, str]: + """Return the Apache binary and the config file matching its distribution.""" + from shutil import which + + apache = which("apache2") + if not apache: + # apache2 lives in sbin, which is not always on PATH for non-root users. + for candidate in ("/usr/lib/apache2/mpm-prefork/apache2", "/usr/sbin/apache2"): + if Path(candidate).exists(): + apache = candidate + break + if apache: + return apache, "apache24ubuntu.conf" + apache = which("httpd") + if not apache: + for candidate in ("/usr/sbin/httpd", "/usr/local/apache2/bin/httpd"): + if Path(candidate).exists(): + apache = candidate + break + if not apache: + raise ValueError("Could not find apache2 or httpd") + return apache, "httpd24fedora.conf" + + +def port_is_open() -> bool: + try: + socket.create_connection(("127.0.0.1", PORT), timeout=1).close() + return True + except OSError: + return False + + +def pid_file(apache_config: str) -> Path: + # The test configs write the master process's pid to the checkout root. + return ROOT / ("httpd.pid" if "httpd" in apache_config else "apache2.pid") + + +def pid_is_our_apache(pid: int, apache_config: str) -> bool: + # A stale pid file may name a pid the OS has reused for an unrelated + # process, so only signal an Apache whose command line carries this + # checkout's config argument. + try: + cmdline = (Path("/proc") / str(pid) / "cmdline").read_bytes() + except OSError: + return False + argv = cmdline.decode("utf-8", "replace").split("\0") + return f"{ROOT}/test/mod_wsgi_test/{apache_config}" in argv + + +def stop_apache(apache: str, apache_config: str, env: dict[str, str] | None = None) -> None: + # Idempotent: safe to call when Apache is not running. + # Prefer the pid file: -k stop re-parses the config, which fails when the + # mod_wsgi .so or the config's env vars are unavailable. Fall back to a + # validated pgrep for a server whose pid file was removed under it. The + # commands run as argument lists, so state-derived values never reach a + # shell. + from shutil import which + + pid = pid_file(apache_config) + if pid.exists(): + try: + victim = int(pid.read_text().strip()) + except ValueError: + victim = 0 + if victim and pid_is_our_apache(victim, apache_config): + try: + os.kill(victim, signal.SIGTERM) + except (ProcessLookupError, PermissionError): + LOGGER.warning("Could not stop Apache via the pid file") + else: + # Stale, reused, or unreadable pid: remove the file and stop + # through the config-validated path instead. + LOGGER.warning(f"Removing stale pid file {pid}") + pid.unlink() + if not pid.exists(): + run_command( + [apache, "-k", "stop", "-f", f"{ROOT}/test/mod_wsgi_test/{apache_config}"], + env=env, + check=False, + ) + if port_is_open(): + # Last resort: find pids whose full command line mentions this + # checkout's absolute config path, then signal only the ones whose + # command line carries it as an exact argument, so a server from + # another checkout (or an unrelated process) is never killed. + config_path = f"{ROOT}/test/mod_wsgi_test/{apache_config}" + pgrep = which("pgrep") + if pgrep: + result = subprocess.run( # noqa: S603 + [pgrep, "-f", config_path], capture_output=True, text=True, check=False + ) + for line in result.stdout.split(): + try: + victim = int(line) + except ValueError: + continue + if pid_is_our_apache(victim, apache_config): + try: + os.kill(victim, signal.SIGTERM) + except (ProcessLookupError, PermissionError): + LOGGER.warning(f"Could not stop Apache pid {victim}") + + # Stopping returns before the port is released, which the next mode needs + # to rebind. + deadline = time.time() + 10 + while time.time() < deadline: + if not port_is_open(): + return + time.sleep(0.5) + raise ValueError(f"Apache did not release port {PORT}") + + +def write_state(state: dict) -> None: + # The cache directory is user-only (0700) and the file is created 0600, so + # no other local user can read or plant this state. + fd = os.open(STATE_FILE, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w") as f: + json.dump(state, f) + + +def read_state() -> dict: + # Lost or corrupt state falls back to discovery in teardown. + try: + return json.loads(STATE_FILE.read_text()) + except (OSError, ValueError): + return {} + + +def setup(sub_test_name: str) -> None: + if sub_test_name == "embedded": + conf = "mod_wsgi_test_embedded.conf" + elif sub_test_name == "standalone": + conf = "mod_wsgi_test.conf" + else: + raise ValueError("mod_wsgi sub test must be either 'standalone' or 'embedded'") + apache, apache_config = find_apache() + # On Fedora/RHEL, httpd embeds the interpreter mod_wsgi was built with and + # needs to be told where the Python installation lives. + env = { + "MOD_WSGI_CONF": conf, + "MOD_WSGI_SO": find_mod_wsgi_so(), + "MOD_WSGI_PYTHON_HOME": sys.base_prefix, + "PROJECT_DIRECTORY": str(ROOT), + } + # Stop any server left behind by a previous failed run so the port is free. + stop_apache(apache, apache_config, env) + run_command( + [apache, "-k", "start", "-f", f"{ROOT}/test/mod_wsgi_test/{apache_config}"], + env=env, + ) + write_state({"apache": apache, "config": apache_config, "env": env}) + + try: + for interp in ("interpreter1", "interpreter2"): + make_request(f"http://localhost:{PORT}/{interp}{ROOT}", 10) + except Exception: + # The Apache error log holds the actual failure. Stop the server so a + # failed setup does not leave it bound to the port. + stop_apache(apache, apache_config, env) + error_log = Path("error_log") + if error_log.exists(): + LOGGER.error(error_log.read_text()) + raise + + +def test() -> None: + sys.path.insert(0, str(ROOT)) + from test.mod_wsgi_test.test_client import main, parse_args + + uri1 = f"http://localhost:{PORT}/interpreter1{ROOT}" + uri2 = f"http://localhost:{PORT}/interpreter2{ROOT}" + try: + main(*parse_args(f"-n 25000 -t 100 parallel {uri1} {uri2}".split())) + main(*parse_args(f"-n 25000 serial {uri1} {uri2}".split())) + except BaseException: + # The test client raises KeyboardInterrupt in the failing path. + error_log = Path("error_log") + if error_log.exists(): + LOGGER.error(error_log.read_text()) + raise + + +def teardown() -> None: + state = read_state() + apache = state.get("apache") + apache_config = state.get("config") + env = state.get("env") + if not apache or not apache_config: + # State lost; fall back to discovery to stop a leftover server. The + # pid file in stop_apache usually makes this unnecessary. + try: + apache, apache_config = find_apache() + except ValueError: + return + env = { + "MOD_WSGI_CONF": "mod_wsgi_test.conf", + "MOD_WSGI_PYTHON_HOME": sys.base_prefix, + "PROJECT_DIRECTORY": str(ROOT), + } + try: + env["MOD_WSGI_SO"] = find_mod_wsgi_so() + except (ImportError, ValueError): + pass + stop_apache(apache, apache_config, env) + + +if __name__ == "__main__": + command = sys.argv[1] if len(sys.argv) > 1 else "" + if command == "setup": + setup(sys.argv[2]) + elif command == "test": + test() + elif command == "teardown": + teardown() + else: + raise SystemExit(f"Usage: {sys.argv[0]} setup |test|teardown") diff --git a/.evergreen/scripts/resync-all-specs.py b/.evergreen/scripts/resync-all-specs.py index 78afe8a4d8..52c276b8b8 100644 --- a/.evergreen/scripts/resync-all-specs.py +++ b/.evergreen/scripts/resync-all-specs.py @@ -28,6 +28,8 @@ def resync_specs(directory: pathlib.Path, errored: dict[str, str]) -> None: ) except CalledProcessError as exc: errored[spec.name] = exc.stderr + # The PR-body summary is lost if PR creation fails. + print(f"Spec sync failed for {spec.name}:\n{exc.stderr}", flush=True) print("Done syncing specs") @@ -57,9 +59,11 @@ def apply_patches(errored): ], check=True, stderr=subprocess.PIPE, + text=True, ) except CalledProcessError as exc: errored[f"{patch}"] = exc.stderr + print(f"Patch application failed for {patch}:\n{exc.stderr}", flush=True) def check_new_spec_directories(directory: pathlib.Path) -> list[str]: @@ -127,9 +131,10 @@ def write_summary(errored: dict[str, str], new: list[str], filename: str | None) if filename is None: print(f"\n{pr_body}") else: + # Plain text: the consumer embeds this file via jq --rawfile, + # which handles the JSON escaping itself. with open(filename, "w") as f: - # replacements made for proper json - f.write(pr_body.replace("\n", "\\n").replace("\t", "\\t")) + f.write(pr_body) def main(args: Namespace): diff --git a/.evergreen/scripts/resync-all-specs.sh b/.evergreen/scripts/resync-all-specs.sh index 41e4a2bc73..11dd2e65ba 100755 --- a/.evergreen/scripts/resync-all-specs.sh +++ b/.evergreen/scripts/resync-all-specs.sh @@ -34,7 +34,12 @@ then # we're running locally python3 ./.evergreen/scripts/resync-all-specs.py else - /opt/devtools/bin/python3.11 ./.evergreen/scripts/resync-all-specs.py --filename "$PR_DESC" + # Use the newest mongodbtoolchain python; the default python3 may be too old. + PY=$(ls -1d /opt/mongodbtoolchain/v*/bin/python3 2>/dev/null | sort -V | tail -1) + [ -n "$PY" ] || PY=python3 + [ "$("$PY" -c 'import sys; print(sys.version_info >= (3, 7))' 2>/dev/null)" = True ] \ + || { echo "Error: $PY is not Python 3.7+, which resync-all-specs.py requires." >&2; exit 1; } + "$PY" ./.evergreen/scripts/resync-all-specs.py --filename "$PR_DESC" if [[ -f $PR_DESC ]]; then # changes were made -> call scrypt to create PR for us .evergreen/scripts/create-spec-pr.sh "$PR_DESC" diff --git a/.evergreen/scripts/run-getdata.sh b/.evergreen/scripts/run-getdata.sh index 9435a5fcc3..416744ac03 100755 --- a/.evergreen/scripts/run-getdata.sh +++ b/.evergreen/scripts/run-getdata.sh @@ -2,6 +2,13 @@ # Get the debug data for an evergreen task. set -eu +HERE=$(dirname ${BASH_SOURCE:-$0}) +HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )" +ROOT=$(dirname "$(dirname $HERE)") + +# Default to the drivers-evergreen-tools submodule when unset. +: "${DRIVERS_TOOLS:=$ROOT/drivers-evergreen-tools}" + . ${DRIVERS_TOOLS}/.evergreen/get-distro.sh || true get_distro || true echo $DISTRO diff --git a/.evergreen/scripts/run_server.py b/.evergreen/scripts/run_server.py index f81964f367..38f817c15d 100644 --- a/.evergreen/scripts/run_server.py +++ b/.evergreen/scripts/run_server.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Any -from utils import DRIVERS_TOOLS, ROOT, get_test_options, run_command +from utils import DRIVERS_TOOLS, ROOT, check_drivers_tools, get_test_options, run_command def set_env(name: str, value: Any = "1") -> None: @@ -14,11 +14,9 @@ def set_env(name: str, value: Any = "1") -> None: def start_server(): - run_mongodb_script = ( - Path(DRIVERS_TOOLS) / ".evergreen" / "run-mongodb.sh" if DRIVERS_TOOLS else None - ) + run_mongodb_script = Path(DRIVERS_TOOLS) / ".evergreen" / "run-mongodb.sh" want_help = bool({"-h", "--help"} & set(sys.argv[1:])) - if want_help and run_mongodb_script and run_mongodb_script.is_file(): + if want_help and run_mongodb_script.is_file(): # Forward straight to run-mongodb.sh's own help, without run_command's # "Running command..." logging noise. subprocess.run( # noqa: S603 @@ -28,13 +26,10 @@ def start_server(): ) return - # DRIVERS_TOOLS is only needed to actually start a server. When it's unset and - # -h/--help was requested, fall through to get_test_options' own argparse help below. - if not want_help and not DRIVERS_TOOLS: - raise ValueError( - "DRIVERS_TOOLS is not set; run `just run-server` from an Evergreen task " - "or set DRIVERS_TOOLS to a drivers-evergreen-tools checkout." - ) + # DRIVERS_TOOLS is only needed to start a server; allow -h/--help to fall + # through to argparse when the submodule is missing. + if not want_help: + check_drivers_tools() opts, extra_opts = get_test_options( "Run a MongoDB server. All given flags will be passed to run-mongodb.sh in DRIVERS_TOOLS.", diff --git a/.evergreen/scripts/setup-dev-env.sh b/.evergreen/scripts/setup-dev-env.sh index 508eb099bc..d9fbef2b04 100755 --- a/.evergreen/scripts/setup-dev-env.sh +++ b/.evergreen/scripts/setup-dev-env.sh @@ -53,6 +53,29 @@ if [ "${CI:-}" != "true" ] && [ "${GITHUB_ACTIONS:-}" != "true" ]; then printf 'export PATH="%s:$PATH"\n' "$PYMONGO_BIN_DIR_POSIX" >> "$_rc" fi +# Initialize the submodule (Evergreen's git.get_project does not); tolerate +# non-git hosts with a warning. +if ! git -C "$ROOT" submodule update --init --recursive; then + echo "WARNING: could not initialize the drivers-evergreen-tools submodule;" \ + "set DRIVERS_TOOLS to a drivers-evergreen-tools checkout instead." +fi + +# Mirror configure-env.sh's uv config boundary; see it for the full explanation. +if [ -d "$ROOT/drivers-evergreen-tools" ] && [ ! -f "$ROOT/drivers-evergreen-tools/uv.toml" ]; then + cat < "$ROOT/drivers-evergreen-tools/uv.toml" +# Written by mongo-python-driver to stop uv's config discovery here; see +# .evergreen/scripts/configure-env.sh. +EOT +fi + +# Keep the boundary out of git status via the submodule's local exclude; +# no-op without git. +if _git_dir=$(git -C "$ROOT/drivers-evergreen-tools" rev-parse --absolute-git-dir 2>/dev/null); then + mkdir -p "${_git_dir}/info" + grep -qxF "uv.toml" "${_git_dir}/info/exclude" 2>/dev/null || + printf "uv.toml\n" >> "${_git_dir}/info/exclude" +fi + # Ensure dependencies are installed. bash $HERE/install-dependencies.sh diff --git a/.evergreen/scripts/setup-uv-python.sh b/.evergreen/scripts/setup-uv-python.sh index e83abce4b9..2d91cc6dc6 100755 --- a/.evergreen/scripts/setup-uv-python.sh +++ b/.evergreen/scripts/setup-uv-python.sh @@ -1,60 +1,131 @@ #!/bin/bash -# Set up the UV_PYTHON variable. -set -eu +# Choose the Python that uv uses, and make sure uv can provide it. +# +# Input: +# UV_PYTHON - a version ("3.10", "3.14t"), an implementation ("pypy3.11"), +# or an interpreter path ("/usr/bin/python3.11"). +# +# Exports: +# UV_PYTHON_SEARCH_PATH - the Python toolchain bin dir for the request, so uv +# uses the toolchain Python instead of downloading one. +# UV_PYTHON_PREFERENCE - "system" so the toolchain wins over managed installs. +# A preference set by the task is left alone. +# UV_PYTHON - the Python interpreter uv uses; defaults to CPython +# 3.10 when the task does not set one. +set -euo pipefail -HERE=$(dirname ${BASH_SOURCE:-$0}) +HERE=$(dirname "${BASH_SOURCE:-$0}") HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )" +ROOT=$(dirname "$(dirname "$HERE")") -# Use min supported version by default. -_python="3.10" +# A .venv whose interpreter was removed breaks every `uv python find` and +# `uv sync`. It is a build artifact, so remove it and let `uv sync` recreate it. +# A removed base interpreter leaves a dangling symlink on Linux/macOS and a +# launcher that will not start on Windows, so probe by running the interpreter. +for _py in "$ROOT/.venv/bin/python3" "$ROOT/.venv/bin/python" "$ROOT/.venv/Scripts/python.exe"; do + if [ -e "$_py" ] || [ -L "$_py" ]; then + if ! "$_py" -c 'import sys' > /dev/null 2>&1; then + echo "Removing broken .venv (its interpreter will not start); uv sync will recreate it." + rm -rf "$ROOT/.venv" + fi + break + fi +done +unset _py # Source the env files to pick up common variables. -if [ -f $HERE/env.sh ]; then - . $HERE/env.sh +if [ -f "$HERE/env.sh" ]; then + . "$HERE/env.sh" fi # Get variables defined in test-env.sh. -if [ -f $HERE/test-env.sh ]; then - . $HERE/test-env.sh +if [ -f "$HERE/test-env.sh" ]; then + . "$HERE/test-env.sh" fi -if [ -z "${UV_PYTHON:-}" ]; then - set -x - # Translate a TOOLCHAIN_VERSION to UV_PYTHON. - if [ -n "${TOOLCHAIN_VERSION:-}" ]; then - _python=$TOOLCHAIN_VERSION - if [ "$(uname -s)" = "Darwin" ]; then - if [[ "$_python" == *"t"* ]]; then - binary_name="python3t" - framework_dir="PythonT" - else - binary_name="python3" - framework_dir="Python" - fi - _python=$(echo "$_python" | sed 's/t//g') - _python="/Library/Frameworks/$framework_dir.Framework/Versions/$_python/bin/$binary_name" - elif [ "Windows_NT" = "${OS:-}" ]; then - _python=$(echo $_python | cut -d. -f1,2 | sed 's/\.//g; s/t//g') - if [[ "$TOOLCHAIN_VERSION" == *"t"* ]]; then - _exe="python${TOOLCHAIN_VERSION}.exe" - else - _exe="python.exe" - fi - if [ -n "${IS_WIN32:-}" ]; then - _win_python="C:/python/32/Python${_python}/${_exe}" - else - _win_python="C:/python/Python${_python}/${_exe}" - fi - # Fall back to the version string if the toolchain path doesn't exist. - # uv will then download and manage Python itself. - if [ -f "$_win_python" ]; then - _python="$_win_python" - else - _python="$TOOLCHAIN_VERSION" - fi - elif [ -d "/opt/python/$_python/bin" ]; then - _python="/opt/python/$_python/bin/python3" +# Default to a known-good Python so behavior is deterministic when a task does +# not pin one. Selecting the uv binary is separate and driven by the required +# uv version in pyproject.toml. +export UV_PYTHON="${UV_PYTHON:-3.10}" + +# Print the Python toolchain bin dir for a version like "3.10" or "3.14t". +function _toolchain_dir() { + local version="$1" dir + # Only plain CPython versions live in the toolchain. + [[ "$version" =~ ^[0-9]+\.[0-9]+t?$ ]] || return 1 + if [ "Windows_NT" = "${OS:-}" ]; then + local winver="${version%t}" + winver="${winver//./}" + if [ -n "${IS_WIN32:-}" ]; then + dir="C:/python/32/Python${winver}" + else + dir="C:/python/Python${winver}" + fi + elif [ "$(uname -s)" = "Darwin" ]; then + if [[ "$version" == *t ]]; then + dir="/Library/Frameworks/PythonT.Framework/Versions/${version%t}/bin" + else + dir="/Library/Frameworks/Python.Framework/Versions/${version}/bin" fi + else + dir="/opt/python/${version}/bin" + fi + [ -d "$dir" ] || return 1 + echo "$dir" +} + +# Make sure uv can provide the requested Python, downloading it if needed. +# If that fails, fall back to any available Python on local hosts; on CI, fail. +function _ensure_python() { + local request="$1" out _fallback + if out=$(uv python find "$request" 2>&1); then + return 0 + fi + # Show why find failed: e.g. an interpreter uv cannot run, or none at all. + echo "Python \"$request\" was not found on this host:" >&2 + sed 's/^/ /' <<<"$out" >&2 + echo "Asking uv to install Python \"$request\"..." + if uv python install "$request"; then + return 0 + fi + echo "uv could not install Python \"$request\"; looking for another interpreter..." + # Ignore UV_PYTHON and friends: they would steer the fallback right back to + # the broken interpreter. Local setup only: the Evergreen agent exports + # CI=true for every task command. + if [ "${CI:-}" != "true" ] && \ + _fallback=$(env -u UV_PYTHON -u UV_PYTHON_SEARCH_PATH -u UV_PYTHON_PREFERENCE \ + uv python find ">=3.9" 2>/dev/null); then + echo "Using fallback interpreter: $_fallback" + export UV_PYTHON="$_fallback" + return 0 fi - export UV_PYTHON="$_python" + echo "ERROR: uv could not find or install Python \"$request\"." >&2 + return 1 +} + +_search_path="" +_skip_ensure="" + +if [[ "$UV_PYTHON" == */* || "$UV_PYTHON" == *\\* ]]; then + # An explicit interpreter path; uv uses it as-is, so probe/install is skipped. + if [ ! -x "$UV_PYTHON" ]; then + echo "ERROR: UV_PYTHON=$UV_PYTHON does not exist or is not executable." >&2 + exit 1 + fi + _skip_ensure=1 +elif _dir=$(_toolchain_dir "$UV_PYTHON"); then + _search_path="$_dir" +fi + +# Point uv at the toolchain Python when there is one and the task did not set +# its own preference: a task-level UV_PYTHON_PREFERENCE wins (PYTHON-6135). +# On CI the toolchain dir is already first on PATH (configure-env.sh), so this +# mainly benefits local hosts and later steps. +if [ -n "$_search_path" ] && [ -z "${UV_PYTHON_PREFERENCE:-}" ]; then + export UV_PYTHON_SEARCH_PATH="$_search_path" + export UV_PYTHON_PREFERENCE="system" +fi + +if [ -z "$_skip_ensure" ]; then + _ensure_python "${UV_PYTHON:-}" fi diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 4f6833e289..45909ab8a4 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -13,10 +13,12 @@ ENV_FILE, HERE, LOGGER, + PERF_PYTHON_VERSION, PLATFORM, ROOT, TEST_SUITE_MAP, Distro, + check_drivers_tools, get_test_options, read_env, run_command, @@ -30,6 +32,8 @@ "MONGODB_API_VERSION", "DEBUG_LOG", "UV_PYTHON", + "UV_PYTHON_SEARCH_PATH", + "UV_PYTHON_PREFERENCE", "REQUIRE_FIPS", "IS_WIN32", ] @@ -50,9 +54,6 @@ # Map the test name to test group. GROUP_MAP = dict(mockupdb="mockupdb", perf="perf") -# The python version used for perf tests. -PERF_PYTHON_VERSION = "3.10.11" - # The libmongocrypt release used when LIBMONGOCRYPT_URL is not set. Must be at # least 1.20.0 for the GA "substring" query type. LIBMONGOCRYPT_VERSION = "1.20.2" @@ -328,8 +329,7 @@ def handle_test_env() -> None: MULTI_MONGOS_LB_URI += "&tls=true" write_env("SINGLE_MONGOS_LB_URI", SINGLE_MONGOS_LB_URI) write_env("MULTI_MONGOS_LB_URI", MULTI_MONGOS_LB_URI) - if not DRIVERS_TOOLS: - raise RuntimeError("Missing DRIVERS_TOOLS") + check_drivers_tools() cmd = f'bash "{DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh" start' run_command(cmd) @@ -359,7 +359,9 @@ def handle_test_env() -> None: # The mock OCSP responder MUST BE started before the mongod as the mongod expects that # a responder will be available upon startup. - version = os.environ.get("VERSION", "latest") + # Default to the newest stable release; the "latest" nightly build is + # downloaded from a private S3 bucket and needs AWS credentials. + version = os.environ.get("VERSION", "latest-stable") cmd = [ "bash", f"{DRIVERS_TOOLS}/.evergreen/run-mongodb.sh", @@ -375,8 +377,7 @@ def handle_test_env() -> None: run_command(cmd, cwd=DRIVERS_TOOLS) if SSL != "nossl": - if not DRIVERS_TOOLS: - raise RuntimeError("Missing DRIVERS_TOOLS") + check_drivers_tools() write_env("CLIENT_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/client.pem") write_env("CA_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem") @@ -428,8 +429,7 @@ def handle_test_env() -> None: # PATH is updated by configure-env.sh for access to mongocryptd. if test_name == "encryption": - if not DRIVERS_TOOLS: - raise RuntimeError("Missing DRIVERS_TOOLS") + check_drivers_tools() csfle_dir = Path(f"{DRIVERS_TOOLS}/.evergreen/csfle") # Opt in to corporate Azure credentials (DRIVERS-3392) os.environ["FLE_AZURE_USE_CORPORATE"] = "YES" @@ -517,8 +517,12 @@ def handle_test_env() -> None: run_command("tar xf single_and_multi_document.tgz", cwd=data_dir) write_env("TEST_PATH", str(data_dir)) write_env("OUTPUT_FILE", str(ROOT / "results.json")) - # Overwrite the UV_PYTHON from the env.sh file. - write_env("UV_PYTHON", "") + # The toolchain build may be this same patch version but is not + # optimized, which regressed CPU-bound benchmarks (PYTHON-6135). + write_env("UV_PYTHON", PERF_PYTHON_VERSION) + write_env("UV_PYTHON_PREFERENCE", "only-managed") + with ENV_FILE.open("a", newline="\n") as fid: + fid.write("unset UV_PYTHON_SEARCH_PATH\n") UV_ARGS.append(f"--python={PERF_PYTHON_VERSION}") diff --git a/.evergreen/scripts/stop-server.sh b/.evergreen/scripts/stop-server.sh index 045a655cbd..a254a2715a 100755 --- a/.evergreen/scripts/stop-server.sh +++ b/.evergreen/scripts/stop-server.sh @@ -4,6 +4,7 @@ set -eu HERE=$(dirname ${BASH_SOURCE:-$0}) HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )" +ROOT=$(dirname "$(dirname $HERE)") # Try to source the env file. if [ -f $HERE/env.sh ]; then @@ -11,4 +12,7 @@ if [ -f $HERE/env.sh ]; then source $HERE/env.sh fi +# Default to the drivers-evergreen-tools submodule when unset. +: "${DRIVERS_TOOLS:=$ROOT/drivers-evergreen-tools}" + bash ${DRIVERS_TOOLS}/.evergreen/run-mongodb.sh stop diff --git a/.evergreen/scripts/utils.py b/.evergreen/scripts/utils.py index 880e28cec1..6bb51cc1f7 100644 --- a/.evergreen/scripts/utils.py +++ b/.evergreen/scripts/utils.py @@ -12,7 +12,11 @@ HERE = Path(__file__).absolute().parent ROOT = HERE.parent.parent -DRIVERS_TOOLS = os.environ.get("DRIVERS_TOOLS", "").replace(os.sep, "/") +# DRIVERS_TOOLS defaults to the drivers-evergreen-tools submodule; an env var +# override wins. +DRIVERS_TOOLS = (os.environ.get("DRIVERS_TOOLS") or str(ROOT / "drivers-evergreen-tools")).replace( + os.sep, "/" +) TMP_DRIVER_FILE = "/tmp/mongo-python-driver.tgz" # noqa: S108 LOGGER = logging.getLogger("test") @@ -82,6 +86,9 @@ class Distro: EXTRA_TESTS = ["aws_lambda", "doctest"] +# The managed Python version used for perf tests (PYTHON-6135). +PERF_PYTHON_VERSION = "3.10.11" + # Tests that do not use run-mongodb directly. NO_RUN_ORCHESTRATION = [ "auth_oidc", @@ -298,8 +305,8 @@ def run_command(cmd: str | list[str], **kwargs: Any) -> None: kwargs.setdefault("check", True) # Prevent overriding the python used by other tools. env = kwargs.pop("env", os.environ).copy() - if "UV_PYTHON" in env: - del env["UV_PYTHON"] + for var in ["UV_PYTHON", "UV_PYTHON_SEARCH_PATH", "UV_PYTHON_PREFERENCE"]: + env.pop(var, None) kwargs["env"] = env try: subprocess.run(shlex.split(cmd), **kwargs) # noqa: PLW1510, S603 @@ -310,6 +317,18 @@ def run_command(cmd: str | list[str], **kwargs: Any) -> None: LOGGER.info("Running command '%s'... done.", cmd) +def check_drivers_tools() -> None: + """Raise a clear error when the drivers-evergreen-tools checkout is missing.""" + # An uninitialized submodule can exist as an empty directory, so a bare + # is_dir() check passes. Require a script every consumer needs instead. + if not (Path(DRIVERS_TOOLS) / ".evergreen" / "run-mongodb.sh").is_file(): + raise RuntimeError( + "The drivers-evergreen-tools checkout is missing or empty; run `just " + "install` to initialize the submodule, or set DRIVERS_TOOLS to a " + "drivers-evergreen-tools checkout." + ) + + def create_archive() -> str: run_command("git add .", cwd=ROOT) run_command('git commit --no-verify -m "add files"', check=False, cwd=ROOT) diff --git a/.evergreen/spec-patch/PYTHON-5993.patch b/.evergreen/spec-patch/PYTHON-5993.patch new file mode 100644 index 0000000000..3880924e83 --- /dev/null +++ b/.evergreen/spec-patch/PYTHON-5993.patch @@ -0,0 +1,94 @@ +diff --git a/test/unified-test-format/valid-pass/operator-gte.json b/test/unified-test-format/valid-pass/operator-gte.json +new file mode 100644 +index 000000000..ae69da5c9 +--- /dev/null ++++ b/test/unified-test-format/valid-pass/operator-gte.json +@@ -0,0 +1,88 @@ ++{ ++ "description": "operator-gte", ++ "schemaVersion": "1.29", ++ "createEntities": [ ++ { ++ "client": { ++ "id": "client0", ++ "observeEvents": [ ++ "commandStartedEvent" ++ ] ++ } ++ }, ++ { ++ "database": { ++ "id": "database0", ++ "client": "client0", ++ "databaseName": "database0Name" ++ } ++ }, ++ { ++ "collection": { ++ "id": "collection0", ++ "database": "database0", ++ "collectionName": "coll0" ++ } ++ } ++ ], ++ "initialData": [ ++ { ++ "collectionName": "coll0", ++ "databaseName": "database0Name", ++ "documents": [] ++ } ++ ], ++ "tests": [ ++ { ++ "description": "special gte matching operator", ++ "operations": [ ++ { ++ "name": "insertOne", ++ "object": "collection0", ++ "arguments": { ++ "document": { ++ "_id": 1, ++ "x": 2, ++ "y": 3, ++ "z": 4 ++ } ++ } ++ } ++ ], ++ "expectEvents": [ ++ { ++ "client": "client0", ++ "events": [ ++ { ++ "commandStartedEvent": { ++ "command": { ++ "insert": "coll0", ++ "documents": [ ++ { ++ "_id": { ++ "$$gte": 1 ++ }, ++ "x": { ++ "$$gte": 1.9 ++ }, ++ "y": { ++ "$$gte": { ++ "$numberLong": "3" ++ } ++ }, ++ "z": { ++ "$$gte": 4 ++ } ++ } ++ ] ++ }, ++ "commandName": "insert", ++ "databaseName": "database0Name" ++ } ++ } ++ ] ++ } ++ ] ++ } ++ ] ++} diff --git a/.github/actions/mod-wsgi-versions/action.yml b/.github/actions/mod-wsgi-versions/action.yml new file mode 100644 index 0000000000..63994a3c14 --- /dev/null +++ b/.github/actions/mod-wsgi-versions/action.yml @@ -0,0 +1,23 @@ +name: Compute Mod WSGI versions +description: Compute the CPython and MongoDB versions for a Mod WSGI job. +inputs: + index: + description: Row of the version matrix to use. + required: true +outputs: + python-version: + description: The CPython version to test with. + value: ${{ steps.versions.outputs.python-version }} + mongodb-version: + description: The MongoDB version to test against. + value: ${{ steps.versions.outputs.mongodb-version }} +runs: + using: composite + steps: + - id: versions + # The versions come from the Evergreen config's version lists. + shell: bash + env: + INDEX: ${{ inputs.index }} + run: | + uv run --no-project --with 'shrub.py>=3.10.0' python .evergreen/scripts/mod_wsgi_matrix.py --index "$INDEX" >> "$GITHUB_OUTPUT" diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 72d1e1e084..603c47d7fe 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -19,3 +19,29 @@ updates: schedule: interval: "weekly" open-pull-requests-limit: 0 + # drivers-evergreen-tools submodule + - package-ecosystem: "gitsubmodule" + directory: "/" + schedule: + interval: "weekly" + cooldown: + default-days: 7 + allow: + - dependency-name: "drivers-evergreen-tools" + # Python (pip). Manages requirements/mod_wsgi.txt only; the other + # requirements files are resolved in uv.lock. + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + exclude-paths: + - "/pyproject.toml" + - "/requirements.txt" + - "/requirements/aws.txt" + - "/requirements/docs.txt" + - "/requirements/encryption.txt" + - "/requirements/gssapi.txt" + - "/requirements/ocsp.txt" + - "/requirements/snappy.txt" + - "/requirements/test.txt" + - "/requirements/zstd.txt" diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml index 819261bd9d..c0856cf2d8 100644 --- a/.github/workflows/dist.yml +++ b/.github/workflows/dist.yml @@ -61,7 +61,7 @@ jobs: - name: Set up QEMU if: runner.os == 'Linux' - uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4.3.0 + uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 with: # setup-qemu-action by default uses `tonistiigi/binfmt:latest` image, # which is out of date. This causes seg faults during build. diff --git a/.github/workflows/sbom.yml b/.github/workflows/sbom.yml index 91af8d840c..62b0ca3ceb 100644 --- a/.github/workflows/sbom.yml +++ b/.github/workflows/sbom.yml @@ -14,6 +14,7 @@ on: - 'requirements/**.txt' - '!requirements/docs.txt' - '!requirements/test.txt' + - '!requirements/mod_wsgi.txt' permissions: contents: write diff --git a/.github/workflows/test-mod-wsgi.yml b/.github/workflows/test-mod-wsgi.yml new file mode 100644 index 0000000000..c10db479cb --- /dev/null +++ b/.github/workflows/test-mod-wsgi.yml @@ -0,0 +1,179 @@ +name: Mod WSGI Tests + +on: + push: + branches: ["main", "v**"] + paths: + - ".evergreen/scripts/mod_wsgi_*" + - ".evergreen/scripts/mod_wsgi.sh" + - ".evergreen/scripts/dispatch.sh" + - ".evergreen/scripts/generate_config_utils.py" + - ".github/workflows/test-mod-wsgi.yml" + - ".github/actions/mod-wsgi-*/**" + - "test/mod_wsgi_test/**" + - "pymongo/**" + - "bson/**" + - "justfile" + - "pyproject.toml" + - "uv.lock" + - "requirements/mod_wsgi.txt" + pull_request: + paths: + - ".evergreen/scripts/mod_wsgi_*" + - ".evergreen/scripts/mod_wsgi.sh" + - ".evergreen/scripts/dispatch.sh" + - ".evergreen/scripts/generate_config_utils.py" + - ".github/workflows/test-mod-wsgi.yml" + - ".github/actions/mod-wsgi-*/**" + - "test/mod_wsgi_test/**" + - "pymongo/**" + - "bson/**" + - "justfile" + - "pyproject.toml" + - "uv.lock" + - "requirements/mod_wsgi.txt" + workflow_dispatch: + +concurrency: + group: mod-wsgi-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash -eux {0} + +permissions: + contents: read + +jobs: + mod_wsgi_min_deps: + # Exercise the minimum dependency versions against Fedora's httpd, so the + # mod_wsgi tests cover httpd in addition to Ubuntu's apache2. + runs-on: ubuntu-latest + name: Mod WSGI Min Deps + container: + # fedora:latest, pinned by digest; bump periodically. + image: fedora@sha256:43b29f65a41eb9c35e1cd5323e3bdf3b655c2357a9f4f1ff2f9c2798e5045d80 + env: + # Install the interpreter where the Apache user can read it. + UV_PYTHON_INSTALL_DIR: /opt/uv-python + # The test rounds run against the lowest direct dependency versions + # installed below; a resync would reconcile them back to the lockfile. + JUST_NO_RESYNC: "1" + steps: + # actions/checkout needs git, which fedora:latest does not include. + - name: Install git + run: dnf install -y git + - uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + - name: Install just + run: | + uv tool install rust-just + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + - name: Install Apache + # jq and curl are needed to pick the MongoDB download. + run: dnf install -y httpd httpd-devel jq curl + - id: versions + uses: $/.github/actions/mod-wsgi-versions + with: + index: 0 + - name: Install Python + env: + PYTHON_VERSION: ${{ steps.versions.outputs.python-version }} + run: | + uv python install "$PYTHON_VERSION" + echo "UV_PYTHON=$PYTHON_VERSION" >> "$GITHUB_ENV" + - name: Install dependencies + run: | + export PYMONGO_C_EXT_MUST_BUILD=1 + just install + # just install skips the sync when uv is already set up. + uv sync --refresh-package pymongo + uv run --no-sync python tools/fail_if_no_c.py + # Exercise the mod_wsgi>=4.9.4 lower bound. + uv pip install -e ".[test]" -r requirements/mod_wsgi.txt --resolution=lowest-direct --force-reinstall + - name: Start MongoDB + # Fedora is not a supported MongoDB platform, so install the RHEL build + # directly instead of using the drivers-evergreen-tools action, which + # relies on actions/setup-python and does not run on Fedora. + env: + MONGODB_VERSION: ${{ steps.versions.outputs.mongodb-version }} + run: | + ARCH=$(uname -m) + URL=$(curl -fsSL https://downloads.mongodb.org/current.json | jq -r --arg v "$MONGODB_VERSION." --arg a "$ARCH" ' + .versions[] | select(.current and .production_release) | select(.version | startswith($v)) | + .downloads[] | select(.target=="rhel90" and .arch==$a) | .archive.url' | grep -v enterprise | head -1) + curl -fsSL "$URL" -o /tmp/mongo.tgz + tar -xz -C /tmp --strip-components=1 -f /tmp/mongo.tgz + mkdir -p /tmp/db + /tmp/bin/mongod --replSet rs0 --bind_ip 127.0.0.1 --port 27017 --dbpath /tmp/db --fork --logpath /tmp/mongod.log + # --no-sync: a sync would reconcile the lowest-direct dependency + # versions installed above back to the lockfile. + uv run --no-sync python -c " + from pymongo import MongoClient + client = MongoClient('127.0.0.1:27017', directConnection=True) + client.admin.command('replSetInitiate', {'_id': 'rs0', 'members': [{'_id': 0, 'host': '127.0.0.1:27017'}]}) + MongoClient().admin.command('hello') + " + - name: Run standalone tests + run: | + just setup-tests mod_wsgi standalone + just run-tests + - name: Teardown standalone + if: always() + run: just teardown-tests + - name: Run embedded tests + run: | + just setup-tests mod_wsgi embedded + just run-tests + - name: Teardown embedded + if: always() + run: just teardown-tests + + mod_wsgi_latest: + runs-on: ubuntu-22.04 + name: Mod WSGI Latest + steps: + - uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + - id: versions + uses: $/.github/actions/mod-wsgi-versions + with: + index: 1 + - name: Install Python tooling + uses: mongodb-labs/drivers-github-tools/python/setup@f137fdd28483af14ebf466ebc5aa789fbf867218 # v3.0.5 + with: + python-version: ${{ steps.versions.outputs.python-version }} + - name: Install Apache + run: sudo apt-get install -y apache2 apache2-dev + - name: Install dependencies + run: | + export PYMONGO_C_EXT_MUST_BUILD=1 + just install + # just install skips the sync when uv is already set up. + uv sync --refresh-package pymongo + uv pip install -r requirements/mod_wsgi.txt + uv run --no-sync python tools/fail_if_no_c.py + - id: setup-mongodb + uses: mongodb-labs/drivers-evergreen-tools@354a81692bfd2b654a38277b681d248507d17800 # v1.2.0 + with: + version: ${{ steps.versions.outputs.mongodb-version }} + topology: "replica_set" + - name: Run standalone tests + run: | + just setup-tests mod_wsgi standalone + just run-tests + - name: Teardown standalone + if: always() + run: just teardown-tests + - name: Run embedded tests + run: | + just setup-tests mod_wsgi embedded + just run-tests + - name: Teardown embedded + if: always() + run: just teardown-tests diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 5e0d89f7a4..8a33ec381c 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -85,7 +85,7 @@ jobs: # The beta Python here relies on the action's prerelease default. python-version: ${{ matrix.python-version }} - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@354a81692bfd2b654a38277b681d248507d17800 # v1.2.0 with: version: "${{ matrix.mongodb-version }}" - name: Run tests @@ -101,12 +101,13 @@ jobs: - uses: actions/checkout@v7.0.1 with: persist-credentials: false + submodules: true - name: Install Python tooling uses: mongodb-labs/drivers-github-tools/python/setup@f137fdd28483af14ebf466ebc5aa789fbf867218 # v3.0.5 with: python-version: "3.10" - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@354a81692bfd2b654a38277b681d248507d17800 # v1.2.0 with: version: "8.0" - name: Setup tests @@ -116,7 +117,7 @@ jobs: - name: Generate xml report run: uv tool run --with "coverage[toml]" coverage xml - name: Upload test results to Codecov - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: token: ${{ secrets.CODECOV_TOKEN }} doctest: @@ -126,6 +127,7 @@ jobs: - uses: actions/checkout@v7.0.1 with: persist-credentials: false + submodules: true - name: Install Python tooling uses: mongodb-labs/drivers-github-tools/python/setup@f137fdd28483af14ebf466ebc5aa789fbf867218 # v3.0.5 with: @@ -133,7 +135,7 @@ jobs: - name: Install dependencies run: just install - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@354a81692bfd2b654a38277b681d248507d17800 # v1.2.0 with: version: "8.0" - name: Run tests @@ -184,6 +186,7 @@ jobs: - uses: actions/checkout@v7.0.1 with: persist-credentials: false + submodules: true - name: Install Python tooling uses: mongodb-labs/drivers-github-tools/python/setup@f137fdd28483af14ebf466ebc5aa789fbf867218 # v3.0.5 with: @@ -191,12 +194,12 @@ jobs: - name: Install dependencies run: just install - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@354a81692bfd2b654a38277b681d248507d17800 # v1.2.0 - name: Run tests run: | just integration-tests - id: setup-mongodb-ssl - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@354a81692bfd2b654a38277b681d248507d17800 # v1.2.0 with: ssl: true - name: Run tests @@ -251,7 +254,7 @@ jobs: # Test sdist on lowest supported Python python-version: "3.9" - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@354a81692bfd2b654a38277b681d248507d17800 # v1.2.0 - name: Run connect test from sdist shell: bash run: | @@ -275,7 +278,7 @@ jobs: with: python-version: "3.9" - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@354a81692bfd2b654a38277b681d248507d17800 # v1.2.0 with: version: "8.0" - name: Run tests diff --git a/.github/zizmor.yml b/.github/zizmor.yml index af9f715dd4..2a69f0dd1a 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -4,4 +4,4 @@ rules: policies: actions/*: ref-pin mongodb-labs/drivers-github-tools/*: hash-pin - mongodb-labs/drivers-evergreen-tools: ref-pin + mongodb-labs/drivers-evergreen-tools: hash-pin diff --git a/.gitignore b/.gitignore index 0c29bdbd48..f65211826f 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,6 @@ build/ doc/_build/ dist/ tools/settings.py -drivers-evergreen-tools pymongo.egg-info/ *.so *.egg* @@ -48,3 +47,10 @@ xunit-results/ coverage.xml server.log .coverage +test-results.json +access_log +error_log + +# AI-assistant review workflow logs (kept untracked) +/REVIEW_STATE.md +/REVIEW.md diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..d89986aa20 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "drivers-evergreen-tools"] + path = drivers-evergreen-tools + url = https://github.com/mongodb-labs/drivers-evergreen-tools.git diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 20b7210736..7f24a3aca3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -203,10 +203,15 @@ the pages will re-render and the browser will automatically refresh. `just install` installs the pinned version of `uv` (from `[tool.uv] required-version`) into `$HOME/.local/bin`, and adds that directory to your shell rc file when missing, so the pinned `uv` takes effect in new shells. If a project `uv` command (e.g. `just test`) runs with a different `uv` version, `uv` fails fast and tells you how to update. + Scripts inside the vendored `drivers-evergreen-tools` submodule are exempt: the setup scripts write a `uv.toml` + boundary file into that checkout (see `.evergreen/scripts/configure-env.sh`), which stops uv's config discovery + there, since the tools' scripts run uv versions this project does not pin. - Ensure you have started the appropriate Mongo Server(s). You can run `just run-server` with optional args to set up the server. All given options will be passed to - [`run-mongodb.sh`](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-mongodb.sh). Run `$DRIVERS_TOOLS/.evergreen/run-mongodb.sh start -h` - for a full list of options. + [`run-mongodb.sh`](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-mongodb.sh). Run `${DRIVERS_TOOLS:-drivers-evergreen-tools}/.evergreen/run-mongodb.sh start -h` + for a full list of options. By default the newest stable server release is used. To test against the + nightly build instead, pass `--version latest`, which is downloaded from a private S3 bucket and requires + an `AWS_PROFILE` with [Drivers test secrets](https://github.com/mongodb-labs/drivers-evergreen-tools/tree/master/.evergreen/secrets_handling#secrets-handling) credentials. - Run `just test` or `pytest` to run all of the tests. - Append `test/.py::::` to run specific tests. You can omit the `` to test a full class @@ -220,10 +225,10 @@ the pages will re-render and the browser will automatically refresh. ### Prerequisites -- Clone `drivers-evergreen-tools`: - `git clone git@github.com:mongodb-labs/drivers-evergreen-tools.git`. -- Run `export DRIVERS_TOOLS=$PWD/drivers-evergreen-tools`. This can be put into a `.bashrc` file - for convenience. +- The `drivers-evergreen-tools` submodule (see + [The drivers-evergreen-tools submodule](#the-drivers-evergreen-tools-submodule)). + `just install` initializes it; no manual clone or `export DRIVERS_TOOLS` is needed. + `DRIVERS_TOOLS` remains an optional override for an existing local checkout. - Some tests require access to [Drivers test secrets](https://github.com/mongodb-labs/drivers-evergreen-tools/tree/master/.evergreen/secrets_handling#secrets-handling). ### Usage @@ -353,10 +358,28 @@ You will need to set up access to the `drivers-test-secrets-role`, see the [Wiki - Run `just setup-tests aws_lambda`. - Run `just run-tests`. +### mod_wsgi tests + +Continuous integration runs the tests on pull requests that change +mod_wsgi-relevant files, in the `test-mod-wsgi.yml` workflow. + +To run the tests by hand, install Apache and mod_wsgi (`sudo apt-get install -y apache2 +apache2-dev` and `uv pip install -r requirements/mod_wsgi.txt` on Ubuntu), then: + +- On Linux, run `TOPOLOGY=replica_set just run-server`. +- Run `just setup-tests mod_wsgi `. +- Run `just run-tests`. +- Run `just teardown-tests`. + +The `mode` can be `standalone` or `embedded`. On non-Linux hosts the same +commands run inside an ubuntu container, which bootstraps its own replica +set (so the `run-server` step does not apply), or use `just smoke-mod-wsgi` +to run both modes. + ### OCSP tests - Export the orchestration file, e.g. `export ORCHESTRATION_FILE=rsa-basic-tls-ocsp-disableStapling.json`. -This corresponds to a config file in `$DRIVERS_TOOLS/.evergreen/orchestration/configs/servers`. +This corresponds to a config file in `${DRIVERS_TOOLS:-drivers-evergreen-tools}/.evergreen/orchestration/configs/servers`. MongoDB servers on MacOS and Windows do not staple OCSP responses and only support RSA. NOTE: because the mock ocsp responder MUST be started prior to the server starting, the ocsp tests start the server as part of `setup-tests`. @@ -372,6 +395,44 @@ If you are running one of the `no-responder` tests, omit the `run-server` step. - Set up the tests with `sync` or `async`: `just setup-tests perf sync`. - Run the tests: `just run-tests`. +## The drivers-evergreen-tools submodule + +The `drivers-evergreen-tools` repository is consumed as a git submodule at the repo root, +pinned to a specific commit. Dependabot bumps the pin weekly. + +### Daily flow + +Nothing to do: `just install` initializes the submodule, and Dependabot keeps it fresh. + +### Manually bumping the submodule + +```bash +git -C drivers-evergreen-tools fetch --tags +git -C drivers-evergreen-tools checkout vX.Y.Z +git add drivers-evergreen-tools +``` + +### Evergreen patches that need tools changes + +Point the submodule at the needed commit and commit the new gitlink in the patch branch; +`configure-env.sh` checks out the recorded SHA on Evergreen hosts (`setup-dev-env.sh` does +the same for local checkouts with `just install`). + +### Using a local checkout instead + +Set `DRIVERS_TOOLS` to the path of a local clone — the environment variable wins over the +submodule default: + +```bash +export DRIVERS_TOOLS=/path/to/drivers-evergreen-tools +``` + +Alternatively, keep a local pin from being reset by `git submodule update`: + +```bash +git config submodule.drivers-evergreen-tools.update none +``` + ## Enable Debug Logs - Use `-o log_cli_level="DEBUG" -o log_cli=1` with `just test` or `pytest` to output all debug logs to the terminal. **Warning**: This will output a huge amount of logs. @@ -403,19 +464,21 @@ To run any of the test suites with minimum supported dependencies, pass `--test- - If there are any services or atlas clusters to teardown, handle them in `.evergreen/scripts/teardown_tests.py`. - Add functions to generate the test variant(s) and task(s) to the `.evergreen/scripts/generate_config.py`. - There are some considerations about the Python version used in the test: - - If a specific version of Python is needed in a task that is running on variants with a toolchain, use -``TOOLCHAIN_VERSION`` (e.g. `TOOLCHAIN_VERSION=3.10`). The actual path lookup needs to be done on the host, since -tasks are host-agnostic. + - To request a specific Python, set `UV_PYTHON` (e.g. `UV_PYTHON=3.10`, `UV_PYTHON=3.14t`, or +`UV_PYTHON=pypy3.11`). Tasks are host-agnostic, so the interpreter lookup happens on the host: for a plain +CPython version whose toolchain dir exists, `UV_PYTHON_SEARCH_PATH` points uv at it, and for anything else +(including PyPy, or a version the toolchain lacks) uv downloads it. - If a specific Python binary is needed (for example on the FIPS host), set `UV_PYTHON=/path/to/python`. - - If a specific Python version is needed and the toolchain will not be available, use `UV_PYTHON` (e.g. `UV_PYTHON=3.11`). - - The default if neither ``TOOLCHAIN_VERSION`` or ``UV_PYTHON`` is set is to use UV to install the minimum - supported version of Python and use that. This ensures a consistent behavior across host types that do not - have the Python toolchain (e.g. Azure VMs), by having a known version of Python with the build headers (`Python.h`) - needed to build the C extensions. + - The default if `UV_PYTHON` is not set is CPython 3.10. This is deterministic across host types, so a task + that does not pin a version always gets the same Python (with the build headers (`Python.h`) needed to build + the C extensions). - The uv binary version is pinned once in `[tool.uv] required-version` in `pyproject.toml`. `.evergreen/scripts/install-dependencies.sh` installs it with `uv tool install`, uv enforces it locally, and `astral-sh/setup-uv` reads it on GitHub. Bump it manually when a newer uv is needed. If uv cannot find the - requested Python, it installs it; if that fails, the task fails. + requested Python, it installs it; if that fails, the task fails. The pin is not enforced for scripts inside + the vendored `drivers-evergreen-tools` submodule: those scripts run uv versions this project does not pin, so + the setup scripts write a `uv.toml` boundary into the submodule checkout that stops uv's config discovery + (and with it the required-version check) at the submodule boundary. - Regenerate the test variants and tasks using `pre-commit run --all-files generate-config`. - Make sure to add instructions for running the test suite to `CONTRIBUTING.md`. diff --git a/doc/changelog.rst b/doc/changelog.rst index dde6388937..6cc34f5981 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -16,6 +16,19 @@ PyMongo 4.19 brings a number of changes including: interpreter remain daemon threads and shutdown behavior is unchanged. Note that because these threads are non-daemon, a subinterpreter may block on teardown until any in-flight monitor work completes. +- Added the ``srv_host_validator`` keyword argument to + :class:`~pymongo.synchronous.mongo_client.MongoClient` and + :class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient`, an alternative to + ``srvAllowedHostsSuffix`` for deployments whose acceptable SRV hosts cannot be + expressed as a single suffix. The callback is invoked once per SRV-returned + host and returns ``True`` to accept it. It is mutually exclusive with + ``srvAllowedHostsSuffix`` and, because it takes a callable, cannot be set in a + connection string. See the + :class:`~pymongo.synchronous.mongo_client.MongoClient` and + :class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient` documentation for + security considerations. +- ``srvAllowedHostsSuffix`` may now be set to a single label reserved for + private or special use, such as ``localhost``, ``test``, or ``internal``. Bug fixes ......... @@ -23,8 +36,25 @@ Bug fixes - Fixed a bug where the synchronous client could permanently deadlock under gevent when a greenlet was killed while checking a connection back into the pool (`PYTHON-6074`_). +- ``MongoClient.append_metadata()`` and ``AsyncMongoClient.append_metadata()`` + now detect duplicates by comparing the whole + :class:`~pymongo.driver_info.DriverInfo` instead of only its name. The + comparison is exact, so drivers that differ in name case or platform are no + longer treated as duplicates (`PYTHON-6040`_). +- ``driver.name`` and ``driver.version`` in the handshake metadata are now + ``|``-delimited lists with 1:1 index correspondence, including empty version + entries for the built-in ``|c`` and ``|async`` name segments + (`PYTHON-6040`_). +- Fixed a bug where truncating the handshake metadata to 512 bytes could leave + ``driver.name`` and ``driver.version`` with different numbers of ``|`` + delimiters (`PYTHON-6040`_). +- :class:`~pymongo.driver_info.DriverInfo` now raises :class:`ValueError` when + any field contains the reserved ``|`` delimiter (`PYTHON-6040`_). +- Fixed a bug in SRV polling where invalid hosts where topology would not be + updated if one returned host was invalid. .. _PYTHON-6074: https://jira.mongodb.org/browse/PYTHON-6074 +.. _PYTHON-6040: https://jira.mongodb.org/browse/PYTHON-6040 Changes in Version 4.18.2 (2026/09/24) -------------------------------------- diff --git a/drivers-evergreen-tools b/drivers-evergreen-tools new file mode 160000 index 0000000000..354a81692b --- /dev/null +++ b/drivers-evergreen-tools @@ -0,0 +1 @@ +Subproject commit 354a81692bfd2b654a38277b681d248507d17800 diff --git a/justfile b/justfile index b6156fe72c..328c4fda8e 100644 --- a/justfile +++ b/justfile @@ -12,9 +12,12 @@ mypy_args := "--install-types --non-interactive" default: @just --list +# Restore the default development environment. Set JUST_NO_RESYNC to skip the +# sync when a caller resolves dependencies differently, such as the +# minimum-dependencies job in test-mod-wsgi.yml. [private] resync: - @uv sync --quiet + @if [ -z "${JUST_NO_RESYNC:-}" ]; then uv sync --quiet; fi # Set up the development environment install: @@ -85,17 +88,17 @@ test-numpy *args="": && resync # Run tests via the Evergreen test runner script [group('test')] run-tests *args: && resync - bash ./.evergreen/run-tests.sh {{args}} + bash .evergreen/scripts/dispatch.sh run-tests {{args}} # Set up the test environment (auth, TLS, etc.) [group('test')] setup-tests *args="": - bash .evergreen/scripts/setup-tests.sh {{args}} + bash .evergreen/scripts/dispatch.sh setup-tests {{args}} # Tear down resources created by setup-tests [group('test')] -teardown-tests: - bash .evergreen/scripts/teardown-tests.sh +teardown-tests *args="": + bash .evergreen/scripts/dispatch.sh teardown-tests {{args}} [group('test')] integration-tests: @@ -124,6 +127,29 @@ coverage-xml: uv tool run --with "coverage[toml]" coverage xml @echo "Coverage report generated in coverage.xml" +# Run the mod_wsgi tests (natively on Linux, else in an ubuntu container) +[group('test')] +smoke-mod-wsgi: + #!/usr/bin/env bash + set -euo pipefail + STATUS=0 + # Tear down on an early exit (a failed setup or teardown), so a partially + # started Apache or container and the dispatch marker never outlive the + # recipe; the per-mode teardown below clears the guard. + CLEANED=1 + cleanup() { + [ "$CLEANED" -eq 1 ] || just teardown-tests || true + } + trap cleanup EXIT + for mode in standalone embedded; do + CLEANED=0 + just setup-tests mod_wsgi $mode + just run-tests || STATUS=$? + just teardown-tests + CLEANED=1 + [ "$STATUS" -eq 0 ] || exit "$STATUS" + done + # Start a MongoDB server via drivers-evergreen-tools [group('server')] run-server *args="": diff --git a/pymongo/_client_bulk_shared.py b/pymongo/_client_bulk_shared.py index f4f7f8fe7f..0019d0c0bb 100644 --- a/pymongo/_client_bulk_shared.py +++ b/pymongo/_client_bulk_shared.py @@ -18,13 +18,22 @@ from __future__ import annotations from collections.abc import Mapping, MutableMapping -from typing import TYPE_CHECKING, Any, NoReturn +from typing import TYPE_CHECKING, Any, NoReturn, Optional, Union +from bson.objectid import ObjectId +from bson.raw_bson import RawBSONDocument +from pymongo.common import ( + validate_is_document_type, + validate_ok_for_replace, + validate_ok_for_update, +) from pymongo.errors import ClientBulkWriteException, OperationFailure from pymongo.helpers_shared import _get_wce_doc +from pymongo.message import _ClientBulkWriteContext +from pymongo.write_concern import WriteConcern if TYPE_CHECKING: - from pymongo.typings import _DocumentOut + from pymongo.typings import _AgnosticMongoClient, _DocumentOut, _Pipeline def _merge_command( @@ -79,3 +88,145 @@ def _throw_client_bulk_write_exception( if isinstance(full_result["error"], BaseException): raise ClientBulkWriteException(full_result, verbose_results) from full_result["error"] raise ClientBulkWriteException(full_result, verbose_results) + + +class _AgnosticClientBulkBase: + """The private guts of the client-level bulk write API.""" + + def __init__( + self, + client: _AgnosticMongoClient, + write_concern: WriteConcern, + ordered: bool = True, + bypass_document_validation: Optional[bool] = None, + comment: Optional[str] = None, + let: Optional[Any] = None, + verbose_results: bool = False, + ) -> None: + """Initialize a _AgnosticClientBulkBase instance.""" + self.client = client + self.write_concern = write_concern + self.let = let + if self.let is not None: + validate_is_document_type("let", self.let) + self.ordered = ordered + self.bypass_doc_val = bypass_document_validation + self.comment = comment + self.verbose_results = verbose_results + self.ops: list[tuple[str, Mapping[str, Any]]] = [] + self.namespaces: list[str] = [] + self.idx_offset: int = 0 + self.total_ops: int = 0 + self.executed = False + self.uses_collation = False + self.uses_array_filters = False + self.is_retryable = self.client.options.retry_writes + self.retrying = False + self.started_retryable_write = False + + @property + def bulk_ctx_class(self) -> type[_ClientBulkWriteContext]: + return _ClientBulkWriteContext + + def add_insert(self, namespace: str, document: _DocumentOut) -> None: + """Add an insert document to the list of ops.""" + validate_is_document_type("document", document) + # Generate ObjectId client side. + if not (isinstance(document, RawBSONDocument) or "_id" in document): + document["_id"] = ObjectId() + cmd = {"insert": -1, "document": document} + self.ops.append(("insert", cmd)) + self.namespaces.append(namespace) + self.total_ops += 1 + + def add_update( + self, + namespace: str, + selector: Mapping[str, Any], + update: Union[Mapping[str, Any], _Pipeline], + multi: bool, + upsert: Optional[bool] = None, + collation: Optional[Mapping[str, Any]] = None, + array_filters: Optional[list[Mapping[str, Any]]] = None, + hint: Union[str, dict[str, Any], None] = None, + sort: Optional[Mapping[str, Any]] = None, + ) -> None: + """Create an update document and add it to the list of ops.""" + validate_ok_for_update(update) + cmd = { + "update": -1, + "filter": selector, + "updateMods": update, + "multi": multi, + } + if upsert is not None: + cmd["upsert"] = upsert + if array_filters is not None: + self.uses_array_filters = True + cmd["arrayFilters"] = array_filters + if hint is not None: + cmd["hint"] = hint + if collation is not None: + self.uses_collation = True + cmd["collation"] = collation + if sort is not None: + cmd["sort"] = sort + if multi: + # A bulk_write containing an update_many is not retryable. + self.is_retryable = False + self.ops.append(("update", cmd)) + self.namespaces.append(namespace) + self.total_ops += 1 + + def add_replace( + self, + namespace: str, + selector: Mapping[str, Any], + replacement: Mapping[str, Any], + upsert: Optional[bool] = None, + collation: Optional[Mapping[str, Any]] = None, + hint: Union[str, dict[str, Any], None] = None, + sort: Optional[Mapping[str, Any]] = None, + ) -> None: + """Create a replace document and add it to the list of ops.""" + validate_ok_for_replace(replacement) + cmd = { + "update": -1, + "filter": selector, + "updateMods": replacement, + "multi": False, + } + if upsert is not None: + cmd["upsert"] = upsert + if hint is not None: + cmd["hint"] = hint + if collation is not None: + self.uses_collation = True + cmd["collation"] = collation + if sort is not None: + cmd["sort"] = sort + self.ops.append(("replace", cmd)) + self.namespaces.append(namespace) + self.total_ops += 1 + + def add_delete( + self, + namespace: str, + selector: Mapping[str, Any], + multi: bool, + collation: Optional[Mapping[str, Any]] = None, + hint: Union[str, dict[str, Any], None] = None, + ) -> None: + """Create a delete document and add it to the list of ops.""" + cmd = {"delete": -1, "filter": selector, "multi": multi} + if hint is not None: + cmd["hint"] = hint + if collation is not None: + self.uses_collation = True + cmd["collation"] = collation + if multi: + # A bulk_write containing a delete_many is not retryable. + self.is_retryable = False + self.ops.append(("delete", cmd)) + self.namespaces.append(namespace) + self.total_ops += 1 diff --git a/pymongo/_psl.py b/pymongo/_psl.py index b8f11f5bd7..e0130188ff 100644 --- a/pymongo/_psl.py +++ b/pymongo/_psl.py @@ -21,6 +21,26 @@ _PUBLIC_SUFFIXES: Optional[tuple[set[str], set[str], set[str]]] = None +# Single labels that srvAllowedHostsSuffix may be set to. See the +# srvAllowedHostsSuffix section of the Initial DNS Seedlist Discovery spec. +SPECIAL_USE_LABELS = frozenset( + [ + # RFC 6761 special use names. + "test", + "localhost", + "invalid", + "example", + # RFC 6762 multicast DNS. + "local", + # Reserved by ICANN for private use. + "internal", + # Not officially reserved by ICANN but commonly used privately. + "corp", + "home", + "mail", + ] +) + def _to_punycode(string: str) -> str: """Convert a string to Punycode.""" @@ -28,7 +48,7 @@ def _to_punycode(string: str) -> str: return string.encode("idna").decode("ascii") except UnicodeError: # Not every input is a valid IDNA label (e.g. empty or over-long - # labels). Leave it as-is and let the caller reject it. + # labels). Return it unchanged -- SRV host will fail verification. return string diff --git a/pymongo/_telemetry.py b/pymongo/_telemetry.py index e6821ef6a9..156352d1e3 100644 --- a/pymongo/_telemetry.py +++ b/pymongo/_telemetry.py @@ -776,6 +776,17 @@ def log_srv_monitor_failure(failure: Exception) -> None: _debug_log(_SDAM_LOGGER, message="SRV monitor check failed", failure=repr(failure)) +def log_srv_monitor_invalid_host(host: str, failure: Exception) -> None: + """Emit a log entry when the SRV monitor skips a host that failed verification.""" + if _is_debug_enabled(_SDAM_LOGGER): + _debug_log( + _SDAM_LOGGER, + message="SRV monitor skipped invalid host", + serverHost=host, + failure=repr(failure), + ) + + def log_command_retry( topology_id: Optional[ObjectId], command_name: str, diff --git a/pymongo/asynchronous/bulk.py b/pymongo/asynchronous/bulk.py index 86b9528b6e..81186499f6 100644 --- a/pymongo/asynchronous/bulk.py +++ b/pymongo/asynchronous/bulk.py @@ -29,9 +29,7 @@ Union, ) -from bson.objectid import ObjectId -from bson.raw_bson import RawBSONDocument -from pymongo import _csot, common +from pymongo import _csot from pymongo._telemetry import _generate_op_id_or_none from pymongo.asynchronous.client_session import AsyncClientSession from pymongo.asynchronous.command_runner import ( @@ -40,17 +38,11 @@ from pymongo.asynchronous.helpers import _handle_reauth from pymongo.bulk_shared import ( _COMMANDS, - _DELETE_ALL, + _AgnosticBulkBase, _merge_command, _raise_bulk_write_error, - _Run, ) from pymongo.client_session_shared import _validate_session_write_concern -from pymongo.common import ( - validate_is_document_type, - validate_ok_for_replace, - validate_ok_for_update, -) from pymongo.errors import ( ConfigurationError, InvalidOperation, @@ -59,7 +51,6 @@ from pymongo.helpers_shared import _RETRYABLE_ERROR_CODES from pymongo.message import ( _DELETE, - _INSERT, _UPDATE, _BulkWriteContext, _EncryptedBulkWriteContext, @@ -71,165 +62,14 @@ from pymongo.asynchronous.collection import AsyncCollection from pymongo.asynchronous.mongo_client import AsyncMongoClient from pymongo.asynchronous.pool import AsyncConnection - from pymongo.typings import _DocumentOut, _DocumentType, _Pipeline _IS_SYNC = False -class _AsyncBulk: +class _AsyncBulk(_AgnosticBulkBase): """The private guts of the bulk write API.""" - def __init__( - self, - collection: AsyncCollection[_DocumentType], - ordered: bool, - bypass_document_validation: Optional[bool], - comment: Optional[str] = None, - let: Optional[Any] = None, - ) -> None: - """Initialize a _AsyncBulk instance.""" - self.collection = collection.with_options( - codec_options=collection.codec_options._replace( - unicode_decode_error_handler="replace", document_class=dict - ) - ) - self.let = let - if self.let is not None: - common.validate_is_document_type("let", self.let) - self.comment: Optional[str] = comment - self.ordered = ordered - self.ops: list[tuple[int, Mapping[str, Any]]] = [] - self.executed = False - self.bypass_doc_val = bypass_document_validation - self.uses_collation = False - self.uses_array_filters = False - self.uses_sort = False - self.is_retryable = True - self.retrying = False - self.started_retryable_write = False - # Extra state so that we know where to pick up on a retry attempt. - self.current_run = None - self.next_run = None - self.is_encrypted = False - - @property - def bulk_ctx_class(self) -> type[_BulkWriteContext]: - encrypter = self.collection.database.client._encrypter - if encrypter and not encrypter._bypass_auto_encryption: - self.is_encrypted = True - return _EncryptedBulkWriteContext - else: - self.is_encrypted = False - return _BulkWriteContext - - def add_insert(self, document: _DocumentOut) -> None: - """Add an insert document to the list of ops.""" - validate_is_document_type("document", document) - # Generate ObjectId client side. - if not (isinstance(document, RawBSONDocument) or "_id" in document): - document["_id"] = ObjectId() - self.ops.append((_INSERT, document)) - - def add_update( - self, - selector: Mapping[str, Any], - update: Union[Mapping[str, Any], _Pipeline], - multi: bool, - upsert: Optional[bool], - collation: Optional[Mapping[str, Any]] = None, - array_filters: Optional[list[Mapping[str, Any]]] = None, - hint: Union[str, dict[str, Any], None] = None, - sort: Optional[Mapping[str, Any]] = None, - ) -> None: - """Create an update document and add it to the list of ops.""" - validate_ok_for_update(update) - cmd: dict[str, Any] = {"q": selector, "u": update, "multi": multi} - if upsert is not None: - cmd["upsert"] = upsert - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if array_filters is not None: - self.uses_array_filters = True - cmd["arrayFilters"] = array_filters - if hint is not None: - cmd["hint"] = hint - if sort is not None: - self.uses_sort = True - cmd["sort"] = sort - if multi: - # A bulk_write containing an update_many is not retryable. - self.is_retryable = False - self.ops.append((_UPDATE, cmd)) - - def add_replace( - self, - selector: Mapping[str, Any], - replacement: Mapping[str, Any], - upsert: Optional[bool], - collation: Optional[Mapping[str, Any]] = None, - hint: Union[str, dict[str, Any], None] = None, - sort: Optional[Mapping[str, Any]] = None, - ) -> None: - """Create a replace document and add it to the list of ops.""" - validate_ok_for_replace(replacement) - cmd: dict[str, Any] = {"q": selector, "u": replacement} - if upsert is not None: - cmd["upsert"] = upsert - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if hint is not None: - cmd["hint"] = hint - if sort is not None: - self.uses_sort = True - cmd["sort"] = sort - self.ops.append((_UPDATE, cmd)) - - def add_delete( - self, - selector: Mapping[str, Any], - limit: int, - collation: Optional[Mapping[str, Any]] = None, - hint: Union[str, dict[str, Any], None] = None, - ) -> None: - """Create a delete document and add it to the list of ops.""" - cmd: dict[str, Any] = {"q": selector, "limit": limit} - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if hint is not None: - cmd["hint"] = hint - if limit == _DELETE_ALL: - # A bulk_write containing a delete_many is not retryable. - self.is_retryable = False - self.ops.append((_DELETE, cmd)) - - def gen_ordered(self) -> Iterator[Optional[_Run]]: - """Generate batches of operations, batched by type of - operation, in the order **provided**. - """ - run = None - for idx, (op_type, operation) in enumerate(self.ops): - if run is None: - run = _Run(op_type) - elif run.op_type != op_type: - yield run - run = _Run(op_type) - run.add(idx, operation) - yield run - - def gen_unordered(self) -> Iterator[_Run]: - """Generate batches of operations, batched by type of - operation, in arbitrary order. - """ - operations = [_Run(_INSERT), _Run(_UPDATE), _Run(_DELETE)] - for idx, (op_type, operation) in enumerate(self.ops): - operations[op_type].add(idx, operation) - - for run in operations: - if run.ops: - yield run + collection: AsyncCollection[Any] @_handle_reauth async def write_command( diff --git a/pymongo/asynchronous/client_bulk.py b/pymongo/asynchronous/client_bulk.py index 2847dcb2c0..bfbe9e2c6b 100644 --- a/pymongo/asynchronous/client_bulk.py +++ b/pymongo/asynchronous/client_bulk.py @@ -29,9 +29,7 @@ Union, ) -from bson.objectid import ObjectId -from bson.raw_bson import RawBSONDocument -from pymongo import _csot, common +from pymongo import _csot from pymongo._telemetry import _generate_op_id_or_none from pymongo.asynchronous.client_session import AsyncClientSession from pymongo.asynchronous.collection import AsyncCollection @@ -46,15 +44,11 @@ from pymongo.asynchronous.mongo_client import AsyncMongoClient from pymongo.asynchronous.pool import AsyncConnection from pymongo._client_bulk_shared import ( + _AgnosticClientBulkBase, _merge_command, _throw_client_bulk_write_exception, ) from pymongo.client_session_shared import _validate_session_write_concern -from pymongo.common import ( - validate_is_document_type, - validate_ok_for_replace, - validate_ok_for_update, -) from pymongo.errors import ( ConfigurationError, ConnectionFailure, @@ -76,152 +70,15 @@ InsertOneResult, UpdateResult, ) -from pymongo.typings import _DocumentOut, _Pipeline from pymongo.write_concern import WriteConcern _IS_SYNC = False -class _AsyncClientBulk: +class _AsyncClientBulk(_AgnosticClientBulkBase): """The private guts of the client-level bulk write API.""" - def __init__( - self, - client: AsyncMongoClient[Any], - write_concern: WriteConcern, - ordered: bool = True, - bypass_document_validation: Optional[bool] = None, - comment: Optional[str] = None, - let: Optional[Any] = None, - verbose_results: bool = False, - ) -> None: - """Initialize a _AsyncClientBulk instance.""" - self.client = client - self.write_concern = write_concern - self.let = let - if self.let is not None: - common.validate_is_document_type("let", self.let) - self.ordered = ordered - self.bypass_doc_val = bypass_document_validation - self.comment = comment - self.verbose_results = verbose_results - self.ops: list[tuple[str, Mapping[str, Any]]] = [] - self.namespaces: list[str] = [] - self.idx_offset: int = 0 - self.total_ops: int = 0 - self.executed = False - self.uses_collation = False - self.uses_array_filters = False - self.is_retryable = self.client.options.retry_writes - self.retrying = False - self.started_retryable_write = False - - @property - def bulk_ctx_class(self) -> type[_ClientBulkWriteContext]: - return _ClientBulkWriteContext - - def add_insert(self, namespace: str, document: _DocumentOut) -> None: - """Add an insert document to the list of ops.""" - validate_is_document_type("document", document) - # Generate ObjectId client side. - if not (isinstance(document, RawBSONDocument) or "_id" in document): - document["_id"] = ObjectId() - cmd = {"insert": -1, "document": document} - self.ops.append(("insert", cmd)) - self.namespaces.append(namespace) - self.total_ops += 1 - - def add_update( - self, - namespace: str, - selector: Mapping[str, Any], - update: Union[Mapping[str, Any], _Pipeline], - multi: bool, - upsert: Optional[bool] = None, - collation: Optional[Mapping[str, Any]] = None, - array_filters: Optional[list[Mapping[str, Any]]] = None, - hint: Union[str, dict[str, Any], None] = None, - sort: Optional[Mapping[str, Any]] = None, - ) -> None: - """Create an update document and add it to the list of ops.""" - validate_ok_for_update(update) - cmd = { - "update": -1, - "filter": selector, - "updateMods": update, - "multi": multi, - } - if upsert is not None: - cmd["upsert"] = upsert - if array_filters is not None: - self.uses_array_filters = True - cmd["arrayFilters"] = array_filters - if hint is not None: - cmd["hint"] = hint - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if sort is not None: - cmd["sort"] = sort - if multi: - # A bulk_write containing an update_many is not retryable. - self.is_retryable = False - self.ops.append(("update", cmd)) - self.namespaces.append(namespace) - self.total_ops += 1 - - def add_replace( - self, - namespace: str, - selector: Mapping[str, Any], - replacement: Mapping[str, Any], - upsert: Optional[bool] = None, - collation: Optional[Mapping[str, Any]] = None, - hint: Union[str, dict[str, Any], None] = None, - sort: Optional[Mapping[str, Any]] = None, - ) -> None: - """Create a replace document and add it to the list of ops.""" - validate_ok_for_replace(replacement) - cmd = { - "update": -1, - "filter": selector, - "updateMods": replacement, - "multi": False, - } - if upsert is not None: - cmd["upsert"] = upsert - if hint is not None: - cmd["hint"] = hint - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if sort is not None: - cmd["sort"] = sort - self.ops.append(("replace", cmd)) - self.namespaces.append(namespace) - self.total_ops += 1 - - def add_delete( - self, - namespace: str, - selector: Mapping[str, Any], - multi: bool, - collation: Optional[Mapping[str, Any]] = None, - hint: Union[str, dict[str, Any], None] = None, - ) -> None: - """Create a delete document and add it to the list of ops.""" - cmd = {"delete": -1, "filter": selector, "multi": multi} - if hint is not None: - cmd["hint"] = hint - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if multi: - # A bulk_write containing an update_many is not retryable. - self.is_retryable = False - self.ops.append(("delete", cmd)) - self.namespaces.append(namespace) - self.total_ops += 1 + client: AsyncMongoClient[Any] @_handle_reauth async def write_command( diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index 5f8f2578f3..3d66dab535 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -470,6 +470,31 @@ def __init__( srvAllowedHostsSuffix=".internal.example.com", ) + - `srv_host_validator`: (callable or None) A callback used in place of the + default parent-domain check for hosts returned by SRV DNS records. It is + called once per returned host with the lowercased hostname as its only + argument, and must return ``True`` to accept the host or ``False`` to + reject it. During initial DNS discovery, rejecting a host raises + :exc:`~pymongo.errors.ConfigurationError`, as does an exception raised by + the callback itself. Use this when the set of acceptable hosts cannot be + expressed as a single suffix:: + + def validator(host: str) -> bool: + return host.endswith((".a.example.com", ".b.example.com")) + + AsyncMongoClient( + "mongodb+srv://cluster.example.com/", + srv_host_validator=validator, + ) + + The callback must not block. It is mutually exclusive with + ``srvAllowedHostsSuffix``. Because this option is a callable, it can + only be passed in as a keyword argument, not through the connection string. + + .. warning:: + + This option replaces the built-in DNS spoofing safeguards. + Please use with caution. | **Write Concern options:** | (Only set if passed. No default values.) @@ -654,6 +679,12 @@ def __init__( .. seealso:: The MongoDB documentation on `connections `_. + .. versionchanged:: 4.19 + Added the ``srv_host_validator`` keyword argument. + + .. versionchanged:: 4.18 + Added the ``srvAllowedHostsSuffix`` URI and keyword argument. + .. versionchanged:: 4.18 Added the ``tracing`` keyword argument. @@ -843,6 +874,7 @@ def __init__( srv_service_name = keyword_opts.get("srvservicename") srv_max_hosts = keyword_opts.get("srvmaxhosts") srv_allowed_hosts_suffix = keyword_opts.get("srvallowedhostssuffix") + srv_host_validator = keyword_opts.get("srv_host_validator") if len([h for h in self._host if "/" in h]) > 1: raise ConfigurationError("host must not contain multiple MongoDB URIs") for entity in self._host: @@ -935,7 +967,11 @@ def __init__( ) self._init_based_on_options( - self._seeds, srv_max_hosts, srv_service_name, srv_allowed_hosts_suffix + self._seeds, + srv_max_hosts, + srv_service_name, + srv_allowed_hosts_suffix, + srv_host_validator, ) self._opened = False @@ -955,6 +991,7 @@ async def _resolve_srv(self) -> None: srv_service_name = keyword_opts.get("srvservicename") srv_max_hosts = keyword_opts.get("srvmaxhosts") srv_allowed_hosts_suffix = keyword_opts.get("srvallowedhostssuffix") + srv_host_validator = keyword_opts.get("srv_host_validator") for entity in self._host: # A hostname can only include a-z, 0-9, '-' and '.'. If we find a '/' # it must be a URI, @@ -976,6 +1013,7 @@ async def _resolve_srv(self) -> None: srv_service_name=srv_service_name, srv_max_hosts=srv_max_hosts, srv_allowed_hosts_suffix=srv_allowed_hosts_suffix, + srv_host_validator=srv_host_validator, ) seeds.update(res["nodelist"]) opts = res["options"] @@ -1020,7 +1058,11 @@ async def _resolve_srv(self) -> None: ) self._init_based_on_options( - seeds, srv_max_hosts, srv_service_name, srv_allowed_hosts_suffix + seeds, + srv_max_hosts, + srv_service_name, + srv_allowed_hosts_suffix, + srv_host_validator, ) def _init_based_on_options( @@ -1029,7 +1071,17 @@ def _init_based_on_options( srv_max_hosts: Any, srv_service_name: Any, srv_allowed_hosts_suffix: Any, + srv_host_validator: Any = None, ) -> None: + if srv_host_validator is not None: + if srv_allowed_hosts_suffix is not None: + raise ConfigurationError( + "Cannot specify both srv_host_validator and srvAllowedHostsSuffix" + ) + if not self._resolve_srv_info["is_srv"]: + raise ConfigurationError( + "The srv_host_validator option is only allowed with 'mongodb+srv://' URIs" + ) self._event_listeners = self._options.pool_options._event_listeners self._topology_settings = TopologySettings( seeds=seeds, @@ -1048,6 +1100,7 @@ def _init_based_on_options( srv_service_name=srv_service_name, srv_max_hosts=srv_max_hosts, srv_allowed_hosts_suffix=srv_allowed_hosts_suffix, + srv_host_validator=srv_host_validator, server_monitoring_mode=self._options.server_monitoring_mode, topology_id=self._topology_settings._topology_id if self._topology_settings else None, ) diff --git a/pymongo/asynchronous/monitor.py b/pymongo/asynchronous/monitor.py index da261e37ea..8ed4078fd7 100644 --- a/pymongo/asynchronous/monitor.py +++ b/pymongo/asynchronous/monitor.py @@ -368,6 +368,7 @@ async def _get_seedlist(self) -> Optional[list[tuple[str, Any]]]: self._settings.pool_options.connect_timeout, self._settings.srv_service_name, srv_allowed_hosts_suffix=self._settings.srv_allowed_hosts_suffix, + srv_host_validator=self._settings.srv_host_validator, ) seedlist, ttl = await resolver.get_hosts_and_min_ttl() if len(seedlist) == 0: diff --git a/pymongo/asynchronous/pool.py b/pymongo/asynchronous/pool.py index 2898cd5ced..f7e408acfc 100644 --- a/pymongo/asynchronous/pool.py +++ b/pymongo/asynchronous/pool.py @@ -106,6 +106,13 @@ _IS_SYNC = False +# Flags recording the counters a checkout has incremented, so a failed +# checkout can restore them exactly once (PYTHON-6136). +_UNDO_OPERATION_COUNT = 1 +_UNDO_REQUESTS = 2 +_UNDO_SOCKETS = 4 +_UNDO_PENDING = 8 + class AsyncConnection(_ConnectionTelemetryInfo): """Store a connection with some metadata. @@ -972,8 +979,12 @@ async def _get_conn( "Attempted to check out a connection from closed connection pool" ) + # Every counter increment sets a flag in ``applied`` so a failed + # checkout can restore them exactly once (PYTHON-6136). + applied = 0 async with self.lock: self.operation_count += 1 + applied |= _UNDO_OPERATION_COUNT # Get a free socket or create one. if _csot.get_timeout(): @@ -983,28 +994,32 @@ async def _get_conn( else: deadline = None - async with self.size_cond: - self._raise_if_not_ready(checkout_started_time, emit_event=True) - while not (self.requests < self.max_pool_size): - timeout = deadline - time.monotonic() if deadline else None - if not await _async_cond_wait(self.size_cond, timeout): - # Timed out, notify the next thread to ensure a - # timeout doesn't consume the condition. - if self.requests < self.max_pool_size: - self.size_cond.notify() - self._raise_wait_queue_timeout(checkout_started_time) + try: + async with self.size_cond: self._raise_if_not_ready(checkout_started_time, emit_event=True) - self.requests += 1 + while not (self.requests < self.max_pool_size): + timeout = deadline - time.monotonic() if deadline else None + if not await _async_cond_wait(self.size_cond, timeout): + # Timed out, notify the next thread to ensure a + # timeout doesn't consume the condition. + if self.requests < self.max_pool_size: + self.size_cond.notify() + self._raise_wait_queue_timeout(checkout_started_time) + self._raise_if_not_ready(checkout_started_time, emit_event=True) + self.requests += 1 + applied |= _UNDO_REQUESTS + except BaseException: + await self._restore_counters(applied) + raise # We've now acquired the semaphore and must release it on error. conn = None - incremented = False emitted_event = False is_new_conn = False try: async with self.lock: self.active_sockets += 1 - incremented = True + applied |= _UNDO_SOCKETS while conn is None: # CMAP: we MUST wait for either maxConnecting OR for a socket # to be checked back into the pool. @@ -1025,6 +1040,7 @@ async def _get_conn( conn = self.conns.popleft() except IndexError: self._pending += 1 + applied |= _UNDO_PENDING if conn: # We got a socket from the pool if await self._perished(conn): conn = None @@ -1036,7 +1052,17 @@ async def _get_conn( finally: async with self._max_connecting_cond: self._pending -= 1 - self._max_connecting_cond.notify() + applied &= ~_UNDO_PENDING + notified = False + try: + self._max_connecting_cond.notify() + notified = True + finally: + if not notified: + # A kill landed inside notify() (a gevent + # yield point); retry so a waiting + # checkout is not stranded (PYTHON-6136). + self._max_connecting_cond.notify() conn.active = True # connect() already adds cancel_context for new connections; only add @@ -1046,27 +1072,14 @@ async def _get_conn( self.active_contexts.add(conn.cancel_context) # Catch KeyboardInterrupt, CancelledError, etc. and cleanup. except BaseException: - if conn: - # We checked out a socket but authentication failed. - await conn.close_conn(ConnectionClosedReason.ERROR) - # Re-apply the accounting if a GreenletExit interrupts - # during the size_cond acquisition; during unwind gevent - # lets the re-acquire complete (PYTHON-6074). - accounted = False try: - async with self.size_cond: - self.requests -= 1 - if incremented: - self.active_sockets -= 1 - accounted = True - self.size_cond.notify() + if conn is not None: + # We checked out a socket but authentication failed. + await conn.close_conn(ConnectionClosedReason.ERROR) finally: - if not accounted: - async with self.size_cond: - self.requests -= 1 - if incremented: - self.active_sockets -= 1 - self.size_cond.notify() + # Restore the counters even if the cleanup above was + # interrupted (PYTHON-6136). + await self._restore_counters(applied) if not emitted_event: self._telemetry.checkout_failed( @@ -1078,6 +1091,51 @@ async def _get_conn( return conn + def _restore_applied(self, applied: int) -> None: + """Restore the counters flagged in ``applied``. Caller holds ``size_cond``.""" + if applied & _UNDO_OPERATION_COUNT: + self.operation_count -= 1 + if applied & _UNDO_REQUESTS: + self.requests -= 1 + if applied & _UNDO_SOCKETS: + self.active_sockets -= 1 + if applied & _UNDO_PENDING: + self._pending -= 1 + + async def _restore_counters(self, applied: int) -> None: + """Restore the counters a failed checkout incremented (PYTHON-6136). + + Gevent grants the re-acquire during unwind (PYTHON-6074). A kill can + also land inside notify() (a yield point), so notifications are + tracked and retried separately from the counter restore. + """ + accounted = False + notified = 0 + try: + async with self.size_cond: + self._restore_applied(applied) + accounted = True + if applied & _UNDO_REQUESTS: + # A pool slot was freed; wake the next waiting thread. + self.size_cond.notify() + notified |= _UNDO_REQUESTS + if applied & _UNDO_PENDING: + # A maxConnecting slot was freed; wake the next waiting thread. + self._max_connecting_cond.notify() + notified |= _UNDO_PENDING + finally: + # Always reacquired: `applied` keeps restore-only flags + # `notified` never tracks, so skipping needs a mask synced + # to the notify sites; uncontended acquires don't yield (PYTHON-6136). + async with self.size_cond: + if not accounted: + self._restore_applied(applied) + missing = applied & ~notified + if missing & _UNDO_REQUESTS: + self.size_cond.notify() + if missing & _UNDO_PENDING: + self._max_connecting_cond.notify() + def _checkin_apply( self, conn: AsyncConnection, txn: bool, cursor: bool, forked: bool ) -> tuple[Optional[str], bool, bool]: @@ -1124,9 +1182,7 @@ async def checkin(self, conn: AsyncConnection) -> None: conn.pinned_cursor = False self._pinned_sockets.discard(conn) forked = self.pid != os.getpid() - # Re-apply the accounting if a gevent GreenletExit interrupts during - # the size_cond acquisition; gevent lets the re-acquire complete while - # unwinding (PYTHON-6074). + # Re-apply the accounting if a BaseException interrupts here (PYTHON-6074). close_conn_reason: Optional[str] = None emit_closed = False accounted = False diff --git a/pymongo/asynchronous/settings.py b/pymongo/asynchronous/settings.py index bdfb334ab7..4bc5f05f28 100644 --- a/pymongo/asynchronous/settings.py +++ b/pymongo/asynchronous/settings.py @@ -18,7 +18,7 @@ import threading from collections.abc import Collection -from typing import Optional +from typing import Callable, Optional from bson.objectid import ObjectId from pymongo import common @@ -51,6 +51,7 @@ def __init__( srv_service_name: str = common.SRV_SERVICE_NAME, srv_max_hosts: int = 0, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, server_monitoring_mode: str = common.SERVER_MONITORING_MODE, topology_id: Optional[ObjectId] = None, ): @@ -77,6 +78,7 @@ def __init__( srv_service_name, srv_max_hosts, srv_allowed_hosts_suffix, + srv_host_validator, server_monitoring_mode, topology_id, ) diff --git a/pymongo/asynchronous/srv_resolver.py b/pymongo/asynchronous/srv_resolver.py index 5a068dbe72..554d226acf 100644 --- a/pymongo/asynchronous/srv_resolver.py +++ b/pymongo/asynchronous/srv_resolver.py @@ -18,9 +18,10 @@ import ipaddress import random -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Optional, Union -from pymongo._psl import is_public_suffix +from pymongo._psl import SPECIAL_USE_LABELS, _to_punycode, is_public_suffix +from pymongo._telemetry import log_srv_monitor_invalid_host from pymongo.common import CONNECT_TIMEOUT from pymongo.errors import ConfigurationError @@ -64,21 +65,40 @@ def __init__( srv_service_name: str, srv_max_hosts: int = 0, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, ): self.__fqdn = fqdn.lower() self.__srv = srv_service_name self.__connect_timeout = connect_timeout or CONNECT_TIMEOUT self.__srv_max_hosts = srv_max_hosts or 0 - self.__srv_allowed_hosts_suffix = ( - "." + srv_allowed_hosts_suffix.lower().strip(".") if srv_allowed_hosts_suffix else None - ) # ensure there's a . at the beginning of the domain - if self.__srv_allowed_hosts_suffix is not None and is_public_suffix( - self.__srv_allowed_hosts_suffix - ): + self.__srv_host_validator = srv_host_validator + # parse_uri() can call this constructor independently of AsyncMongoClient. + if srv_host_validator is not None and srv_allowed_hosts_suffix is not None: raise ConfigurationError( - f"srvAllowedHostsSuffix must not be a public suffix, got: {srv_allowed_hosts_suffix}" + "Cannot specify both srv_host_validator and srvAllowedHostsSuffix" ) - # Validate the fully qualified domain name. + self.__srv_allowed_hosts_suffix = None + if srv_allowed_hosts_suffix is not None: + suffix = srv_allowed_hosts_suffix.strip(".") + if not suffix: + raise ConfigurationError( + f"srvAllowedHostsSuffix must not be empty, got: {srv_allowed_hosts_suffix!r}" + ) + suffix = _to_punycode(suffix).lower() + + is_special_use = suffix in SPECIAL_USE_LABELS + if len(suffix.split(".")) < 2 and not is_special_use: + raise ConfigurationError( + "srvAllowedHostsSuffix must contain at least two '.' separated labels, " + f"got: {srv_allowed_hosts_suffix}" + ) + + if not is_special_use and is_public_suffix(suffix): + raise ConfigurationError( + f"srvAllowedHostsSuffix must not be a public suffix, got: {srv_allowed_hosts_suffix}" + ) + self.__srv_allowed_hosts_suffix = "." + suffix + try: ipaddress.ip_address(fqdn) raise ConfigurationError(_INVALID_HOST_MSG % ("an IP address",)) @@ -106,23 +126,51 @@ async def get_options(self) -> Optional[str]: raise ConfigurationError("Only one TXT record is supported") return (b"&".join([b"".join(res.strings) for res in results])).decode("utf-8") # type: ignore[attr-defined] - async def _resolve_uri(self, encapsulate_errors: bool) -> resolver.Answer: + async def _resolve_uri(self, is_polling: bool) -> resolver.Answer: try: results = await _resolve( "_" + self.__srv + "._tcp." + self.__fqdn, "SRV", lifetime=self.__connect_timeout ) except Exception as exc: - if not encapsulate_errors: + if is_polling: # Raise the original error. raise # Else, raise all errors as ConfigurationError. raise ConfigurationError(str(exc)) from exc return results + def _validate_host(self, srv_host: str) -> None: + """Raise ConfigurationError if an SRV-returned host fails verification.""" + if self.__srv_host_validator is not None: + try: + allowed = self.__srv_host_validator(srv_host) + except Exception as exc: + raise ConfigurationError( + f"srv_host_validator raised an exception for SRV host {srv_host}: {exc}" + ) from exc + if not allowed: + raise ConfigurationError( + f"Invalid SRV host: {srv_host} was rejected by srv_host_validator" + ) + elif self.__srv_allowed_hosts_suffix is not None: + if not srv_host.endswith(self.__srv_allowed_hosts_suffix): + raise ConfigurationError(f"Invalid SRV host: {srv_host}") + else: + if self.__fqdn == srv_host and self.nparts < 3: + raise ConfigurationError( + "Invalid SRV host: return address is identical to SRV hostname" + ) + try: + nlist = srv_host.split(".")[1:][-self.__slen :] + except Exception as exc: + raise ConfigurationError(f"Invalid SRV host: {srv_host}") from exc + if self.__plist != nlist: + raise ConfigurationError(f"Invalid SRV host: {srv_host}") + async def _get_srv_response_and_hosts( - self, encapsulate_errors: bool + self, is_polling: bool ) -> tuple[resolver.Answer, list[tuple[str, Any]]]: - results = await self._resolve_uri(encapsulate_errors) + results = await self._resolve_uri(is_polling) # Construct address tuples nodes = [ @@ -130,33 +178,28 @@ async def _get_srv_response_and_hosts( for res in results ] - # Validate hosts + # Validate hosts. + valid_nodes = [] for node in nodes: - srv_host = node[0].lower() - if self.__fqdn == srv_host and self.nparts < 3: - raise ConfigurationError( - "Invalid SRV host: return address is identical to SRV hostname" - ) - if self.__srv_allowed_hosts_suffix is not None: - if not srv_host.endswith(self.__srv_allowed_hosts_suffix): - raise ConfigurationError(f"Invalid SRV host: {node[0]}") + try: + self._validate_host(node[0]) + except ConfigurationError as exc: + if not is_polling: + raise + log_srv_monitor_invalid_host(node[0], exc) else: - try: - nlist = srv_host.split(".")[1:][-self.__slen :] - except Exception as exc: - raise ConfigurationError(f"Invalid SRV host: {node[0]}") from exc - if self.__plist != nlist: - raise ConfigurationError(f"Invalid SRV host: {node[0]}") + valid_nodes.append(node) + nodes = valid_nodes if self.__srv_max_hosts: nodes = random.sample(nodes, min(self.__srv_max_hosts, len(nodes))) return results, nodes async def get_hosts(self) -> list[tuple[str, Any]]: - _, nodes = await self._get_srv_response_and_hosts(True) + _, nodes = await self._get_srv_response_and_hosts(is_polling=False) return nodes async def get_hosts_and_min_ttl(self) -> tuple[list[tuple[str, Any]], int]: - results, nodes = await self._get_srv_response_and_hosts(False) + results, nodes = await self._get_srv_response_and_hosts(is_polling=True) rrset = results.rrset ttl = rrset.ttl if rrset else 0 return nodes, ttl diff --git a/pymongo/asynchronous/uri_parser.py b/pymongo/asynchronous/uri_parser.py index 24d3427843..c42add6ae8 100644 --- a/pymongo/asynchronous/uri_parser.py +++ b/pymongo/asynchronous/uri_parser.py @@ -17,7 +17,7 @@ from __future__ import annotations -from typing import Any, Optional +from typing import Any, Callable, Optional from pymongo.asynchronous.srv_resolver import _SrvResolver from pymongo.common import SRV_SERVICE_NAME, _CaseInsensitiveDictionary @@ -48,6 +48,7 @@ async def parse_uri( srv_service_name: Optional[str] = None, srv_max_hosts: Optional[int] = None, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, ) -> dict[str, Any]: """Parse and validate a MongoDB URI. @@ -117,6 +118,7 @@ async def parse_uri( srv_service_name, srv_max_hosts, srv_allowed_hosts_suffix, + srv_host_validator, ) ) result["options"] = _make_options_case_sensitive(result["options"]) @@ -133,6 +135,7 @@ async def _parse_srv( srv_service_name: Optional[str] = None, srv_max_hosts: Optional[int] = None, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, ) -> dict[str, Any]: if uri.startswith(SCHEME): is_srv = False @@ -168,7 +171,12 @@ async def _parse_srv( # argument overrides the same option passed in the connection string. connect_timeout = connect_timeout or options.get("connectTimeoutMS") dns_resolver = _SrvResolver( - fqdn, connect_timeout, srv_service_name, srv_max_hosts, srv_allowed_hosts_suffix + fqdn, + connect_timeout, + srv_service_name, + srv_max_hosts, + srv_allowed_hosts_suffix, + srv_host_validator, ) nodes = await dns_resolver.get_hosts() dns_options = await dns_resolver.get_options() diff --git a/pymongo/bulk_shared.py b/pymongo/bulk_shared.py index a086611be5..54f0d4cd22 100644 --- a/pymongo/bulk_shared.py +++ b/pymongo/bulk_shared.py @@ -17,19 +17,28 @@ from __future__ import annotations -from collections.abc import Mapping, MutableMapping -from typing import TYPE_CHECKING, Any, NoReturn +from collections.abc import Iterator, Mapping, MutableMapping +from typing import TYPE_CHECKING, Any, NoReturn, Optional, Union +from bson.objectid import ObjectId +from bson.raw_bson import RawBSONDocument +from pymongo.common import ( + validate_is_document_type, + validate_ok_for_replace, + validate_ok_for_update, +) from pymongo.errors import BulkWriteError, OperationFailure from pymongo.helpers_shared import _get_wce_doc from pymongo.message import ( _DELETE, _INSERT, _UPDATE, + _BulkWriteContext, + _EncryptedBulkWriteContext, ) if TYPE_CHECKING: - from pymongo.typings import _DocumentOut + from pymongo.typings import _AgnosticCollection, _DocumentOut, _DocumentType, _Pipeline _DELETE_ALL: int = 0 @@ -131,3 +140,159 @@ def _raise_bulk_write_error(full_result: _DocumentOut) -> NoReturn: ) raise OperationFailure(errmsg, code, full_result) raise BulkWriteError(full_result) + + +class _AgnosticBulkBase: + """The private guts of the bulk write API.""" + + def __init__( + self, + collection: _AgnosticCollection[_DocumentType], + ordered: bool, + bypass_document_validation: Optional[bool], + comment: Optional[str] = None, + let: Optional[Any] = None, + ) -> None: + """Initialize an _AgnosticBulkBase instance.""" + self.collection = collection.with_options( + codec_options=collection.codec_options._replace( + unicode_decode_error_handler="replace", document_class=dict + ) + ) + self.let = let + if self.let is not None: + validate_is_document_type("let", self.let) + self.comment: Optional[str] = comment + self.ordered = ordered + self.ops: list[tuple[int, Mapping[str, Any]]] = [] + self.executed = False + self.bypass_doc_val = bypass_document_validation + self.uses_collation = False + self.uses_array_filters = False + self.uses_sort = False + self.is_retryable = True + self.retrying = False + self.started_retryable_write = False + # Extra state so that we know where to pick up on a retry attempt. + self.current_run = None + self.next_run = None + self.is_encrypted = False + + @property + def bulk_ctx_class(self) -> type[_BulkWriteContext]: + encrypter = self.collection.database.client._encrypter + if encrypter and not encrypter._bypass_auto_encryption: + self.is_encrypted = True + return _EncryptedBulkWriteContext + else: + self.is_encrypted = False + return _BulkWriteContext + + def add_insert(self, document: _DocumentOut) -> None: + """Add an insert document to the list of ops.""" + validate_is_document_type("document", document) + # Generate ObjectId client side. + if not (isinstance(document, RawBSONDocument) or "_id" in document): + document["_id"] = ObjectId() + self.ops.append((_INSERT, document)) + + def add_update( + self, + selector: Mapping[str, Any], + update: Union[Mapping[str, Any], _Pipeline], + multi: bool, + upsert: Optional[bool], + collation: Optional[Mapping[str, Any]] = None, + array_filters: Optional[list[Mapping[str, Any]]] = None, + hint: Union[str, dict[str, Any], None] = None, + sort: Optional[Mapping[str, Any]] = None, + ) -> None: + """Create an update document and add it to the list of ops.""" + validate_ok_for_update(update) + cmd: dict[str, Any] = {"q": selector, "u": update, "multi": multi} + if upsert is not None: + cmd["upsert"] = upsert + if collation is not None: + self.uses_collation = True + cmd["collation"] = collation + if array_filters is not None: + self.uses_array_filters = True + cmd["arrayFilters"] = array_filters + if hint is not None: + cmd["hint"] = hint + if sort is not None: + self.uses_sort = True + cmd["sort"] = sort + if multi: + # A bulk_write containing an update_many is not retryable. + self.is_retryable = False + self.ops.append((_UPDATE, cmd)) + + def add_replace( + self, + selector: Mapping[str, Any], + replacement: Mapping[str, Any], + upsert: Optional[bool], + collation: Optional[Mapping[str, Any]] = None, + hint: Union[str, dict[str, Any], None] = None, + sort: Optional[Mapping[str, Any]] = None, + ) -> None: + """Create a replace document and add it to the list of ops.""" + validate_ok_for_replace(replacement) + cmd: dict[str, Any] = {"q": selector, "u": replacement} + if upsert is not None: + cmd["upsert"] = upsert + if collation is not None: + self.uses_collation = True + cmd["collation"] = collation + if hint is not None: + cmd["hint"] = hint + if sort is not None: + self.uses_sort = True + cmd["sort"] = sort + self.ops.append((_UPDATE, cmd)) + + def add_delete( + self, + selector: Mapping[str, Any], + limit: int, + collation: Optional[Mapping[str, Any]] = None, + hint: Union[str, dict[str, Any], None] = None, + ) -> None: + """Create a delete document and add it to the list of ops.""" + cmd: dict[str, Any] = {"q": selector, "limit": limit} + if collation is not None: + self.uses_collation = True + cmd["collation"] = collation + if hint is not None: + cmd["hint"] = hint + if limit == _DELETE_ALL: + # A bulk_write containing a delete_many is not retryable. + self.is_retryable = False + self.ops.append((_DELETE, cmd)) + + def gen_ordered(self) -> Iterator[Optional[_Run]]: + """Generate batches of operations, batched by type of + operation, in the order **provided**. + """ + run = None + for idx, (op_type, operation) in enumerate(self.ops): + if run is None: + run = _Run(op_type) + elif run.op_type != op_type: + yield run + run = _Run(op_type) + run.add(idx, operation) + yield run + + def gen_unordered(self) -> Iterator[_Run]: + """Generate batches of operations, batched by type of + operation, in arbitrary order. + """ + operations = [_Run(_INSERT), _Run(_UPDATE), _Run(_DELETE)] + for idx, (op_type, operation) in enumerate(self.ops): + operations[op_type].add(idx, operation) + + for run in operations: + if run.ops: + yield run diff --git a/pymongo/common.py b/pymongo/common.py index c068758b04..ebab02f47c 100644 --- a/pymongo/common.py +++ b/pymongo/common.py @@ -810,6 +810,7 @@ def validate_server_monitoring_mode(option: str, value: str) -> str: "username": validate_string_or_none, "password": validate_string_or_none, "server_selector": validate_is_callable_or_none, + "srv_host_validator": validate_is_callable_or_none, "auto_encryption_opts": validate_auto_encryption_opts_or_none, "authoidcallowedhosts": validate_list, "max_adaptive_retries": validate_non_negative_integer, diff --git a/pymongo/driver_info.py b/pymongo/driver_info.py index 18a51ae638..a325ea1493 100644 --- a/pymongo/driver_info.py +++ b/pymongo/driver_info.py @@ -29,8 +29,14 @@ class DriverInfo(namedtuple("DriverInfo", ["name", "version", "platform"])): The MongoDB server logs PyMongo's name, version, and platform whenever PyMongo establishes a connection. A driver implemented on top of PyMongo can add its own info to this log message. Initialize with three strings - like 'MyDriver', '1.2.3', 'some platform info'. Any of these strings may be - None to accept PyMongo's default. + like 'MyDriver', '1.2.3', 'some platform info'. Any of these strings may + be None. A None or empty name or version appends an empty metadata + segment, keeping the ``driver.name`` and ``driver.version`` segment + counts aligned. A None or empty platform omits the platform. + + The ``|`` character is the reserved delimiter used to join appended + metadata, so it must not appear in any of the fields. A + :class:`ValueError` is raised if it does. """ def __new__( @@ -42,5 +48,7 @@ def __new__( raise TypeError( f"Wrong type for DriverInfo {key} option, value must be an instance of str, not {type(value)}" ) + if value and "|" in value: + raise ValueError(f"DriverInfo {key} must not contain the '|' delimiter") return self diff --git a/pymongo/pool_options.py b/pymongo/pool_options.py index 8b26b4baf2..dd95db525e 100644 --- a/pymongo/pool_options.py +++ b/pymongo/pool_options.py @@ -24,6 +24,7 @@ import platform import sys from collections.abc import MutableMapping +from contextlib import AbstractContextManager, nullcontext from pathlib import Path from typing import TYPE_CHECKING, Any, Optional @@ -37,6 +38,7 @@ WAIT_QUEUE_TIMEOUT, has_c, ) +from pymongo.lock import _create_lock if TYPE_CHECKING: from pymongo.auth_shared import MongoCredential @@ -200,60 +202,117 @@ def _metadata_env() -> dict[str, Any]: _MAX_METADATA_SIZE = 512 +def _truncate_utf8(content: str, overflow: int) -> str: + """Trim `overflow` UTF-8 bytes from the end of content, keeping a valid prefix.""" + if overflow <= 0: + return content + data = content.encode("utf-8") + if len(data) <= overflow: + return "" + return data[: len(data) - overflow].decode("utf-8", errors="ignore") + + +def _normalize_driver(driver: DriverInfo) -> DriverInfo: + """Treat None and "" as equivalent unset fields for deduplication.""" + return driver._replace( + name=driver.name or "", + version=driver.version or "", + platform=driver.platform or "", + ) + + +def _element_size(key: str, value: Any) -> int: + """Size in bytes of the BSON element for ``key``, excluding doc overhead.""" + # bson.encode({key: value}) is 4 bytes of document header plus 1 byte of + # document terminator on top of the element itself. + return len(bson.encode({key: value})) - 5 + + # See: https://github.com/mongodb/specifications/blob/master/source/mongodb-handshake/handshake.md#limitations def _truncate_metadata(metadata: MutableMapping[str, Any]) -> None: """Perform metadata truncation.""" - if len(bson.encode(metadata)) <= _MAX_METADATA_SIZE: + # The only full encode is the initial size check; each step then shrinks + # the tracked size by the exact bytes its change removes. + size = len(bson.encode(metadata)) + if size <= _MAX_METADATA_SIZE: return # 1. Omit fields from env except env.name. env_name = metadata.get("env", {}).get("name") if env_name: - metadata["env"] = {"name": env_name} - if len(bson.encode(metadata)) <= _MAX_METADATA_SIZE: + env = {"name": env_name} + size += _element_size("env", env) - _element_size("env", metadata["env"]) + metadata["env"] = env + if size <= _MAX_METADATA_SIZE: return # 2. Omit fields from os except os.type. os_type = metadata.get("os", {}).get("type") if os_type: - metadata["os"] = {"type": os_type} - if len(bson.encode(metadata)) <= _MAX_METADATA_SIZE: + old_os = metadata["os"] + new_os = {"type": os_type} + size += _element_size("os", new_os) - _element_size("os", old_os) + metadata["os"] = new_os + if size <= _MAX_METADATA_SIZE: return # 3. Omit the env document entirely. - metadata.pop("env", None) - encoded_size = len(bson.encode(metadata)) - if encoded_size <= _MAX_METADATA_SIZE: + env = metadata.pop("env", None) + if env is not None: + size -= _element_size("env", env) + if size <= _MAX_METADATA_SIZE: return # 4. Truncate platform. - overflow = encoded_size - _MAX_METADATA_SIZE + overflow = size - _MAX_METADATA_SIZE plat = metadata.get("platform", "") if plat: - plat = plat[:-overflow] - if plat: - metadata["platform"] = plat + truncated = _truncate_utf8(plat, overflow) + if truncated: + size += _element_size("platform", truncated) - _element_size("platform", plat) + else: + size -= _element_size("platform", plat) + if truncated: + metadata["platform"] = truncated + else: + del metadata["platform"] else: - metadata.pop("platform", None) - encoded_size = len(bson.encode(metadata)) - if encoded_size <= _MAX_METADATA_SIZE: - return - # 5. Truncate driver info. - overflow = encoded_size - _MAX_METADATA_SIZE + # The platform field may be present but empty. + plat = metadata.pop("platform", None) + if plat is not None: + size -= _element_size("platform", plat) + # 5. Truncate driver info, keeping name and version 1:1 index-aligned. driver = metadata.get("driver", {}) if driver: - # Truncate driver version. - driver_version = driver.get("version")[:-overflow] - if len(driver_version) >= len(_METADATA["driver"]["version"]): - metadata["driver"]["version"] = driver_version - else: - metadata["driver"]["version"] = _METADATA["driver"]["version"] - encoded_size = len(bson.encode(metadata)) - if encoded_size <= _MAX_METADATA_SIZE: - return - # Truncate driver name. - overflow = encoded_size - _MAX_METADATA_SIZE - driver_name = driver.get("name")[:-overflow] - if len(driver_name) >= len(_METADATA["driver"]["name"]): - metadata["driver"]["name"] = driver_name - else: - metadata["driver"]["name"] = _METADATA["driver"]["name"] + # Keep the name and version segments paired so they stay 1:1 aligned, + # trimming wrapper content and dropping paired segments only as a + # last resort. Blank pairs (for example from a platform-only appended + # driver) are dropped first since they cost nothing. + name_str = driver.get("name", "") + version_str = driver.get("version", "") + raw_pairs = list(zip(name_str.split("|"), version_str.split("|"))) + pairs = [(name, version) for name, version in raw_pairs if name or version] + new_name = "|".join(name for name, _ in pairs) + new_version = "|".join(version for _, version in pairs) + size += (len(new_name.encode("utf-8")) - len(name_str.encode("utf-8"))) + ( + len(new_version.encode("utf-8")) - len(version_str.encode("utf-8")) + ) + driver["name"] = new_name + driver["version"] = new_version + overflow = size - _MAX_METADATA_SIZE + while overflow > 0 and len(pairs) > 1: + # A single remaining pair never exceeds the limit: it is the base + # driver pair, which steps 1-4 left small enough to fit. + last_name, last_version = pairs[-1] + if last_version: + new_version = _truncate_utf8(last_version, overflow) + overflow -= len(last_version.encode("utf-8")) - len(new_version.encode("utf-8")) + pairs[-1] = (last_name, new_version) + elif last_name: + new_name = _truncate_utf8(last_name, overflow) + overflow -= len(last_name.encode("utf-8")) - len(new_name.encode("utf-8")) + pairs[-1] = (new_name, last_version) + else: + pairs.pop() + overflow -= len(f"|{last_name}|{last_version}".encode()) + driver["name"] = "|".join(name for name, _ in pairs) + driver["version"] = "|".join(version for _, version in pairs) # If the first getaddrinfo call of this interpreter's life is on a thread, @@ -277,6 +336,7 @@ class PoolOptions: """ __slots__ = ( + "__appended_drivers", "__appname", "__compression_settings", "__connect_timeout", @@ -288,6 +348,7 @@ class PoolOptions: "__max_idle_time_seconds", "__max_pool_size", "__metadata", + "__metadata_lock", "__min_pool_size", "__pause_enabled", "__server_api", @@ -336,6 +397,11 @@ def __init__( self.__load_balanced = load_balanced self.__credentials = credentials self.__metadata = copy.deepcopy(_METADATA) + self.__appended_drivers: set[DriverInfo] = set() + # Only the synchronous client can append metadata from multiple threads. + self.__metadata_lock: AbstractContextManager[bool | None] = ( + _create_lock() if is_sync else nullcontext() + ) if appname: self.__metadata["application"] = {"name": appname} @@ -353,11 +419,19 @@ def __init__( self.__metadata["driver"]["name"], "c", ) + self.__metadata["driver"]["version"] = "{}|{}".format( + self.__metadata["driver"]["version"], + "", + ) if not is_sync: self.__metadata["driver"]["name"] = "{}|{}".format( self.__metadata["driver"]["name"], "async", ) + self.__metadata["driver"]["version"] = "{}|{}".format( + self.__metadata["driver"]["version"], + "", + ) if driver: self._update_metadata(driver) @@ -368,28 +442,42 @@ def __init__( _truncate_metadata(self.__metadata) def _update_metadata(self, driver: DriverInfo) -> None: - """Updates the client's metadata""" - if driver.name and driver.name.lower() in self.__metadata["driver"]["name"].lower().split( - "|" - ): - return - - metadata = copy.deepcopy(self.__metadata) - - if driver.name: - metadata["driver"]["name"] = "{}|{}".format( - metadata["driver"]["name"], - driver.name, - ) - if driver.version: + """Updates the client's metadata.""" + with self.__metadata_lock: + driver = _normalize_driver(driver) + if driver in self.__appended_drivers: + return + + # Only the top-level keys and the "driver" document are mutated, + # so shallow copies of those two are enough. + metadata = {**self.__metadata, "driver": dict(self.__metadata["driver"])} + + metadata["driver"]["name"] = "{}|{}".format(metadata["driver"]["name"], driver.name) metadata["driver"]["version"] = "{}|{}".format( - metadata["driver"]["version"], - driver.version, + metadata["driver"]["version"], driver.version ) - if driver.platform: - metadata["platform"] = "{}|{}".format(metadata["platform"], driver.platform) - - self.__metadata = metadata + if driver.platform: + if "platform" in metadata: + metadata["platform"] = "{}|{}".format(metadata["platform"], driver.platform) + else: + metadata["platform"] = driver.platform + + _truncate_metadata(metadata) + + self.__metadata = metadata + + # Only track drivers whose appended name/version pair survived + # truncation, so __appended_drivers stays bounded. Truncation + # keeps name and version index-aligned, so strict=True never + # raises. + names = metadata["driver"]["name"].split("|") + versions = metadata["driver"]["version"].split("|") + if sys.version_info >= (3, 10): + pairs = zip(names, versions, strict=True) + else: + pairs = zip(names, versions) + if (driver.name, driver.version) in pairs: + self.__appended_drivers.add(driver) @property def _credentials(self) -> Optional[MongoCredential]: diff --git a/pymongo/public_suffix_list.dat b/pymongo/public_suffix_list.dat index 78da70c4d4..0f09a04291 100644 --- a/pymongo/public_suffix_list.dat +++ b/pymongo/public_suffix_list.dat @@ -6962,7 +6962,6 @@ ltd.ua a2hosted.com cpserver.com activetrail.biz -adaptable.app myaddr.dev myaddr.io dyn.addr.tools @@ -6982,7 +6981,7 @@ africa.com *.auiusercontent.com beep.pl aiven.app -aivencloud.com +*.aivencloud.com akadns.net akamai.net akamai-staging.net @@ -7857,6 +7856,9 @@ eero-stage.online opentunnel.xyz antagonist.cloud claude.app +claudeusercontent.com +frame.claudeusercontent.com +*.cursorusercontent.com apigee.io panel.dev siiites.com @@ -8069,6 +8071,8 @@ sch.ac dev.cv store.cv codeberg.page +codepen.app +codepen.dev csb.app preview.csb.app co.nl @@ -8948,6 +8952,7 @@ hepforge.org onhercules.app hercules-app.com hercules-dev.com +here.now herokuapp.com heyflow.page heyflow.site @@ -9043,6 +9048,7 @@ botdash.gg botdash.net botda.sh botdash.xyz +online-server.cloud apps-1and1.com live-website.com webspace-host.com @@ -9882,6 +9888,9 @@ scbl.pl-waw.scw.cloud scalebook.scw.cloud smartlabeling.scw.cloud dedibox.fr +scw.site +ams.scw.site +waw.scw.site schokokeks.net gov.scot service.gov.scot @@ -10175,9 +10184,9 @@ toolforge.org wmcloud.org beta.wmcloud.org wmflabs.org -vps.hrsn.au hrsn.dev is-a.dev +vps.hrsn.net localcert.net windsurf.app windsurf.build diff --git a/pymongo/settings_shared.py b/pymongo/settings_shared.py index fbe87697c1..bf45c40334 100644 --- a/pymongo/settings_shared.py +++ b/pymongo/settings_shared.py @@ -19,7 +19,7 @@ import threading import traceback from collections.abc import Collection -from typing import Any, Generic, Optional, TypeVar, Union +from typing import Any, Callable, Generic, Optional, TypeVar, Union from bson.objectid import ObjectId from pymongo import common @@ -52,6 +52,7 @@ def __init__( srv_service_name: str = common.SRV_SERVICE_NAME, srv_max_hosts: int = 0, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, server_monitoring_mode: str = common.SERVER_MONITORING_MODE, topology_id: Optional[ObjectId] = None, ): @@ -79,6 +80,7 @@ def __init__( self._srv_service_name = srv_service_name self._srv_max_hosts = srv_max_hosts or 0 self._srv_allowed_hosts_suffix = srv_allowed_hosts_suffix + self._srv_host_validator = srv_host_validator self._server_monitoring_mode = server_monitoring_mode if topology_id is not None: self._topology_id = topology_id @@ -161,6 +163,11 @@ def srv_allowed_hosts_suffix(self) -> Optional[str]: """The srvAllowedHostsSuffix.""" return self._srv_allowed_hosts_suffix + @property + def srv_host_validator(self) -> Optional[Callable[[str], bool]]: + """The srv_host_validator callback.""" + return self._srv_host_validator + @property def server_monitoring_mode(self) -> str: """The serverMonitoringMode.""" diff --git a/pymongo/synchronous/bulk.py b/pymongo/synchronous/bulk.py index be12a72127..291965fca1 100644 --- a/pymongo/synchronous/bulk.py +++ b/pymongo/synchronous/bulk.py @@ -29,23 +29,15 @@ Union, ) -from bson.objectid import ObjectId -from bson.raw_bson import RawBSONDocument -from pymongo import _csot, common +from pymongo import _csot from pymongo._telemetry import _generate_op_id_or_none from pymongo.bulk_shared import ( _COMMANDS, - _DELETE_ALL, + _AgnosticBulkBase, _merge_command, _raise_bulk_write_error, - _Run, ) from pymongo.client_session_shared import _validate_session_write_concern -from pymongo.common import ( - validate_is_document_type, - validate_ok_for_replace, - validate_ok_for_update, -) from pymongo.errors import ( ConfigurationError, InvalidOperation, @@ -54,7 +46,6 @@ from pymongo.helpers_shared import _RETRYABLE_ERROR_CODES from pymongo.message import ( _DELETE, - _INSERT, _UPDATE, _BulkWriteContext, _EncryptedBulkWriteContext, @@ -71,165 +62,14 @@ from pymongo.synchronous.collection import Collection from pymongo.synchronous.mongo_client import MongoClient from pymongo.synchronous.pool import Connection - from pymongo.typings import _DocumentOut, _DocumentType, _Pipeline _IS_SYNC = True -class _Bulk: +class _Bulk(_AgnosticBulkBase): """The private guts of the bulk write API.""" - def __init__( - self, - collection: Collection[_DocumentType], - ordered: bool, - bypass_document_validation: Optional[bool], - comment: Optional[str] = None, - let: Optional[Any] = None, - ) -> None: - """Initialize a _Bulk instance.""" - self.collection = collection.with_options( - codec_options=collection.codec_options._replace( - unicode_decode_error_handler="replace", document_class=dict - ) - ) - self.let = let - if self.let is not None: - common.validate_is_document_type("let", self.let) - self.comment: Optional[str] = comment - self.ordered = ordered - self.ops: list[tuple[int, Mapping[str, Any]]] = [] - self.executed = False - self.bypass_doc_val = bypass_document_validation - self.uses_collation = False - self.uses_array_filters = False - self.uses_sort = False - self.is_retryable = True - self.retrying = False - self.started_retryable_write = False - # Extra state so that we know where to pick up on a retry attempt. - self.current_run = None - self.next_run = None - self.is_encrypted = False - - @property - def bulk_ctx_class(self) -> type[_BulkWriteContext]: - encrypter = self.collection.database.client._encrypter - if encrypter and not encrypter._bypass_auto_encryption: - self.is_encrypted = True - return _EncryptedBulkWriteContext - else: - self.is_encrypted = False - return _BulkWriteContext - - def add_insert(self, document: _DocumentOut) -> None: - """Add an insert document to the list of ops.""" - validate_is_document_type("document", document) - # Generate ObjectId client side. - if not (isinstance(document, RawBSONDocument) or "_id" in document): - document["_id"] = ObjectId() - self.ops.append((_INSERT, document)) - - def add_update( - self, - selector: Mapping[str, Any], - update: Union[Mapping[str, Any], _Pipeline], - multi: bool, - upsert: Optional[bool], - collation: Optional[Mapping[str, Any]] = None, - array_filters: Optional[list[Mapping[str, Any]]] = None, - hint: Union[str, dict[str, Any], None] = None, - sort: Optional[Mapping[str, Any]] = None, - ) -> None: - """Create an update document and add it to the list of ops.""" - validate_ok_for_update(update) - cmd: dict[str, Any] = {"q": selector, "u": update, "multi": multi} - if upsert is not None: - cmd["upsert"] = upsert - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if array_filters is not None: - self.uses_array_filters = True - cmd["arrayFilters"] = array_filters - if hint is not None: - cmd["hint"] = hint - if sort is not None: - self.uses_sort = True - cmd["sort"] = sort - if multi: - # A bulk_write containing an update_many is not retryable. - self.is_retryable = False - self.ops.append((_UPDATE, cmd)) - - def add_replace( - self, - selector: Mapping[str, Any], - replacement: Mapping[str, Any], - upsert: Optional[bool], - collation: Optional[Mapping[str, Any]] = None, - hint: Union[str, dict[str, Any], None] = None, - sort: Optional[Mapping[str, Any]] = None, - ) -> None: - """Create a replace document and add it to the list of ops.""" - validate_ok_for_replace(replacement) - cmd: dict[str, Any] = {"q": selector, "u": replacement} - if upsert is not None: - cmd["upsert"] = upsert - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if hint is not None: - cmd["hint"] = hint - if sort is not None: - self.uses_sort = True - cmd["sort"] = sort - self.ops.append((_UPDATE, cmd)) - - def add_delete( - self, - selector: Mapping[str, Any], - limit: int, - collation: Optional[Mapping[str, Any]] = None, - hint: Union[str, dict[str, Any], None] = None, - ) -> None: - """Create a delete document and add it to the list of ops.""" - cmd: dict[str, Any] = {"q": selector, "limit": limit} - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if hint is not None: - cmd["hint"] = hint - if limit == _DELETE_ALL: - # A bulk_write containing a delete_many is not retryable. - self.is_retryable = False - self.ops.append((_DELETE, cmd)) - - def gen_ordered(self) -> Iterator[Optional[_Run]]: - """Generate batches of operations, batched by type of - operation, in the order **provided**. - """ - run = None - for idx, (op_type, operation) in enumerate(self.ops): - if run is None: - run = _Run(op_type) - elif run.op_type != op_type: - yield run - run = _Run(op_type) - run.add(idx, operation) - yield run - - def gen_unordered(self) -> Iterator[_Run]: - """Generate batches of operations, batched by type of - operation, in arbitrary order. - """ - operations = [_Run(_INSERT), _Run(_UPDATE), _Run(_DELETE)] - for idx, (op_type, operation) in enumerate(self.ops): - operations[op_type].add(idx, operation) - - for run in operations: - if run.ops: - yield run + collection: Collection[Any] @_handle_reauth def write_command( diff --git a/pymongo/synchronous/client_bulk.py b/pymongo/synchronous/client_bulk.py index 5e5d1a76e8..549c9d59ca 100644 --- a/pymongo/synchronous/client_bulk.py +++ b/pymongo/synchronous/client_bulk.py @@ -29,9 +29,7 @@ Union, ) -from bson.objectid import ObjectId -from bson.raw_bson import RawBSONDocument -from pymongo import _csot, common +from pymongo import _csot from pymongo._telemetry import _generate_op_id_or_none from pymongo.synchronous.client_session import ClientSession from pymongo.synchronous.collection import Collection @@ -46,15 +44,11 @@ from pymongo.synchronous.mongo_client import MongoClient from pymongo.synchronous.pool import Connection from pymongo._client_bulk_shared import ( + _AgnosticClientBulkBase, _merge_command, _throw_client_bulk_write_exception, ) from pymongo.client_session_shared import _validate_session_write_concern -from pymongo.common import ( - validate_is_document_type, - validate_ok_for_replace, - validate_ok_for_update, -) from pymongo.errors import ( ConfigurationError, ConnectionFailure, @@ -76,152 +70,15 @@ InsertOneResult, UpdateResult, ) -from pymongo.typings import _DocumentOut, _Pipeline from pymongo.write_concern import WriteConcern _IS_SYNC = True -class _ClientBulk: +class _ClientBulk(_AgnosticClientBulkBase): """The private guts of the client-level bulk write API.""" - def __init__( - self, - client: MongoClient[Any], - write_concern: WriteConcern, - ordered: bool = True, - bypass_document_validation: Optional[bool] = None, - comment: Optional[str] = None, - let: Optional[Any] = None, - verbose_results: bool = False, - ) -> None: - """Initialize a _ClientBulk instance.""" - self.client = client - self.write_concern = write_concern - self.let = let - if self.let is not None: - common.validate_is_document_type("let", self.let) - self.ordered = ordered - self.bypass_doc_val = bypass_document_validation - self.comment = comment - self.verbose_results = verbose_results - self.ops: list[tuple[str, Mapping[str, Any]]] = [] - self.namespaces: list[str] = [] - self.idx_offset: int = 0 - self.total_ops: int = 0 - self.executed = False - self.uses_collation = False - self.uses_array_filters = False - self.is_retryable = self.client.options.retry_writes - self.retrying = False - self.started_retryable_write = False - - @property - def bulk_ctx_class(self) -> type[_ClientBulkWriteContext]: - return _ClientBulkWriteContext - - def add_insert(self, namespace: str, document: _DocumentOut) -> None: - """Add an insert document to the list of ops.""" - validate_is_document_type("document", document) - # Generate ObjectId client side. - if not (isinstance(document, RawBSONDocument) or "_id" in document): - document["_id"] = ObjectId() - cmd = {"insert": -1, "document": document} - self.ops.append(("insert", cmd)) - self.namespaces.append(namespace) - self.total_ops += 1 - - def add_update( - self, - namespace: str, - selector: Mapping[str, Any], - update: Union[Mapping[str, Any], _Pipeline], - multi: bool, - upsert: Optional[bool] = None, - collation: Optional[Mapping[str, Any]] = None, - array_filters: Optional[list[Mapping[str, Any]]] = None, - hint: Union[str, dict[str, Any], None] = None, - sort: Optional[Mapping[str, Any]] = None, - ) -> None: - """Create an update document and add it to the list of ops.""" - validate_ok_for_update(update) - cmd = { - "update": -1, - "filter": selector, - "updateMods": update, - "multi": multi, - } - if upsert is not None: - cmd["upsert"] = upsert - if array_filters is not None: - self.uses_array_filters = True - cmd["arrayFilters"] = array_filters - if hint is not None: - cmd["hint"] = hint - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if sort is not None: - cmd["sort"] = sort - if multi: - # A bulk_write containing an update_many is not retryable. - self.is_retryable = False - self.ops.append(("update", cmd)) - self.namespaces.append(namespace) - self.total_ops += 1 - - def add_replace( - self, - namespace: str, - selector: Mapping[str, Any], - replacement: Mapping[str, Any], - upsert: Optional[bool] = None, - collation: Optional[Mapping[str, Any]] = None, - hint: Union[str, dict[str, Any], None] = None, - sort: Optional[Mapping[str, Any]] = None, - ) -> None: - """Create a replace document and add it to the list of ops.""" - validate_ok_for_replace(replacement) - cmd = { - "update": -1, - "filter": selector, - "updateMods": replacement, - "multi": False, - } - if upsert is not None: - cmd["upsert"] = upsert - if hint is not None: - cmd["hint"] = hint - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if sort is not None: - cmd["sort"] = sort - self.ops.append(("replace", cmd)) - self.namespaces.append(namespace) - self.total_ops += 1 - - def add_delete( - self, - namespace: str, - selector: Mapping[str, Any], - multi: bool, - collation: Optional[Mapping[str, Any]] = None, - hint: Union[str, dict[str, Any], None] = None, - ) -> None: - """Create a delete document and add it to the list of ops.""" - cmd = {"delete": -1, "filter": selector, "multi": multi} - if hint is not None: - cmd["hint"] = hint - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if multi: - # A bulk_write containing an update_many is not retryable. - self.is_retryable = False - self.ops.append(("delete", cmd)) - self.namespaces.append(namespace) - self.total_ops += 1 + client: MongoClient[Any] @_handle_reauth def write_command( diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index 391eeeb26b..79412c11de 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -471,6 +471,31 @@ def __init__( srvAllowedHostsSuffix=".internal.example.com", ) + - `srv_host_validator`: (callable or None) A callback used in place of the + default parent-domain check for hosts returned by SRV DNS records. It is + called once per returned host with the lowercased hostname as its only + argument, and must return ``True`` to accept the host or ``False`` to + reject it. During initial DNS discovery, rejecting a host raises + :exc:`~pymongo.errors.ConfigurationError`, as does an exception raised by + the callback itself. Use this when the set of acceptable hosts cannot be + expressed as a single suffix:: + + def validator(host: str) -> bool: + return host.endswith((".a.example.com", ".b.example.com")) + + MongoClient( + "mongodb+srv://cluster.example.com/", + srv_host_validator=validator, + ) + + The callback must not block. It is mutually exclusive with + ``srvAllowedHostsSuffix``. Because this option is a callable, it can + only be passed in as a keyword argument, not through the connection string. + + .. warning:: + + This option replaces the built-in DNS spoofing safeguards. + Please use with caution. | **Write Concern options:** | (Only set if passed. No default values.) @@ -655,6 +680,12 @@ def __init__( .. seealso:: The MongoDB documentation on `connections `_. + .. versionchanged:: 4.19 + Added the ``srv_host_validator`` keyword argument. + + .. versionchanged:: 4.18 + Added the ``srvAllowedHostsSuffix`` URI and keyword argument. + .. versionchanged:: 4.18 Added the ``tracing`` keyword argument. @@ -844,6 +875,7 @@ def __init__( srv_service_name = keyword_opts.get("srvservicename") srv_max_hosts = keyword_opts.get("srvmaxhosts") srv_allowed_hosts_suffix = keyword_opts.get("srvallowedhostssuffix") + srv_host_validator = keyword_opts.get("srv_host_validator") if len([h for h in self._host if "/" in h]) > 1: raise ConfigurationError("host must not contain multiple MongoDB URIs") for entity in self._host: @@ -936,7 +968,11 @@ def __init__( ) self._init_based_on_options( - self._seeds, srv_max_hosts, srv_service_name, srv_allowed_hosts_suffix + self._seeds, + srv_max_hosts, + srv_service_name, + srv_allowed_hosts_suffix, + srv_host_validator, ) self._opened = False @@ -956,6 +992,7 @@ def _resolve_srv(self) -> None: srv_service_name = keyword_opts.get("srvservicename") srv_max_hosts = keyword_opts.get("srvmaxhosts") srv_allowed_hosts_suffix = keyword_opts.get("srvallowedhostssuffix") + srv_host_validator = keyword_opts.get("srv_host_validator") for entity in self._host: # A hostname can only include a-z, 0-9, '-' and '.'. If we find a '/' # it must be a URI, @@ -977,6 +1014,7 @@ def _resolve_srv(self) -> None: srv_service_name=srv_service_name, srv_max_hosts=srv_max_hosts, srv_allowed_hosts_suffix=srv_allowed_hosts_suffix, + srv_host_validator=srv_host_validator, ) seeds.update(res["nodelist"]) opts = res["options"] @@ -1021,7 +1059,11 @@ def _resolve_srv(self) -> None: ) self._init_based_on_options( - seeds, srv_max_hosts, srv_service_name, srv_allowed_hosts_suffix + seeds, + srv_max_hosts, + srv_service_name, + srv_allowed_hosts_suffix, + srv_host_validator, ) def _init_based_on_options( @@ -1030,7 +1072,17 @@ def _init_based_on_options( srv_max_hosts: Any, srv_service_name: Any, srv_allowed_hosts_suffix: Any, + srv_host_validator: Any = None, ) -> None: + if srv_host_validator is not None: + if srv_allowed_hosts_suffix is not None: + raise ConfigurationError( + "Cannot specify both srv_host_validator and srvAllowedHostsSuffix" + ) + if not self._resolve_srv_info["is_srv"]: + raise ConfigurationError( + "The srv_host_validator option is only allowed with 'mongodb+srv://' URIs" + ) self._event_listeners = self._options.pool_options._event_listeners self._topology_settings = TopologySettings( seeds=seeds, @@ -1049,6 +1101,7 @@ def _init_based_on_options( srv_service_name=srv_service_name, srv_max_hosts=srv_max_hosts, srv_allowed_hosts_suffix=srv_allowed_hosts_suffix, + srv_host_validator=srv_host_validator, server_monitoring_mode=self._options.server_monitoring_mode, topology_id=self._topology_settings._topology_id if self._topology_settings else None, ) diff --git a/pymongo/synchronous/monitor.py b/pymongo/synchronous/monitor.py index 3a1e1a49d0..cbc5796a8a 100644 --- a/pymongo/synchronous/monitor.py +++ b/pymongo/synchronous/monitor.py @@ -366,6 +366,7 @@ def _get_seedlist(self) -> Optional[list[tuple[str, Any]]]: self._settings.pool_options.connect_timeout, self._settings.srv_service_name, srv_allowed_hosts_suffix=self._settings.srv_allowed_hosts_suffix, + srv_host_validator=self._settings.srv_host_validator, ) seedlist, ttl = resolver.get_hosts_and_min_ttl() if len(seedlist) == 0: diff --git a/pymongo/synchronous/pool.py b/pymongo/synchronous/pool.py index 9d47a4f031..f881e31046 100644 --- a/pymongo/synchronous/pool.py +++ b/pymongo/synchronous/pool.py @@ -106,6 +106,13 @@ _IS_SYNC = True +# Flags recording the counters a checkout has incremented, so a failed +# checkout can restore them exactly once (PYTHON-6136). +_UNDO_OPERATION_COUNT = 1 +_UNDO_REQUESTS = 2 +_UNDO_SOCKETS = 4 +_UNDO_PENDING = 8 + class Connection(_ConnectionTelemetryInfo): """Store a connection with some metadata. @@ -968,8 +975,12 @@ def _get_conn( "Attempted to check out a connection from closed connection pool" ) + # Every counter increment sets a flag in ``applied`` so a failed + # checkout can restore them exactly once (PYTHON-6136). + applied = 0 with self.lock: self.operation_count += 1 + applied |= _UNDO_OPERATION_COUNT # Get a free socket or create one. if _csot.get_timeout(): @@ -979,28 +990,32 @@ def _get_conn( else: deadline = None - with self.size_cond: - self._raise_if_not_ready(checkout_started_time, emit_event=True) - while not (self.requests < self.max_pool_size): - timeout = deadline - time.monotonic() if deadline else None - if not _cond_wait(self.size_cond, timeout): - # Timed out, notify the next thread to ensure a - # timeout doesn't consume the condition. - if self.requests < self.max_pool_size: - self.size_cond.notify() - self._raise_wait_queue_timeout(checkout_started_time) + try: + with self.size_cond: self._raise_if_not_ready(checkout_started_time, emit_event=True) - self.requests += 1 + while not (self.requests < self.max_pool_size): + timeout = deadline - time.monotonic() if deadline else None + if not _cond_wait(self.size_cond, timeout): + # Timed out, notify the next thread to ensure a + # timeout doesn't consume the condition. + if self.requests < self.max_pool_size: + self.size_cond.notify() + self._raise_wait_queue_timeout(checkout_started_time) + self._raise_if_not_ready(checkout_started_time, emit_event=True) + self.requests += 1 + applied |= _UNDO_REQUESTS + except BaseException: + self._restore_counters(applied) + raise # We've now acquired the semaphore and must release it on error. conn = None - incremented = False emitted_event = False is_new_conn = False try: with self.lock: self.active_sockets += 1 - incremented = True + applied |= _UNDO_SOCKETS while conn is None: # CMAP: we MUST wait for either maxConnecting OR for a socket # to be checked back into the pool. @@ -1021,6 +1036,7 @@ def _get_conn( conn = self.conns.popleft() except IndexError: self._pending += 1 + applied |= _UNDO_PENDING if conn: # We got a socket from the pool if self._perished(conn): conn = None @@ -1032,7 +1048,17 @@ def _get_conn( finally: with self._max_connecting_cond: self._pending -= 1 - self._max_connecting_cond.notify() + applied &= ~_UNDO_PENDING + notified = False + try: + self._max_connecting_cond.notify() + notified = True + finally: + if not notified: + # A kill landed inside notify() (a gevent + # yield point); retry so a waiting + # checkout is not stranded (PYTHON-6136). + self._max_connecting_cond.notify() conn.active = True # connect() already adds cancel_context for new connections; only add @@ -1042,27 +1068,14 @@ def _get_conn( self.active_contexts.add(conn.cancel_context) # Catch KeyboardInterrupt, CancelledError, etc. and cleanup. except BaseException: - if conn: - # We checked out a socket but authentication failed. - conn.close_conn(ConnectionClosedReason.ERROR) - # Re-apply the accounting if a GreenletExit interrupts - # during the size_cond acquisition; during unwind gevent - # lets the re-acquire complete (PYTHON-6074). - accounted = False try: - with self.size_cond: - self.requests -= 1 - if incremented: - self.active_sockets -= 1 - accounted = True - self.size_cond.notify() + if conn is not None: + # We checked out a socket but authentication failed. + conn.close_conn(ConnectionClosedReason.ERROR) finally: - if not accounted: - with self.size_cond: - self.requests -= 1 - if incremented: - self.active_sockets -= 1 - self.size_cond.notify() + # Restore the counters even if the cleanup above was + # interrupted (PYTHON-6136). + self._restore_counters(applied) if not emitted_event: self._telemetry.checkout_failed( @@ -1074,6 +1087,51 @@ def _get_conn( return conn + def _restore_applied(self, applied: int) -> None: + """Restore the counters flagged in ``applied``. Caller holds ``size_cond``.""" + if applied & _UNDO_OPERATION_COUNT: + self.operation_count -= 1 + if applied & _UNDO_REQUESTS: + self.requests -= 1 + if applied & _UNDO_SOCKETS: + self.active_sockets -= 1 + if applied & _UNDO_PENDING: + self._pending -= 1 + + def _restore_counters(self, applied: int) -> None: + """Restore the counters a failed checkout incremented (PYTHON-6136). + + Gevent grants the re-acquire during unwind (PYTHON-6074). A kill can + also land inside notify() (a yield point), so notifications are + tracked and retried separately from the counter restore. + """ + accounted = False + notified = 0 + try: + with self.size_cond: + self._restore_applied(applied) + accounted = True + if applied & _UNDO_REQUESTS: + # A pool slot was freed; wake the next waiting thread. + self.size_cond.notify() + notified |= _UNDO_REQUESTS + if applied & _UNDO_PENDING: + # A maxConnecting slot was freed; wake the next waiting thread. + self._max_connecting_cond.notify() + notified |= _UNDO_PENDING + finally: + # Always reacquired: `applied` keeps restore-only flags + # `notified` never tracks, so skipping needs a mask synced + # to the notify sites; uncontended acquires don't yield (PYTHON-6136). + with self.size_cond: + if not accounted: + self._restore_applied(applied) + missing = applied & ~notified + if missing & _UNDO_REQUESTS: + self.size_cond.notify() + if missing & _UNDO_PENDING: + self._max_connecting_cond.notify() + def _checkin_apply( self, conn: Connection, txn: bool, cursor: bool, forked: bool ) -> tuple[Optional[str], bool, bool]: @@ -1120,9 +1178,7 @@ def checkin(self, conn: Connection) -> None: conn.pinned_cursor = False self._pinned_sockets.discard(conn) forked = self.pid != os.getpid() - # Re-apply the accounting if a gevent GreenletExit interrupts during - # the size_cond acquisition; gevent lets the re-acquire complete while - # unwinding (PYTHON-6074). + # Re-apply the accounting if a BaseException interrupts here (PYTHON-6074). close_conn_reason: Optional[str] = None emit_closed = False accounted = False diff --git a/pymongo/synchronous/settings.py b/pymongo/synchronous/settings.py index 2e8c6ddcd8..3416bbd130 100644 --- a/pymongo/synchronous/settings.py +++ b/pymongo/synchronous/settings.py @@ -18,7 +18,7 @@ import threading from collections.abc import Collection -from typing import Optional +from typing import Callable, Optional from bson.objectid import ObjectId from pymongo import common @@ -51,6 +51,7 @@ def __init__( srv_service_name: str = common.SRV_SERVICE_NAME, srv_max_hosts: int = 0, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, server_monitoring_mode: str = common.SERVER_MONITORING_MODE, topology_id: Optional[ObjectId] = None, ): @@ -77,6 +78,7 @@ def __init__( srv_service_name, srv_max_hosts, srv_allowed_hosts_suffix, + srv_host_validator, server_monitoring_mode, topology_id, ) diff --git a/pymongo/synchronous/srv_resolver.py b/pymongo/synchronous/srv_resolver.py index 0e9745eb08..18083c8a0c 100644 --- a/pymongo/synchronous/srv_resolver.py +++ b/pymongo/synchronous/srv_resolver.py @@ -18,9 +18,10 @@ import ipaddress import random -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Optional, Union -from pymongo._psl import is_public_suffix +from pymongo._psl import SPECIAL_USE_LABELS, _to_punycode, is_public_suffix +from pymongo._telemetry import log_srv_monitor_invalid_host from pymongo.common import CONNECT_TIMEOUT from pymongo.errors import ConfigurationError @@ -64,21 +65,40 @@ def __init__( srv_service_name: str, srv_max_hosts: int = 0, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, ): self.__fqdn = fqdn.lower() self.__srv = srv_service_name self.__connect_timeout = connect_timeout or CONNECT_TIMEOUT self.__srv_max_hosts = srv_max_hosts or 0 - self.__srv_allowed_hosts_suffix = ( - "." + srv_allowed_hosts_suffix.lower().strip(".") if srv_allowed_hosts_suffix else None - ) # ensure there's a . at the beginning of the domain - if self.__srv_allowed_hosts_suffix is not None and is_public_suffix( - self.__srv_allowed_hosts_suffix - ): + self.__srv_host_validator = srv_host_validator + # parse_uri() can call this constructor independently of MongoClient. + if srv_host_validator is not None and srv_allowed_hosts_suffix is not None: raise ConfigurationError( - f"srvAllowedHostsSuffix must not be a public suffix, got: {srv_allowed_hosts_suffix}" + "Cannot specify both srv_host_validator and srvAllowedHostsSuffix" ) - # Validate the fully qualified domain name. + self.__srv_allowed_hosts_suffix = None + if srv_allowed_hosts_suffix is not None: + suffix = srv_allowed_hosts_suffix.strip(".") + if not suffix: + raise ConfigurationError( + f"srvAllowedHostsSuffix must not be empty, got: {srv_allowed_hosts_suffix!r}" + ) + suffix = _to_punycode(suffix).lower() + + is_special_use = suffix in SPECIAL_USE_LABELS + if len(suffix.split(".")) < 2 and not is_special_use: + raise ConfigurationError( + "srvAllowedHostsSuffix must contain at least two '.' separated labels, " + f"got: {srv_allowed_hosts_suffix}" + ) + + if not is_special_use and is_public_suffix(suffix): + raise ConfigurationError( + f"srvAllowedHostsSuffix must not be a public suffix, got: {srv_allowed_hosts_suffix}" + ) + self.__srv_allowed_hosts_suffix = "." + suffix + try: ipaddress.ip_address(fqdn) raise ConfigurationError(_INVALID_HOST_MSG % ("an IP address",)) @@ -106,23 +126,51 @@ def get_options(self) -> Optional[str]: raise ConfigurationError("Only one TXT record is supported") return (b"&".join([b"".join(res.strings) for res in results])).decode("utf-8") # type: ignore[attr-defined] - def _resolve_uri(self, encapsulate_errors: bool) -> resolver.Answer: + def _resolve_uri(self, is_polling: bool) -> resolver.Answer: try: results = _resolve( "_" + self.__srv + "._tcp." + self.__fqdn, "SRV", lifetime=self.__connect_timeout ) except Exception as exc: - if not encapsulate_errors: + if is_polling: # Raise the original error. raise # Else, raise all errors as ConfigurationError. raise ConfigurationError(str(exc)) from exc return results + def _validate_host(self, srv_host: str) -> None: + """Raise ConfigurationError if an SRV-returned host fails verification.""" + if self.__srv_host_validator is not None: + try: + allowed = self.__srv_host_validator(srv_host) + except Exception as exc: + raise ConfigurationError( + f"srv_host_validator raised an exception for SRV host {srv_host}: {exc}" + ) from exc + if not allowed: + raise ConfigurationError( + f"Invalid SRV host: {srv_host} was rejected by srv_host_validator" + ) + elif self.__srv_allowed_hosts_suffix is not None: + if not srv_host.endswith(self.__srv_allowed_hosts_suffix): + raise ConfigurationError(f"Invalid SRV host: {srv_host}") + else: + if self.__fqdn == srv_host and self.nparts < 3: + raise ConfigurationError( + "Invalid SRV host: return address is identical to SRV hostname" + ) + try: + nlist = srv_host.split(".")[1:][-self.__slen :] + except Exception as exc: + raise ConfigurationError(f"Invalid SRV host: {srv_host}") from exc + if self.__plist != nlist: + raise ConfigurationError(f"Invalid SRV host: {srv_host}") + def _get_srv_response_and_hosts( - self, encapsulate_errors: bool + self, is_polling: bool ) -> tuple[resolver.Answer, list[tuple[str, Any]]]: - results = self._resolve_uri(encapsulate_errors) + results = self._resolve_uri(is_polling) # Construct address tuples nodes = [ @@ -130,33 +178,28 @@ def _get_srv_response_and_hosts( for res in results ] - # Validate hosts + # Validate hosts. + valid_nodes = [] for node in nodes: - srv_host = node[0].lower() - if self.__fqdn == srv_host and self.nparts < 3: - raise ConfigurationError( - "Invalid SRV host: return address is identical to SRV hostname" - ) - if self.__srv_allowed_hosts_suffix is not None: - if not srv_host.endswith(self.__srv_allowed_hosts_suffix): - raise ConfigurationError(f"Invalid SRV host: {node[0]}") + try: + self._validate_host(node[0]) + except ConfigurationError as exc: + if not is_polling: + raise + log_srv_monitor_invalid_host(node[0], exc) else: - try: - nlist = srv_host.split(".")[1:][-self.__slen :] - except Exception as exc: - raise ConfigurationError(f"Invalid SRV host: {node[0]}") from exc - if self.__plist != nlist: - raise ConfigurationError(f"Invalid SRV host: {node[0]}") + valid_nodes.append(node) + nodes = valid_nodes if self.__srv_max_hosts: nodes = random.sample(nodes, min(self.__srv_max_hosts, len(nodes))) return results, nodes def get_hosts(self) -> list[tuple[str, Any]]: - _, nodes = self._get_srv_response_and_hosts(True) + _, nodes = self._get_srv_response_and_hosts(is_polling=False) return nodes def get_hosts_and_min_ttl(self) -> tuple[list[tuple[str, Any]], int]: - results, nodes = self._get_srv_response_and_hosts(False) + results, nodes = self._get_srv_response_and_hosts(is_polling=True) rrset = results.rrset ttl = rrset.ttl if rrset else 0 return nodes, ttl diff --git a/pymongo/synchronous/uri_parser.py b/pymongo/synchronous/uri_parser.py index e4f521e3a9..92608b5efe 100644 --- a/pymongo/synchronous/uri_parser.py +++ b/pymongo/synchronous/uri_parser.py @@ -17,7 +17,7 @@ from __future__ import annotations -from typing import Any, Optional +from typing import Any, Callable, Optional from pymongo.common import SRV_SERVICE_NAME, _CaseInsensitiveDictionary from pymongo.errors import ConfigurationError, InvalidURI @@ -48,6 +48,7 @@ def parse_uri( srv_service_name: Optional[str] = None, srv_max_hosts: Optional[int] = None, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, ) -> dict[str, Any]: """Parse and validate a MongoDB URI. @@ -117,6 +118,7 @@ def parse_uri( srv_service_name, srv_max_hosts, srv_allowed_hosts_suffix, + srv_host_validator, ) ) result["options"] = _make_options_case_sensitive(result["options"]) @@ -133,6 +135,7 @@ def _parse_srv( srv_service_name: Optional[str] = None, srv_max_hosts: Optional[int] = None, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, ) -> dict[str, Any]: if uri.startswith(SCHEME): is_srv = False @@ -168,7 +171,12 @@ def _parse_srv( # argument overrides the same option passed in the connection string. connect_timeout = connect_timeout or options.get("connectTimeoutMS") dns_resolver = _SrvResolver( - fqdn, connect_timeout, srv_service_name, srv_max_hosts, srv_allowed_hosts_suffix + fqdn, + connect_timeout, + srv_service_name, + srv_max_hosts, + srv_allowed_hosts_suffix, + srv_host_validator, ) nodes = dns_resolver.get_hosts() dns_options = dns_resolver.get_options() diff --git a/pymongo/typings.py b/pymongo/typings.py index 0281c57c59..e0777e07e1 100644 --- a/pymongo/typings.py +++ b/pymongo/typings.py @@ -70,7 +70,6 @@ def strip_optional(elem: Optional[_T]) -> _T: __all__ = [ "_Address", - "_AgnosticMongoClient", "_CollationIn", "_DocumentOut", "_DocumentType", diff --git a/pyproject.toml b/pyproject.toml index fce3ad3445..1aec34beb9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,6 +94,11 @@ validate-bump = false [tool.hatch.build.targets.wheel] packages = ["bson","gridfs", "pymongo"] +[tool.hatch.build.targets.sdist] +# The drivers-evergreen-tools submodule must not ship in the sdist; hatchling's +# walk would otherwise pick it up when the submodule is populated locally. +exclude = ["drivers-evergreen-tools"] + [tool.hatch.metadata.hooks.requirements_txt] files = ["requirements.txt"] diff --git a/requirements/mod_wsgi.txt b/requirements/mod_wsgi.txt new file mode 100644 index 0000000000..31bb5dfda8 --- /dev/null +++ b/requirements/mod_wsgi.txt @@ -0,0 +1,3 @@ +# Test-only; kept out of pyproject.toml so the uv lock never has to +# resolve it (building its sdist needs Apache). +mod_wsgi>=4.9.4 diff --git a/test/asynchronous/test_client.py b/test/asynchronous/test_client.py index 0c95ba978b..563d29cbad 100644 --- a/test/asynchronous/test_client.py +++ b/test/asynchronous/test_client.py @@ -90,7 +90,13 @@ WriteConcernError, ) from pymongo.monitoring import ServerHeartbeatListener, ServerHeartbeatStartedEvent -from pymongo.pool_options import _MAX_METADATA_SIZE, _METADATA, ENV_VAR_K8S, PoolOptions +from pymongo.pool_options import ( + _MAX_METADATA_SIZE, + _METADATA, + ENV_VAR_K8S, + PoolOptions, + _truncate_metadata, +) from pymongo.read_preferences import ReadPreference from pymongo.server_description import ServerDescription from pymongo.server_selectors import readable_server_selector, writable_server_selector @@ -124,6 +130,8 @@ NTHREADS, CMAPListener, FunctionCallRecorder, + _driver_version, + _metadata_with_appended_driver, delay, gevent_monkey_patched, is_greenthread_patched, @@ -387,6 +395,9 @@ async def test_metadata(self): metadata["driver"]["name"] = "PyMongo|c|async" else: metadata["driver"]["name"] = "PyMongo|async" + metadata["driver"]["version"] = _driver_version( + _METADATA["driver"]["version"], metadata["driver"]["name"] + ) metadata["application"] = {"name": "foobar"} client = self.simple_client("mongodb://foo:27017/?appname=foobar&connect=false") options = client.options @@ -398,6 +409,8 @@ async def test_metadata(self): self.simple_client(appname="x" * 128) with self.assertRaises(ValueError): self.simple_client(appname="x" * 129) + + async def test_metadata_bad_driver_options(self): # Bad "driver" options. self.assertRaises(TypeError, DriverInfo, "Foo", 1, "a") self.assertRaises(TypeError, DriverInfo, version="1", platform="a") @@ -408,12 +421,9 @@ async def test_metadata(self): self.simple_client(driver="abc") with self.assertRaises(TypeError): self.simple_client(driver=("Foo", "1", "a")) - # Test appending to driver info. - if has_c(): - metadata["driver"]["name"] = "PyMongo|c|async|FooDriver" - else: - metadata["driver"]["name"] = "PyMongo|async|FooDriver" - metadata["driver"]["version"] = "{}|1.2.3".format(_METADATA["driver"]["version"]) + + async def test_metadata_appends_driver_info(self): + metadata = _metadata_with_appended_driver(_IS_SYNC, "FooDriver", "1.2.3") client = self.simple_client( "foo", 27017, @@ -421,9 +431,9 @@ async def test_metadata(self): driver=DriverInfo("FooDriver", "1.2.3", None), connect=False, ) - options = client.options - self.assertEqual(options.pool_options.metadata, metadata) - metadata["platform"] = "{}|FooPlatform".format(_METADATA["platform"]) + self.assertEqual(client.options.pool_options.metadata, metadata) + + metadata = _metadata_with_appended_driver(_IS_SYNC, "FooDriver", "1.2.3", "FooPlatform") client = self.simple_client( "foo", 27017, @@ -431,27 +441,131 @@ async def test_metadata(self): driver=DriverInfo("FooDriver", "1.2.3", "FooPlatform"), connect=False, ) - options = client.options - self.assertEqual(options.pool_options.metadata, metadata) - # Test truncating driver info metadata. + self.assertEqual(client.options.pool_options.metadata, metadata) + + async def test_metadata_truncates_driver_info(self): + # Truncated driver info must stay within the limit and keep name and + # version index-aligned. client = self.simple_client( driver=DriverInfo(name="s" * _MAX_METADATA_SIZE), connect=False, ) options = client.options + truncated = options.pool_options.metadata["driver"] self.assertLessEqual( len(bson.encode(options.pool_options.metadata)), _MAX_METADATA_SIZE, ) + self.assertEqual( + truncated["name"].count("|"), + truncated["version"].count("|"), + ) client = self.simple_client( driver=DriverInfo(name="s" * _MAX_METADATA_SIZE, version="s" * _MAX_METADATA_SIZE), connect=False, ) options = client.options + truncated = options.pool_options.metadata["driver"] self.assertLessEqual( len(bson.encode(options.pool_options.metadata)), _MAX_METADATA_SIZE, ) + self.assertEqual( + truncated["name"].count("|"), + truncated["version"].count("|"), + ) + # An oversized wrapper name with no version must retain a truncated + # name rather than collapse to the base entry. + client = self.simple_client( + driver=DriverInfo(name="x" * (_MAX_METADATA_SIZE * 2), version=None), + connect=False, + ) + truncated = client.options.pool_options.metadata["driver"] + self.assertLessEqual( + len(bson.encode(client.options.pool_options.metadata)), + _MAX_METADATA_SIZE, + ) + self.assertIn("xxxx", truncated["name"]) + self.assertEqual( + truncated["name"].count("|"), + truncated["version"].count("|"), + ) + + async def test_metadata_truncation_drops_blank_pairs_first(self): + # A platform-only driver appends a blank name/version pair. Truncation + # must drop that pair before trimming a real driver's content. + def metadata_for(name_len: int) -> dict[str, Any]: + name = "PyMongo||" + "W" * name_len # blank pair in the middle + return {"driver": {"name": name, "version": "1.0||1.0"}} + + # Size the name so the document is exactly two bytes over the limit: + # dropping the blank pair alone must make it fit. + name_len = next( + n + for n in range(1, 2 * _MAX_METADATA_SIZE) + if len(bson.encode(metadata_for(n))) == _MAX_METADATA_SIZE + 2 + ) + metadata = metadata_for(name_len) + _truncate_metadata(metadata) + self.assertEqual(len(bson.encode(metadata)), _MAX_METADATA_SIZE) + self.assertEqual( + metadata["driver"], + {"name": "PyMongo|" + "W" * name_len, "version": "1.0|1.0"}, + ) + + async def test_metadata_append_is_bounded(self): + # Successive appends must stay within the limit and keep name and + # version index-aligned after truncation. Once the metadata saturates, + # further appends must not grow the dedup tracking list. + client = self.simple_client(connect=False) + for i in range(300): + client.append_metadata(DriverInfo(name=f"D{i}", version=f"1.{i}")) + pool = client.options.pool_options + self.assertLessEqual(len(bson.encode(pool.metadata)), _MAX_METADATA_SIZE) + self.assertEqual( + pool.metadata["driver"]["name"].count("|"), + pool.metadata["driver"]["version"].count("|"), + ) + count = len(pool._PoolOptions__appended_drivers) + for i in range(300, 600): + client.append_metadata(DriverInfo(name=f"D{i}", version=f"1.{i}")) + self.assertEqual(len(pool._PoolOptions__appended_drivers), count) + # Platform-only appends (empty name/version) stay bounded the same way. + client = self.simple_client(connect=False) + for i in range(300): + client.append_metadata(DriverInfo(name="", version="", platform=f"P{i}")) + pool = client.options.pool_options + self.assertLessEqual(len(bson.encode(pool.metadata)), _MAX_METADATA_SIZE) + count = len(pool._PoolOptions__appended_drivers) + for i in range(300, 600): + client.append_metadata(DriverInfo(name="", version="", platform=f"P{i}")) + self.assertEqual(len(pool._PoolOptions__appended_drivers), count) + + async def test_metadata_recreates_platform_after_truncation(self): + # Appending a platform after truncation has dropped it recreates the field. + client = self.simple_client(connect=False) + for i in range(300): + client.append_metadata(DriverInfo(name="", version="", platform=f"Q{i}")) + pool = client.options.pool_options + self.assertLess(len(pool._PoolOptions__appended_drivers), 300) + client.append_metadata(DriverInfo(name="Wrapper", version="1.0", platform="Recreated")) + self.assertLessEqual(len(bson.encode(pool.metadata)), _MAX_METADATA_SIZE) + self.assertEqual( + pool.metadata["driver"]["name"].count("|"), + pool.metadata["driver"]["version"].count("|"), + ) + + async def test_metadata_deduplicates_none_and_empty(self): + # Empty strings are treated as unset, so a duplicate differing only in + # None vs "" is a no-op. + client = self.simple_client(connect=False) + client.append_metadata(DriverInfo("library", None, "Library Platform")) + names = client.options.pool_options.metadata["driver"]["name"] + vers = client.options.pool_options.metadata["driver"]["version"] + client.append_metadata(DriverInfo("library", "", "Library Platform")) + metadata = client.options.pool_options.metadata + self.assertEqual(metadata["driver"]["name"], names) + self.assertEqual(metadata["driver"]["version"], vers) @mock.patch.dict("os.environ", {ENV_VAR_K8S: "1"}) def test_container_metadata(self): @@ -2226,6 +2340,9 @@ async def _test_handshake(self, env_vars, expected_env): metadata["driver"]["name"] = "PyMongo|c|async" else: metadata["driver"]["name"] = "PyMongo|async" + metadata["driver"]["version"] = _driver_version( + _METADATA["driver"]["version"], metadata["driver"]["name"] + ) if expected_env is not None: metadata["env"] = expected_env @@ -2627,20 +2744,18 @@ def test_gevent_kill_churn_deadlock(self): import gevent.thread as _gthread from gevent import Timeout, spawn - # AMPLIFY_RACE=1 widens gevent's brief sleep on a contended lock - # so a kill lands there reliably. Only the bare sleep() is - # widened; timed sleeps pass through. The unfixed test then - # deadlocks within seconds. + # AMPLIFY_RACE=1 widens the kill windows below; keep AMPLIFY_SECONDS + # small or ops starve. + amplify_seconds = 0.0 if os.environ.get("AMPLIFY_RACE", "0") == "1": - _AMPLIFY_SECONDS = float(os.environ.get("AMPLIFY_SECONDS", "0.02")) + amplify_seconds = float(os.environ.get("AMPLIFY_SECONDS", "0.002")) _orig_thread_sleep = _gthread.sleep def _amplified_sleep(*args): - if not args: # bare sleep(): the courtesy yield on a failed - # non-blocking acquire (Condition.notify -> _is_owned -> acquire(False)) - _orig_thread_sleep(_AMPLIFY_SECONDS) - else: # sleep(0.001), sleep(2), etc.: passthrough + if not args: # bare sleep(): checkin courtesy yield + _orig_thread_sleep(amplify_seconds) + else: # timed sleeps pass through _orig_thread_sleep(*args) _gthread.sleep = _amplified_sleep @@ -2650,6 +2765,31 @@ def _amplified_sleep(*args): coll = client.pymongo_test.coll coll.insert_one({}) + pool = async_get_pool(client) # type:ignore + # Widen the post-gate checkout windows (PYTHON-6136). + if amplify_seconds: + + class _AmplifiedCondition: + def __init__(self, cond, seconds): + self._cond = cond + self._seconds = seconds + + def __enter__(self): + self._cond.__enter__() + return self + + def __exit__(self, *args): + self._cond.__exit__(*args) + time.sleep(self._seconds) + + def __getattr__(self, name): + return getattr(self._cond, name) + + pool.size_cond = _AmplifiedCondition(pool.size_cond, amplify_seconds) + pool._max_connecting_cond = _AmplifiedCondition( + pool._max_connecting_cond, amplify_seconds + ) + op_count = [0] running = [True] workers: list = [] @@ -2663,9 +2803,12 @@ def worker(): except Exception: return + # Scale the kill cadence or ops starve. + reaper_interval = max(0.003, amplify_seconds * 8) + def reaper(): while running[0]: - time.sleep(0.003) + time.sleep(reaper_interval) if not workers: continue idx = random.randrange(len(workers)) @@ -2706,11 +2849,6 @@ def reaper(): coll.find_one({}) except Timeout: self.fail("Pool gate saturated (PYTHON-6074)") - # Deterministic check: a saturated size gate pins the pool's - # checkout counters at maxPoolSize (PYTHON-6074). - pool = async_get_pool(client) # type:ignore - self.assertLess(pool.requests, pool.max_pool_size) - self.assertLess(pool.active_sockets, pool.max_pool_size) self.assertGreater(op_count[0], 0) finally: running[0] = False @@ -2728,6 +2866,14 @@ def reaper(): client.close() except Timeout: pass + # Post-settle: the counters must have fully drained. Note close() + # does not zero operation_count (only a fork does), so a leaked + # increment survives and is caught here. + time.sleep(1.0) + self.assertEqual(pool.requests, 0) + self.assertEqual(pool._pending, 0) + self.assertEqual(pool.active_sockets, 0) + self.assertEqual(pool.operation_count, 0) class TestClientLazyConnect(AsyncIntegrationTest): diff --git a/test/asynchronous/test_client_metadata.py b/test/asynchronous/test_client_metadata.py index 1a07e835a8..a0438fb72a 100644 --- a/test/asynchronous/test_client_metadata.py +++ b/test/asynchronous/test_client_metadata.py @@ -18,7 +18,7 @@ import pathlib import time import unittest -from typing import Any, Optional +from typing import Any, Optional, cast import pytest @@ -99,20 +99,14 @@ async def check_metadata_added( new_name, new_version, new_platform, new_metadata = await self.send_ping_and_get_metadata( client, True ) - if add_name is not None and add_name.lower() in name.lower().split("|"): - self.assertEqual(name, new_name) - self.assertEqual(version, new_version) - self.assertEqual(platform, new_platform) - else: - self.assertEqual(new_name, f"{name}|{add_name}" if add_name is not None else name) - self.assertEqual( - new_version, - f"{version}|{add_version}" if add_version is not None else version, - ) - self.assertEqual( - new_platform, - f"{platform}|{add_platform}" if add_platform is not None else platform, - ) + # Name and version always get a delimiter (empty string if None) to + # preserve 1:1 index correspondence. + self.assertEqual(new_name, f"{name}|{add_name or ''}") + self.assertEqual(new_version, f"{version}|{add_version or ''}") + self.assertEqual( + new_platform, + f"{platform}|{add_platform}" if add_platform is not None else platform, + ) metadata.pop("driver") metadata.pop("platform") @@ -120,7 +114,7 @@ async def check_metadata_added( new_metadata.pop("platform") self.assertEqual(metadata, new_metadata) - async def test_append_metadata(self): + async def test_1_test_that_the_driver_updates_metadata(self): client = await self.async_rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -128,7 +122,7 @@ async def test_append_metadata(self): ) await self.check_metadata_added(client, "framework", "2.0", "Framework Platform") - async def test_append_metadata_platform_none(self): + async def test_1_test_that_the_driver_updates_metadata_platform_none(self): client = await self.async_rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -136,7 +130,7 @@ async def test_append_metadata_platform_none(self): ) await self.check_metadata_added(client, "framework", "2.0", None) - async def test_append_metadata_version_none(self): + async def test_1_test_that_the_driver_updates_metadata_version_none(self): client = await self.async_rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -144,7 +138,7 @@ async def test_append_metadata_version_none(self): ) await self.check_metadata_added(client, "framework", None, "Framework Platform") - async def test_append_metadata_platform_version_none(self): + async def test_1_test_that_the_driver_updates_metadata_platform_version_none(self): client = await self.async_rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -152,14 +146,14 @@ async def test_append_metadata_platform_version_none(self): ) await self.check_metadata_added(client, "framework", None, None) - async def test_multiple_successive_metadata_updates(self): + async def test_2_multiple_successive_metadata_updates(self): client = await self.async_rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, connect=False ) client.append_metadata(DriverInfo("library", "1.2", "Library Platform")) await self.check_metadata_added(client, "framework", "2.0", "Framework Platform") - async def test_multiple_successive_metadata_updates_platform_none(self): + async def test_2_multiple_successive_metadata_updates_platform_none(self): client = await self.async_rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -167,7 +161,7 @@ async def test_multiple_successive_metadata_updates_platform_none(self): client.append_metadata(DriverInfo("library", "1.2", "Library Platform")) await self.check_metadata_added(client, "framework", "2.0", None) - async def test_multiple_successive_metadata_updates_version_none(self): + async def test_2_multiple_successive_metadata_updates_version_none(self): client = await self.async_rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -175,7 +169,7 @@ async def test_multiple_successive_metadata_updates_version_none(self): client.append_metadata(DriverInfo("library", "1.2", "Library Platform")) await self.check_metadata_added(client, "framework", None, "Framework Platform") - async def test_multiple_successive_metadata_updates_platform_version_none(self): + async def test_2_multiple_successive_metadata_updates_platform_version_none(self): client = await self.async_rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -216,10 +210,16 @@ async def test_duplicate_driver_name_no_op(self): await self.check_metadata_added(client, "framework", None, None) # wait for connection to become idle await asyncio.sleep(0.005) - # add same metadata again - await self.check_metadata_added(client, "Framework", None, None) + # Append the exact same DriverInfo again: no-op. + name, version, platform, _ = await self.send_ping_and_get_metadata(client, True) + await asyncio.sleep(0.005) + client.append_metadata(DriverInfo("framework", None, None)) + new_name, new_version, new_platform, _ = await self.send_ping_and_get_metadata(client, True) + self.assertEqual(new_name, name) + self.assertEqual(new_version, version) + self.assertEqual(new_platform, platform) - async def test_handshake_documents_include_backpressure(self): + async def test_9_handshake_documents_include_backpressure(self): # Create a `MongoClient` that is configured to record all handshake documents sent to the server as a part of # connection establishment. client = await self.async_rs_or_single_client("mongodb://" + self.server.address_string) @@ -232,6 +232,131 @@ async def test_handshake_documents_include_backpressure(self): # the document has a field `backpressure` whose value is `"2"`. self.assertEqual(self.handshake_req["backpressure"], "2") + async def test_10_entries_in_driver_name_and_driver_version_correspond_by_index(self): + cases = [ + ("Gap in middle (name)", [(None, None), ("F2", None)], "||F2", "||"), + ("Gap in middle (version)", [("F1", None), ("F2", "2.0")], "|F1|F2", "||2.0"), + ("Trailing delimiter retained", [("F1", None)], "|F1", "|"), + ( + "Equal versions do not collapse", + [("F1", "{driver_version}")], + "|F1", + "|{driver_version}", + ), + ( + "Equal names do not collapse", + [("{driver_name}", "1.0")], + "|{driver_name}", + "|1.0", + ), + ("Duplicates deduplicate", [("F1", "1.0"), ("F1", "1.0")], "|F1", "|1.0"), + ("All versions absent", [("F1", None), ("F2", None)], "|F1|F2", "||"), + ("All names absent", [(None, "1.0"), (None, "2.0")], "||", "|1.0|2.0"), + ( + "Non-adjacent duplicate", + [("F1", "1.0"), ("F2", "2.0"), ("F1", "1.0")], + "|F1|F2", + "|1.0|2.0", + ), + ( + "Platform-only difference is not a duplicate", + [("F1", "1.0", "P1"), ("F1", "1.0", "P2")], + "|F1|F1", + "|1.0|1.0", + ), + ( + "Wrapper matching the driver's own identity", + [("{driver_name}", "{driver_version}")], + "|{driver_name}", + "|{driver_version}", + ), + ( + "Duplicates with an unset field deduplicate", + [("F1", None), ("F1", None)], + "|F1", + "|", + ), + ] + for ( + description, + appended, + expected_name_suffix, + expected_version_suffix, + ) in cases: + with self.subTest(description=description): + client = await self.async_rs_or_single_client( + "mongodb://" + self.server.address_string, + maxIdleTimeMS=1, + ) + self.addAsyncCleanup(client.close) + # Capture the driver's own name and version from the first handshake. + name0, version0, _, _ = await self.send_ping_and_get_metadata(client, True) + await asyncio.sleep(0.005) + + self.assertIsNotNone(name0) + self.assertIsNotNone(version0) + version0 = cast(str, version0) + driver_name = name0.split("|")[0] + driver_version = version0.split("|")[0] + + def resolve(value: Optional[str]) -> Optional[str]: + if value is None: + return None + return value.format(driver_name=driver_name, driver_version=driver_version) + + # Append each DriverInfo in order. + for opts in appended: + d_name = resolve(opts[0]) if len(opts) > 0 else None + d_version = resolve(opts[1]) if len(opts) > 1 else None + d_platform = resolve(opts[2]) if len(opts) > 2 else None + client.append_metadata(DriverInfo(d_name or "", d_version, d_platform)) + + # New handshake with the appended metadata. + name1, version1, _, _ = await self.send_ping_and_get_metadata(client, True) + + self.assertEqual( + name1, + name0 + + expected_name_suffix.format( + driver_name=driver_name, driver_version=driver_version + ), + ) + self.assertEqual( + version1, + version0 + + expected_version_suffix.format( + driver_name=driver_name, driver_version=driver_version + ), + ) + + async def test_11_appending_metadata_containing_the_delimiter_raises_an_error(self): + cases = [ + ("frame|work", "2.0", "Framework Platform"), + ("framework", "2|0", "Framework Platform"), + ("framework", "2.0", "Framework|Platform"), + ] + for name, version, platform in cases: + with self.subTest(name=name, version=version, platform=platform): + client = await self.async_rs_or_single_client( + "mongodb://" + self.server.address_string, + maxIdleTimeMS=1, + driver=DriverInfo("library", "1.2", "Library Platform"), + ) + self.addAsyncCleanup(client.close) + # Send initial handshake. + name0, version0, platform0, _metadata = await self.send_ping_and_get_metadata( + client, True + ) + await asyncio.sleep(0.005) + # Constructing metadata containing the delimiter raises. + with self.assertRaises(ValueError): + DriverInfo(name, version, platform) + # Metadata is unchanged on the next handshake. + name1, version1, platform1, _ = await self.send_ping_and_get_metadata(client, True) + self.assertEqual(name1, name0) + self.assertEqual(version1, version0) + self.assertEqual(platform1, platform0) + if __name__ == "__main__": unittest.main() diff --git a/test/asynchronous/test_dns.py b/test/asynchronous/test_dns.py index 6796ecff55..befa119b46 100644 --- a/test/asynchronous/test_dns.py +++ b/test/asynchronous/test_dns.py @@ -209,30 +209,40 @@ async def test_connect_case_insensitive(self): class TestInitialDnsSeedlistDiscovery(AsyncPyMongoTestCase): """ Initial DNS Seedlist Discovery prose tests - https://github.com/mongodb/specifications/blob/0a7a8b5/source/initial-dns-seedlist-discovery/tests/README.md#prose-tests + https://github.com/mongodb/specifications/blob/5036f26/source/initial-dns-seedlist-discovery/tests/README.md#prose-tests + + Numbered tests correspond to the numbered prose tests in the spec. The + unnumbered tests are PyMongo-specific additions with no spec counterpart. """ + async def _parse(self, srv_hostname, mock_target, **kwargs): + """Resolve mongodb+srv:// with SRV records naming mock_target.""" + with patch("dns.asyncresolver.resolve") as mock_resolver: + + async def mock_resolve(query, record_type, *args, **kwargs): + mock_srv = MagicMock() + # Mirror dnspython: the wire form keeps the root label, and the + # caller strips it via omit_final_dot. + mock_srv.target.to_text.side_effect = lambda omit_final_dot=False: ( + mock_target.rstrip(".") if omit_final_dot else mock_target + ) + return [mock_srv] + + mock_resolver.side_effect = mock_resolve + return await parse_uri(f"mongodb+srv://{srv_hostname}", **kwargs) + async def run_initial_dns_seedlist_discovery_prose_tests(self, test_cases): for case in test_cases: - with patch("dns.asyncresolver.resolve") as mock_resolver: - - async def mock_resolve(query, record_type, *args, **kwargs): - mock_srv = MagicMock() - mock_srv.target.to_text.return_value = case["mock_target"] - return [mock_srv] - - mock_resolver.side_effect = mock_resolve - domain = case["query"].split("._tcp.")[1] - connection_string = f"mongodb+srv://{domain}" - if "expected_error" not in case: - await parse_uri(connection_string) + domain = case["query"].split("._tcp.")[1] + if "expected_error" not in case: + await self._parse(domain, case["mock_target"]) + else: + try: + await self._parse(domain, case["mock_target"]) + except ConfigurationError as e: + self.assertIn(case["expected_error"], str(e)) else: - try: - await parse_uri(connection_string) - except ConfigurationError as e: - self.assertIn(case["expected_error"], str(e)) - else: - self.fail(f"ConfigurationError was not raised for query: {case['query']}") + self.fail(f"ConfigurationError was not raised for query: {case['query']}") async def test_1_allow_srv_hosts_with_fewer_than_three_dot_separated_parts(self): with patch("dns.asyncresolver.resolve"): @@ -252,8 +262,8 @@ async def test_2_throw_when_return_address_does_not_end_with_srv_domain(self): "expected_error": "Invalid SRV host", }, { - "query": "_mongodb._tcp.blogs.mongo.local", - "mock_target": "test_1.evil.com", + "query": "_mongodb._tcp.mongo.local", + "mock_target": "test_1.evil.local", "expected_error": "Invalid SRV host", }, ] @@ -296,7 +306,103 @@ async def test_4_throw_when_return_address_does_not_contain_dot_separating_share ] await self.run_initial_dns_seedlist_discovery_prose_tests(test_cases) - async def test_5_when_srv_hostname_has_three_or_more_dot_separated_parts_it_is_valid_for_the_returned_hostname_to_be_identical( + async def test_5_srv_host_validator_accepts_a_host_the_default_verification_would_reject(self): + # "blogs.evil.com" does not share a parent domain with the seed, so the + # default check rejects it; the callback overrides that decision. + res = await self._parse( + "blogs.mongodb.com", "blogs.evil.com", srv_host_validator=lambda host: True + ) + self.assertEqual(["blogs.evil.com"], [node[0] for node in res["nodelist"]]) + + # "mongo.local" does not add a domain level to an SRV hostname with fewer + # than three "." separated parts, which the default check also rejects. + res = await self._parse("mongo.local", "mongo.local", srv_host_validator=lambda host: True) + self.assertEqual(["mongo.local"], [node[0] for node in res["nodelist"]]) + + async def test_6_reject_a_host_the_default_verification_would_accept(self): + with self.assertRaisesRegex(ConfigurationError, "rejected by srv_host_validator"): + await self._parse( + "blogs.mongodb.com", "cluster.mongodb.com", srv_host_validator=lambda host: False + ) + + async def test_7_the_validator_receives_the_normalized_host_name(self): + seen = [] + + def validator(host): + seen.append(host) + return True + + await self._parse("blogs.mongodb.com", "CLUSTER.MONGODB.COM.", srv_host_validator=validator) + self.assertEqual(["cluster.mongodb.com"], seen) + + async def test_8_wrap_an_error_raised_by_the_validator(self): + original_exc = Exception("validator_error") + + def validator(host): + raise original_exc + + with self.assertRaisesRegex( + ConfigurationError, "srv_host_validator raised an exception" + ) as ctx: + await self._parse( + "blogs.mongodb.com", "cluster.mongodb.com", srv_host_validator=validator + ) + # The wrapping error must retain the error raised by the validator. + self.assertIs(original_exc, ctx.exception.__cause__) + self.assertIn("validator_error", str(ctx.exception)) + + async def test_9_throw_when_both_srv_allowed_hosts_suffix_and_srv_host_validator_are_configured( + self, + ): + # Rejected by the client + with self.assertRaisesRegex(ConfigurationError, "Cannot specify both"): + self.simple_client( + "mongodb+srv://blogs.mongodb.com", + srv_host_validator=lambda host: True, + srvAllowedHostsSuffix=".mongodb.com", + connect=False, + ) + + # Rejected by the resolver + with self.assertRaisesRegex(ConfigurationError, "Cannot specify both"): + await self._parse( + "blogs.mongodb.com", + "cluster.mongodb.com", + srv_host_validator=lambda host: True, + srv_allowed_hosts_suffix=".mongodb.com", + ) + + async def test_10_accept_a_mixed_case_returned_address_with_srv_allowed_hosts_suffix(self): + # Returned addresses are normalized before the suffix comparison, so + # the case DNS happens to use must not affect the result. + res = await self._parse( + "blogs.mongodb.com", "CLUSTER.MONGODB.COM.", srv_allowed_hosts_suffix=".mongodb.com" + ) + self.assertEqual(["cluster.mongodb.com"], [node[0] for node in res["nodelist"]]) + + async def test_11_throw_when_srv_host_validator_is_not_callable(self): + with self.assertRaisesRegex(ValueError, "must be a callable"): + self.simple_client("mongodb+srv://blogs.mongodb.com", srv_host_validator="notacallable") + + async def test_12_accept_a_reserved_single_label_as_srv_allowed_hosts_suffix(self): + # A single label is a public suffix under the Public Suffix List's "*" + # rule, but the reserved names are accepted despite that. + res = await self._parse( + "cluster.localhost", "db.cluster.localhost", srv_allowed_hosts_suffix="localhost" + ) + self.assertEqual(["db.cluster.localhost"], [node[0] for node in res["nodelist"]]) + + async def test_13_throw_when_srv_host_validator_is_used_with_a_non_srv_uri(self): + with self.assertRaisesRegex( + ConfigurationError, "only allowed with 'mongodb\\+srv://' URIs" + ): + self.simple_client( + "mongodb://localhost:27017", + srv_host_validator=lambda host: True, + connect=False, + ) + + async def test_srv_hostname_with_three_or_more_parts_may_equal_the_returned_hostname( self, ): test_cases = [ diff --git a/test/asynchronous/test_pooling.py b/test/asynchronous/test_pooling.py index 661bd4e3d2..25da5134b9 100644 --- a/test/asynchronous/test_pooling.py +++ b/test/asynchronous/test_pooling.py @@ -306,6 +306,7 @@ def notify(): # Accounting was applied exactly once. self.assertEqual(0, cx_pool.requests) self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool.operation_count) async def test_checkout_error_accounting_on_kill_during_acquire(self): # PYTHON-6074: an exception delivered while the checkout error @@ -341,6 +342,98 @@ async def __aexit__(self, *args): # The fallback applied the accounting exactly once. self.assertEqual(0, cx_pool.requests) self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool.operation_count) + + async def test_checkout_error_accounting_on_connect_keyboard_interrupt(self): + # PYTHON-6136: a KeyboardInterrupt from connect() must roll back the + # size gate, the maxConnecting gate, and active_sockets. + cx_pool = await self.create_pool(max_pool_size=1) + + with patch.object(cx_pool, "connect", side_effect=KeyboardInterrupt()): + with self.assertRaises(KeyboardInterrupt): + async with cx_pool.checkout(): + pass + + self.assertEqual(0, cx_pool.requests) + self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool._pending) + self.assertEqual(0, cx_pool.operation_count) + + async def test_checkout_error_accounting_on_kill_during_pending_cleanup(self): + # PYTHON-6136: an interruption during the pending-gate cleanup must + # not skip the counter restore, which must wake threads waiting at + # the maxConnecting gate. + cx_pool = await self.create_pool(max_pool_size=1) + + class _InterruptOnSecondEnter(type(cx_pool._max_connecting_cond)): + def __init__(self, lock): + super().__init__(lock) + self.enters = 0 + self.notifies = 0 + + async def __aenter__(self): + self.enters += 1 + if self.enters == 2: + # First enter is the checkout wait, second is connect()'s + # cleanup. Simulate a kill delivered while waiting there. + raise KeyboardInterrupt() + return await super().__aenter__() + + async def __aexit__(self, *args): + return await super().__aexit__(*args) + + def notify(self, n=1): + # The counter restore wakes the maxConnecting gate, then is + # itself killed. + self.notifies += 1 + raise KeyboardInterrupt() + + cond = _InterruptOnSecondEnter(cx_pool._max_connecting_cond._lock) + cx_pool._max_connecting_cond = cond + + with patch.object(cx_pool, "connect", side_effect=asyncio.CancelledError()): + with self.assertRaises(KeyboardInterrupt): + async with cx_pool.checkout(): + pass + + self.assertEqual(0, cx_pool.requests) + self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool._pending) + self.assertEqual(0, cx_pool.operation_count) + # The restore notified the maxConnecting gate; the notify interrupted + # by the kill was retried. + self.assertEqual(2, cond.notifies) + + async def test_checkout_error_accounting_on_kill_during_pending_notify(self): + # PYTHON-6136: a kill landing inside the cleanup notify must not + # strand a checkout waiting at the maxConnecting gate. + cx_pool = await self.create_pool(max_pool_size=1) + + class _InterruptOnFirstNotify(type(cx_pool._max_connecting_cond)): + def __init__(self, lock): + super().__init__(lock) + self.notifies = 0 + + def notify(self, n=1): + self.notifies += 1 + if self.notifies == 1: + # Simulate a kill delivered inside notify(). + raise KeyboardInterrupt() + + cond = _InterruptOnFirstNotify(cx_pool._max_connecting_cond._lock) + cx_pool._max_connecting_cond = cond + + with patch.object(cx_pool, "connect", side_effect=asyncio.CancelledError()): + with self.assertRaises(KeyboardInterrupt): + async with cx_pool.checkout(): + pass + + # The cleanup notify was interrupted, then retried. + self.assertEqual(2, cond.notifies) + self.assertEqual(0, cx_pool.requests) + self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool._pending) + self.assertEqual(0, cx_pool.operation_count) async def test_pool_removes_closed_socket(self): # Test that Pool removes explicitly closed socket. @@ -463,6 +556,8 @@ async def test_wait_queue_timeout(self): 1, f"Waited {duration:.2f} seconds for a socket, expected {wait_queue_timeout:f}", ) + # The load metric must not be inflated by the failed checkout. + self.assertEqual(0, pool.operation_count) async def test_no_wait_queue_timeout(self): # Verify get_socket() with no wait_queue_timeout blocks forever. diff --git a/test/asynchronous/test_srv_polling.py b/test/asynchronous/test_srv_polling.py index b8b9129925..3d98a6fdc8 100644 --- a/test/asynchronous/test_srv_polling.py +++ b/test/asynchronous/test_srv_polling.py @@ -19,7 +19,9 @@ import asyncio import sys import time +from types import SimpleNamespace from typing import Any +from unittest.mock import patch from test.utils_shared import FunctionCallRecorder @@ -37,6 +39,19 @@ WAIT_TIME = 0.1 +class _FakeSrvAnswer(list): + """A minimal stand-in for the dnspython answer to an SRV query.""" + + def __init__(self, hosts, ttl=60): + import dns.name + + hosts = [(h, 27017) if isinstance(h, str) else h for h in hosts] + super().__init__( + SimpleNamespace(target=dns.name.from_text(h), port=port) for h, port in hosts + ) + self.rrset = SimpleNamespace(ttl=ttl) + + class SrvPollingKnobs: def __init__( self, @@ -64,9 +79,15 @@ def enable(self): async def mock_get_hosts_and_min_ttl(resolver, *args): assert self.old_dns_resolver_response is not None - nodes, ttl = await self.old_dns_resolver_response(resolver) - if self.nodelist_callback is not None: - nodes = self.nodelist_callback() + if self.nodelist_callback is None: + nodes, ttl = await self.old_dns_resolver_response(resolver) + else: + + async def mock_resolve_uri(is_polling): + return _FakeSrvAnswer(self.nodelist_callback()) + + with patch.object(resolver, "_resolve_uri", mock_resolve_uri): + nodes, ttl = await self.old_dns_resolver_response(resolver) if self.ttl_time is not None: ttl = self.ttl_time return nodes, ttl @@ -357,8 +378,92 @@ def nodelist_callback(): ) await client.aconnect() with SrvPollingKnobs(nodelist_callback=nodelist_callback): + expected = [(host.rstrip("."), port) for host, port in response] + await self.assert_nodelist_change(expected, client) + + async def test_14_the_validator_is_consulted_when_srv_records_are_rescanned(self): + seen = [] + + def validator(host): + seen.append(host) + return True + + response = self.BASE_SRV_RESPONSE[:] + response.append(("localhost.test.build.10gen.cc", 27019)) + + with SrvPollingKnobs(ttl_time=WAIT_TIME, min_srv_rescan_interval=WAIT_TIME): + client = self.simple_client(self.CONNECTION_STRING, srv_host_validator=validator) + await client.aconnect() + await self.assert_nodelist_change(self.BASE_SRV_RESPONSE, client) + seen.clear() + with SrvPollingKnobs(nodelist_callback=lambda: response): await self.assert_nodelist_change(response, client) + self.assertIn("localhost.test.build.10gen.cc", seen) + + async def test_15_a_rejecting_or_raising_validator_does_not_raise_or_stop_polling(self): + def rejecting(host): + return False + + def raising(host): + raise RuntimeError("boom") + + response = self.BASE_SRV_RESPONSE[:] + response.append(("localhost.test.build.10gen.cc", 27019)) + + for failing in (rejecting, raising): + with self.subTest(validator=failing.__name__): + # Accept everything until the client is connected, then start failing. + state = {"validator": lambda host: True} + + with SrvPollingKnobs(ttl_time=WAIT_TIME, min_srv_rescan_interval=WAIT_TIME): + client = self.simple_client( + self.CONNECTION_STRING, + srv_host_validator=lambda host: state["validator"](host), + ) + await client.aconnect() + await self.assert_nodelist_change(self.BASE_SRV_RESPONSE, client) + + # The rescan sees the new record but the validator fails: no + # error reaches the application and the topology is unchanged. + state["validator"] = failing + with SrvPollingKnobs( + nodelist_callback=lambda: response, count_resolver_calls=True + ): + await self.assert_nodelist_nochange(self.BASE_SRV_RESPONSE, client) + + # Polling was not stopped by the failures: once the validator + # accepts again, the new host is picked up. + state["validator"] = lambda host: True + with SrvPollingKnobs(nodelist_callback=lambda: response): + await self.assert_nodelist_change(response, client) + + await client.close() + + async def test_rescan_skips_only_the_hosts_that_fail_verification(self): + # Per the polling spec, a host that fails verification is left out of + # the topology without discarding the other hosts from the same rescan. + good_host = "localhost.test.build.10gen.cc" + bad_host = "bad.evil.com" + + def raising(host): + if host == bad_host: + raise RuntimeError("some error") + return True + + def rejecting(host): + return host != bad_host + + for validator in (rejecting, raising): + with self.subTest(validator=validator.__name__): + resolver = pymongo.asynchronous.srv_resolver._SrvResolver( + "test1.test.build.10gen.cc", None, "mongodb", srv_host_validator=validator + ) + answer = _FakeSrvAnswer([good_host, bad_host]) + with patch("dns.asyncresolver.resolve", return_value=answer): + nodes, _ = await resolver.get_hosts_and_min_ttl() + self.assertEqual([(good_host, 27017)], nodes) + async def test_srv_waits_to_poll(self): modified = [("localhost.test.build.10gen.cc", 27019)] diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index 5d16bc5918..2ee4e68657 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -23,6 +23,7 @@ import binascii import copy import functools +import gc import os import platform import re @@ -539,6 +540,11 @@ def setUpClass(cls) -> None: @classmethod def tearDownClass(cls) -> None: cls.knobs.disable() + # Workaround for sockets leaked by tests (PYTHON-3923): force garbage + # collection so they are closed here, rather than during a later test + # where their ResourceWarnings could cause failures. Remove once the + # leaks are fixed. + gc.collect() async def asyncSetUp(self): # super call creates internal client cls.client diff --git a/test/bson_corpus/dbref.json b/test/bson_corpus/dbref.json index 41c0b09d0e..cf54ccf538 100644 --- a/test/bson_corpus/dbref.json +++ b/test/bson_corpus/dbref.json @@ -22,6 +22,21 @@ "canonical_bson": "4400000003646272656600380000000224726566000b000000636f6c6c656374696f6e00072469640058921b3e6e32ab156a22b59e02666f6f0004000000626172000000", "canonical_extjson": "{\"dbref\": {\"$ref\": \"collection\", \"$id\": {\"$oid\": \"58921b3e6e32ab156a22b59e\"}, \"foo\": \"bar\"}}" }, + { + "description": "DBRef with dotted collection name", + "canonical_bson": "3a000000036462726566002e0000000224726566000e00000064622e636f6c6c656374696f6e00072469640058921b3e6e32ab156a22b59e0000", + "canonical_extjson": "{\"dbref\": {\"$ref\": \"db.collection\", \"$id\": {\"$oid\": \"58921b3e6e32ab156a22b59e\"}}}" + }, + { + "description": "DBRef with dotted collection name and database", + "canonical_bson": "4b000000036462726566003f0000000224726566000e00000064622e636f6c6c656374696f6e00072469640058921b3e6e32ab156a22b59e0224646200080000006f746865726462000000", + "canonical_extjson": "{\"dbref\": {\"$ref\": \"db.collection\", \"$id\": {\"$oid\": \"58921b3e6e32ab156a22b59e\"}, \"$db\": \"otherdb\"}}" + }, + { + "description": "DBRef with multiple dots in collection name", + "canonical_bson": "32000000036462726566002600000002247265660006000000612e622e6300072469640058921b3e6e32ab156a22b59e0000", + "canonical_extjson": "{\"dbref\": {\"$ref\": \"a.b.c\", \"$id\": {\"$oid\": \"58921b3e6e32ab156a22b59e\"}}}" + }, { "description": "Document with key names similar to those of a DBRef", "canonical_bson": "3e0000000224726566000c0000006e6f742d612d646272656600072469640058921b3e6e32ab156a22b59e022462616e616e6100050000007065656c0000", diff --git a/test/mockupdb/test_handshake.py b/test/mockupdb/test_handshake.py index 2772e6f77a..e3a3fc0562 100644 --- a/test/mockupdb/test_handshake.py +++ b/test/mockupdb/test_handshake.py @@ -49,9 +49,11 @@ def _check_handshake_data(request): assert data["application"] == {"name": "my app"} if has_c(): name = "PyMongo|c" + version = pymongo_version + "|" else: name = "PyMongo" - assert data["driver"] == {"name": name, "version": pymongo_version} + version = pymongo_version + assert data["driver"] == {"name": name, "version": version} # Keep it simple, just check these fields exist. assert "os" in data diff --git a/test/mod_wsgi_test/README.rst b/test/mod_wsgi_test/README.rst index e96db9406c..5a9348feb2 100644 --- a/test/mod_wsgi_test/README.rst +++ b/test/mod_wsgi_test/README.rst @@ -13,44 +13,39 @@ mod_wsgi creates. Test Matrix ----------- -PyMongo should be tested with several versions of mod_wsgi and a selection -of Python versions. Each combination of mod_wsgi and Python version should -be tested with a standalone and a replica set. ``mod_wsgi_test.py`` -detects if the deployment is a replica set and connects to the whole set. +Continuous integration tests the oldest supported CPython against MongoDB +6.0, the oldest version that installs on the GitHub runners, with minimum +dependencies, and the newest supported CPython against the latest MongoDB, in +both daemon and embedded mode against a replica set. The Python and MongoDB +versions come from ``generate_config_utils.py`` in ``.evergreen/scripts``. +Other combinations can be tested manually. Setup ----- -Compile Python +Install Apache .............. -We need a Python interpreter built as a shared library. Download the -source tarball for each Python version tested, untar it, and run:: +On Ubuntu, install Apache and the headers used to build mod_wsgi:: - ./configure --prefix=/some/directory --enable-shared LDFLAGS="-Wl,--rpath=/some/directory/lib" - make - make install + sudo apt-get install -y apache2 apache2-dev -This results in an executable named "python" or "python3" and a shared -library named something like "libpython2.7.so.1.0" or "libpython3.3m.so.1.0". +On Fedora/RHEL, install httpd and its headers instead:: -Compile mod_wsgi -................ + sudo dnf install -y httpd httpd-devel + +The test harness detects ``apache2`` or ``httpd`` automatically and starts it +with the matching config file (``apache24ubuntu.conf`` or +``httpd24fedora.conf``). -Compile mod_wsgi for each combination for Python and mod_wsgi version in the -test matrix. For example, to compile mod_wsgi 3.4 for Python 2.7 on a -RedHat-like Linux:: +Install mod_wsgi +................ - sudo yum install -y httpd httpd-devel - wget https://modwsgi.googlecode.com/files/mod_wsgi-3.4.tar.gz - tar xzf mod_wsgi-3.4.tar.gz - cd mod_wsgi-3.4 - ./configure --with-python=/some/directory/bin/python LDFLAGS="-Wl,--rpath=/some/directory/lib" - make - make install +mod_wsgi is only needed by these tests, so it is not a project dependency. +Its constraint lives in ``requirements/mod_wsgi.txt``. pip builds mod_wsgi +against the interpreter it is installed with:: -To ease testing of several matrix combinations, copy the resulting -``mod_wsgi.so`` to a safe place. + uv pip install -r requirements/mod_wsgi.txt Start mongod ............ @@ -61,12 +56,21 @@ listening on port 27017. Configure Apache ................ -Set a MOD_WSGI_SO environment variable so our ``mod_wsgi_test.conf`` -can find and load mod_wsgi.so:: +The Apache configs expand environment variables, so export them before +starting Apache directly. All configs need ``MOD_WSGI_SO``, the path to +mod_wsgi.so, and include the mode config named by ``MOD_WSGI_CONF`` +(``mod_wsgi_test.conf`` for standalone or ``mod_wsgi_test_embedded.conf`` +for embedded mode) from the checkout at ``PROJECT_DIRECTORY``. The Fedora +config also needs ``MOD_WSGI_PYTHON_HOME``, the prefix of the interpreter +mod_wsgi was built with:: - export MOD_WSGI_SO=/path/to/mod_wsgi.so + export MOD_WSGI_SO=$(find .venv -name "mod_wsgi*.so") + export MOD_WSGI_CONF=mod_wsgi_test.conf + export PROJECT_DIRECTORY=$PWD + export MOD_WSGI_PYTHON_HOME=$(.venv/bin/python -c "import sys; print(sys.base_prefix)") -Start Apache with one of the config files in this directory. +``mod_wsgi_tester.py`` exports these variables itself, so ``just setup-tests +mod_wsgi `` (see `Automation`_) is the easier path. Run the test ------------ @@ -104,7 +108,13 @@ the workaround added in `PYTHON-569 Automation ---------- -At MongoDB, Inc. we use a continuous integration job that tests each -combination in the matrix. The job starts up Apache, starts a single server -or replica set, and runs ``test_client.py`` with the proper arguments. -See `run-mod-wsgi-tests.sh `_ +Continuous integration runs the test on pull requests that change +mod_wsgi-relevant files. The minimum dependency job runs in a Fedora +container against ``httpd`` and the latest dependency job runs on Ubuntu +against ``apache2`` (see the `test-mod-wsgi workflow +`_). +To run the same steps locally, use ``just setup-tests mod_wsgi ``, +``just run-tests``, and ``just teardown-tests``. On Linux they run against the +host's Apache, either ``apache2`` or ``httpd``; on other hosts they run in an +ubuntu container with Apache, a single-node replica set, and both test modes. +``just smoke-mod-wsgi`` runs both modes with one command. diff --git a/test/mod_wsgi_test/apache22amazon.conf b/test/mod_wsgi_test/apache22amazon.conf deleted file mode 100644 index 7755336b07..0000000000 --- a/test/mod_wsgi_test/apache22amazon.conf +++ /dev/null @@ -1,34 +0,0 @@ -# This is a minimal httpd.conf file written for Apache 2.2 on Amazon Linux - -# The modules directory is here by default. -# ServerRoot "/etc/httpd" -DocumentRoot ${PWD} -PidFile ${PWD}/apache2.pid - -User nobody -Group nobody - -# Bind to localhost only, don't require sudo. -Listen 127.0.0.1:8080 - -# Required modules. -LoadModule authz_host_module modules/mod_authz_host.so -# Needed so we can set a custom log location. -LoadModule log_config_module modules/mod_log_config.so - -ErrorLog ${PWD}/error_log -CustomLog ${PWD}/access_log combined - - - AllowOverride None - Order Deny,Allow - Deny from All - - - - AllowOverride None - Order Allow,Deny - Allow from All - - -Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${MOD_WSGI_CONF} diff --git a/test/mod_wsgi_test/apache22ubuntu1204.conf b/test/mod_wsgi_test/apache22ubuntu1204.conf deleted file mode 100644 index 9fa4b2060b..0000000000 --- a/test/mod_wsgi_test/apache22ubuntu1204.conf +++ /dev/null @@ -1,29 +0,0 @@ -# This is a minimal httpd.conf file written for Apache 2.2 on Ubuntu 12.04 - -# The modules directory is here on Ubuntu. -ServerRoot "/usr/lib/apache2" -DocumentRoot ${PWD} -PidFile ${PWD}/apache2.pid - -# Bind to localhost only, don't require sudo. -Listen 127.0.0.1:8080 - -# Required modules. -LoadModule authz_host_module modules/mod_authz_host.so - -ErrorLog ${PWD}/error_log -CustomLog ${PWD}/access_log combined - - - AllowOverride None - Order Deny,Allow - Deny from All - - - - AllowOverride None - Order Allow,Deny - Allow from All - - -Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${MOD_WSGI_CONF} diff --git a/test/mod_wsgi_test/apache24ubuntu.conf b/test/mod_wsgi_test/apache24ubuntu.conf new file mode 100644 index 0000000000..63161de86b --- /dev/null +++ b/test/mod_wsgi_test/apache24ubuntu.conf @@ -0,0 +1,47 @@ +# This is a minimal httpd.conf file written for Apache 2.4 on Ubuntu. + +# The modules directory is here on Ubuntu. +ServerRoot "/usr/lib/apache2" + +# mod_wsgi 6 destroys the interpreter as child processes exit. pymongo shuts +# its monitor threads down during interpreter finalization, so the default +# lifecycle (WSGIDestroyInterpreter On) is safe to use. +DocumentRoot ${PWD} +PidFile ${PWD}/apache2.pid + +# Bind to localhost only, don't require sudo. +Listen 127.0.0.1:8080 + +# Required modules. +LoadModule mpm_prefork_module modules/mod_mpm_prefork.so +LoadModule authz_core_module modules/mod_authz_core.so +# Loaded before the mod_wsgi directives in the included test config. +LoadModule wsgi_module ${MOD_WSGI_SO} + +# Bound the number of worker processes. mod_wsgi embedded mode loads a full +# Python interpreter and PyMongo in every worker, so the parallel test's +# concurrent requests would otherwise ramp prefork up to its default +# MaxRequestWorkers (256) and exhaust the container's memory. + + StartServers 2 + MinSpareServers 2 + MaxSpareServers 4 + ServerLimit 16 + MaxRequestWorkers 16 + MaxConnectionsPerChild 0 + + +ErrorLog ${PWD}/error_log +CustomLog ${PWD}/access_log combined + + + AllowOverride None + Require all denied + + + + AllowOverride None + Require all granted + + +Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${MOD_WSGI_CONF} diff --git a/test/mod_wsgi_test/apache24ubuntu161404.conf b/test/mod_wsgi_test/apache24ubuntu161404.conf deleted file mode 100644 index eb5414f0f7..0000000000 --- a/test/mod_wsgi_test/apache24ubuntu161404.conf +++ /dev/null @@ -1,28 +0,0 @@ -# This is a minimal httpd.conf file written for Apache 2.4 on Ubuntu 14.04/16.04 - -# The modules directory is here on Ubuntu. -ServerRoot "/usr/lib/apache2" -DocumentRoot ${PWD} -PidFile ${PWD}/apache2.pid - -# Bind to localhost only, don't require sudo. -Listen 127.0.0.1:8080 - -# Required modules. -LoadModule mpm_prefork_module modules/mod_mpm_prefork.so -LoadModule authz_core_module modules/mod_authz_core.so - -ErrorLog ${PWD}/error_log -CustomLog ${PWD}/access_log combined - - - AllowOverride None - Require all denied - - - - AllowOverride None - Require all granted - - -Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${MOD_WSGI_CONF} diff --git a/test/mod_wsgi_test/httpd24fedora.conf b/test/mod_wsgi_test/httpd24fedora.conf new file mode 100644 index 0000000000..6693b75447 --- /dev/null +++ b/test/mod_wsgi_test/httpd24fedora.conf @@ -0,0 +1,58 @@ +# This is a minimal httpd.conf file written for Apache 2.4 on Fedora/RHEL. + +# The modules directory is here on Fedora/RHEL. +ServerRoot "/etc/httpd" +DefaultRuntimeDir ${PWD} + +DocumentRoot ${PWD} +PidFile ${PWD}/httpd.pid + +# Bind to localhost only, don't require sudo. +Listen 127.0.0.1:8080 + +# Required modules. +LoadModule mpm_prefork_module modules/mod_mpm_prefork.so +LoadModule authz_core_module modules/mod_authz_core.so +LoadModule unixd_module modules/mod_unixd.so +LoadModule log_config_module modules/mod_log_config.so +# Loaded before any mod_wsgi directive below. +LoadModule wsgi_module ${MOD_WSGI_SO} + +# mod_wsgi is built against the interpreter used to install it. When Apache is +# started by a different executable than that interpreter, the embedded Python +# cannot locate its standard library, so point it at the installation. +WSGIPythonHome "${MOD_WSGI_PYTHON_HOME}" + +# Bound the number of worker processes. mod_wsgi embedded mode loads a full +# Python interpreter and PyMongo in every worker, so the parallel test's +# concurrent requests would otherwise ramp prefork up to its default +# MaxRequestWorkers (256) and exhaust the container's memory. + + StartServers 2 + MinSpareServers 2 + MaxSpareServers 4 + ServerLimit 16 + MaxRequestWorkers 16 + MaxConnectionsPerChild 0 + + +ServerName localhost + +ErrorLog ${PWD}/error_log +CustomLog ${PWD}/access_log combined + +# Apache must not run as root. The httpd package creates this user. +User apache +Group apache + + + AllowOverride None + Require all denied + + + + AllowOverride None + Require all granted + + +Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${MOD_WSGI_CONF} diff --git a/test/mod_wsgi_test/mod_wsgi_test.conf b/test/mod_wsgi_test/mod_wsgi_test.conf index a5b09e437f..b4cbaa083e 100644 --- a/test/mod_wsgi_test/mod_wsgi_test.conf +++ b/test/mod_wsgi_test/mod_wsgi_test.conf @@ -14,8 +14,6 @@ # Minimal test of PyMongo in a WSGI application, see bug PYTHON-353 -LoadModule wsgi_module ${MOD_WSGI_SO} - # Avoid permissions issues WSGISocketPrefix /tmp/ diff --git a/test/mod_wsgi_test/mod_wsgi_test_embedded.conf b/test/mod_wsgi_test/mod_wsgi_test_embedded.conf index 306dab4ab6..970803e586 100644 --- a/test/mod_wsgi_test/mod_wsgi_test_embedded.conf +++ b/test/mod_wsgi_test/mod_wsgi_test_embedded.conf @@ -14,8 +14,6 @@ # Minimal test of PyMongo in an *Embedded mode* WSGI application. -LoadModule wsgi_module ${MOD_WSGI_SO} - # Avoid permissions issues WSGISocketPrefix /tmp/ diff --git a/test/test_bson.py b/test/test_bson.py index 191a706388..0800806b84 100644 --- a/test/test_bson.py +++ b/test/test_bson.py @@ -75,6 +75,25 @@ _NUMPY_AVAILABLE = importlib.util.find_spec("numpy") is not None +def _total_memory(): + """Return total physical memory in bytes, or None if it cannot be determined.""" + try: + return os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") + except (ValueError, OSError, AttributeError): + return None + + +# PYTHON-6140: on Linux, skip if the host cannot allocate the ~4GiB this test +# needs. On other platforms, run the test and skip on MemoryError. +_TOTAL_MEMORY = _total_memory() +_ENCODE_SIZE_LIMIT_MIN_MEMORY = 6 << 30 +_SKIP_ENCODE_SIZE_LIMIT = ( + sys.platform == "linux" + and _TOTAL_MEMORY is not None + and _TOTAL_MEMORY < _ENCODE_SIZE_LIMIT_MIN_MEMORY +) + + class NotADict(abc.MutableMapping): """Non-dict type that implements the mapping protocol.""" @@ -692,12 +711,22 @@ def test_overflow(self): self.assertRaises(OverflowError, encode, {"x": -9223372036854775809}) @unittest.skipUnless(bson.has_c(), "This test requires the C extension") + @unittest.skipIf( + _SKIP_ENCODE_SIZE_LIMIT, + "Test requires at least 6GiB of memory (PYTHON-6140)", + ) def test_encode_size_limit(self): # PYTHON-5996: encoding must raise when a document's encoded size # exceeds the BSON size limit. - big_value = "a" * (1 << 30) - with self.assertRaises(ValueError): - encode({"a": big_value, "b": big_value, "c": big_value}) + try: + big_value = "a" * (1 << 30) + with self.assertRaises(ValueError): + encode({"a": big_value, "b": big_value, "c": big_value}) + except MemoryError: + # PYTHON-6140: some hosts report enough installed memory but + # cannot actually commit it (pagefile limits, mongod running on + # the same host). Skip rather than fail. + self.skipTest("Could not allocate enough memory (PYTHON-6140)") def test_small_long_encode_decode(self): encoded1 = encode({"x": 256}) diff --git a/test/test_client.py b/test/test_client.py index ee492e9b9b..0c8d0e7595 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -81,7 +81,13 @@ WriteConcernError, ) from pymongo.monitoring import ServerHeartbeatListener, ServerHeartbeatStartedEvent -from pymongo.pool_options import _MAX_METADATA_SIZE, _METADATA, ENV_VAR_K8S, PoolOptions +from pymongo.pool_options import ( + _MAX_METADATA_SIZE, + _METADATA, + ENV_VAR_K8S, + PoolOptions, + _truncate_metadata, +) from pymongo.read_preferences import ReadPreference from pymongo.server_description import ServerDescription from pymongo.server_selectors import readable_server_selector, writable_server_selector @@ -123,6 +129,8 @@ NTHREADS, CMAPListener, FunctionCallRecorder, + _driver_version, + _metadata_with_appended_driver, delay, gevent_monkey_patched, is_greenthread_patched, @@ -380,6 +388,9 @@ def test_metadata(self): metadata["driver"]["name"] = "PyMongo|c" else: metadata["driver"]["name"] = "PyMongo" + metadata["driver"]["version"] = _driver_version( + _METADATA["driver"]["version"], metadata["driver"]["name"] + ) metadata["application"] = {"name": "foobar"} client = self.simple_client("mongodb://foo:27017/?appname=foobar&connect=false") options = client.options @@ -391,6 +402,8 @@ def test_metadata(self): self.simple_client(appname="x" * 128) with self.assertRaises(ValueError): self.simple_client(appname="x" * 129) + + def test_metadata_bad_driver_options(self): # Bad "driver" options. self.assertRaises(TypeError, DriverInfo, "Foo", 1, "a") self.assertRaises(TypeError, DriverInfo, version="1", platform="a") @@ -401,12 +414,9 @@ def test_metadata(self): self.simple_client(driver="abc") with self.assertRaises(TypeError): self.simple_client(driver=("Foo", "1", "a")) - # Test appending to driver info. - if has_c(): - metadata["driver"]["name"] = "PyMongo|c|FooDriver" - else: - metadata["driver"]["name"] = "PyMongo|FooDriver" - metadata["driver"]["version"] = "{}|1.2.3".format(_METADATA["driver"]["version"]) + + def test_metadata_appends_driver_info(self): + metadata = _metadata_with_appended_driver(_IS_SYNC, "FooDriver", "1.2.3") client = self.simple_client( "foo", 27017, @@ -414,9 +424,9 @@ def test_metadata(self): driver=DriverInfo("FooDriver", "1.2.3", None), connect=False, ) - options = client.options - self.assertEqual(options.pool_options.metadata, metadata) - metadata["platform"] = "{}|FooPlatform".format(_METADATA["platform"]) + self.assertEqual(client.options.pool_options.metadata, metadata) + + metadata = _metadata_with_appended_driver(_IS_SYNC, "FooDriver", "1.2.3", "FooPlatform") client = self.simple_client( "foo", 27017, @@ -424,27 +434,131 @@ def test_metadata(self): driver=DriverInfo("FooDriver", "1.2.3", "FooPlatform"), connect=False, ) - options = client.options - self.assertEqual(options.pool_options.metadata, metadata) - # Test truncating driver info metadata. + self.assertEqual(client.options.pool_options.metadata, metadata) + + def test_metadata_truncates_driver_info(self): + # Truncated driver info must stay within the limit and keep name and + # version index-aligned. client = self.simple_client( driver=DriverInfo(name="s" * _MAX_METADATA_SIZE), connect=False, ) options = client.options + truncated = options.pool_options.metadata["driver"] self.assertLessEqual( len(bson.encode(options.pool_options.metadata)), _MAX_METADATA_SIZE, ) + self.assertEqual( + truncated["name"].count("|"), + truncated["version"].count("|"), + ) client = self.simple_client( driver=DriverInfo(name="s" * _MAX_METADATA_SIZE, version="s" * _MAX_METADATA_SIZE), connect=False, ) options = client.options + truncated = options.pool_options.metadata["driver"] self.assertLessEqual( len(bson.encode(options.pool_options.metadata)), _MAX_METADATA_SIZE, ) + self.assertEqual( + truncated["name"].count("|"), + truncated["version"].count("|"), + ) + # An oversized wrapper name with no version must retain a truncated + # name rather than collapse to the base entry. + client = self.simple_client( + driver=DriverInfo(name="x" * (_MAX_METADATA_SIZE * 2), version=None), + connect=False, + ) + truncated = client.options.pool_options.metadata["driver"] + self.assertLessEqual( + len(bson.encode(client.options.pool_options.metadata)), + _MAX_METADATA_SIZE, + ) + self.assertIn("xxxx", truncated["name"]) + self.assertEqual( + truncated["name"].count("|"), + truncated["version"].count("|"), + ) + + def test_metadata_truncation_drops_blank_pairs_first(self): + # A platform-only driver appends a blank name/version pair. Truncation + # must drop that pair before trimming a real driver's content. + def metadata_for(name_len: int) -> dict[str, Any]: + name = "PyMongo||" + "W" * name_len # blank pair in the middle + return {"driver": {"name": name, "version": "1.0||1.0"}} + + # Size the name so the document is exactly two bytes over the limit: + # dropping the blank pair alone must make it fit. + name_len = next( + n + for n in range(1, 2 * _MAX_METADATA_SIZE) + if len(bson.encode(metadata_for(n))) == _MAX_METADATA_SIZE + 2 + ) + metadata = metadata_for(name_len) + _truncate_metadata(metadata) + self.assertEqual(len(bson.encode(metadata)), _MAX_METADATA_SIZE) + self.assertEqual( + metadata["driver"], + {"name": "PyMongo|" + "W" * name_len, "version": "1.0|1.0"}, + ) + + def test_metadata_append_is_bounded(self): + # Successive appends must stay within the limit and keep name and + # version index-aligned after truncation. Once the metadata saturates, + # further appends must not grow the dedup tracking list. + client = self.simple_client(connect=False) + for i in range(300): + client.append_metadata(DriverInfo(name=f"D{i}", version=f"1.{i}")) + pool = client.options.pool_options + self.assertLessEqual(len(bson.encode(pool.metadata)), _MAX_METADATA_SIZE) + self.assertEqual( + pool.metadata["driver"]["name"].count("|"), + pool.metadata["driver"]["version"].count("|"), + ) + count = len(pool._PoolOptions__appended_drivers) + for i in range(300, 600): + client.append_metadata(DriverInfo(name=f"D{i}", version=f"1.{i}")) + self.assertEqual(len(pool._PoolOptions__appended_drivers), count) + # Platform-only appends (empty name/version) stay bounded the same way. + client = self.simple_client(connect=False) + for i in range(300): + client.append_metadata(DriverInfo(name="", version="", platform=f"P{i}")) + pool = client.options.pool_options + self.assertLessEqual(len(bson.encode(pool.metadata)), _MAX_METADATA_SIZE) + count = len(pool._PoolOptions__appended_drivers) + for i in range(300, 600): + client.append_metadata(DriverInfo(name="", version="", platform=f"P{i}")) + self.assertEqual(len(pool._PoolOptions__appended_drivers), count) + + def test_metadata_recreates_platform_after_truncation(self): + # Appending a platform after truncation has dropped it recreates the field. + client = self.simple_client(connect=False) + for i in range(300): + client.append_metadata(DriverInfo(name="", version="", platform=f"Q{i}")) + pool = client.options.pool_options + self.assertLess(len(pool._PoolOptions__appended_drivers), 300) + client.append_metadata(DriverInfo(name="Wrapper", version="1.0", platform="Recreated")) + self.assertLessEqual(len(bson.encode(pool.metadata)), _MAX_METADATA_SIZE) + self.assertEqual( + pool.metadata["driver"]["name"].count("|"), + pool.metadata["driver"]["version"].count("|"), + ) + + def test_metadata_deduplicates_none_and_empty(self): + # Empty strings are treated as unset, so a duplicate differing only in + # None vs "" is a no-op. + client = self.simple_client(connect=False) + client.append_metadata(DriverInfo("library", None, "Library Platform")) + names = client.options.pool_options.metadata["driver"]["name"] + vers = client.options.pool_options.metadata["driver"]["version"] + client.append_metadata(DriverInfo("library", "", "Library Platform")) + metadata = client.options.pool_options.metadata + self.assertEqual(metadata["driver"]["name"], names) + self.assertEqual(metadata["driver"]["version"], vers) @mock.patch.dict("os.environ", {ENV_VAR_K8S: "1"}) def test_container_metadata(self): @@ -2179,6 +2293,9 @@ def _test_handshake(self, env_vars, expected_env): metadata["driver"]["name"] = "PyMongo|c" else: metadata["driver"]["name"] = "PyMongo" + metadata["driver"]["version"] = _driver_version( + _METADATA["driver"]["version"], metadata["driver"]["name"] + ) if expected_env is not None: metadata["env"] = expected_env @@ -2578,20 +2695,18 @@ def test_gevent_kill_churn_deadlock(self): import gevent.thread as _gthread from gevent import Timeout, spawn - # AMPLIFY_RACE=1 widens gevent's brief sleep on a contended lock - # so a kill lands there reliably. Only the bare sleep() is - # widened; timed sleeps pass through. The unfixed test then - # deadlocks within seconds. + # AMPLIFY_RACE=1 widens the kill windows below; keep AMPLIFY_SECONDS + # small or ops starve. + amplify_seconds = 0.0 if os.environ.get("AMPLIFY_RACE", "0") == "1": - _AMPLIFY_SECONDS = float(os.environ.get("AMPLIFY_SECONDS", "0.02")) + amplify_seconds = float(os.environ.get("AMPLIFY_SECONDS", "0.002")) _orig_thread_sleep = _gthread.sleep def _amplified_sleep(*args): - if not args: # bare sleep(): the courtesy yield on a failed - # non-blocking acquire (Condition.notify -> _is_owned -> acquire(False)) - _orig_thread_sleep(_AMPLIFY_SECONDS) - else: # sleep(0.001), sleep(2), etc.: passthrough + if not args: # bare sleep(): checkin courtesy yield + _orig_thread_sleep(amplify_seconds) + else: # timed sleeps pass through _orig_thread_sleep(*args) _gthread.sleep = _amplified_sleep @@ -2601,6 +2716,31 @@ def _amplified_sleep(*args): coll = client.pymongo_test.coll coll.insert_one({}) + pool = get_pool(client) # type:ignore + # Widen the post-gate checkout windows (PYTHON-6136). + if amplify_seconds: + + class _AmplifiedCondition: + def __init__(self, cond, seconds): + self._cond = cond + self._seconds = seconds + + def __enter__(self): + self._cond.__enter__() + return self + + def __exit__(self, *args): + self._cond.__exit__(*args) + time.sleep(self._seconds) + + def __getattr__(self, name): + return getattr(self._cond, name) + + pool.size_cond = _AmplifiedCondition(pool.size_cond, amplify_seconds) + pool._max_connecting_cond = _AmplifiedCondition( + pool._max_connecting_cond, amplify_seconds + ) + op_count = [0] running = [True] workers: list = [] @@ -2614,9 +2754,12 @@ def worker(): except Exception: return + # Scale the kill cadence or ops starve. + reaper_interval = max(0.003, amplify_seconds * 8) + def reaper(): while running[0]: - time.sleep(0.003) + time.sleep(reaper_interval) if not workers: continue idx = random.randrange(len(workers)) @@ -2657,11 +2800,6 @@ def reaper(): coll.find_one({}) except Timeout: self.fail("Pool gate saturated (PYTHON-6074)") - # Deterministic check: a saturated size gate pins the pool's - # checkout counters at maxPoolSize (PYTHON-6074). - pool = get_pool(client) # type:ignore - self.assertLess(pool.requests, pool.max_pool_size) - self.assertLess(pool.active_sockets, pool.max_pool_size) self.assertGreater(op_count[0], 0) finally: running[0] = False @@ -2679,6 +2817,14 @@ def reaper(): client.close() except Timeout: pass + # Post-settle: the counters must have fully drained. Note close() + # does not zero operation_count (only a fork does), so a leaked + # increment survives and is caught here. + time.sleep(1.0) + self.assertEqual(pool.requests, 0) + self.assertEqual(pool._pending, 0) + self.assertEqual(pool.active_sockets, 0) + self.assertEqual(pool.operation_count, 0) class TestClientLazyConnect(IntegrationTest): diff --git a/test/test_client_metadata.py b/test/test_client_metadata.py index f5ec92f2f3..b35e3f0bc7 100644 --- a/test/test_client_metadata.py +++ b/test/test_client_metadata.py @@ -18,7 +18,7 @@ import pathlib import time import unittest -from typing import Any, Optional +from typing import Any, Optional, cast import pytest @@ -99,20 +99,14 @@ def check_metadata_added( new_name, new_version, new_platform, new_metadata = self.send_ping_and_get_metadata( client, True ) - if add_name is not None and add_name.lower() in name.lower().split("|"): - self.assertEqual(name, new_name) - self.assertEqual(version, new_version) - self.assertEqual(platform, new_platform) - else: - self.assertEqual(new_name, f"{name}|{add_name}" if add_name is not None else name) - self.assertEqual( - new_version, - f"{version}|{add_version}" if add_version is not None else version, - ) - self.assertEqual( - new_platform, - f"{platform}|{add_platform}" if add_platform is not None else platform, - ) + # Name and version always get a delimiter (empty string if None) to + # preserve 1:1 index correspondence. + self.assertEqual(new_name, f"{name}|{add_name or ''}") + self.assertEqual(new_version, f"{version}|{add_version or ''}") + self.assertEqual( + new_platform, + f"{platform}|{add_platform}" if add_platform is not None else platform, + ) metadata.pop("driver") metadata.pop("platform") @@ -120,7 +114,7 @@ def check_metadata_added( new_metadata.pop("platform") self.assertEqual(metadata, new_metadata) - def test_append_metadata(self): + def test_1_test_that_the_driver_updates_metadata(self): client = self.rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -128,7 +122,7 @@ def test_append_metadata(self): ) self.check_metadata_added(client, "framework", "2.0", "Framework Platform") - def test_append_metadata_platform_none(self): + def test_1_test_that_the_driver_updates_metadata_platform_none(self): client = self.rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -136,7 +130,7 @@ def test_append_metadata_platform_none(self): ) self.check_metadata_added(client, "framework", "2.0", None) - def test_append_metadata_version_none(self): + def test_1_test_that_the_driver_updates_metadata_version_none(self): client = self.rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -144,7 +138,7 @@ def test_append_metadata_version_none(self): ) self.check_metadata_added(client, "framework", None, "Framework Platform") - def test_append_metadata_platform_version_none(self): + def test_1_test_that_the_driver_updates_metadata_platform_version_none(self): client = self.rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -152,14 +146,14 @@ def test_append_metadata_platform_version_none(self): ) self.check_metadata_added(client, "framework", None, None) - def test_multiple_successive_metadata_updates(self): + def test_2_multiple_successive_metadata_updates(self): client = self.rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, connect=False ) client.append_metadata(DriverInfo("library", "1.2", "Library Platform")) self.check_metadata_added(client, "framework", "2.0", "Framework Platform") - def test_multiple_successive_metadata_updates_platform_none(self): + def test_2_multiple_successive_metadata_updates_platform_none(self): client = self.rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -167,7 +161,7 @@ def test_multiple_successive_metadata_updates_platform_none(self): client.append_metadata(DriverInfo("library", "1.2", "Library Platform")) self.check_metadata_added(client, "framework", "2.0", None) - def test_multiple_successive_metadata_updates_version_none(self): + def test_2_multiple_successive_metadata_updates_version_none(self): client = self.rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -175,7 +169,7 @@ def test_multiple_successive_metadata_updates_version_none(self): client.append_metadata(DriverInfo("library", "1.2", "Library Platform")) self.check_metadata_added(client, "framework", None, "Framework Platform") - def test_multiple_successive_metadata_updates_platform_version_none(self): + def test_2_multiple_successive_metadata_updates_platform_version_none(self): client = self.rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -216,10 +210,16 @@ def test_duplicate_driver_name_no_op(self): self.check_metadata_added(client, "framework", None, None) # wait for connection to become idle time.sleep(0.005) - # add same metadata again - self.check_metadata_added(client, "Framework", None, None) + # Append the exact same DriverInfo again: no-op. + name, version, platform, _ = self.send_ping_and_get_metadata(client, True) + time.sleep(0.005) + client.append_metadata(DriverInfo("framework", None, None)) + new_name, new_version, new_platform, _ = self.send_ping_and_get_metadata(client, True) + self.assertEqual(new_name, name) + self.assertEqual(new_version, version) + self.assertEqual(new_platform, platform) - def test_handshake_documents_include_backpressure(self): + def test_9_handshake_documents_include_backpressure(self): # Create a `MongoClient` that is configured to record all handshake documents sent to the server as a part of # connection establishment. client = self.rs_or_single_client("mongodb://" + self.server.address_string) @@ -232,6 +232,131 @@ def test_handshake_documents_include_backpressure(self): # the document has a field `backpressure` whose value is `"2"`. self.assertEqual(self.handshake_req["backpressure"], "2") + def test_10_entries_in_driver_name_and_driver_version_correspond_by_index(self): + cases = [ + ("Gap in middle (name)", [(None, None), ("F2", None)], "||F2", "||"), + ("Gap in middle (version)", [("F1", None), ("F2", "2.0")], "|F1|F2", "||2.0"), + ("Trailing delimiter retained", [("F1", None)], "|F1", "|"), + ( + "Equal versions do not collapse", + [("F1", "{driver_version}")], + "|F1", + "|{driver_version}", + ), + ( + "Equal names do not collapse", + [("{driver_name}", "1.0")], + "|{driver_name}", + "|1.0", + ), + ("Duplicates deduplicate", [("F1", "1.0"), ("F1", "1.0")], "|F1", "|1.0"), + ("All versions absent", [("F1", None), ("F2", None)], "|F1|F2", "||"), + ("All names absent", [(None, "1.0"), (None, "2.0")], "||", "|1.0|2.0"), + ( + "Non-adjacent duplicate", + [("F1", "1.0"), ("F2", "2.0"), ("F1", "1.0")], + "|F1|F2", + "|1.0|2.0", + ), + ( + "Platform-only difference is not a duplicate", + [("F1", "1.0", "P1"), ("F1", "1.0", "P2")], + "|F1|F1", + "|1.0|1.0", + ), + ( + "Wrapper matching the driver's own identity", + [("{driver_name}", "{driver_version}")], + "|{driver_name}", + "|{driver_version}", + ), + ( + "Duplicates with an unset field deduplicate", + [("F1", None), ("F1", None)], + "|F1", + "|", + ), + ] + for ( + description, + appended, + expected_name_suffix, + expected_version_suffix, + ) in cases: + with self.subTest(description=description): + client = self.rs_or_single_client( + "mongodb://" + self.server.address_string, + maxIdleTimeMS=1, + ) + self.addCleanup(client.close) + # Capture the driver's own name and version from the first handshake. + name0, version0, _, _ = self.send_ping_and_get_metadata(client, True) + time.sleep(0.005) + + self.assertIsNotNone(name0) + self.assertIsNotNone(version0) + version0 = cast(str, version0) + driver_name = name0.split("|")[0] + driver_version = version0.split("|")[0] + + def resolve(value: Optional[str]) -> Optional[str]: + if value is None: + return None + return value.format(driver_name=driver_name, driver_version=driver_version) + + # Append each DriverInfo in order. + for opts in appended: + d_name = resolve(opts[0]) if len(opts) > 0 else None + d_version = resolve(opts[1]) if len(opts) > 1 else None + d_platform = resolve(opts[2]) if len(opts) > 2 else None + client.append_metadata(DriverInfo(d_name or "", d_version, d_platform)) + + # New handshake with the appended metadata. + name1, version1, _, _ = self.send_ping_and_get_metadata(client, True) + + self.assertEqual( + name1, + name0 + + expected_name_suffix.format( + driver_name=driver_name, driver_version=driver_version + ), + ) + self.assertEqual( + version1, + version0 + + expected_version_suffix.format( + driver_name=driver_name, driver_version=driver_version + ), + ) + + def test_11_appending_metadata_containing_the_delimiter_raises_an_error(self): + cases = [ + ("frame|work", "2.0", "Framework Platform"), + ("framework", "2|0", "Framework Platform"), + ("framework", "2.0", "Framework|Platform"), + ] + for name, version, platform in cases: + with self.subTest(name=name, version=version, platform=platform): + client = self.rs_or_single_client( + "mongodb://" + self.server.address_string, + maxIdleTimeMS=1, + driver=DriverInfo("library", "1.2", "Library Platform"), + ) + self.addCleanup(client.close) + # Send initial handshake. + name0, version0, platform0, _metadata = self.send_ping_and_get_metadata( + client, True + ) + time.sleep(0.005) + # Constructing metadata containing the delimiter raises. + with self.assertRaises(ValueError): + DriverInfo(name, version, platform) + # Metadata is unchanged on the next handshake. + name1, version1, platform1, _ = self.send_ping_and_get_metadata(client, True) + self.assertEqual(name1, name0) + self.assertEqual(version1, version0) + self.assertEqual(platform1, platform0) + if __name__ == "__main__": unittest.main() diff --git a/test/test_dns.py b/test/test_dns.py index e631420643..89dee6f3a7 100644 --- a/test/test_dns.py +++ b/test/test_dns.py @@ -207,30 +207,40 @@ def test_connect_case_insensitive(self): class TestInitialDnsSeedlistDiscovery(PyMongoTestCase): """ Initial DNS Seedlist Discovery prose tests - https://github.com/mongodb/specifications/blob/0a7a8b5/source/initial-dns-seedlist-discovery/tests/README.md#prose-tests + https://github.com/mongodb/specifications/blob/5036f26/source/initial-dns-seedlist-discovery/tests/README.md#prose-tests + + Numbered tests correspond to the numbered prose tests in the spec. The + unnumbered tests are PyMongo-specific additions with no spec counterpart. """ + def _parse(self, srv_hostname, mock_target, **kwargs): + """Resolve mongodb+srv:// with SRV records naming mock_target.""" + with patch("dns.resolver.resolve") as mock_resolver: + + def mock_resolve(query, record_type, *args, **kwargs): + mock_srv = MagicMock() + # Mirror dnspython: the wire form keeps the root label, and the + # caller strips it via omit_final_dot. + mock_srv.target.to_text.side_effect = lambda omit_final_dot=False: ( + mock_target.rstrip(".") if omit_final_dot else mock_target + ) + return [mock_srv] + + mock_resolver.side_effect = mock_resolve + return parse_uri(f"mongodb+srv://{srv_hostname}", **kwargs) + def run_initial_dns_seedlist_discovery_prose_tests(self, test_cases): for case in test_cases: - with patch("dns.resolver.resolve") as mock_resolver: - - def mock_resolve(query, record_type, *args, **kwargs): - mock_srv = MagicMock() - mock_srv.target.to_text.return_value = case["mock_target"] - return [mock_srv] - - mock_resolver.side_effect = mock_resolve - domain = case["query"].split("._tcp.")[1] - connection_string = f"mongodb+srv://{domain}" - if "expected_error" not in case: - parse_uri(connection_string) + domain = case["query"].split("._tcp.")[1] + if "expected_error" not in case: + self._parse(domain, case["mock_target"]) + else: + try: + self._parse(domain, case["mock_target"]) + except ConfigurationError as e: + self.assertIn(case["expected_error"], str(e)) else: - try: - parse_uri(connection_string) - except ConfigurationError as e: - self.assertIn(case["expected_error"], str(e)) - else: - self.fail(f"ConfigurationError was not raised for query: {case['query']}") + self.fail(f"ConfigurationError was not raised for query: {case['query']}") def test_1_allow_srv_hosts_with_fewer_than_three_dot_separated_parts(self): with patch("dns.resolver.resolve"): @@ -250,8 +260,8 @@ def test_2_throw_when_return_address_does_not_end_with_srv_domain(self): "expected_error": "Invalid SRV host", }, { - "query": "_mongodb._tcp.blogs.mongo.local", - "mock_target": "test_1.evil.com", + "query": "_mongodb._tcp.mongo.local", + "mock_target": "test_1.evil.local", "expected_error": "Invalid SRV host", }, ] @@ -294,7 +304,101 @@ def test_4_throw_when_return_address_does_not_contain_dot_separating_shared_part ] self.run_initial_dns_seedlist_discovery_prose_tests(test_cases) - def test_5_when_srv_hostname_has_three_or_more_dot_separated_parts_it_is_valid_for_the_returned_hostname_to_be_identical( + def test_5_srv_host_validator_accepts_a_host_the_default_verification_would_reject(self): + # "blogs.evil.com" does not share a parent domain with the seed, so the + # default check rejects it; the callback overrides that decision. + res = self._parse( + "blogs.mongodb.com", "blogs.evil.com", srv_host_validator=lambda host: True + ) + self.assertEqual(["blogs.evil.com"], [node[0] for node in res["nodelist"]]) + + # "mongo.local" does not add a domain level to an SRV hostname with fewer + # than three "." separated parts, which the default check also rejects. + res = self._parse("mongo.local", "mongo.local", srv_host_validator=lambda host: True) + self.assertEqual(["mongo.local"], [node[0] for node in res["nodelist"]]) + + def test_6_reject_a_host_the_default_verification_would_accept(self): + with self.assertRaisesRegex(ConfigurationError, "rejected by srv_host_validator"): + self._parse( + "blogs.mongodb.com", "cluster.mongodb.com", srv_host_validator=lambda host: False + ) + + def test_7_the_validator_receives_the_normalized_host_name(self): + seen = [] + + def validator(host): + seen.append(host) + return True + + self._parse("blogs.mongodb.com", "CLUSTER.MONGODB.COM.", srv_host_validator=validator) + self.assertEqual(["cluster.mongodb.com"], seen) + + def test_8_wrap_an_error_raised_by_the_validator(self): + original_exc = Exception("validator_error") + + def validator(host): + raise original_exc + + with self.assertRaisesRegex( + ConfigurationError, "srv_host_validator raised an exception" + ) as ctx: + self._parse("blogs.mongodb.com", "cluster.mongodb.com", srv_host_validator=validator) + # The wrapping error must retain the error raised by the validator. + self.assertIs(original_exc, ctx.exception.__cause__) + self.assertIn("validator_error", str(ctx.exception)) + + def test_9_throw_when_both_srv_allowed_hosts_suffix_and_srv_host_validator_are_configured( + self, + ): + # Rejected by the client + with self.assertRaisesRegex(ConfigurationError, "Cannot specify both"): + self.simple_client( + "mongodb+srv://blogs.mongodb.com", + srv_host_validator=lambda host: True, + srvAllowedHostsSuffix=".mongodb.com", + connect=False, + ) + + # Rejected by the resolver + with self.assertRaisesRegex(ConfigurationError, "Cannot specify both"): + self._parse( + "blogs.mongodb.com", + "cluster.mongodb.com", + srv_host_validator=lambda host: True, + srv_allowed_hosts_suffix=".mongodb.com", + ) + + def test_10_accept_a_mixed_case_returned_address_with_srv_allowed_hosts_suffix(self): + # Returned addresses are normalized before the suffix comparison, so + # the case DNS happens to use must not affect the result. + res = self._parse( + "blogs.mongodb.com", "CLUSTER.MONGODB.COM.", srv_allowed_hosts_suffix=".mongodb.com" + ) + self.assertEqual(["cluster.mongodb.com"], [node[0] for node in res["nodelist"]]) + + def test_11_throw_when_srv_host_validator_is_not_callable(self): + with self.assertRaisesRegex(ValueError, "must be a callable"): + self.simple_client("mongodb+srv://blogs.mongodb.com", srv_host_validator="notacallable") + + def test_12_accept_a_reserved_single_label_as_srv_allowed_hosts_suffix(self): + # A single label is a public suffix under the Public Suffix List's "*" + # rule, but the reserved names are accepted despite that. + res = self._parse( + "cluster.localhost", "db.cluster.localhost", srv_allowed_hosts_suffix="localhost" + ) + self.assertEqual(["db.cluster.localhost"], [node[0] for node in res["nodelist"]]) + + def test_13_throw_when_srv_host_validator_is_used_with_a_non_srv_uri(self): + with self.assertRaisesRegex( + ConfigurationError, "only allowed with 'mongodb\\+srv://' URIs" + ): + self.simple_client( + "mongodb://localhost:27017", + srv_host_validator=lambda host: True, + connect=False, + ) + + def test_srv_hostname_with_three_or_more_parts_may_equal_the_returned_hostname( self, ): test_cases = [ diff --git a/test/test_pooling.py b/test/test_pooling.py index a3f0eaf589..3a81bc796f 100644 --- a/test/test_pooling.py +++ b/test/test_pooling.py @@ -306,6 +306,7 @@ def notify(): # Accounting was applied exactly once. self.assertEqual(0, cx_pool.requests) self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool.operation_count) def test_checkout_error_accounting_on_kill_during_acquire(self): # PYTHON-6074: an exception delivered while the checkout error @@ -341,6 +342,98 @@ def __exit__(self, *args): # The fallback applied the accounting exactly once. self.assertEqual(0, cx_pool.requests) self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool.operation_count) + + def test_checkout_error_accounting_on_connect_keyboard_interrupt(self): + # PYTHON-6136: a KeyboardInterrupt from connect() must roll back the + # size gate, the maxConnecting gate, and active_sockets. + cx_pool = self.create_pool(max_pool_size=1) + + with patch.object(cx_pool, "connect", side_effect=KeyboardInterrupt()): + with self.assertRaises(KeyboardInterrupt): + with cx_pool.checkout(): + pass + + self.assertEqual(0, cx_pool.requests) + self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool._pending) + self.assertEqual(0, cx_pool.operation_count) + + def test_checkout_error_accounting_on_kill_during_pending_cleanup(self): + # PYTHON-6136: an interruption during the pending-gate cleanup must + # not skip the counter restore, which must wake threads waiting at + # the maxConnecting gate. + cx_pool = self.create_pool(max_pool_size=1) + + class _InterruptOnSecondEnter(type(cx_pool._max_connecting_cond)): + def __init__(self, lock): + super().__init__(lock) + self.enters = 0 + self.notifies = 0 + + def __enter__(self): + self.enters += 1 + if self.enters == 2: + # First enter is the checkout wait, second is connect()'s + # cleanup. Simulate a kill delivered while waiting there. + raise KeyboardInterrupt() + return super().__enter__() + + def __exit__(self, *args): + return super().__exit__(*args) + + def notify(self, n=1): + # The counter restore wakes the maxConnecting gate, then is + # itself killed. + self.notifies += 1 + raise KeyboardInterrupt() + + cond = _InterruptOnSecondEnter(cx_pool._max_connecting_cond._lock) + cx_pool._max_connecting_cond = cond + + with patch.object(cx_pool, "connect", side_effect=asyncio.CancelledError()): + with self.assertRaises(KeyboardInterrupt): + with cx_pool.checkout(): + pass + + self.assertEqual(0, cx_pool.requests) + self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool._pending) + self.assertEqual(0, cx_pool.operation_count) + # The restore notified the maxConnecting gate; the notify interrupted + # by the kill was retried. + self.assertEqual(2, cond.notifies) + + def test_checkout_error_accounting_on_kill_during_pending_notify(self): + # PYTHON-6136: a kill landing inside the cleanup notify must not + # strand a checkout waiting at the maxConnecting gate. + cx_pool = self.create_pool(max_pool_size=1) + + class _InterruptOnFirstNotify(type(cx_pool._max_connecting_cond)): + def __init__(self, lock): + super().__init__(lock) + self.notifies = 0 + + def notify(self, n=1): + self.notifies += 1 + if self.notifies == 1: + # Simulate a kill delivered inside notify(). + raise KeyboardInterrupt() + + cond = _InterruptOnFirstNotify(cx_pool._max_connecting_cond._lock) + cx_pool._max_connecting_cond = cond + + with patch.object(cx_pool, "connect", side_effect=asyncio.CancelledError()): + with self.assertRaises(KeyboardInterrupt): + with cx_pool.checkout(): + pass + + # The cleanup notify was interrupted, then retried. + self.assertEqual(2, cond.notifies) + self.assertEqual(0, cx_pool.requests) + self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool._pending) + self.assertEqual(0, cx_pool.operation_count) def test_pool_removes_closed_socket(self): # Test that Pool removes explicitly closed socket. @@ -463,6 +556,8 @@ def test_wait_queue_timeout(self): 1, f"Waited {duration:.2f} seconds for a socket, expected {wait_queue_timeout:f}", ) + # The load metric must not be inflated by the failed checkout. + self.assertEqual(0, pool.operation_count) def test_no_wait_queue_timeout(self): # Verify get_socket() with no wait_queue_timeout blocks forever. diff --git a/test/test_srv_polling.py b/test/test_srv_polling.py index e01ae1da7f..11ea739a55 100644 --- a/test/test_srv_polling.py +++ b/test/test_srv_polling.py @@ -19,7 +19,9 @@ import asyncio import sys import time +from types import SimpleNamespace from typing import Any +from unittest.mock import patch from test.utils_shared import FunctionCallRecorder @@ -37,6 +39,19 @@ WAIT_TIME = 0.1 +class _FakeSrvAnswer(list): + """A minimal stand-in for the dnspython answer to an SRV query.""" + + def __init__(self, hosts, ttl=60): + import dns.name + + hosts = [(h, 27017) if isinstance(h, str) else h for h in hosts] + super().__init__( + SimpleNamespace(target=dns.name.from_text(h), port=port) for h, port in hosts + ) + self.rrset = SimpleNamespace(ttl=ttl) + + class SrvPollingKnobs: def __init__( self, @@ -64,9 +79,15 @@ def enable(self): def mock_get_hosts_and_min_ttl(resolver, *args): assert self.old_dns_resolver_response is not None - nodes, ttl = self.old_dns_resolver_response(resolver) - if self.nodelist_callback is not None: - nodes = self.nodelist_callback() + if self.nodelist_callback is None: + nodes, ttl = self.old_dns_resolver_response(resolver) + else: + + def mock_resolve_uri(is_polling): + return _FakeSrvAnswer(self.nodelist_callback()) + + with patch.object(resolver, "_resolve_uri", mock_resolve_uri): + nodes, ttl = self.old_dns_resolver_response(resolver) if self.ttl_time is not None: ttl = self.ttl_time return nodes, ttl @@ -357,8 +378,92 @@ def nodelist_callback(): ) client._connect() with SrvPollingKnobs(nodelist_callback=nodelist_callback): + expected = [(host.rstrip("."), port) for host, port in response] + self.assert_nodelist_change(expected, client) + + def test_14_the_validator_is_consulted_when_srv_records_are_rescanned(self): + seen = [] + + def validator(host): + seen.append(host) + return True + + response = self.BASE_SRV_RESPONSE[:] + response.append(("localhost.test.build.10gen.cc", 27019)) + + with SrvPollingKnobs(ttl_time=WAIT_TIME, min_srv_rescan_interval=WAIT_TIME): + client = self.simple_client(self.CONNECTION_STRING, srv_host_validator=validator) + client._connect() + self.assert_nodelist_change(self.BASE_SRV_RESPONSE, client) + seen.clear() + with SrvPollingKnobs(nodelist_callback=lambda: response): self.assert_nodelist_change(response, client) + self.assertIn("localhost.test.build.10gen.cc", seen) + + def test_15_a_rejecting_or_raising_validator_does_not_raise_or_stop_polling(self): + def rejecting(host): + return False + + def raising(host): + raise RuntimeError("boom") + + response = self.BASE_SRV_RESPONSE[:] + response.append(("localhost.test.build.10gen.cc", 27019)) + + for failing in (rejecting, raising): + with self.subTest(validator=failing.__name__): + # Accept everything until the client is connected, then start failing. + state = {"validator": lambda host: True} + + with SrvPollingKnobs(ttl_time=WAIT_TIME, min_srv_rescan_interval=WAIT_TIME): + client = self.simple_client( + self.CONNECTION_STRING, + srv_host_validator=lambda host: state["validator"](host), + ) + client._connect() + self.assert_nodelist_change(self.BASE_SRV_RESPONSE, client) + + # The rescan sees the new record but the validator fails: no + # error reaches the application and the topology is unchanged. + state["validator"] = failing + with SrvPollingKnobs( + nodelist_callback=lambda: response, count_resolver_calls=True + ): + self.assert_nodelist_nochange(self.BASE_SRV_RESPONSE, client) + + # Polling was not stopped by the failures: once the validator + # accepts again, the new host is picked up. + state["validator"] = lambda host: True + with SrvPollingKnobs(nodelist_callback=lambda: response): + self.assert_nodelist_change(response, client) + + client.close() + + def test_rescan_skips_only_the_hosts_that_fail_verification(self): + # Per the polling spec, a host that fails verification is left out of + # the topology without discarding the other hosts from the same rescan. + good_host = "localhost.test.build.10gen.cc" + bad_host = "bad.evil.com" + + def raising(host): + if host == bad_host: + raise RuntimeError("some error") + return True + + def rejecting(host): + return host != bad_host + + for validator in (rejecting, raising): + with self.subTest(validator=validator.__name__): + resolver = pymongo.synchronous.srv_resolver._SrvResolver( + "test1.test.build.10gen.cc", None, "mongodb", srv_host_validator=validator + ) + answer = _FakeSrvAnswer([good_host, bad_host]) + with patch("dns.resolver.resolve", return_value=answer): + nodes, _ = resolver.get_hosts_and_min_ttl() + self.assertEqual([(good_host, 27017)], nodes) + def test_srv_waits_to_poll(self): modified = [("localhost.test.build.10gen.cc", 27019)] diff --git a/test/unified_format.py b/test/unified_format.py index 56ea563e6d..657d7c0f22 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -23,6 +23,7 @@ import binascii import copy import functools +import gc import os import platform import re @@ -538,6 +539,11 @@ def setUpClass(cls) -> None: @classmethod def tearDownClass(cls) -> None: cls.knobs.disable() + # Workaround for sockets leaked by tests (PYTHON-3923): force garbage + # collection so they are closed here, rather than during a later test + # where their ResourceWarnings could cause failures. Remove once the + # leaks are fixed. + gc.collect() def setUp(self): # super call creates internal client cls.client diff --git a/test/utils_shared.py b/test/utils_shared.py index 96f3b1f6ed..fb209d69a0 100644 --- a/test/utils_shared.py +++ b/test/utils_shared.py @@ -31,9 +31,11 @@ from collections import abc, defaultdict from functools import partial from inspect import iscoroutinefunction +from typing import Any from bson.objectid import ObjectId from pymongo import monitoring, operations, read_preferences +from pymongo.common import has_c from pymongo.cursor_shared import CursorType from pymongo.errors import OperationFailure from pymongo.helpers_shared import _SENSITIVE_COMMANDS @@ -51,6 +53,7 @@ PoolCreatedEvent, PoolReadyEvent, ) +from pymongo.pool_options import _METADATA from pymongo.pool_shared import _CancellationContext, _PoolGeneration from pymongo.read_concern import ReadConcern from pymongo.server_type import SERVER_TYPE @@ -791,3 +794,36 @@ def pack_msg_header(length: int, request_id: int, response_to: int, op_code: int production header-packing never does. """ return struct.pack(" str: + """Build a metadata driver version aligned 1:1 with ``name`` segments. + + The ``|c`` and ``|async`` name segments always have an empty version entry, + so the version string has one delimiter per name delimiter. ``last_version`` + is used when the final segment carries a wrapped driver's version. + """ + segments = [""] * name.count("|") + if last_version is not None: + segments[-1] = last_version + return "|".join([base_version, *segments]) + + +def _metadata_with_appended_driver( + is_sync: bool, name: str, version: str, platform: str | None = None +) -> dict[str, Any]: + """Build the expected client metadata after appending a driver.""" + driver_name = "PyMongo" + if has_c(): + driver_name += "|c" + if not is_sync: + driver_name += "|async" + metadata = copy.deepcopy(_METADATA) + metadata["driver"]["name"] = driver_name + f"|{name}" + metadata["driver"]["version"] = _driver_version( + _METADATA["driver"]["version"], metadata["driver"]["name"], last_version=version + ) + metadata["application"] = {"name": "foobar"} + if platform is not None: + metadata["platform"] = "{}|{}".format(_METADATA["platform"], platform) + return metadata diff --git a/tools/generate_sbom_requirements.py b/tools/generate_sbom_requirements.py index 59389bee5a..aa8d5579ca 100644 --- a/tools/generate_sbom_requirements.py +++ b/tools/generate_sbom_requirements.py @@ -3,7 +3,7 @@ - Includes repo_root/requirements.txt if present - Includes all files matching repo_root/requirements/**/*.txt -- Excludes docs.txt and test.txt in the requirements folder +- Excludes docs.txt, test.txt, and mod_wsgi.txt in the requirements folder - Writes output to sbom-requirements.txt (overwrites) """ @@ -11,7 +11,7 @@ from pathlib import Path -EXCLUDED_NAMES = {"docs.txt", "test.txt"} +EXCLUDED_NAMES = {"docs.txt", "test.txt", "mod_wsgi.txt"} def collect_files(root: Path) -> list[Path]: From 5482643a238da0048c6f038ebdacd8a6582f61f8 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 8 Oct 2026 21:20:04 -0500 Subject: [PATCH 6/8] Address review feedback and fix the otel min-deps variant - Never record a literal 0 for db.mongodb.cursor_id: an exhausted getMore reply records the cursor id sent in the command, and a cursor-creating command whose reply exhausts the cursor omits the attribute. - Preserve the construction-time tracing environment snapshot when an SRV client rebuilds its ClientOptions. - Key the benchmark's persisted pre-OTel comparison by API so the sync and async rows both survive. - Interpret --data-dir and --output-dir as absolute paths before launching benchmark children that run in a different cwd. - Pin opentelemetry-sdk>=1.20.0 for the test SDK so lowest-direct resolution cannot drag opentelemetry-api below the driver's floor, and skip the tracing env-deferral tests when opentelemetry is not installed. --- .evergreen/scripts/setup_tests.py | 7 ++- pymongo/_otel.py | 24 +++++++- pymongo/_telemetry.py | 2 +- pymongo/asynchronous/mongo_client.py | 4 ++ pymongo/synchronous/mongo_client.py | 4 ++ test/asynchronous/test_otel.py | 91 +++++++++++++++++++++++++++- test/test_otel.py | 89 ++++++++++++++++++++++++++- tools/otel_bench.py | 11 +++- tools/synchro.py | 1 + 9 files changed, 224 insertions(+), 9 deletions(-) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 45909ab8a4..724556997e 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -506,7 +506,12 @@ def handle_test_env() -> None: if test_name == "otel": # The SDK is test-only tooling (for the in-memory span exporter); the driver # itself must not depend on it, only on opentelemetry-api (the "opentelemetry" extra). - UV_ARGS.append("--with opentelemetry-sdk") + # The lower bound matters: under --resolution=lowest-direct an unpinned + # --with dependency resolves to its oldest version (1.0.0), which pins + # opentelemetry-api==1.0.0 and drags the API below the driver's floor + # (>=1.20.0 in requirements/opentelemetry.txt). Keep the SDK's floor in + # sync with that file when bumping the supported API version. + UV_ARGS.append("--with opentelemetry-sdk>=1.20.0") if test_name == "perf": data_dir = ROOT / "specifications/source/benchmarking/data" diff --git a/pymongo/_otel.py b/pymongo/_otel.py index 0476f3b993..15acc631b7 100644 --- a/pymongo/_otel.py +++ b/pymongo/_otel.py @@ -334,13 +334,31 @@ def start_command_span( return span -def end_command_span_success(span: Optional[Span], reply: _DocumentOut) -> None: - """Set the cursor id (if any) and end the span.""" +def end_command_span_success( + span: Optional[Span], + cmd: Mapping[str, Any], + command_name: str, + reply: _DocumentOut, +) -> None: + """Set the cursor id (if any) and end the span. + + The ``db.mongodb.cursor_id`` rules (spec: "db.mongodb.cursor_id"): a + non-zero reply cursor id is recorded as-is; a zero reply id means the + cursor is exhausted, so the attribute is omitted unless the command + operated on an existing cursor (getMore), in which case the cursor id the + command sent is recorded; a literal zero is never recorded. + """ if span is None: return cursor = reply.get("cursor") if isinstance(cursor, Mapping) and "id" in cursor: - span.set_attribute("db.mongodb.cursor_id", cursor["id"]) + cursor_id = cursor["id"] + if cursor_id: + span.set_attribute("db.mongodb.cursor_id", cursor_id) + elif command_name == _GET_MORE: + sent_id = cmd.get(_GET_MORE) + if sent_id: + span.set_attribute("db.mongodb.cursor_id", sent_id) span.end() diff --git a/pymongo/_telemetry.py b/pymongo/_telemetry.py index 156352d1e3..e5f0c2c9ff 100644 --- a/pymongo/_telemetry.py +++ b/pymongo/_telemetry.py @@ -226,7 +226,7 @@ def succeeded( database_name=self._dbname, ) if self._span is not None: - _otel.end_command_span_success(self._span, reply) + _otel.end_command_span_success(self._span, self._cmd, self._name, reply) def failed( self, diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index 3d66dab535..d749f4236b 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -1053,6 +1053,10 @@ async def _resolve_srv(self) -> None: # Username and password passed as kwargs override user info in URI. username = opts.get("username", self._resolve_srv_info["username"]) password = opts.get("password", self._resolve_srv_info["password"]) + # The tracing option was resolved against the environment when the + # client was constructed; carry that snapshot into the rebuilt + # options rather than re-reading the (possibly changed) environment. + opts["tracing"] = self._options.tracing self._options = ClientOptions( username, password, self._resolve_srv_info["dbase"], opts, _IS_SYNC ) diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index 79412c11de..811b0ffd8f 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -1054,6 +1054,10 @@ def _resolve_srv(self) -> None: # Username and password passed as kwargs override user info in URI. username = opts.get("username", self._resolve_srv_info["username"]) password = opts.get("password", self._resolve_srv_info["password"]) + # The tracing option was resolved against the environment when the + # client was constructed; carry that snapshot into the rebuilt + # options rather than re-reading the (possibly changed) environment. + opts["tracing"] = self._options.tracing self._options = ClientOptions( username, password, self._resolve_srv_info["dbase"], opts, _IS_SYNC ) diff --git a/test/asynchronous/test_otel.py b/test/asynchronous/test_otel.py index 6977bf19cb..7c7dcd7e26 100644 --- a/test/asynchronous/test_otel.py +++ b/test/asynchronous/test_otel.py @@ -30,6 +30,7 @@ import pymongo._otel as _otel from pymongo import common from pymongo.errors import ConfigurationError, OperationFailure +from pymongo.monitoring import CommandListener from pymongo.operations import InsertOne from pymongo.typings import _Address from test.asynchronous import AsyncIntegrationTest, async_client_context, unittest @@ -133,6 +134,66 @@ async def test_span_created_for_get_more(self): self.assertEqual(span.attributes["db.collection.name"], "test_otel_getmore") self.assertEqual(span.attributes["db.command.name"], "getMore") + async def test_cursor_id_omitted_when_first_batch_exhausts_cursor(self): + # A find whose reply carries a cursor id of 0 (everything fit in the + # first batch) must not record db.mongodb.cursor_id: the spec forbids + # a literal 0 (spec: "db.mongodb.cursor_id"). + client = await self.async_rs_or_single_client(tracing={"enabled": True}) + coll = client[self.db.name].test_otel_cursor_id + await coll.drop() + await coll.insert_one({"x": 1}) + self.exporter.clear() + + docs = await coll.find({}).to_list() + self.assertEqual(len(docs), 1) + + find_spans = self.spans("find") + self.assertEqual(len(find_spans), 1) + self.assertNotIn("db.mongodb.cursor_id", find_spans[0].attributes) + + async def test_get_more_records_sent_cursor_id_when_reply_exhausts(self): + # The final getMore's reply carries a cursor id of 0 (cursor exhausted); + # its span must record the nonzero cursor id the command sent instead + # (spec: "db.mongodb.cursor_id"). + class StartedListener(CommandListener): + def __init__(self): + self.sent_ids: list = [] + + def started(self, event): + if event.command_name == "getMore": + self.sent_ids.append(event.command["getMore"]) + + def succeeded(self, event): + pass + + def failed(self, event): + pass + + listener = StartedListener() + client = await self.async_rs_or_single_client( + tracing={"enabled": True}, event_listeners=[listener] + ) + coll = client[self.db.name].test_otel_cursor_id + await coll.drop() + await coll.insert_many([{"x": i} for i in range(5)]) + self.exporter.clear() + + docs = await coll.find({}, batch_size=2).to_list() + self.assertEqual(len(docs), 5) + + get_more_spans = self.spans("getMore") + self.assertGreater(len(get_more_spans), 0) + self.assertEqual(len(get_more_spans), len(listener.sent_ids)) + # Every getMore span records the cursor id the command sent; the final + # one exhausts the cursor (reply id 0) but must not record 0. + for span, sent_id in zip(get_more_spans, listener.sent_ids): + self.assertEqual(span.attributes["db.mongodb.cursor_id"], sent_id) + self.assertNotEqual(listener.sent_ids[-1], 0) + # The find's reply carried a nonzero cursor id, which its span records. + find_spans = self.spans("find") + self.assertEqual(len(find_spans), 1) + self.assertEqual(find_spans[0].attributes["db.mongodb.cursor_id"], listener.sent_ids[0]) + async def test_explain_retains_collection_name(self): # explain wraps the real command ({"explain": {"find": "coll", ...}}), the # same shape as getMore's indirection, so it needs the same handling. @@ -177,7 +238,7 @@ class _FakeUnixConn: "ping", False, ) - _otel.end_command_span_success(span, {"ok": 1}) + _otel.end_command_span_success(span, {"ping": 1}, "ping", {"ok": 1}) spans = self.spans("ping") self.assertEqual(len(spans), 1) @@ -323,6 +384,26 @@ async def test_unset_enabled_defers_to_env_var(self): await client.admin.command("ping") self.assertIn("ping", [s.name for s in self.spans()]) + async def test_srv_resolution_preserves_construction_time_snapshot(self): + # An SRV client rebuilds its ClientOptions at first use (_resolve_srv); + # the rebuilt options must keep the tracing values resolved at + # construction rather than re-reading the (possibly changed) environment. + with ( + patch("pymongo.asynchronous.srv_resolver._SrvResolver.get_hosts") as mock_hosts, + patch("pymongo.asynchronous.srv_resolver._SrvResolver.get_options", return_value=None), + ): + mock_hosts.return_value = [("localhost", 27017)] + with patch.dict(os.environ, {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED": "true"}): + client = self.unmanaged_simple_client( + "mongodb+srv://test.example.com", connect=False + ) + self.addAsyncCleanup(client.close) + self.assertEqual(client.options.tracing["enabled"], True) + # The environment changes after construction; the snapshot must not. + os.environ["OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED"] = "false" + await client._get_topology() + self.assertEqual(client.options.tracing["enabled"], True) + async def test_query_text_truncation_shrinks_oversized_field_values(self): client = await self.async_rs_or_single_client( tracing={"enabled": True, "query_text_max_length": 200} @@ -495,6 +576,9 @@ def test_none_without_env(self): {"enabled": False, "query_text_max_length": 0}, ) + # The environment-deferral semantics only exist when opentelemetry is + # importable; without it _resolve_tracing_options always disables tracing. + @unittest.skipUnless(_otel._HAS_OPENTELEMETRY, "opentelemetry is not installed") def test_unset_defers_to_env(self): with patch.dict(os.environ, {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED": "true"}): self.assertTrue( @@ -511,6 +595,8 @@ def test_env_false(self): ] ) + # Explicit values only win over the environment when tracing is available. + @unittest.skipUnless(_otel._HAS_OPENTELEMETRY, "opentelemetry is not installed") def test_explicit_enabled_overrides_env(self): env = {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED": "true"} with patch.dict(os.environ, env): @@ -523,6 +609,9 @@ def test_explicit_enabled_overrides_env(self): ) self.assertIs(resolved["enabled"], True) + # The environment-deferral semantics only exist when opentelemetry is + # importable; without it _resolve_tracing_options always omits query text. + @unittest.skipUnless(_otel._HAS_OPENTELEMETRY, "opentelemetry is not installed") def test_query_text_max_length_from_env(self): with patch.dict( os.environ, {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_QUERY_TEXT_MAX_LENGTH": "1024"} diff --git a/test/test_otel.py b/test/test_otel.py index 76adc91e9d..f6dbe76a8f 100644 --- a/test/test_otel.py +++ b/test/test_otel.py @@ -30,6 +30,7 @@ import pymongo._otel as _otel from pymongo import common from pymongo.errors import ConfigurationError, OperationFailure +from pymongo.monitoring import CommandListener from pymongo.operations import InsertOne from pymongo.typings import _Address from test import IntegrationTest, client_context, unittest @@ -133,6 +134,64 @@ def test_span_created_for_get_more(self): self.assertEqual(span.attributes["db.collection.name"], "test_otel_getmore") self.assertEqual(span.attributes["db.command.name"], "getMore") + def test_cursor_id_omitted_when_first_batch_exhausts_cursor(self): + # A find whose reply carries a cursor id of 0 (everything fit in the + # first batch) must not record db.mongodb.cursor_id: the spec forbids + # a literal 0 (spec: "db.mongodb.cursor_id"). + client = self.rs_or_single_client(tracing={"enabled": True}) + coll = client[self.db.name].test_otel_cursor_id + coll.drop() + coll.insert_one({"x": 1}) + self.exporter.clear() + + docs = coll.find({}).to_list() + self.assertEqual(len(docs), 1) + + find_spans = self.spans("find") + self.assertEqual(len(find_spans), 1) + self.assertNotIn("db.mongodb.cursor_id", find_spans[0].attributes) + + def test_get_more_records_sent_cursor_id_when_reply_exhausts(self): + # The final getMore's reply carries a cursor id of 0 (cursor exhausted); + # its span must record the nonzero cursor id the command sent instead + # (spec: "db.mongodb.cursor_id"). + class StartedListener(CommandListener): + def __init__(self): + self.sent_ids: list = [] + + def started(self, event): + if event.command_name == "getMore": + self.sent_ids.append(event.command["getMore"]) + + def succeeded(self, event): + pass + + def failed(self, event): + pass + + listener = StartedListener() + client = self.rs_or_single_client(tracing={"enabled": True}, event_listeners=[listener]) + coll = client[self.db.name].test_otel_cursor_id + coll.drop() + coll.insert_many([{"x": i} for i in range(5)]) + self.exporter.clear() + + docs = coll.find({}, batch_size=2).to_list() + self.assertEqual(len(docs), 5) + + get_more_spans = self.spans("getMore") + self.assertGreater(len(get_more_spans), 0) + self.assertEqual(len(get_more_spans), len(listener.sent_ids)) + # Every getMore span records the cursor id the command sent; the final + # one exhausts the cursor (reply id 0) but must not record 0. + for span, sent_id in zip(get_more_spans, listener.sent_ids): + self.assertEqual(span.attributes["db.mongodb.cursor_id"], sent_id) + self.assertNotEqual(listener.sent_ids[-1], 0) + # The find's reply carried a nonzero cursor id, which its span records. + find_spans = self.spans("find") + self.assertEqual(len(find_spans), 1) + self.assertEqual(find_spans[0].attributes["db.mongodb.cursor_id"], listener.sent_ids[0]) + def test_explain_retains_collection_name(self): # explain wraps the real command ({"explain": {"find": "coll", ...}}), the # same shape as getMore's indirection, so it needs the same handling. @@ -177,7 +236,7 @@ class _FakeUnixConn: "ping", False, ) - _otel.end_command_span_success(span, {"ok": 1}) + _otel.end_command_span_success(span, {"ping": 1}, "ping", {"ok": 1}) spans = self.spans("ping") self.assertEqual(len(spans), 1) @@ -319,6 +378,26 @@ def test_unset_enabled_defers_to_env_var(self): client.admin.command("ping") self.assertIn("ping", [s.name for s in self.spans()]) + def test_srv_resolution_preserves_construction_time_snapshot(self): + # An SRV client rebuilds its ClientOptions at first use (_resolve_srv); + # the rebuilt options must keep the tracing values resolved at + # construction rather than re-reading the (possibly changed) environment. + with ( + patch("pymongo.synchronous.srv_resolver._SrvResolver.get_hosts") as mock_hosts, + patch("pymongo.synchronous.srv_resolver._SrvResolver.get_options", return_value=None), + ): + mock_hosts.return_value = [("localhost", 27017)] + with patch.dict(os.environ, {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED": "true"}): + client = self.unmanaged_simple_client( + "mongodb+srv://test.example.com", connect=False + ) + self.addCleanup(client.close) + self.assertEqual(client.options.tracing["enabled"], True) + # The environment changes after construction; the snapshot must not. + os.environ["OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED"] = "false" + client._get_topology() + self.assertEqual(client.options.tracing["enabled"], True) + def test_query_text_truncation_shrinks_oversized_field_values(self): client = self.rs_or_single_client(tracing={"enabled": True, "query_text_max_length": 200}) coll = client[self.db.name].test_otel @@ -489,6 +568,9 @@ def test_none_without_env(self): {"enabled": False, "query_text_max_length": 0}, ) + # The environment-deferral semantics only exist when opentelemetry is + # importable; without it _resolve_tracing_options always disables tracing. + @unittest.skipUnless(_otel._HAS_OPENTELEMETRY, "opentelemetry is not installed") def test_unset_defers_to_env(self): with patch.dict(os.environ, {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED": "true"}): self.assertTrue( @@ -505,6 +587,8 @@ def test_env_false(self): ] ) + # Explicit values only win over the environment when tracing is available. + @unittest.skipUnless(_otel._HAS_OPENTELEMETRY, "opentelemetry is not installed") def test_explicit_enabled_overrides_env(self): env = {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_ENABLED": "true"} with patch.dict(os.environ, env): @@ -517,6 +601,9 @@ def test_explicit_enabled_overrides_env(self): ) self.assertIs(resolved["enabled"], True) + # The environment-deferral semantics only exist when opentelemetry is + # importable; without it _resolve_tracing_options always omits query text. + @unittest.skipUnless(_otel._HAS_OPENTELEMETRY, "opentelemetry is not installed") def test_query_text_max_length_from_env(self): with patch.dict( os.environ, {"OTEL_PYTHON_INSTRUMENTATION_MONGODB_QUERY_TEXT_MAX_LENGTH": "1024"} diff --git a/tools/otel_bench.py b/tools/otel_bench.py index 5b0d44eca6..71aca28bae 100755 --- a/tools/otel_bench.py +++ b/tools/otel_bench.py @@ -249,7 +249,7 @@ def report( "reps": args.reps, "fast": args.fast, "configs": {}, - "pre_otel_compare": None, + "pre_otel_compare": {}, } for api in ("sync", "async"): @@ -288,7 +288,8 @@ def report( "off_median_mb_s": off_median, "overhead_pct": (1 - off_median / main_median) * 100, } - summary["pre_otel_compare"] = {"api": api, **row} + # Key by api: both rows must survive into the persisted results. + summary["pre_otel_compare"][api] = row overheads = ", ".join(f"{task}: {row[task]['overhead_pct']:+.1f}%" for task in TASKS) print(f"\nPre-OTel comparison (off on this branch vs main, {api}): {overheads}") @@ -342,6 +343,12 @@ def main() -> None: if args.reps < 1: raise SystemExit("--reps must be at least 1") + # Children run with a different working directory (the repo root, or the + # main worktree); interpret user-supplied relative paths against the + # harness's cwd once, so the harness's own file operations and the paths + # passed to children in the environment agree. + args.data_dir = args.data_dir.absolute() + args.output_dir = args.output_dir.absolute() if not args.data_dir.exists(): raise SystemExit( f"Dataset directory not found: {args.data_dir}\n" diff --git a/tools/synchro.py b/tools/synchro.py index b5c6b2873a..2060b2a448 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -140,6 +140,7 @@ "async_simple_test_client": "simple_test_client", "_async_create_connection": "_create_connection", "pymongo.asynchronous.srv_resolver._SrvResolver.get_hosts": "pymongo.synchronous.srv_resolver._SrvResolver.get_hosts", + "pymongo.asynchronous.srv_resolver._SrvResolver.get_options": "pymongo.synchronous.srv_resolver._SrvResolver.get_options", "dns.asyncresolver.resolve": "dns.resolver.resolve", "__aenter__": "__enter__", "__aexit__": "__exit__", From 0fa4b111d652a6116d829958dd0baf321fdd2a09 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 9 Oct 2026 05:10:29 -0500 Subject: [PATCH 7/8] Set error.type on failed spans and gate CPU timing - Set error.type on failed command spans: the server response code alongside db.response.status_code for server failures, otherwise the exception class name, so server and transport failures are distinguishable. - Only collect CPU time in the performance tests when PERF_CPU_TIME is enabled, so the standard suite no longer pays clock calls inside the measured window. --- pymongo/_otel.py | 13 ++++++++-- test/asynchronous/test_otel.py | 37 +++++++++++++++++++++++++++++ test/performance/async_perf_test.py | 9 ++++--- test/performance/perf_test.py | 9 ++++--- test/test_otel.py | 37 +++++++++++++++++++++++++++++ 5 files changed, 97 insertions(+), 8 deletions(-) diff --git a/pymongo/_otel.py b/pymongo/_otel.py index 15acc631b7..af115bdb68 100644 --- a/pymongo/_otel.py +++ b/pymongo/_otel.py @@ -367,12 +367,21 @@ def end_command_span_failure( failure: _DocumentOut, exc: BaseException, ) -> None: - """Record the exception, set the error status, and end the span.""" + """Record the exception, set the error status, and end the span. + + ``error.type`` (OpenTelemetry convention): the server response code when + the failure came from a server response, otherwise the exception class + name, so server and transport failures are distinguishable. + """ if span is None: return span.record_exception(exc) code = failure.get("code") if code is not None: - span.set_attribute("db.response.status_code", str(code)) + code = str(code) + span.set_attribute("db.response.status_code", code) + span.set_attribute("error.type", code) + else: + span.set_attribute("error.type", type(exc).__name__) span.set_status(Status(StatusCode.ERROR, description=failure.get("errmsg"))) span.end() diff --git a/test/asynchronous/test_otel.py b/test/asynchronous/test_otel.py index 7c7dcd7e26..5a15be20f0 100644 --- a/test/asynchronous/test_otel.py +++ b/test/asynchronous/test_otel.py @@ -280,8 +280,45 @@ async def test_failure_records_exception_and_status_code(self): span = spans[0] self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) self.assertIn("db.response.status_code", span.attributes) + # error.type mirrors the server response code for server failures. + self.assertEqual(span.attributes["error.type"], span.attributes["db.response.status_code"]) self.assertTrue(any(event.name == "exception" for event in span.events)) + async def test_transport_failure_records_exception_class_as_error_type(self): + class _FakeConn: + id = 1 + server_connection_id: Optional[int] = None + address: _Address = ("localhost", 27017) + service_id = None + _otel_connection_attributes: tuple[Optional[int], dict[str, Any]] = (None, {}) + + options = {"enabled": True, "query_text_max_length": None} + self.exporter.clear() + + # Transport failure: no server response, error.type is the exception + # class name and db.response.status_code is absent. + span = _otel.start_command_span(options, _FakeConn(), {"ping": 1}, "admin", "ping", False) + _otel.end_command_span_failure(span, {}, TimeoutError("timed out")) + spans = self.spans() + self.assertEqual(len(spans), 1) + attrs = spans[0].attributes + self.assertEqual(spans[0].status.status_code, trace.StatusCode.ERROR) + self.assertNotIn("db.response.status_code", attrs) + self.assertEqual(attrs["error.type"], "TimeoutError") + + # Server failure: error.type mirrors db.response.status_code. + self.exporter.clear() + span = _otel.start_command_span(options, _FakeConn(), {"ping": 1}, "admin", "ping", False) + _otel.end_command_span_failure( + span, {"code": 26, "errmsg": "not found"}, OperationFailure("") + ) + spans = self.spans() + self.assertEqual(len(spans), 1) + attrs = spans[0].attributes + self.assertEqual(spans[0].status.status_code, trace.StatusCode.ERROR) + self.assertEqual(attrs["db.response.status_code"], "26") + self.assertEqual(attrs["error.type"], "26") + async def test_tracing_disabled_by_default(self): client = await self.async_rs_or_single_client() self.exporter.clear() diff --git a/test/performance/async_perf_test.py b/test/performance/async_perf_test.py index 5aa5e8d4ba..fb2069c429 100644 --- a/test/performance/async_perf_test.py +++ b/test/performance/async_perf_test.py @@ -107,13 +107,15 @@ def tearDownModule(): class Timer: def __enter__(self): self.start = time.monotonic() - self.cpu_start = time.process_time() + if RECORD_CPU_TIME: + self.cpu_start = time.process_time() return self def __exit__(self, *args): self.end = time.monotonic() self.interval = self.end - self.start - self.cpu_interval = time.process_time() - self.cpu_start + if RECORD_CPU_TIME: + self.cpu_interval = time.process_time() - self.cpu_start async def concurrent(n_tasks, func): @@ -212,7 +214,8 @@ async def runTest(self): await concurrent(self.n_tasks, self.do_task) await self.after() results.append(timer.interval) - cpu_results.append(timer.cpu_interval) + if RECORD_CPU_TIME: + cpu_results.append(timer.cpu_interval) duration = time.monotonic() - start if duration > MIN_ITERATION_TIME and i >= NUM_ITERATIONS: break diff --git a/test/performance/perf_test.py b/test/performance/perf_test.py index 7464e21b13..1b8109225f 100644 --- a/test/performance/perf_test.py +++ b/test/performance/perf_test.py @@ -109,13 +109,15 @@ def tearDownModule(): class Timer: def __enter__(self): self.start = time.monotonic() - self.cpu_start = time.process_time() + if RECORD_CPU_TIME: + self.cpu_start = time.process_time() return self def __exit__(self, *args): self.end = time.monotonic() self.interval = self.end - self.start - self.cpu_interval = time.process_time() - self.cpu_start + if RECORD_CPU_TIME: + self.cpu_interval = time.process_time() - self.cpu_start def threaded(n_threads, func): @@ -219,7 +221,8 @@ def runTest(self): threaded(self.n_threads, self.do_task) self.after() results.append(timer.interval) - cpu_results.append(timer.cpu_interval) + if RECORD_CPU_TIME: + cpu_results.append(timer.cpu_interval) duration = time.monotonic() - start if duration > MIN_ITERATION_TIME and i >= NUM_ITERATIONS: break diff --git a/test/test_otel.py b/test/test_otel.py index f6dbe76a8f..ac4a81742b 100644 --- a/test/test_otel.py +++ b/test/test_otel.py @@ -278,8 +278,45 @@ def test_failure_records_exception_and_status_code(self): span = spans[0] self.assertEqual(span.status.status_code, trace.StatusCode.ERROR) self.assertIn("db.response.status_code", span.attributes) + # error.type mirrors the server response code for server failures. + self.assertEqual(span.attributes["error.type"], span.attributes["db.response.status_code"]) self.assertTrue(any(event.name == "exception" for event in span.events)) + def test_transport_failure_records_exception_class_as_error_type(self): + class _FakeConn: + id = 1 + server_connection_id: Optional[int] = None + address: _Address = ("localhost", 27017) + service_id = None + _otel_connection_attributes: tuple[Optional[int], dict[str, Any]] = (None, {}) + + options = {"enabled": True, "query_text_max_length": None} + self.exporter.clear() + + # Transport failure: no server response, error.type is the exception + # class name and db.response.status_code is absent. + span = _otel.start_command_span(options, _FakeConn(), {"ping": 1}, "admin", "ping", False) + _otel.end_command_span_failure(span, {}, TimeoutError("timed out")) + spans = self.spans() + self.assertEqual(len(spans), 1) + attrs = spans[0].attributes + self.assertEqual(spans[0].status.status_code, trace.StatusCode.ERROR) + self.assertNotIn("db.response.status_code", attrs) + self.assertEqual(attrs["error.type"], "TimeoutError") + + # Server failure: error.type mirrors db.response.status_code. + self.exporter.clear() + span = _otel.start_command_span(options, _FakeConn(), {"ping": 1}, "admin", "ping", False) + _otel.end_command_span_failure( + span, {"code": 26, "errmsg": "not found"}, OperationFailure("") + ) + spans = self.spans() + self.assertEqual(len(spans), 1) + attrs = spans[0].attributes + self.assertEqual(spans[0].status.status_code, trace.StatusCode.ERROR) + self.assertEqual(attrs["db.response.status_code"], "26") + self.assertEqual(attrs["error.type"], "26") + def test_tracing_disabled_by_default(self): client = self.rs_or_single_client() self.exporter.clear() From 1d83c6b0ac4f0d8d4eccd463d3e3e4cb3570a14f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 9 Oct 2026 06:16:42 -0500 Subject: [PATCH 8/8] Implement operation spans for benchmarking Prototype the spec's operation-span layer in the driver: one CLIENT span per driver operation, started before server selection, with the operation's command spans nested beneath it via the current OTel context. Wired at the _run_operation (queries) and _retryable_write funnels with db/collection plumbed from the public call sites; getMore and cursor iteration remain PYTHON-5947 work. On failure the span records the exception and sets error.type to the exception class name. This replaces the bench-side prototype: the benchmark's existing tracing configurations now measure command and operation spans together, and otel_bench --verify asserts the nesting. Also annotate the failure-path test's options dict as TracingOptions, fixing 4 mypy arg-type errors from the previous commit. --- pymongo/_otel.py | 88 +++++++++++++++++++++++++++ pymongo/_telemetry.py | 15 +++++ pymongo/asynchronous/bulk.py | 2 + pymongo/asynchronous/client_bulk.py | 3 + pymongo/asynchronous/collection.py | 60 ++++++++++++++++--- pymongo/asynchronous/database.py | 4 +- pymongo/asynchronous/mongo_client.py | 90 +++++++++++++++++----------- pymongo/synchronous/bulk.py | 2 + pymongo/synchronous/client_bulk.py | 3 + pymongo/synchronous/collection.py | 62 ++++++++++++++++--- pymongo/synchronous/database.py | 2 +- pymongo/synchronous/mongo_client.py | 88 ++++++++++++++++----------- test/asynchronous/test_otel.py | 59 +++++++++++++++++- test/test_otel.py | 59 +++++++++++++++++- tools/otel_bench.py | 5 ++ 15 files changed, 454 insertions(+), 88 deletions(-) diff --git a/pymongo/_otel.py b/pymongo/_otel.py index af115bdb68..b43f831418 100644 --- a/pymongo/_otel.py +++ b/pymongo/_otel.py @@ -31,6 +31,7 @@ from pymongo.logger import _HELLO_COMMANDS, _JSON_OPTIONS, _SENSITIVE_COMMANDS try: + from opentelemetry import context as otel_context from opentelemetry import trace from opentelemetry.trace import SpanKind, Status, StatusCode @@ -385,3 +386,90 @@ def end_command_span_failure( span.set_attribute("error.type", type(exc).__name__) span.set_status(Status(StatusCode.ERROR, description=failure.get("errmsg"))) span.end() + + +class _NoOperationSpan: + """No-op context manager used by :func:`operation_span` when tracing is off. + + A module-level singleton: the operation funnels enter it unconditionally, + so the disabled-tracing path pays only the ``with`` statement itself. + """ + + __slots__ = () + + def __enter__(self) -> None: + return None + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + return None + + +_NO_OPERATION_SPAN = _NoOperationSpan() + + +class _OperationSpanContext: + """Context manager for one operation span. + + Makes the span current while the operation runs, so the operation's + command spans (created with ``Tracer.start_span``, which parents from the + current context) nest beneath it. Ends the span on exit; on exception, + records the exception and sets the spec's ``error.type`` attribute first. + """ + + __slots__ = ("_span", "_token") + + def __init__(self, span: Span) -> None: + self._span = span + self._token: Optional[object] = None + + def __enter__(self) -> Span: + self._token = otel_context.attach(trace.set_span_in_context(self._span)) + return self._span + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + otel_context.detach(self._token) + span = self._span + if exc_value is not None: + span.record_exception(exc_value) + # The spec's operation-span error.type: the name of the exception + # class raised to the application (a failed command does not carry + # its server code over: a retry of the same operation may succeed). + span.set_attribute("error.type", type(exc_value).__name__) + span.set_status(Status(StatusCode.ERROR, description=str(exc_value))) + span.end() + + +def operation_span( + tracing_options: Optional[TracingOptions], + operation_name: str, + dbname: Optional[str], + collection: Optional[str], +) -> _OperationSpanContext | _NoOperationSpan: + """Start an operation span and return a context manager that ends it. + + Per the spec ("Instrumenting Driver Operations"): one CLIENT-kind span per + driver operation, started before server selection so its duration reflects + all of the driver's activities, named ``operation_name db.collection_name`` + (or ``operation_name db`` when there is no collection) and carrying the + spec's required attributes. Returns the :data:`_NO_OPERATION_SPAN` + singleton when tracing is disabled so the disabled path stays at parity. + """ + if not _is_tracing_enabled(tracing_options): + return _NO_OPERATION_SPAN + if collection: + summary = f"{operation_name} {dbname}.{collection}" + else: + summary = f"{operation_name} {dbname}" + attributes: dict[str, Any] = { + "db.system.name": "mongodb", + "db.operation.name": operation_name, + "db.operation.summary": summary, + } + if dbname: + attributes["db.namespace"] = dbname + if collection: + attributes["db.collection.name"] = collection + assert _TRACER is not None # _is_tracing_enabled already checked _HAS_OPENTELEMETRY + return _OperationSpanContext( + _TRACER.start_span(summary, kind=_SPAN_KIND_CLIENT, attributes=attributes) + ) diff --git a/pymongo/_telemetry.py b/pymongo/_telemetry.py index e5f0c2c9ff..2295f3bd80 100644 --- a/pymongo/_telemetry.py +++ b/pymongo/_telemetry.py @@ -71,6 +71,21 @@ def _generate_op_id_or_none(listeners: Optional[_EventListeners]) -> Optional[in ) +def operation_span( + tracing_options: Optional[_otel.TracingOptions], + operation_name: str, + dbname: Optional[str], + collection: Optional[str], +) -> Any: + """Start the spec's operation span for one driver operation, as a context manager. + + Thin wrapper over :func:`pymongo._otel.operation_span`, kept here so + callers already importing :mod:`pymongo._telemetry` need no new import + edges. See the ``_otel`` function for the span's shape. + """ + return _otel.operation_span(tracing_options, operation_name, dbname, collection) + + class _CommandTelemetry: """Combines structured logging and APM event publishing for a single command. diff --git a/pymongo/asynchronous/bulk.py b/pymongo/asynchronous/bulk.py index 81186499f6..436df3902c 100644 --- a/pymongo/asynchronous/bulk.py +++ b/pymongo/asynchronous/bulk.py @@ -312,6 +312,8 @@ async def retryable_bulk( retryable_bulk, session, operation, + db_name=self.collection.database.name, + coll_name=self.collection.name, bulk=self, # type: ignore[arg-type] operation_id=op_id, ) diff --git a/pymongo/asynchronous/client_bulk.py b/pymongo/asynchronous/client_bulk.py index bfbe9e2c6b..4340046de9 100644 --- a/pymongo/asynchronous/client_bulk.py +++ b/pymongo/asynchronous/client_bulk.py @@ -404,6 +404,9 @@ async def retryable_bulk( retryable_bulk, session, operation, + # The spec's client bulkWrite spans run against admin with no + # single collection. + db_name="admin", bulk=self, operation_id=op_id, ) diff --git a/pymongo/asynchronous/collection.py b/pymongo/asynchronous/collection.py index 924635d7dc..9de95fe6b2 100644 --- a/pymongo/asynchronous/collection.py +++ b/pymongo/asynchronous/collection.py @@ -650,7 +650,9 @@ async def inner( session=session, ) - await self.database.client._retryable_write(False, inner, session, _Op.CREATE) + await self.database.client._retryable_write( + False, inner, session, _Op.CREATE, db_name=self._database.name, coll_name=self._name + ) async def _create( self, @@ -813,7 +815,12 @@ async def _insert_command( _check_write_command_response(result) await self._database.client._retryable_write( - acknowledged, _insert_command, session, operation=_Op.INSERT + acknowledged, + _insert_command, + session, + operation=_Op.INSERT, + db_name=self._database.name, + coll_name=self._name, ) if not isinstance(doc, RawBSONDocument): @@ -1100,6 +1107,8 @@ async def _update( _update, session, operation, + db_name=self._database.name, + coll_name=self._name, ) async def replace_one( @@ -1562,6 +1571,8 @@ async def _delete( _delete, session, operation=_Op.DELETE, + db_name=self._database.name, + coll_name=self._name, ) async def delete_one( @@ -2251,7 +2262,12 @@ def gen_indexes() -> Iterator[Mapping[str, Any]]: return names return await self.database.client._retryable_write( - False, inner, session, _Op.CREATE_INDEXES + False, + inner, + session, + _Op.CREATE_INDEXES, + db_name=self._database.name, + coll_name=self._name, ) async def create_index( @@ -2485,7 +2501,14 @@ async def inner( session=session, ) - await self.database.client._retryable_write(False, inner, session, _Op.DROP_INDEXES) + await self.database.client._retryable_write( + False, + inner, + session, + _Op.DROP_INDEXES, + db_name=self._database.name, + coll_name=self._name, + ) async def list_indexes( self, @@ -2763,7 +2786,12 @@ async def inner( return [index["name"] for index in resp["indexesCreated"]] return await self.database.client._retryable_write( - False, inner, session, _Op.CREATE_SEARCH_INDEXES + False, + inner, + session, + _Op.CREATE_SEARCH_INDEXES, + db_name=self._database.name, + coll_name=self._name, ) async def drop_search_index( @@ -2804,7 +2832,14 @@ async def inner( session=session, ) - await self.database.client._retryable_write(False, inner, session, _Op.DROP_SEARCH_INDEXES) + await self.database.client._retryable_write( + False, + inner, + session, + _Op.DROP_SEARCH_INDEXES, + db_name=self._database.name, + coll_name=self._name, + ) async def update_search_index( self, @@ -2846,7 +2881,14 @@ async def inner( session=session, ) - await self.database.client._retryable_write(False, inner, session, _Op.UPDATE_SEARCH_INDEX) + await self.database.client._retryable_write( + False, + inner, + session, + _Op.UPDATE_SEARCH_INDEX, + db_name=self._database.name, + coll_name=self._name, + ) async def options( self, @@ -3142,7 +3184,9 @@ async def inner( client=client, ) - return await client._retryable_write(False, inner, session, _Op.RENAME) + return await client._retryable_write( + False, inner, session, _Op.RENAME, db_name=self._database.name, coll_name=self._name + ) async def distinct( self, diff --git a/pymongo/asynchronous/database.py b/pymongo/asynchronous/database.py index 88ca9cab5b..f9455f1a73 100644 --- a/pymongo/asynchronous/database.py +++ b/pymongo/asynchronous/database.py @@ -1268,7 +1268,9 @@ async def inner( session=session, ) - return await self.client._retryable_write(False, inner, session, _Op.DROP) + return await self.client._retryable_write( + False, inner, session, _Op.DROP, db_name=self.name + ) @_csot.apply async def drop_collection( diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index 91bf902f29..d423ca920a 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -56,7 +56,7 @@ from bson.codec_options import DEFAULT_CODEC_OPTIONS, CodecOptions, TypeRegistry from bson.timestamp import Timestamp from pymongo import _csot, _op_id, common, helpers_shared, periodic_executor -from pymongo._telemetry import _generate_op_id_or_none, log_command_retry +from pymongo._telemetry import _generate_op_id_or_none, log_command_retry, operation_span from pymongo.asynchronous import client_session, database, uri_parser from pymongo.asynchronous.change_stream import AsyncChangeStream, AsyncClusterChangeStream from pymongo.asynchronous.client_bulk import _AsyncClientBulk @@ -1992,43 +1992,51 @@ async def _run_operation( :param address: Optional address when sending a message to a specific server, used for getMore. """ - if operation.conn_mgr: - server = await self._select_server( + # The spec's operation span: one span per driver operation, started + # before server selection, with the operation's command spans nested + # beneath it. getMores are cursor territory (PYTHON-5947): no + # operation span yet. + # getMores are cursor territory (PYTHON-5947): no operation span yet. + tracing = self.options.tracing if not isinstance(operation, _GetMore) else None + op_span = operation_span(tracing, operation.name, operation.db, operation.coll) + with op_span: + if operation.conn_mgr: + server = await self._select_server( + operation.read_preference, + operation.session, # type: ignore[arg-type] + operation.name, + address=address, + ) + + async with operation.conn_mgr._lock: + async with _ClientCheckout.for_existing_conn( + self, + server, + operation.session, # type: ignore[arg-type] + operation.conn_mgr.conn, + ): + return await run_with_conn( + operation.conn_mgr.conn, operation, operation.read_preference + ) + + async def _cmd( + _session: Optional[AsyncClientSession], + _server: Server, + conn: AsyncConnection, + read_preference: _ServerMode, + ) -> Response: + operation.reset() # Reset op in case of retry. + return await run_with_conn(conn, operation, read_preference) + + return await self._retryable_read( + _cmd, operation.read_preference, operation.session, # type: ignore[arg-type] - operation.name, address=address, + retryable=isinstance(operation, _Query), + operation=operation.name, ) - async with operation.conn_mgr._lock: - async with _ClientCheckout.for_existing_conn( - self, - server, - operation.session, # type: ignore[arg-type] - operation.conn_mgr.conn, - ): - return await run_with_conn( - operation.conn_mgr.conn, operation, operation.read_preference - ) - - async def _cmd( - _session: Optional[AsyncClientSession], - _server: Server, - conn: AsyncConnection, - read_preference: _ServerMode, - ) -> Response: - operation.reset() # Reset op in case of retry. - return await run_with_conn(conn, operation, read_preference) - - return await self._retryable_read( - _cmd, - operation.read_preference, - operation.session, # type: ignore[arg-type] - address=address, - retryable=isinstance(operation, _Query), - operation=operation.name, - ) - async def _retry_with_session( self, retryable: bool, @@ -2164,6 +2172,8 @@ async def _retryable_write( operation: str, bulk: Optional[Union[_AsyncBulk, _AsyncClientBulk]] = None, operation_id: Optional[int] = None, + db_name: Optional[str] = None, + coll_name: Optional[str] = None, ) -> T: """Execute an operation with consecutive retries if possible @@ -2178,9 +2188,21 @@ async def _retryable_write( :param operation: The name of the operation that the server is being selected for :param bulk: bulk abstraction to execute operations in bulk, defaults to None :param operation_id: Stable operation id shared across retries, defaults to None + :param db_name: The database the operation targets, for the spec's + operation span. Defaults to None (attribute omitted). + :param coll_name: The collection the operation targets, for the spec's + operation span. Defaults to None (attribute omitted). """ async with self._tmp_session(session) as s: - return await self._retry_with_session(retryable, func, s, bulk, operation, operation_id) + with operation_span( + self.options.tracing, + getattr(operation, "value", operation), + db_name, + coll_name, + ): + return await self._retry_with_session( + retryable, func, s, bulk, operation, operation_id + ) def _cleanup_cursor_no_lock( self, diff --git a/pymongo/synchronous/bulk.py b/pymongo/synchronous/bulk.py index 291965fca1..5a71879750 100644 --- a/pymongo/synchronous/bulk.py +++ b/pymongo/synchronous/bulk.py @@ -312,6 +312,8 @@ def retryable_bulk( retryable_bulk, session, operation, + db_name=self.collection.database.name, + coll_name=self.collection.name, bulk=self, # type: ignore[arg-type] operation_id=op_id, ) diff --git a/pymongo/synchronous/client_bulk.py b/pymongo/synchronous/client_bulk.py index 549c9d59ca..546474f9ea 100644 --- a/pymongo/synchronous/client_bulk.py +++ b/pymongo/synchronous/client_bulk.py @@ -402,6 +402,9 @@ def retryable_bulk( retryable_bulk, session, operation, + # The spec's client bulkWrite spans run against admin with no + # single collection. + db_name="admin", bulk=self, operation_id=op_id, ) diff --git a/pymongo/synchronous/collection.py b/pymongo/synchronous/collection.py index 931001e179..1984a34d44 100644 --- a/pymongo/synchronous/collection.py +++ b/pymongo/synchronous/collection.py @@ -652,7 +652,9 @@ def inner( session=session, ) - self.database.client._retryable_write(False, inner, session, _Op.CREATE) + self.database.client._retryable_write( + False, inner, session, _Op.CREATE, db_name=self._database.name, coll_name=self._name + ) def _create( self, @@ -813,7 +815,12 @@ def _insert_command( _check_write_command_response(result) self._database.client._retryable_write( - acknowledged, _insert_command, session, operation=_Op.INSERT + acknowledged, + _insert_command, + session, + operation=_Op.INSERT, + db_name=self._database.name, + coll_name=self._name, ) if not isinstance(doc, RawBSONDocument): @@ -1100,6 +1107,8 @@ def _update( _update, session, operation, + db_name=self._database.name, + coll_name=self._name, ) def replace_one( @@ -1562,6 +1571,8 @@ def _delete( _delete, session, operation=_Op.DELETE, + db_name=self._database.name, + coll_name=self._name, ) def delete_one( @@ -2248,7 +2259,14 @@ def gen_indexes() -> Iterator[Mapping[str, Any]]: ) return names - return self.database.client._retryable_write(False, inner, session, _Op.CREATE_INDEXES) + return self.database.client._retryable_write( + False, + inner, + session, + _Op.CREATE_INDEXES, + db_name=self._database.name, + coll_name=self._name, + ) def create_index( self, @@ -2481,7 +2499,14 @@ def inner( session=session, ) - self.database.client._retryable_write(False, inner, session, _Op.DROP_INDEXES) + self.database.client._retryable_write( + False, + inner, + session, + _Op.DROP_INDEXES, + db_name=self._database.name, + coll_name=self._name, + ) def list_indexes( self, @@ -2759,7 +2784,12 @@ def inner( return [index["name"] for index in resp["indexesCreated"]] return self.database.client._retryable_write( - False, inner, session, _Op.CREATE_SEARCH_INDEXES + False, + inner, + session, + _Op.CREATE_SEARCH_INDEXES, + db_name=self._database.name, + coll_name=self._name, ) def drop_search_index( @@ -2800,7 +2830,14 @@ def inner( session=session, ) - self.database.client._retryable_write(False, inner, session, _Op.DROP_SEARCH_INDEXES) + self.database.client._retryable_write( + False, + inner, + session, + _Op.DROP_SEARCH_INDEXES, + db_name=self._database.name, + coll_name=self._name, + ) def update_search_index( self, @@ -2842,7 +2879,14 @@ def inner( session=session, ) - self.database.client._retryable_write(False, inner, session, _Op.UPDATE_SEARCH_INDEX) + self.database.client._retryable_write( + False, + inner, + session, + _Op.UPDATE_SEARCH_INDEX, + db_name=self._database.name, + coll_name=self._name, + ) def options( self, @@ -3136,7 +3180,9 @@ def inner( client=client, ) - return client._retryable_write(False, inner, session, _Op.RENAME) + return client._retryable_write( + False, inner, session, _Op.RENAME, db_name=self._database.name, coll_name=self._name + ) def distinct( self, diff --git a/pymongo/synchronous/database.py b/pymongo/synchronous/database.py index 65c5d2a7f0..aa4da7452f 100644 --- a/pymongo/synchronous/database.py +++ b/pymongo/synchronous/database.py @@ -1265,7 +1265,7 @@ def inner( session=session, ) - return self.client._retryable_write(False, inner, session, _Op.DROP) + return self.client._retryable_write(False, inner, session, _Op.DROP, db_name=self.name) @_csot.apply def drop_collection( diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index 775c56847b..6dcc20cf8c 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -56,7 +56,7 @@ from bson.codec_options import DEFAULT_CODEC_OPTIONS, CodecOptions, TypeRegistry from bson.timestamp import Timestamp from pymongo import _csot, _op_id, common, helpers_shared, periodic_executor -from pymongo._telemetry import _generate_op_id_or_none, log_command_retry +from pymongo._telemetry import _generate_op_id_or_none, log_command_retry, operation_span from pymongo.client_options import ClientOptions from pymongo.client_session_shared import SessionOptions, TransactionOptions, _EmptyServerSession from pymongo.driver_info import DriverInfo @@ -1987,43 +1987,51 @@ def _run_operation( :param address: Optional address when sending a message to a specific server, used for getMore. """ - if operation.conn_mgr: - server = self._select_server( + # The spec's operation span: one span per driver operation, started + # before server selection, with the operation's command spans nested + # beneath it. getMores are cursor territory (PYTHON-5947): no + # operation span yet. + # getMores are cursor territory (PYTHON-5947): no operation span yet. + tracing = self.options.tracing if not isinstance(operation, _GetMore) else None + op_span = operation_span(tracing, operation.name, operation.db, operation.coll) + with op_span: + if operation.conn_mgr: + server = self._select_server( + operation.read_preference, + operation.session, # type: ignore[arg-type] + operation.name, + address=address, + ) + + with operation.conn_mgr._lock: + with _ClientCheckout.for_existing_conn( + self, + server, + operation.session, # type: ignore[arg-type] + operation.conn_mgr.conn, + ): + return run_with_conn( + operation.conn_mgr.conn, operation, operation.read_preference + ) + + def _cmd( + _session: Optional[ClientSession], + _server: Server, + conn: Connection, + read_preference: _ServerMode, + ) -> Response: + operation.reset() # Reset op in case of retry. + return run_with_conn(conn, operation, read_preference) + + return self._retryable_read( + _cmd, operation.read_preference, operation.session, # type: ignore[arg-type] - operation.name, address=address, + retryable=isinstance(operation, _Query), + operation=operation.name, ) - with operation.conn_mgr._lock: - with _ClientCheckout.for_existing_conn( - self, - server, - operation.session, # type: ignore[arg-type] - operation.conn_mgr.conn, - ): - return run_with_conn( - operation.conn_mgr.conn, operation, operation.read_preference - ) - - def _cmd( - _session: Optional[ClientSession], - _server: Server, - conn: Connection, - read_preference: _ServerMode, - ) -> Response: - operation.reset() # Reset op in case of retry. - return run_with_conn(conn, operation, read_preference) - - return self._retryable_read( - _cmd, - operation.read_preference, - operation.session, # type: ignore[arg-type] - address=address, - retryable=isinstance(operation, _Query), - operation=operation.name, - ) - def _retry_with_session( self, retryable: bool, @@ -2159,6 +2167,8 @@ def _retryable_write( operation: str, bulk: Optional[Union[_Bulk, _ClientBulk]] = None, operation_id: Optional[int] = None, + db_name: Optional[str] = None, + coll_name: Optional[str] = None, ) -> T: """Execute an operation with consecutive retries if possible @@ -2173,9 +2183,19 @@ def _retryable_write( :param operation: The name of the operation that the server is being selected for :param bulk: bulk abstraction to execute operations in bulk, defaults to None :param operation_id: Stable operation id shared across retries, defaults to None + :param db_name: The database the operation targets, for the spec's + operation span. Defaults to None (attribute omitted). + :param coll_name: The collection the operation targets, for the spec's + operation span. Defaults to None (attribute omitted). """ with self._tmp_session(session) as s: - return self._retry_with_session(retryable, func, s, bulk, operation, operation_id) + with operation_span( + self.options.tracing, + getattr(operation, "value", operation), + db_name, + coll_name, + ): + return self._retry_with_session(retryable, func, s, bulk, operation, operation_id) def _cleanup_cursor_no_lock( self, diff --git a/test/asynchronous/test_otel.py b/test/asynchronous/test_otel.py index 5a15be20f0..c2d7777252 100644 --- a/test/asynchronous/test_otel.py +++ b/test/asynchronous/test_otel.py @@ -292,7 +292,7 @@ class _FakeConn: service_id = None _otel_connection_attributes: tuple[Optional[int], dict[str, Any]] = (None, {}) - options = {"enabled": True, "query_text_max_length": None} + options: _otel.TracingOptions = {"enabled": True, "query_text_max_length": None} self.exporter.clear() # Transport failure: no server response, error.type is the exception @@ -319,6 +319,63 @@ class _FakeConn: self.assertEqual(attrs["db.response.status_code"], "26") self.assertEqual(attrs["error.type"], "26") + async def test_operation_span_wraps_command_span(self): + client = await self.async_rs_or_single_client(tracing={"enabled": True}) + coll = client[self.db.name].test_otel_operation + await coll.drop() + await coll.insert_one({"x": 1}) + self.exporter.clear() + + await coll.insert_one({"x": 2}) + + # One operation span per public-API call, named and attributed per the + # spec, with the operation's command span nested beneath it. + op_name = f"insert {self.db.name}.test_otel_operation" + op_spans = self.spans(op_name) + self.assertEqual(len(op_spans), 1) + op_span = op_spans[0] + attrs = op_span.attributes + self.assertEqual(attrs["db.system.name"], "mongodb") + self.assertEqual(attrs["db.namespace"], self.db.name) + self.assertEqual(attrs["db.collection.name"], "test_otel_operation") + self.assertEqual(attrs["db.operation.name"], "insert") + self.assertEqual(attrs["db.operation.summary"], op_name) + + cmd_spans = self.spans("insert") + self.assertEqual(len(cmd_spans), 1) + self.assertEqual(cmd_spans[0].parent.span_id, op_span.context.span_id) + + async def test_operation_failure_records_exception_class_as_error_type(self): + client = await self.async_rs_or_single_client(tracing={"enabled": True}) + coll = client[self.db.name].test_otel_operation + await coll.drop() + await coll.create_index("x", unique=True) + await coll.insert_one({"x": 1}) + self.exporter.clear() + + with self.assertRaises(OperationFailure): + await coll.insert_one({"x": 1}) + + # The operation span's error.type is the exception class raised to the + # application (the specific subclass pymongo raises); the failed + # command span keeps the server response code. + op_span = self.spans(f"insert {self.db.name}.test_otel_operation")[0] + self.assertEqual(op_span.status.status_code, trace.StatusCode.ERROR) + self.assertEqual(op_span.attributes["error.type"], "DuplicateKeyError") + # The duplicate key arrived as writeErrors in an ok:1 reply: the + # command succeeded at the protocol level, so its span carries no + # error; the failure belongs to the operation span alone. + cmd_spans = self.spans("insert") + self.assertEqual(len(cmd_spans), 1) + self.assertEqual(cmd_spans[0].status.status_code, trace.StatusCode.UNSET) + + async def test_operation_span_absent_when_tracing_disabled(self): + client = await self.async_rs_or_single_client() + coll = client[self.db.name].test_otel_operation + await coll.drop() + await coll.insert_one({"x": 1}) + self.assertEqual(self.spans(), []) + async def test_tracing_disabled_by_default(self): client = await self.async_rs_or_single_client() self.exporter.clear() diff --git a/test/test_otel.py b/test/test_otel.py index ac4a81742b..b4be281931 100644 --- a/test/test_otel.py +++ b/test/test_otel.py @@ -290,7 +290,7 @@ class _FakeConn: service_id = None _otel_connection_attributes: tuple[Optional[int], dict[str, Any]] = (None, {}) - options = {"enabled": True, "query_text_max_length": None} + options: _otel.TracingOptions = {"enabled": True, "query_text_max_length": None} self.exporter.clear() # Transport failure: no server response, error.type is the exception @@ -317,6 +317,63 @@ class _FakeConn: self.assertEqual(attrs["db.response.status_code"], "26") self.assertEqual(attrs["error.type"], "26") + def test_operation_span_wraps_command_span(self): + client = self.rs_or_single_client(tracing={"enabled": True}) + coll = client[self.db.name].test_otel_operation + coll.drop() + coll.insert_one({"x": 1}) + self.exporter.clear() + + coll.insert_one({"x": 2}) + + # One operation span per public-API call, named and attributed per the + # spec, with the operation's command span nested beneath it. + op_name = f"insert {self.db.name}.test_otel_operation" + op_spans = self.spans(op_name) + self.assertEqual(len(op_spans), 1) + op_span = op_spans[0] + attrs = op_span.attributes + self.assertEqual(attrs["db.system.name"], "mongodb") + self.assertEqual(attrs["db.namespace"], self.db.name) + self.assertEqual(attrs["db.collection.name"], "test_otel_operation") + self.assertEqual(attrs["db.operation.name"], "insert") + self.assertEqual(attrs["db.operation.summary"], op_name) + + cmd_spans = self.spans("insert") + self.assertEqual(len(cmd_spans), 1) + self.assertEqual(cmd_spans[0].parent.span_id, op_span.context.span_id) + + def test_operation_failure_records_exception_class_as_error_type(self): + client = self.rs_or_single_client(tracing={"enabled": True}) + coll = client[self.db.name].test_otel_operation + coll.drop() + coll.create_index("x", unique=True) + coll.insert_one({"x": 1}) + self.exporter.clear() + + with self.assertRaises(OperationFailure): + coll.insert_one({"x": 1}) + + # The operation span's error.type is the exception class raised to the + # application (the specific subclass pymongo raises); the failed + # command span keeps the server response code. + op_span = self.spans(f"insert {self.db.name}.test_otel_operation")[0] + self.assertEqual(op_span.status.status_code, trace.StatusCode.ERROR) + self.assertEqual(op_span.attributes["error.type"], "DuplicateKeyError") + # The duplicate key arrived as writeErrors in an ok:1 reply: the + # command succeeded at the protocol level, so its span carries no + # error; the failure belongs to the operation span alone. + cmd_spans = self.spans("insert") + self.assertEqual(len(cmd_spans), 1) + self.assertEqual(cmd_spans[0].status.status_code, trace.StatusCode.UNSET) + + def test_operation_span_absent_when_tracing_disabled(self): + client = self.rs_or_single_client() + coll = client[self.db.name].test_otel_operation + coll.drop() + coll.insert_one({"x": 1}) + self.assertEqual(self.spans(), []) + def test_tracing_disabled_by_default(self): client = self.rs_or_single_client() self.exporter.clear() diff --git a/tools/otel_bench.py b/tools/otel_bench.py index 71aca28bae..afa4666899 100755 --- a/tools/otel_bench.py +++ b/tools/otel_bench.py @@ -235,6 +235,11 @@ def verify() -> None: attrs = dict(span.attributes or {}) assert attrs.get("db.system.name") == "mongodb", attrs assert attrs.get("db.command.name") == "insert", attrs + # The operation span nests the command span beneath it. + op_span = next(s for s in spans if s.name == "insert perftest_otel_verify.coll") + assert span.parent is not None + assert span.parent.span_id == op_span.context.span_id, names + assert op_span.attributes.get("db.operation.name") == "insert", dict(op_span.attributes or {}) print(f"OK: {len(names)} span(s) recorded: {names}")