From 79067435bb6b168ffb7a7d7dcb5af696b9d60988 Mon Sep 17 00:00:00 2001 From: serversidehannes Date: Wed, 8 Jul 2026 17:08:36 +0200 Subject: [PATCH 1/5] fix: do not defer signature for presigned PUTs (#116 regression) Scylla Manager uploads SSTs via presigned URLs (X-Amz-Signature in query). PR #116 treated missing x-amz-content-sha256 as deferred header-auth, which routed presigned PUTs through prepare_header_auth and failed with 403. Co-authored-by: Cursor --- s3proxy/request_handler.py | 32 +++++++++++----- tests/unit/test_request_handler_body.py | 51 +++++++++++++++++++++---- 2 files changed, 67 insertions(+), 16 deletions(-) diff --git a/s3proxy/request_handler.py b/s3proxy/request_handler.py index d2e7577..273c80b 100644 --- a/s3proxy/request_handler.py +++ b/s3proxy/request_handler.py @@ -58,13 +58,20 @@ async def _release_after_stream(iterator, reserved: int): await concurrency.release_memory(reserved) -def _needs_body_for_signature(headers: dict[str, str]) -> bool: - """Body is needed only when x-amz-content-sha256 is absent. +def _is_presigned_request(query: dict[str, list[str]]) -> bool: + """Presigned URLs sign UNSIGNED-PAYLOAD; body is not needed for verification.""" + return "X-Amz-Signature" in query or "Signature" in query + + +def _needs_body_for_signature(headers: dict[str, str], query: dict[str, list[str]]) -> bool: + """Body is needed only when x-amz-content-sha256 is absent (header auth). The verifier uses that header as the payload hash verbatim and only rehashes - the body as a fallback when it is missing. Buffering it otherwise just pins - the whole part in memory. + the body as a fallback when it is missing. Presigned URLs always use + UNSIGNED-PAYLOAD in the canonical request, so the body is never required. """ + if _is_presigned_request(query): + return False return headers.get("x-amz-content-sha256", "") == "" @@ -75,9 +82,14 @@ def _parse_content_length(headers: dict[str, str]) -> int: return 0 -def _defer_signature_for_body(headers: dict[str, str], content_length: int) -> bool: - """Large bodies without x-amz-content-sha256 are hashed while streaming.""" - return _needs_body_for_signature(headers) and content_length > crypto.MAX_BUFFER_SIZE +def _defer_signature_for_body( + headers: dict[str, str], content_length: int, query: dict[str, list[str]] +) -> bool: + """Large header-auth bodies without x-amz-content-sha256 are hashed while streaming.""" + return ( + _needs_body_for_signature(headers, query) + and content_length > crypto.MAX_BUFFER_SIZE + ) def _signature_path(request: Request) -> str: @@ -222,10 +234,12 @@ async def _handle_proxy_request_impl( content_length = _parse_content_length(headers) defer_sig = request.method in ("PUT", "POST") and _defer_signature_for_body( - headers, content_length + headers, content_length, query ) - needs_body = request.method in ("PUT", "POST") and _needs_body_for_signature(headers) + needs_body = request.method in ("PUT", "POST") and _needs_body_for_signature( + headers, query + ) body = b"" if needs_body and not defer_sig: body = await request.body() diff --git a/tests/unit/test_request_handler_body.py b/tests/unit/test_request_handler_body.py index 3853eb3..89167f0 100644 --- a/tests/unit/test_request_handler_body.py +++ b/tests/unit/test_request_handler_body.py @@ -17,19 +17,26 @@ class TestDeferSignatureForBody: def test_defers_large_put_without_payload_hash(self): - assert _defer_signature_for_body({}, 179 * MB) is True + assert _defer_signature_for_body({}, 179 * MB, {}) is True def test_no_defer_with_unsigned_payload(self): assert ( - _defer_signature_for_body({"x-amz-content-sha256": "UNSIGNED-PAYLOAD"}, 179 * MB) + _defer_signature_for_body({"x-amz-content-sha256": "UNSIGNED-PAYLOAD"}, 179 * MB, {}) is False ) def test_no_defer_with_explicit_hash(self): - assert _defer_signature_for_body({"x-amz-content-sha256": "abc"}, 179 * MB) is False + assert _defer_signature_for_body({"x-amz-content-sha256": "abc"}, 179 * MB, {}) is False def test_no_defer_small_put_without_header(self): - assert _defer_signature_for_body({}, 4 * MB) is False + assert _defer_signature_for_body({}, 4 * MB, {}) is False + + def test_no_defer_presigned_large_put(self): + query = { + "X-Amz-Algorithm": ["AWS4-HMAC-SHA256"], + "X-Amz-Signature": ["abc123"], + } + assert _defer_signature_for_body({}, 179 * MB, query) is False def _make_request( @@ -37,17 +44,31 @@ def _make_request( method: str = "PUT", content_length: int, content_sha: str | None = None, + query: str = "", + presigned: bool = False, ) -> MagicMock: request = MagicMock(spec=Request) request.method = method request.url = MagicMock() request.url.path = "/bucket/scylla-backup/sst/big-Data.db" - request.url.query = "" + if presigned: + request.url.query = ( + "X-Amz-Algorithm=AWS4-HMAC-SHA256" + "&X-Amz-Credential=key/20260708/us-east-1/s3/aws4_request" + "&X-Amz-Date=20260708T150228Z" + "&X-Amz-Expires=900" + "&X-Amz-SignedHeaders=content-md5%3Bcontent-type%3Bhost" + "&X-Amz-Signature=deadbeef" + ) + else: + request.url.query = query headers: dict[str, str] = { "content-length": str(content_length), - "authorization": "AWS4-HMAC-SHA256 Credential=x, SignedHeaders=host, Signature=sig", - "x-amz-date": "20260708T080000Z", + "content-type": "application/octet-stream", } + if not presigned: + headers["authorization"] = "AWS4-HMAC-SHA256 Credential=x, SignedHeaders=host, Signature=sig" + headers["x-amz-date"] = "20260708T080000Z" if content_sha is not None: headers["x-amz-content-sha256"] = content_sha request.headers = headers @@ -74,6 +95,22 @@ async def test_large_put_without_header_defers_signature(): verifier.verify.assert_not_called() +@pytest.mark.asyncio +async def test_large_presigned_put_verifies_without_body_preload(): + """Scylla Manager uses presigned PUTs; must not enter deferred header-auth path.""" + request = _make_request(content_length=179 * MB, presigned=True) + verifier = MagicMock() + verifier.verify = MagicMock(return_value=(True, MagicMock(), "")) + + with patch("s3proxy.request_handler.RequestDispatcher") as dispatcher_cls: + dispatcher_cls.return_value.dispatch = AsyncMock(return_value=None) + await _handle_proxy_request_impl(request, MagicMock(), verifier) + + request.body.assert_not_awaited() + verifier.verify.assert_called_once() + verifier.prepare_header_auth.assert_not_called() + + @pytest.mark.asyncio async def test_large_put_unsigned_payload_verifies_immediately(): request = _make_request(content_length=179 * MB, content_sha="UNSIGNED-PAYLOAD") From 3c25d247953ebe493099f1feb8139e76946d6d38 Mon Sep 17 00:00:00 2001 From: serversidehannes Date: Wed, 8 Jul 2026 17:11:42 +0200 Subject: [PATCH 2/5] style: fix ruff E501 in presigned PUT test helper Co-authored-by: Cursor --- tests/unit/test_request_handler_body.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_request_handler_body.py b/tests/unit/test_request_handler_body.py index 89167f0..0c045fe 100644 --- a/tests/unit/test_request_handler_body.py +++ b/tests/unit/test_request_handler_body.py @@ -67,7 +67,9 @@ def _make_request( "content-type": "application/octet-stream", } if not presigned: - headers["authorization"] = "AWS4-HMAC-SHA256 Credential=x, SignedHeaders=host, Signature=sig" + headers["authorization"] = ( + "AWS4-HMAC-SHA256 Credential=x, SignedHeaders=host, Signature=sig" + ) headers["x-amz-date"] = "20260708T080000Z" if content_sha is not None: headers["x-amz-content-sha256"] = content_sha From fe37de3c266783360e722bb49fbefac5da9e5bd4 Mon Sep 17 00:00:00 2001 From: serversidehannes Date: Wed, 8 Jul 2026 17:14:52 +0200 Subject: [PATCH 3/5] fix: update routing tests and format request_handler for CI Pass query dict to _needs_body_for_signature in test_routing.py and add presigned coverage. Ruff-format request_handler.py for format check. Co-authored-by: Cursor --- s3proxy/request_handler.py | 9 ++------- tests/unit/test_routing.py | 14 +++++++++----- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/s3proxy/request_handler.py b/s3proxy/request_handler.py index 273c80b..fe17de8 100644 --- a/s3proxy/request_handler.py +++ b/s3proxy/request_handler.py @@ -86,10 +86,7 @@ def _defer_signature_for_body( headers: dict[str, str], content_length: int, query: dict[str, list[str]] ) -> bool: """Large header-auth bodies without x-amz-content-sha256 are hashed while streaming.""" - return ( - _needs_body_for_signature(headers, query) - and content_length > crypto.MAX_BUFFER_SIZE - ) + return _needs_body_for_signature(headers, query) and content_length > crypto.MAX_BUFFER_SIZE def _signature_path(request: Request) -> str: @@ -237,9 +234,7 @@ async def _handle_proxy_request_impl( headers, content_length, query ) - needs_body = request.method in ("PUT", "POST") and _needs_body_for_signature( - headers, query - ) + needs_body = request.method in ("PUT", "POST") and _needs_body_for_signature(headers, query) body = b"" if needs_body and not defer_sig: body = await request.body() diff --git a/tests/unit/test_routing.py b/tests/unit/test_routing.py index 7dc985f..f5b9be5 100644 --- a/tests/unit/test_routing.py +++ b/tests/unit/test_routing.py @@ -45,19 +45,23 @@ class TestNeedsBodyForSignature: """Body is buffered for signature only when x-amz-content-sha256 is absent.""" def test_unsigned_payload(self): - assert _needs_body_for_signature({"x-amz-content-sha256": "UNSIGNED-PAYLOAD"}) is False + assert _needs_body_for_signature({"x-amz-content-sha256": "UNSIGNED-PAYLOAD"}, {}) is False def test_streaming_payload(self): headers = {"x-amz-content-sha256": "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"} - assert _needs_body_for_signature(headers) is False + assert _needs_body_for_signature(headers, {}) is False def test_signed_payload_skips_body_regardless_of_size(self): headers = {"x-amz-content-sha256": "abc123def456", "content-length": str(16 * 1024 * 1024)} - assert _needs_body_for_signature(headers) is False + assert _needs_body_for_signature(headers, {}) is False def test_missing_header_needs_body(self): - assert _needs_body_for_signature({}) is True - assert _needs_body_for_signature({"x-amz-content-sha256": ""}) is True + assert _needs_body_for_signature({}, {}) is True + assert _needs_body_for_signature({"x-amz-content-sha256": ""}, {}) is True + + def test_presigned_skips_body(self): + query = {"X-Amz-Signature": ["abc123"]} + assert _needs_body_for_signature({}, query) is False class TestQueryConstants: From a929eed20fe927a4184bb5229b0e11390b61ada4 Mon Sep 17 00:00:00 2001 From: serversidehannes Date: Wed, 8 Jul 2026 17:33:00 +0200 Subject: [PATCH 4/5] test: E2E presigned PUT for SigV4 and legacy V2 Full-stack integration tests via generate_presigned_url (s3v4, Scylla shape) and hand-crafted V2 URLs. Covers small/large PUT roundtrips and bad-signature 403 for both query-string auth styles. Co-authored-by: Cursor --- Makefile | 1 + tests/integration/test_presigned_put_e2e.py | 294 ++++++++++++++++++++ 2 files changed, 295 insertions(+) create mode 100644 tests/integration/test_presigned_put_e2e.py diff --git a/Makefile b/Makefile index 089b688..5718daa 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,7 @@ INTEGRATION_core_TESTS = \ tests/integration/test_handlers.py \ tests/integration/test_concurrent_operations.py \ tests/integration/test_per_key_encryption.py \ + tests/integration/test_presigned_put_e2e.py \ tests/integration/test_redis_coordination.py INTEGRATION_multipart_TESTS = \ tests/integration/test_upload_part_copy.py \ diff --git a/tests/integration/test_presigned_put_e2e.py b/tests/integration/test_presigned_put_e2e.py new file mode 100644 index 0000000..8635233 --- /dev/null +++ b/tests/integration/test_presigned_put_e2e.py @@ -0,0 +1,294 @@ +"""E2E: presigned PUT through real s3proxy + MinIO. + +Covers both presigned URL styles the proxy recognizes: +- SigV4 (X-Amz-Signature): Scylla Manager / boto3 with signature_version=s3v4 +- Legacy V2 (Signature=): simple query-only presigns matching our verifier + +PR #116 wrongly routed large presigned PUTs into deferred header-auth → 403. +""" + +from __future__ import annotations + +import base64 +import contextlib +import hashlib +import hmac +import http.client +import socket +import uuid +from datetime import UTC, datetime +from urllib.parse import quote, urlparse + +import boto3 +import pytest +from botocore.config import Config + +from tests.integration.conftest import minio_backend, run_s3proxy + +MB = 1024 * 1024 +SCYLLA_SST_MB = 179 +ACCESS_KEY = "minioadmin" +SECRET_KEY = "minioadmin" + +pytestmark = pytest.mark.xdist_group("presigned_put") + + +def _find_free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + return s.getsockname()[1] + + +class _ZeroBody: + """Stream size bytes of zeros without allocating the full buffer.""" + + def __init__(self, size: int, chunk: int = MB): + self._size = size + self._chunk = chunk + self._pos = 0 + + def read(self, amt: int = -1) -> bytes: + if self._pos >= self._size: + return b"" + if amt is None or amt < 0: + amt = self._chunk + take = min(amt, self._chunk, self._size - self._pos) + self._pos += take + return b"\x00" * take + + def __len__(self) -> int: + return self._size + + +def _s3_client_v4(endpoint: str): + return boto3.client( + "s3", + endpoint_url=endpoint, + aws_access_key_id=ACCESS_KEY, + aws_secret_access_key=SECRET_KEY, + region_name="us-east-1", + config=Config(signature_version="s3v4"), + ) + + +def _md5_zeros(size: int, chunk: int = MB) -> str: + h = hashlib.md5() + block = b"\x00" * chunk + full, rem = divmod(size, chunk) + for _ in range(full): + h.update(block) + if rem: + h.update(block[:rem]) + return base64.b64encode(h.digest()).decode() + + +def _presigned_url_v4( + endpoint: str, + *, + bucket: str, + key: str, + content_md5: str, + mtime: str = "1700000000", +) -> str: + """boto3 SigV4 presign — matches Scylla Manager (X-Amz-Signature in query).""" + client = _s3_client_v4(endpoint) + return client.generate_presigned_url( + "put_object", + Params={ + "Bucket": bucket, + "Key": key, + "ContentType": "application/octet-stream", + "ContentMD5": content_md5, + "ACL": "private", + "Metadata": {"mtime": mtime}, + }, + ExpiresIn=900, + HttpMethod="PUT", + ) + + +def _presigned_url_v2(endpoint: str, *, bucket: str, key: str) -> str: + """Legacy V2 presign (AWSAccessKeyId + Expires + Signature only). + + boto3's default V2 presign adds content-type/content-md5 to the query string, + which our simplified V2 verifier does not cover. Hand-craft the URL the + verifier expects: PUT\\n\\n\\n{expires}\\n{path} + """ + expires = str(int(datetime.now(UTC).timestamp()) + 900) + path = f"/{bucket}/{key}" + string_to_sign = f"PUT\n\n\n{expires}\n{path}" + signature = base64.b64encode( + hmac.new(SECRET_KEY.encode(), string_to_sign.encode(), hashlib.sha1).digest() + ).decode() + return ( + f"{endpoint}{path}" + f"?AWSAccessKeyId={quote(ACCESS_KEY)}" + f"&Expires={expires}" + f"&Signature={quote(signature)}" + ) + + +def _http_put_presigned( + url: str, + *, + size: int, + content_md5: str | None = None, + mtime: str | None = None, +) -> None: + parsed = urlparse(url) + path_query = parsed.path + if parsed.query: + path_query = f"{path_query}?{parsed.query}" + headers: dict[str, str] = { + "Content-Type": "application/octet-stream", + "Content-Length": str(size), + } + if content_md5 is not None: + headers["Content-MD5"] = content_md5 + if mtime is not None: + headers["x-amz-acl"] = "private" + headers["x-amz-meta-mtime"] = mtime + conn = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=600) + conn.request("PUT", path_query, body=_ZeroBody(size), headers=headers) + resp = conn.getresponse() + if resp.status not in (200, 201): + body = resp.read() + raise AssertionError(f"presigned PUT failed: {resp.status} {body!r}") + resp.read() + + +@pytest.fixture(scope="module") +def presigned_server(): + port = _find_free_port() + with ( + minio_backend() as minio_host, + run_s3proxy(port, log_output=False, S3PROXY_HOST=minio_host) as (endpoint, proc), + ): + yield endpoint, proc + + +@pytest.fixture +def presigned_bucket(presigned_server): + endpoint, _ = presigned_server + client = _s3_client_v4(endpoint) + bucket = f"presigned-e2e-{uuid.uuid4().hex[:8]}" + with contextlib.suppress(client.exceptions.BucketAlreadyOwnedByYou): + client.create_bucket(Bucket=bucket) + yield endpoint, bucket, client + try: + resp = client.list_objects_v2(Bucket=bucket) + if contents := resp.get("Contents"): + client.delete_objects( + Bucket=bucket, + Delete={"Objects": [{"Key": o["Key"]} for o in contents]}, + ) + client.delete_bucket(Bucket=bucket) + except Exception: + pass + + +@pytest.mark.e2e +class TestPresignedPutV4E2E: + """SigV4 presigned PUT (Scylla Manager shape).""" + + def test_small_roundtrip(self, presigned_bucket, presigned_server): + endpoint, bucket, client = presigned_bucket + _, proc = presigned_server + key = "backup/sst/small-Data.db" + size = 1 * MB + content_md5 = _md5_zeros(size) + url = _presigned_url_v4(endpoint, bucket=bucket, key=key, content_md5=content_md5) + _http_put_presigned(url, size=size, content_md5=content_md5, mtime="1700000000") + assert proc.poll() is None + assert client.head_object(Bucket=bucket, Key=key)["ContentLength"] == size + + def test_large_scylla_sst_shape(self, presigned_bucket, presigned_server): + endpoint, bucket, client = presigned_bucket + _, proc = presigned_server + key = ( + "backup/sst/cluster/test/dc/eu-north-1/node/test/" + "keyspace/ai_center/table/job_titles/uuid/me-big-Data.db" + ) + size = SCYLLA_SST_MB * MB + content_md5 = _md5_zeros(size) + url = _presigned_url_v4(endpoint, bucket=bucket, key=key, content_md5=content_md5) + _http_put_presigned(url, size=size, content_md5=content_md5, mtime="1700000000") + assert proc.poll() is None + assert client.head_object(Bucket=bucket, Key=key)["ContentLength"] == size + + def test_bad_signature_rejected(self, presigned_bucket): + endpoint, bucket, client = presigned_bucket + key = "backup/sst/bad-sig-v4-Data.db" + size = 10 * MB + content_md5 = _md5_zeros(size) + url = _presigned_url_v4(endpoint, bucket=bucket, key=key, content_md5=content_md5) + url = url.replace("X-Amz-Signature=", "X-Amz-Signature=deadbeef", 1) + parsed = urlparse(url) + path_query = parsed.path + if parsed.query: + path_query = f"{path_query}?{parsed.query}" + conn = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=120) + conn.request( + "PUT", + path_query, + body=_ZeroBody(size), + headers={ + "Content-Type": "application/octet-stream", + "Content-MD5": content_md5, + "Content-Length": str(size), + }, + ) + resp = conn.getresponse() + assert resp.status == 403, resp.read() + with pytest.raises(client.exceptions.ClientError): + client.head_object(Bucket=bucket, Key=key) + + +@pytest.mark.e2e +class TestPresignedPutV2E2E: + """Legacy V2 presigned PUT (Signature= in query).""" + + def test_small_roundtrip(self, presigned_bucket, presigned_server): + endpoint, bucket, client = presigned_bucket + _, proc = presigned_server + key = "legacy/small.bin" + size = 1 * MB + url = _presigned_url_v2(endpoint, bucket=bucket, key=key) + _http_put_presigned(url, size=size) + assert proc.poll() is None + assert client.head_object(Bucket=bucket, Key=key)["ContentLength"] == size + + def test_large_put(self, presigned_bucket, presigned_server): + endpoint, bucket, client = presigned_bucket + _, proc = presigned_server + key = "legacy/large.bin" + size = SCYLLA_SST_MB * MB + url = _presigned_url_v2(endpoint, bucket=bucket, key=key) + _http_put_presigned(url, size=size) + assert proc.poll() is None + assert client.head_object(Bucket=bucket, Key=key)["ContentLength"] == size + + def test_bad_signature_rejected(self, presigned_bucket): + endpoint, bucket, client = presigned_bucket + key = "legacy/bad-sig.bin" + size = 10 * MB + url = _presigned_url_v2(endpoint, bucket=bucket, key=key) + url = url.replace("Signature=", "Signature=deadbeef", 1) + parsed = urlparse(url) + path_query = parsed.path + if parsed.query: + path_query = f"{path_query}?{parsed.query}" + conn = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=120) + conn.request( + "PUT", + path_query, + body=_ZeroBody(size), + headers={ + "Content-Type": "application/octet-stream", + "Content-Length": str(size), + }, + ) + resp = conn.getresponse() + assert resp.status == 403, resp.read() + with pytest.raises(client.exceptions.ClientError): + client.head_object(Bucket=bucket, Key=key) From 86935cf06ea3317ed4c2be66da42ddd11d51def0 Mon Sep 17 00:00:00 2001 From: serversidehannes Date: Wed, 8 Jul 2026 17:59:51 +0200 Subject: [PATCH 5/5] test: fix memory_copy CI MinIO disk exhaustion Heavy copy e2e tests now use isolated throwaway MinIO containers instead of filling the shared compose instance. Concurrent passthrough flood uses the 96MB source (pipeline/memory test, not 1280MB scale). Give compose MinIO a 4GB tmpfs. Co-authored-by: Cursor --- tests/docker-compose.yml | 2 ++ tests/integration/conftest.py | 19 ++++++++++++------- tests/integration/passthrough_verify.py | 2 +- .../integration/test_copy_per_part_metrics.py | 2 +- tests/integration/test_presigned_put_e2e.py | 2 +- .../test_upload_part_copy_passthrough_e2e.py | 3 ++- 6 files changed, 19 insertions(+), 11 deletions(-) diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index f39f02a..7412f8c 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -15,6 +15,8 @@ services: MINIO_ROOT_USER: minioadmin MINIO_ROOT_PASSWORD: minioadmin command: server /data --console-address ":9001" + tmpfs: + - /data:size=4g s3proxy: profiles: ["oom"] diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 6bf0d87..a51fadc 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -42,13 +42,18 @@ def _find_free_port() -> int: @contextlib.contextmanager -def minio_backend() -> Generator[str]: - """Yield a MinIO HTTP endpoint, starting a throwaway container if needed.""" - for port in (9000, 19000, 19001): - endpoint = f"http://localhost:{port}" - if _is_minio(endpoint): - yield endpoint - return +def minio_backend(*, isolated: bool = False) -> Generator[str]: + """Yield a MinIO HTTP endpoint, starting a throwaway container if needed. + + When *isolated* is True, always start a dedicated container (and remove it on + exit). Use for heavy e2e shards so tests do not fill the shared compose MinIO. + """ + if not isolated: + for port in (9000, 19000, 19001): + endpoint = f"http://localhost:{port}" + if _is_minio(endpoint): + yield endpoint + return port = 19000 while port < 19100: diff --git a/tests/integration/passthrough_verify.py b/tests/integration/passthrough_verify.py index 36d4ecc..8780d94 100644 --- a/tests/integration/passthrough_verify.py +++ b/tests/integration/passthrough_verify.py @@ -225,7 +225,7 @@ def verify_session( log_path = log_path or Path(tempfile.gettempdir()) / f"s3proxy-passthrough-verify-{port}.log" bucket = f"verify-{uuid.uuid4().hex[:8]}" - with minio_backend() as minio_host, open(log_path, "w") as log_file: + with minio_backend(isolated=True) as minio_host, open(log_path, "w") as log_file: env = os.environ.copy() env.update( { diff --git a/tests/integration/test_copy_per_part_metrics.py b/tests/integration/test_copy_per_part_metrics.py index 122d8f6..f9c4dbe 100644 --- a/tests/integration/test_copy_per_part_metrics.py +++ b/tests/integration/test_copy_per_part_metrics.py @@ -46,7 +46,7 @@ def _find_free_port() -> int: def metrics_server(): port = _find_free_port() with ( - minio_backend() as minio_host, + minio_backend(isolated=True) as minio_host, run_s3proxy( port, log_output=False, diff --git a/tests/integration/test_presigned_put_e2e.py b/tests/integration/test_presigned_put_e2e.py index 8635233..2988ce6 100644 --- a/tests/integration/test_presigned_put_e2e.py +++ b/tests/integration/test_presigned_put_e2e.py @@ -161,7 +161,7 @@ def _http_put_presigned( def presigned_server(): port = _find_free_port() with ( - minio_backend() as minio_host, + minio_backend(isolated=True) as minio_host, run_s3proxy(port, log_output=False, S3PROXY_HOST=minio_host) as (endpoint, proc), ): yield endpoint, proc diff --git a/tests/integration/test_upload_part_copy_passthrough_e2e.py b/tests/integration/test_upload_part_copy_passthrough_e2e.py index 15fc448..161ef7d 100644 --- a/tests/integration/test_upload_part_copy_passthrough_e2e.py +++ b/tests/integration/test_upload_part_copy_passthrough_e2e.py @@ -103,7 +103,8 @@ def test_scylla_scale_passthrough(self, passthrough_env): def test_ten_concurrent_passthrough_copies(self, passthrough_env): ctx, proc, _ = passthrough_env ctx.failures.clear() - check_concurrent_flood(ctx, "sst/source-1280mb.bin", n=10) + # 96MB source: flood tests pipeline/memory, not 1280MB scale (saves ~12GB on CI). + check_concurrent_flood(ctx, "sst/source-96mb.bin", n=10) assert proc.poll() is None _assert_check(ctx, "all copies succeeded") _assert_check(ctx, "peak memory under prod budget")