From cf44c1dcddf2ac3a98443225460bac30d622446a Mon Sep 17 00:00:00 2001 From: Sophie <49928680+sophiecarreras@users.noreply.github.com> Date: Mon, 19 Oct 2026 23:36:24 -0600 Subject: [PATCH 1/2] Add nineteen sdkharness conformance checks: large files, copy, URLs, locks, encryption, keys, bucket configuration Slices 4 and 5 of the Java conformance plan, completing one check for each of the 27 capabilities this SDK claims: files.server_side_copy, large.multipart, large.concurrent_parts, large.resume, large.compose_mixed, urls.native_download, client.progress, lock.bucket_default, lock.per_file_retention, lock.legal_hold, lock.bypass_governance, enc.sse_b2, enc.sse_c, keys.crud, bucket.cors, bucket.lifecycle, bucket.notification_rules, bucket.inbound_webhook, bucket.replication_config. Support.java gains an executor helper, an in-flight request counter on the recording client, Object Lock and key cleanup, a second client for a key the check created, and a loopback-only raw GET for share links (covered by GuardSelfTest). The README table lists the new scenarios. Both run-conformance and run-resilience now pass -encoding UTF-8 to javac, so a check source with a non-ASCII literal compiles under LC_ALL=C on a JDK before 18 (the harness's daily run exports LC_ALL=C). --- .sdkharness/README.md | 32 ++- .sdkharness/tests.tsv | 19 ++ .sdkharness/tests/conformance/BucketCors.java | 82 +++++++ .../conformance/BucketInboundWebhook.java | 105 +++++++++ .../tests/conformance/BucketLifecycle.java | 74 +++++++ .../conformance/BucketNotificationRules.java | 67 ++++++ .../conformance/BucketReplicationConfig.java | 75 +++++++ .../tests/conformance/ClientProgress.java | 94 ++++++++ .sdkharness/tests/conformance/EncSseB2.java | 67 ++++++ .sdkharness/tests/conformance/EncSseC.java | 70 ++++++ .../conformance/FilesServerSideCopy.java | 72 +++++++ .../tests/conformance/GuardSelfTest.java | 27 ++- .sdkharness/tests/conformance/KeysCrud.java | 96 +++++++++ .../tests/conformance/LargeComposeMixed.java | 76 +++++++ .../conformance/LargeConcurrentParts.java | 132 ++++++++++++ .../tests/conformance/LargeMultipart.java | 63 ++++++ .../tests/conformance/LargeResume.java | 201 ++++++++++++++++++ .../tests/conformance/LockBucketDefault.java | 59 +++++ .../conformance/LockBypassGovernance.java | 55 +++++ .../tests/conformance/LockLegalHold.java | 56 +++++ .../conformance/LockPerFileRetention.java | 59 +++++ .sdkharness/tests/conformance/Support.java | 201 +++++++++++++++++- .../tests/conformance/UrlsNativeDownload.java | 71 +++++++ .sdkharness/tests/run-conformance | 21 +- .sdkharness/tests/run-resilience | 2 +- 25 files changed, 1870 insertions(+), 6 deletions(-) create mode 100644 .sdkharness/tests/conformance/BucketCors.java create mode 100644 .sdkharness/tests/conformance/BucketInboundWebhook.java create mode 100644 .sdkharness/tests/conformance/BucketLifecycle.java create mode 100644 .sdkharness/tests/conformance/BucketNotificationRules.java create mode 100644 .sdkharness/tests/conformance/BucketReplicationConfig.java create mode 100644 .sdkharness/tests/conformance/ClientProgress.java create mode 100644 .sdkharness/tests/conformance/EncSseB2.java create mode 100644 .sdkharness/tests/conformance/EncSseC.java create mode 100644 .sdkharness/tests/conformance/FilesServerSideCopy.java create mode 100644 .sdkharness/tests/conformance/KeysCrud.java create mode 100644 .sdkharness/tests/conformance/LargeComposeMixed.java create mode 100644 .sdkharness/tests/conformance/LargeConcurrentParts.java create mode 100644 .sdkharness/tests/conformance/LargeMultipart.java create mode 100644 .sdkharness/tests/conformance/LargeResume.java create mode 100644 .sdkharness/tests/conformance/LockBucketDefault.java create mode 100644 .sdkharness/tests/conformance/LockBypassGovernance.java create mode 100644 .sdkharness/tests/conformance/LockLegalHold.java create mode 100644 .sdkharness/tests/conformance/LockPerFileRetention.java create mode 100644 .sdkharness/tests/conformance/UrlsNativeDownload.java diff --git a/.sdkharness/README.md b/.sdkharness/README.md index fe6b78d92..16a9a08bc 100644 --- a/.sdkharness/README.md +++ b/.sdkharness/README.md @@ -5,7 +5,7 @@ assertions in `tests/`; the harness builds this checkout, starts a fresh local B2 simulator, and retains and reports the evidence. -Scenarios so far: `health/golden-path` and eight `conformance` and sixteen `resilience` scenarios (below). `tests/run-health` runs +Scenarios so far: `health/golden-path` and 27 `conformance` (one per capability the SDK claims) and sixteen `resilience` scenarios (below). `tests/run-health` runs `tests/health/GoldenPath.java` as a single-file Java program that authorizes, uploads a small file, downloads it and compares the bytes, lists it, deletes it and confirms it is gone, then prints one result line: @@ -35,6 +35,36 @@ deletion, and a deletion that fails turns the verdict into a `FAIL` (`cleanup -- | `files.hide` | hide marker is a new version; not in the default listing, visible in the version listing; 404 by name; the original still downloads by id | | `bucket.crud` | create, duplicate name refused, list, update type and info (revision grows), delete refused while non-empty, delete | | `files.metadata` | by id (`b2_get_file_info`) and by name (an HTTP HEAD of the download URL: the cell is `partial`); the check records the SDK's own requests and asserts that boundary | +| `files.server_side_copy` | `b2_copy_file` sends no bytes (no `b2_upload_file`), same sha1 under a new fileId, the COPY and REPLACE metadata directives, a ranged copy | +| `large.multipart` | `uploadLargeFile` of three parts' worth: one start, more than one `b2_upload_part`, one finish, contentSha1 `none`, `large_file_sha1` carried, bytes intact | +| `large.concurrent_parts` | the same eight-part file on a one-thread and a four-thread executor: the SDK's own HTTP client counts `b2_upload_part` requests in flight (1 with one worker, more than 1 with four; wall time is only a `NOTE`) | +| `large.resume` | a first upload interrupted after two parts, then `finishUploadingLargeFile` on the unfinished file: no new `b2_start_large_file`, only the missing parts, same fileId, bytes intact; run with and without a known sha1 (see the note below) | +| `large.compose_mixed` | one large file from local bytes, a copied byte range and a whole copied file (`storeLargeFile` with `B2UploadingPartStorer` and `B2CopyingPartStorer`), equal to their ordered concatenation | +| `urls.native_download` | `getDownloadByNameUrl` is a native `/file//` URL (no `X-Amz-*`); 401 without a token, 200 with a download authorization, 401 outside its prefix and after it expires | +| `client.progress` | a `B2UploadListener` on a large upload (one worker): it fires, every part's bytes and the running total never go down, every part ends SUCCEEDED, the total equals the byte count | +| `lock.bucket_default` | a default retention of governance, 1 day on a lock-enabled bucket: read back, and an uploaded file inherits a retention of about now + 1 day | +| `lock.per_file_retention` | governance retention now + 2 days on a file: read back, and a delete without bypass is refused (401 `access_denied`) and leaves the file | +| `lock.legal_hold` | hold on: read back and the delete is refused; hold off: read back and the delete succeeds | +| `lock.bypass_governance` | under governance retention a delete is refused (401 `access_denied`), `setBypassGovernance(true)` deletes the version | +| `enc.sse_b2` | an SSE-B2 upload reports SSE-B2 / AES256 and downloads as plaintext; a bucket default of SSE-B2 applies to a plain upload | +| `enc.sse_c` | an SSE-C upload reports SSE-C / AES256; download with the key returns the plaintext and the key's MD5; without it and with another key it is refused (403 for a wrong key) | +| `keys.crud` | a bucket-restricted key is created (secret returned), listed (no secret anywhere in what the SDK returns), used by a second client, deleted (gone, and its secret no longer authorizes) | +| `bucket.cors` | typed `B2CorsRule`: set at create, replaced by update, cleared; read back field for field | +| `bucket.lifecycle` | `daysFromHidingToDeleting` 1 round trips with no other field populated; a replacement list replaces; an empty list clears | +| `bucket.notification_rules` | a webhook rule with a signing secret is set, read back (the secret unchanged) and deleted | +| `bucket.inbound_webhook` | offline: a payload signed in the check with the documented HMAC is accepted and parsed by `B2EventNotification.parse`; a wrong secret, a changed payload and a malformed signature are refused; a test event parses | +| `bucket.replication_config` | a source rule (destination bucket, prefix, priority, flags) with a replication key is set and read back field for field | + +`large.resume` records one SDK behavior on a `NOTE ... FINDING` line. When the content source knows its sha1, the +SDK stores it as the file info `large_file_sha1`; `finishUploadingLargeFile` then compares the request's file info +with the stored one after removing the key `X-Bz-Info-large_file_sha1` (`B2Headers.LARGE_FILE_SHA1`, +`B2LargeFileUploader.throwIfLargeFileVersionDoesntSeemToMatchRequest`), but the stored key is +`large_file_sha1` (`B2Headers.LARGE_FILE_SHA1_INFO_NAME`), so the same request that started the file is refused with a +`mismatch` error. The check then resumes with `large_file_sha1` added to the request's custom fields, which the SDK +accepts, and asserts the rest of the scenario; if the SDK is fixed the first attempt succeeds and the note disappears. + +Every check that locks files releases them on exit (legal hold off, governance retention bypassed); none uses +compliance mode, which nothing could lift. A failing scenario prints one `SDKHARNESS_RESULT ... FAIL` line with the step and the reason. Details are formatted to one capped printable line from exception class names, status codes and the check's own diff --git a/.sdkharness/tests.tsv b/.sdkharness/tests.tsv index 5dc227f1b..87e929c20 100644 --- a/.sdkharness/tests.tsv +++ b/.sdkharness/tests.tsv @@ -8,6 +8,25 @@ conformance files.delete_version simulator ./.sdkharness/tests/run-conformance conformance files.hide simulator ./.sdkharness/tests/run-conformance conformance bucket.crud simulator ./.sdkharness/tests/run-conformance conformance files.metadata simulator ./.sdkharness/tests/run-conformance +conformance files.server_side_copy simulator ./.sdkharness/tests/run-conformance +conformance large.multipart simulator ./.sdkharness/tests/run-conformance +conformance large.concurrent_parts simulator ./.sdkharness/tests/run-conformance +conformance large.resume simulator ./.sdkharness/tests/run-conformance +conformance large.compose_mixed simulator ./.sdkharness/tests/run-conformance +conformance urls.native_download simulator ./.sdkharness/tests/run-conformance +conformance client.progress simulator ./.sdkharness/tests/run-conformance +conformance lock.bucket_default simulator ./.sdkharness/tests/run-conformance +conformance lock.per_file_retention simulator ./.sdkharness/tests/run-conformance +conformance lock.legal_hold simulator ./.sdkharness/tests/run-conformance +conformance lock.bypass_governance simulator ./.sdkharness/tests/run-conformance +conformance enc.sse_b2 simulator ./.sdkharness/tests/run-conformance +conformance enc.sse_c simulator ./.sdkharness/tests/run-conformance +conformance keys.crud simulator ./.sdkharness/tests/run-conformance +conformance bucket.cors simulator ./.sdkharness/tests/run-conformance +conformance bucket.lifecycle simulator ./.sdkharness/tests/run-conformance +conformance bucket.notification_rules simulator ./.sdkharness/tests/run-conformance +conformance bucket.inbound_webhook simulator ./.sdkharness/tests/run-conformance +conformance bucket.replication_config simulator ./.sdkharness/tests/run-conformance resilience upload.retry_503 simulator ./.sdkharness/tests/run-resilience resilience upload.retry_500 simulator ./.sdkharness/tests/run-resilience resilience upload.retry_408 simulator ./.sdkharness/tests/run-resilience diff --git a/.sdkharness/tests/conformance/BucketCors.java b/.sdkharness/tests/conformance/BucketCors.java new file mode 100644 index 000000000..ad262a1f9 --- /dev/null +++ b/.sdkharness/tests/conformance/BucketCors.java @@ -0,0 +1,82 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2CorsRule; +import com.backblaze.b2.client.structures.B2UpdateBucketRequest; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +/** + * bucket.cors. Run by ../run-conformance (see ../../README.md). + * + *

Every field sent is a documented rule field (corsRuleName, allowedOrigins, allowedOperations, allowedHeaders, + * exposeHeaders, maxAgeSeconds), and b2_download_file_by_name and b2_download_file_by_id are documented + * allowedOperations values. Rules can be "set when you create the bucket with b2_create_bucket, or updated on an + * existing bucket using b2_update_bucket", and b2_list_buckets returns corsRules + * (https://www.backblaze.com/docs/cloud-storage-cross-origin-resource-sharing-rules, + * https://www.backblaze.com/apidocs/b2-update-bucket, https://www.backblaze.com/apidocs/b2-list-buckets). + * + *

This SDK models a rule as the typed {@code B2CorsRule} (the card's cell is "yes", not "partial"). Asserted: a + * bucket created with one rule reports it, field for field, when listed; updating the bucket to a different pair of + * rules replaces the set (the read-back is exactly the new pair); updating to an empty list removes every rule. + */ +public class BucketCors { + public static void main(String[] args) { + Support.run("bucket.cors", () -> { + final B2CorsRule first = B2CorsRule.builder("sdkharness-first", Arrays.asList("https://example.com", "https://app.example.com"), + new HashSet<>(Arrays.asList("b2_download_file_by_name", "b2_download_file_by_id")), 3600) + .setAllowedHeaders(Arrays.asList("range", "authorization")) + .setExposeHeaders(Arrays.asList("x-bz-content-sha1")) + .build(); + + Support.step("create with a rule"); + final B2Bucket bucket = Support.bucket(false, b -> b.setCorsRules(Collections.singletonList(first))); + Support.step("read back the created rule"); + assertRules("rules on create", Collections.singletonList(first), bucket.getCorsRules()); + assertRules("rules when listed", Collections.singletonList(first), listed(bucket).getCorsRules()); + + Support.step("replace the rules"); + final B2CorsRule second = B2CorsRule.builder("sdkharness-second", Collections.singletonList("*"), + new HashSet<>(Collections.singletonList("b2_download_file_by_name")), 60).build(); + final B2CorsRule third = B2CorsRule.builder("sdkharness-third", Collections.singletonList("https://other.example.com"), + new HashSet<>(Collections.singletonList("b2_download_file_by_id")), 120) + .setAllowedHeaders(Collections.singletonList("*")).build(); + final B2Bucket replaced = Support.client.updateBucket(B2UpdateBucketRequest.builder(bucket).setCorsRules(Arrays.asList(second, third)).build()); + assertRules("rules after the update", Arrays.asList(second, third), listed(bucket).getCorsRules()); + + Support.step("remove the rules"); + // updateBucket is optimistic: each update names the revision it was made from, so it starts from the last response. + Support.client.updateBucket(B2UpdateBucketRequest.builder(replaced).setCorsRules(Collections.emptyList()).build()); + Support.eq("rules after clearing", 0, listed(bucket).getCorsRules().size()); + }); + } + + private static B2Bucket listed(B2Bucket bucket) throws Exception { + final B2Bucket found = Support.client.getBucketOrNullByName(bucket.getBucketName()); + Support.check(found != null, "the bucket is not listed"); + return found; + } + + /** Field for field, in order. */ + private static void assertRules(String what, List expected, List actual) { + Support.eq(what + ": number of rules", expected.size(), actual.size()); + for (int i = 0; i < expected.size(); i++) { + final B2CorsRule e = expected.get(i); + final B2CorsRule a = actual.get(i); + Support.eq(what + ": corsRuleName", e.getCorsRuleName(), a.getCorsRuleName()); + Support.eq(what + ": allowedOrigins of " + e.getCorsRuleName(), e.getAllowedOrigins(), a.getAllowedOrigins()); + Support.eq(what + ": allowedOperations of " + e.getCorsRuleName(), new HashSet<>(e.getAllowedOperations()), new HashSet<>(a.getAllowedOperations())); + Support.eq(what + ": allowedHeaders of " + e.getCorsRuleName(), nullToEmpty(e.getAllowedHeaders()), nullToEmpty(a.getAllowedHeaders())); + Support.eq(what + ": exposeHeaders of " + e.getCorsRuleName(), nullToEmpty(e.getExposeHeaders()), nullToEmpty(a.getExposeHeaders())); + Support.eq(what + ": maxAgeSeconds of " + e.getCorsRuleName(), e.getMaxAgeSeconds(), a.getMaxAgeSeconds()); + } + } + + private static List nullToEmpty(List list) { + return list == null ? Collections.emptyList() : list; + } +} diff --git a/.sdkharness/tests/conformance/BucketInboundWebhook.java b/.sdkharness/tests/conformance/BucketInboundWebhook.java new file mode 100644 index 000000000..c51064f40 --- /dev/null +++ b/.sdkharness/tests/conformance/BucketInboundWebhook.java @@ -0,0 +1,105 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.exceptions.B2SignatureVerificationException; +import com.backblaze.b2.client.structures.B2EventNotification; +import com.backblaze.b2.client.structures.B2EventNotificationEvent; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import java.nio.charset.StandardCharsets; + +/** + * bucket.inbound_webhook. Run by ../run-conformance (see ../../README.md). + * + *

The X-Bz-Event-Notification-Signature value "comprises the signature version, always v1, followed by an equals + * sign =, followed by the lowercase hex-encoded signature value", and to verify it "create an HMAC SHA-256 digest + * using the hmacSha256SigningSecret ... as the key, with the request payload as the data. The resulting hex-encoded + * HMAC digest value must match the signature in the header." The payload's event fields are the documented ones + * (https://www.backblaze.com/docs/cloud-storage-event-notifications-reference-guide). + * + *

The SDK's part of this capability is parsing and verifying what a service receives, so nothing here needs a + * server round trip: a payload shaped like the documented example is signed in this check with its own HMAC code (not + * the SDK's), then given to {@code B2EventNotification.parse}. Asserted: with the right secret the payload is accepted + * and every documented event field is read back; with another secret, with the payload changed after signing, and + * with a signature that is not v1 hex, it is refused with {@code B2SignatureVerificationException}; a test event (no + * object fields) parses. + */ +public class BucketInboundWebhook { + private static final String PAYLOAD = "{\"events\":[{" + + "\"accountId\":\"sdkharness-account\",\"bucketId\":\"sdkharness-bucket-id\",\"bucketName\":\"sdkharness-conf-bucket\"," + + "\"eventId\":\"sdkharness-event-1\",\"eventTimestamp\":1761000000000,\"eventType\":\"b2:ObjectCreated:Upload\"," + + "\"eventVersion\":1,\"matchedRuleName\":\"sdkharness-rule\",\"objectName\":\"st/new.txt\",\"objectSize\":1234," + + "\"objectVersionId\":\"sdkharness-version-1\"}]}"; + private static final String TEST_PAYLOAD = "{\"events\":[{" + + "\"accountId\":\"sdkharness-account\",\"bucketId\":\"sdkharness-bucket-id\",\"bucketName\":\"sdkharness-conf-bucket\"," + + "\"eventId\":\"sdkharness-event-2\",\"eventTimestamp\":1761000000001,\"eventType\":\"b2:TestEvent\"," + + "\"eventVersion\":1,\"matchedRuleName\":\"sdkharness-rule\"}]}"; + private static final String SECRET = "*******"; + + public static void main(String[] args) { + Support.run("bucket.inbound_webhook", () -> { + Support.step("accept a correctly signed payload"); + final byte[] body = PAYLOAD.getBytes(StandardCharsets.UTF_8); + final B2EventNotification accepted = B2EventNotification.parse(body, sign(body, SECRET), SECRET); + Support.eq("number of events", 1, accepted.getEvents().size()); + final B2EventNotificationEvent event = accepted.getEvents().get(0); + Support.eq("accountId", "sdkharness-account", event.getAccountId()); + Support.eq("bucketId", "sdkharness-bucket-id", event.getBucketId()); + Support.eq("bucketName", "sdkharness-conf-bucket", event.getBucketName()); + Support.eq("eventId", "sdkharness-event-1", event.getEventId()); + Support.eq("eventTimestamp", 1761000000000L, event.getEventTimestamp()); + Support.eq("eventType", "b2:ObjectCreated:Upload", event.getEventType()); + Support.eq("eventVersion", 1, event.getEventVersion()); + Support.eq("matchedRuleName", "sdkharness-rule", event.getMatchedRuleName()); + Support.eq("objectName", "st/new.txt", event.getObjectName()); + Support.eq("objectSize", 1234L, event.getObjectSize()); + Support.eq("objectVersionId", "sdkharness-version-1", event.getObjectVersionId()); + + Support.step("the string overload agrees"); + Support.eq("event id through the string overload", "sdkharness-event-1", + B2EventNotification.parse(PAYLOAD, sign(body, SECRET), SECRET).getEvents().get(0).getEventId()); + + Support.step("refuse a wrong secret"); + refused("a payload verified with another secret", () -> B2EventNotification.parse(body, sign(body, SECRET), "another-secret")); + + Support.step("refuse a changed payload"); + final byte[] changed = PAYLOAD.replace("st/new.txt", "st/evil.txt").getBytes(StandardCharsets.UTF_8); + refused("a payload changed after signing", () -> B2EventNotification.parse(changed, sign(body, SECRET), SECRET)); + + Support.step("refuse a signature that is not v1 hex"); + refused("a signature without the v1= prefix", () -> B2EventNotification.parse(body, sign(body, SECRET).substring(3), SECRET)); + refused("an empty signature", () -> B2EventNotification.parse(body, "", SECRET)); + + Support.step("a test event"); + final byte[] test = TEST_PAYLOAD.getBytes(StandardCharsets.UTF_8); + final B2EventNotificationEvent testEvent = B2EventNotification.parse(test, sign(test, SECRET), SECRET).getEvents().get(0); + Support.eq("test event type", "b2:TestEvent", testEvent.getEventType()); + Support.eq("test event object name", null, testEvent.getObjectName()); + }); + } + + private interface Parse { + void run() throws Exception; + } + + private static void refused(String what, Parse parse) throws Exception { + try { + parse.run(); + } catch (B2SignatureVerificationException expected) { + return; + } + Support.fail(what + " was accepted"); + } + + /** The documented signature: "v1=" and the lowercase hex of HMAC-SHA256(secret, payload). Not the SDK's code. */ + private static String sign(byte[] payload, String secret) throws Exception { + final Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(secret.getBytes(StandardCharsets.UTF_8), "HmacSHA256")); + final StringBuilder hex = new StringBuilder("v1="); + for (byte b : mac.doFinal(payload)) { + hex.append(String.format("%02x", b)); + } + return hex.toString(); + } +} diff --git a/.sdkharness/tests/conformance/BucketLifecycle.java b/.sdkharness/tests/conformance/BucketLifecycle.java new file mode 100644 index 000000000..fa52fa82f --- /dev/null +++ b/.sdkharness/tests/conformance/BucketLifecycle.java @@ -0,0 +1,74 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2LifecycleRule; +import com.backblaze.b2.client.structures.B2UpdateBucketRequest; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * bucket.lifecycle. Run by ../run-conformance (see ../../README.md). + * + *

daysFromHidingToDeleting and fileNamePrefix are documented rule fields; B2's own example rule is + * daysFromHidingToDeleting 1 with daysFromUploadingToHiding null. "Unspecified properties are treated as having a + * null value", which is the expected value of every field not set. b2_update_bucket lifecycleRules: "If specified, + * the existing lifecycle rules will be replaced with this new list." + * (https://www.backblaze.com/docs/cloud-storage-lifecycle-rules, https://www.backblaze.com/apidocs/b2-update-bucket). + * + *

Asserted: a rule {prefix st/, daysFromHidingToDeleting 1} set on an existing bucket is read back with the same + * values and no other field populated; setting a different list replaces the first (the read-back is exactly the new + * list, all three day fields included); an empty list removes every rule. + */ +public class BucketLifecycle { + public static void main(String[] args) { + Support.run("bucket.lifecycle", () -> { + Support.step("bucket"); + final B2Bucket bucket = Support.bucket(); + Support.eq("rules on a new bucket", 0, bucket.getLifecycleRules().size()); + + Support.step("set one rule"); + final B2LifecycleRule rule = B2LifecycleRule.builder("st/").setDaysFromHidingToDeleting(1).build(); + final B2Bucket withRule = Support.client.updateBucket(B2UpdateBucketRequest.builder(bucket).setLifecycleRules(Collections.singletonList(rule)).build()); + final List read = listed(bucket).getLifecycleRules(); + Support.eq("number of rules", 1, read.size()); + Support.eq("fileNamePrefix", "st/", read.get(0).getFileNamePrefix()); + Support.eq("daysFromHidingToDeleting", 1, read.get(0).getDaysFromHidingToDeleting()); + Support.eq("daysFromUploadingToHiding (not set)", null, read.get(0).getDaysFromUploadingToHiding()); + Support.eq("daysFromStartingToCancelingUnfinishedLargeFiles (not set)", null, read.get(0).getDaysFromStartingToCancelingUnfinishedLargeFiles()); + + Support.step("replace the rules"); + final B2LifecycleRule hide = B2LifecycleRule.builder("logs/").setDaysFromUploadingToHiding(30).setDaysFromHidingToDeleting(7).build(); + final B2LifecycleRule cancel = B2LifecycleRule.builder("big/").setDaysFromStartingToCancelingUnfinishedLargeFiles(2).build(); + // updateBucket is optimistic: each update names the revision it was made from, so it starts from the last response. + final B2Bucket replacedBucket = Support.client.updateBucket(B2UpdateBucketRequest.builder(withRule).setLifecycleRules(Arrays.asList(hide, cancel)).build()); + final List replaced = listed(bucket).getLifecycleRules(); + Support.eq("number of rules after the replacement", 2, replaced.size()); + for (B2LifecycleRule expected : Arrays.asList(hide, cancel)) { + B2LifecycleRule actual = null; + for (B2LifecycleRule candidate : replaced) { + if (expected.getFileNamePrefix().equals(candidate.getFileNamePrefix())) { + actual = candidate; + } + } + Support.check(actual != null, "no rule for prefix " + expected.getFileNamePrefix() + " after the replacement"); + Support.eq(expected.getFileNamePrefix() + " daysFromUploadingToHiding", expected.getDaysFromUploadingToHiding(), actual.getDaysFromUploadingToHiding()); + Support.eq(expected.getFileNamePrefix() + " daysFromHidingToDeleting", expected.getDaysFromHidingToDeleting(), actual.getDaysFromHidingToDeleting()); + Support.eq(expected.getFileNamePrefix() + " daysFromStartingToCancelingUnfinishedLargeFiles", + expected.getDaysFromStartingToCancelingUnfinishedLargeFiles(), actual.getDaysFromStartingToCancelingUnfinishedLargeFiles()); + } + + Support.step("remove the rules"); + Support.client.updateBucket(B2UpdateBucketRequest.builder(replacedBucket).setLifecycleRules(Collections.emptyList()).build()); + Support.eq("rules after clearing", 0, listed(bucket).getLifecycleRules().size()); + }); + } + + private static B2Bucket listed(B2Bucket bucket) throws Exception { + final B2Bucket found = Support.client.getBucketOrNullByName(bucket.getBucketName()); + Support.check(found != null, "the bucket is not listed"); + return found; + } +} diff --git a/.sdkharness/tests/conformance/BucketNotificationRules.java b/.sdkharness/tests/conformance/BucketNotificationRules.java new file mode 100644 index 000000000..ad42d111d --- /dev/null +++ b/.sdkharness/tests/conformance/BucketNotificationRules.java @@ -0,0 +1,67 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2EventNotificationRule; +import com.backblaze.b2.client.structures.B2SetBucketNotificationRulesRequest; +import com.backblaze.b2.client.structures.B2WebhookConfiguration; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.TreeSet; + +/** + * bucket.notification_rules. Run by ../run-conformance (see ../../README.md). + * + *

b2_get_bucket_notification_rules lists hmacSha256SigningSecret as a response field and its example response + * carries it unredacted, so the secret is expected back unchanged + * (https://www.backblaze.com/apidocs/b2-get-bucket-notification-rules); b2_set_bucket_notification_rules replaces the + * whole rule set (https://www.backblaze.com/apidocs/b2-set-bucket-notification-rules). The webhook URL is never + * called: the rule only has to be stored and read back. + * + *

Asserted: a rule with a webhook target and a signing secret is set; the SDK's read-back (and the set call's own + * response) carries the same name, event types, object name prefix, enabled flag, target URL and signing secret; + * setting an empty list deletes it. The secret is generated here, compared and never printed. + */ +public class BucketNotificationRules { + private static final String WEBHOOK_URL = "https://example.com/sdkharness-conf"; + + public static void main(String[] args) { + Support.run("bucket.notification_rules", () -> { + Support.step("bucket"); + final B2Bucket bucket = Support.bucket(); + Support.eq("rules on a new bucket", 0, Support.client.getBucketNotificationRules(bucket.getBucketId()).getEventNotificationRules().size()); + + Support.step("set a rule"); + // 32 alphanumeric characters, the documented shape of a signing secret. + final String secret = "*******" + bucket.getBucketName().substring(bucket.getBucketName().length() - 12) + "0123456789abcdef"; + final B2EventNotificationRule rule = new B2EventNotificationRule("sdkharness-conf", + new TreeSet<>(Arrays.asList("b2:ObjectCreated:*", "b2:ObjectDeleted:*")), "st/", + new B2WebhookConfiguration(WEBHOOK_URL, secret), true); + final List set = Support.client.setBucketNotificationRules( + B2SetBucketNotificationRulesRequest.builder(bucket.getBucketId(), Collections.singletonList(rule)).build()).getEventNotificationRules(); + assertRule("set response", rule, set, secret); + + Support.step("read back"); + assertRule("read-back", rule, Support.client.getBucketNotificationRules(bucket.getBucketId()).getEventNotificationRules(), secret); + + Support.step("delete"); + Support.client.setBucketNotificationRules(B2SetBucketNotificationRulesRequest.builder(bucket.getBucketId(), Collections.emptyList()).build()); + Support.eq("rules after deleting", 0, Support.client.getBucketNotificationRules(bucket.getBucketId()).getEventNotificationRules().size()); + }); + } + + private static void assertRule(String what, B2EventNotificationRule expected, List actual, String secret) { + Support.eq(what + ": number of rules", 1, actual.size()); + final B2EventNotificationRule rule = actual.get(0); + Support.eq(what + ": name", expected.getName(), rule.getName()); + Support.eq(what + ": eventTypes", expected.getEventTypes(), rule.getEventTypes()); + Support.eq(what + ": objectNamePrefix", expected.getObjectNamePrefix(), rule.getObjectNamePrefix()); + Support.eq(what + ": isEnabled", true, rule.isEnabled()); + Support.check(rule.getTargetConfiguration() instanceof B2WebhookConfiguration, what + ": the target is not a webhook"); + final B2WebhookConfiguration target = (B2WebhookConfiguration) rule.getTargetConfiguration(); + Support.eq(what + ": webhook url", WEBHOOK_URL, target.getUrl()); + Support.check(secret.equals(target.getHmacSha256SigningSecret()), what + ": the signing secret did not come back unchanged"); + } +} diff --git a/.sdkharness/tests/conformance/BucketReplicationConfig.java b/.sdkharness/tests/conformance/BucketReplicationConfig.java new file mode 100644 index 000000000..11e0e319d --- /dev/null +++ b/.sdkharness/tests/conformance/BucketReplicationConfig.java @@ -0,0 +1,75 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2BucketReplicationConfiguration; +import com.backblaze.b2.client.structures.B2CreateKeyRequest; +import com.backblaze.b2.client.structures.B2CreatedApplicationKey; +import com.backblaze.b2.client.structures.B2ReplicationRule; +import com.backblaze.b2.client.structures.B2UpdateBucketRequest; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +/** + * bucket.replication_config. Run by ../run-conformance (see ../../README.md). + * + *

b2_update_bucket takes replicationConfiguration; "At least one of the asReplicationSource or + * asReplicationDestination parameters is required". Every rule field set here (replicationRuleName, + * destinationBucketId, fileNamePrefix, includeExistingFiles, isEnabled, priority) plus sourceApplicationKeyId is a + * documented field, and "Call b2_list_buckets to retrieve all of the replication rules associated with one or more + * buckets." (https://www.backblaze.com/apidocs/b2-update-bucket, + * https://www.backblaze.com/docs/cloud-storage-create-a-cloud-replication-rule-with-the-native-api). + * + *

Two buckets and a replication-capable key (the capabilities the documentation lists for a source) are created + * and one source rule is set on the first bucket, naming the second as the destination. Asserted: the read-back of + * the source bucket carries the source key id and the rule, field for field, and no destination side; the + * destination bucket reports no source configuration. Nothing is replicated: the check only stores and reads back + * the configuration, which is the whole of this capability (the SDK offers the structures, not a helper that runs + * replication). + */ +public class BucketReplicationConfig { + public static void main(String[] args) { + Support.run("bucket.replication_config", () -> { + Support.step("buckets and a key"); + final B2Bucket source = Support.bucket(); + final B2Bucket destination = Support.bucket(); + final B2CreatedApplicationKey key = Support.client.createKey(B2CreateKeyRequest.builder(new HashSet<>(Arrays.asList( + "listBuckets", "listFiles", "readFiles", "shareFiles", "writeFiles", "deleteFiles", + "readFileLegalHolds", "readFileRetentions", "readBucketReplications", "writeBucketReplications")), + "sdkharness-" + source.getBucketName().substring(source.getBucketName().length() - 8)).build()); + Support.trackKey(key.getApplicationKeyId()); + + Support.step("set a source rule"); + final B2ReplicationRule rule = new B2ReplicationRule("sdkharness-conf", destination.getBucketId(), 1, "st/", true, false); + Support.client.updateBucket(B2UpdateBucketRequest.builder(source) + .setReplicationConfiguration(B2BucketReplicationConfiguration.createForSource(key.getApplicationKeyId(), Collections.singletonList(rule))).build()); + + Support.step("read back"); + final B2BucketReplicationConfiguration read = listed(source).getReplicationConfiguration(); + Support.eq("source application key id", key.getApplicationKeyId(), read.getSourceApplicationKeyIdOrNull()); + final List rules = read.getReplicationRulesOrNull(); + Support.check(rules != null && rules.size() == 1, "expected one replication rule, got " + (rules == null ? "none" : rules.size())); + final B2ReplicationRule actual = rules.get(0); + Support.eq("replicationRuleName", rule.getReplicationRuleName(), actual.getReplicationRuleName()); + Support.eq("destinationBucketId", rule.getDestinationBucketId(), actual.getDestinationBucketId()); + Support.eq("priority", rule.getPriority(), actual.getPriority()); + Support.eq("fileNamePrefix", rule.getFileNamePrefix(), actual.getFileNamePrefix()); + Support.eq("isEnabled", rule.isEnabled(), actual.isEnabled()); + Support.eq("includeExistingFiles", rule.includeExistingFiles(), actual.includeExistingFiles()); + Support.eq("a source-only bucket has no destination mapping", null, read.getSourceToDestinationKeyMappingOrNull()); + + Support.step("the destination bucket is not a source"); + final B2BucketReplicationConfiguration other = listed(destination).getReplicationConfiguration(); + Support.check(other == null || other.getReplicationRulesOrNull() == null, "the destination bucket reports replication rules it was never given"); + }); + } + + private static B2Bucket listed(B2Bucket bucket) throws Exception { + final B2Bucket found = Support.client.getBucketOrNullByName(bucket.getBucketName()); + Support.check(found != null, "the bucket is not listed"); + return found; + } +} diff --git a/.sdkharness/tests/conformance/ClientProgress.java b/.sdkharness/tests/conformance/ClientProgress.java new file mode 100644 index 000000000..f9777b27c --- /dev/null +++ b/.sdkharness/tests/conformance/ClientProgress.java @@ -0,0 +1,94 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.contentSources.B2ByteArrayContentSource; +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2UploadFileRequest; +import com.backblaze.b2.client.structures.B2UploadListener; +import com.backblaze.b2.client.structures.B2UploadProgress; +import com.backblaze.b2.client.structures.B2UploadState; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/** + * client.progress. Run by ../run-conformance (see ../../README.md). + * + *

NO DOCUMENTED CONTRACT: progress reporting is the SDK's own ergonomics (B2UploadListener); the B2 API has no + * progress surface, so the expected values come from the scenario, not from a Backblaze page. Scenario: a large + * object is uploaded with a listener attached; the callbacks fire, are monotonically non-decreasing, and the final + * value equals the total byte count. + * + *

One worker, deliberately: with several, the per-part reports of concurrent parts legitimately interleave, and + * "non-decreasing" would test the thread pool, not the listener. The SDK reports per part (part index, bytes so far, + * state), so the check sums the latest report of every part into one running total. Asserted: callbacks fire; each + * part's own bytes-so-far never goes down; the running total never goes down; every part ends SUCCEEDED with + * bytes-so-far equal to its length; the final total equals the byte count of the object and the part lengths add up + * to it. + */ +public class ClientProgress { + public static void main(String[] args) { + Support.run("client.progress", () -> { + Support.step("bucket"); + final B2Bucket bucket = Support.bucket(); + final byte[] data = Support.payload((int) (4 * Support.partSize() + 5), 71); + + Support.step("upload with a listener"); + final List reports = Collections.synchronizedList(new ArrayList<>()); + final B2UploadListener listener = reports::add; + final B2FileVersion file = Support.client.uploadLargeFile( + B2UploadFileRequest.builder(bucket.getBucketId(), "st/progress/large.bin", "application/octet-stream", + B2ByteArrayContentSource.build(data)).setListener(listener).build(), + Support.executor(1)); + Support.eq("uploaded contentLength", (long) data.length, file.getContentLength()); + + Support.step("callbacks fired"); + Support.check(!reports.isEmpty(), "the listener was never called"); + + Support.step("monotonic"); + final Map latest = new TreeMap<>(); + final Map lengths = new TreeMap<>(); + long previousTotal = 0; + synchronized (reports) { + for (B2UploadProgress report : reports) { + final Long before = latest.get(report.getPartIndex()); + Support.check(before == null || report.getBytesSoFar() >= before, + "part " + report.getPartIndex() + " went backwards from " + before + " to " + report.getBytesSoFar() + " (" + report.getState() + ")"); + latest.put(report.getPartIndex(), report.getBytesSoFar()); + lengths.put(report.getPartIndex(), report.getLength()); + long total = 0; + for (long bytes : latest.values()) { + total += bytes; + } + Support.check(total >= previousTotal, "the running total went backwards from " + previousTotal + " to " + total); + previousTotal = total; + } + } + + Support.step("final value"); + Support.check(latest.size() > 1, "expected progress for more than one part, saw " + latest.size()); + Support.eq("final running total", (long) data.length, previousTotal); + long lengthSum = 0; + for (long length : lengths.values()) { + lengthSum += length; + } + Support.eq("sum of the parts' lengths", (long) data.length, lengthSum); + for (int part : latest.keySet()) { + B2UploadProgress last = null; + synchronized (reports) { + for (B2UploadProgress report : reports) { + if (report.getPartIndex() == part) { + last = report; + } + } + } + Support.eq("part " + part + " final state", B2UploadState.SUCCEEDED, last.getState()); + Support.eq("part " + part + " final bytes", last.getLength(), last.getBytesSoFar()); + } + }); + } +} diff --git a/.sdkharness/tests/conformance/EncSseB2.java b/.sdkharness/tests/conformance/EncSseB2.java new file mode 100644 index 000000000..e89b13ff7 --- /dev/null +++ b/.sdkharness/tests/conformance/EncSseB2.java @@ -0,0 +1,67 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.contentSources.B2ByteArrayContentSource; +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2BucketServerSideEncryption; +import com.backblaze.b2.client.structures.B2FileSseForRequest; +import com.backblaze.b2.client.structures.B2FileSseForResponse; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2ServerSideEncryptionMode; +import com.backblaze.b2.client.structures.B2UploadFileRequest; + +import java.util.Arrays; + +/** + * enc.sse_b2. Run by ../run-conformance (see ../../README.md). + * + *

b2_upload_file's X-Bz-Server-Side-Encryption: "B2 will encrypt the uploaded data before storing the file using + * ... (SSE-B2) with the algorithm specified in this header. Currently, the only supported value for this header is + * AES256." Its response serverSideEncryption carries "the mode ("SSE-B2" or "SSE-C") and algorithm" + * (https://www.backblaze.com/apidocs/b2-upload-file). A bucket default: "All uploads to that bucket, from the time + * default encryption is enabled onward, will then be encrypted with SSE-B2 by default unless you explicitly specify + * SSE-C encryption for a given file at upload time." (https://www.backblaze.com/docs/cloud-storage-server-side-encryption); + * defaultServerSideEncryption is "The default server-side encryption settings for new files uploaded to this + * bucket", example {"mode": "SSE-B2", "algorithm": "AES256"} (https://www.backblaze.com/apidocs/b2-create-bucket). + * + *

Asserted: a file uploaded requesting SSE-B2 reports SSE-B2 / AES256 in the upload response and in its metadata, + * the download returns the plaintext and its response says AES256; a bucket created with SSE-B2 as its default + * reports that default, and a file uploaded to it without asking reports SSE-B2 too. + */ +public class EncSseB2 { + public static void main(String[] args) { + Support.run("enc.sse_b2", () -> { + Support.step("bucket"); + final B2Bucket bucket = Support.bucket(); + + Support.step("upload requesting SSE-B2"); + final byte[] data = Support.payload(1024, 91); + final B2FileVersion uploaded = Support.client.uploadSmallFile( + B2UploadFileRequest.builder(bucket.getBucketId(), "st/enc/sse-b2.bin", "application/octet-stream", B2ByteArrayContentSource.build(data)) + .setServerSideEncryption(B2FileSseForRequest.createSseB2Aes256()).build()); + assertSseB2("upload response", uploaded.getServerSideEncryption()); + + Support.step("metadata"); + assertSseB2("file metadata", Support.client.getFileInfo(uploaded.getFileId()).getServerSideEncryption()); + + Support.step("download"); + final Support.Downloaded got = Support.downloadById(uploaded.getFileId()); + Support.check(Arrays.equals(data, got.bytes), "the download is not the plaintext"); + Support.eq("X-Bz-Server-Side-Encryption on the download", "AES256", got.headers.getServerSideEncryptionOrNull()); + + Support.step("bucket default"); + final B2Bucket defaultBucket = Support.bucket(false, b -> b.setDefaultServerSideEncryption(B2BucketServerSideEncryption.createSseB2Aes256())); + Support.eq("bucket default mode", B2ServerSideEncryptionMode.SSE_B2, defaultBucket.getDefaultServerSideEncryption().getMode()); + Support.eq("bucket default algorithm", "AES256", defaultBucket.getDefaultServerSideEncryption().getAlgorithm()); + final B2FileVersion inherited = Support.upload(defaultBucket, "st/enc/default.bin", data, "application/octet-stream", null); + assertSseB2("a file uploaded to a bucket with default encryption", inherited.getServerSideEncryption()); + Support.check(Arrays.equals(data, Support.downloadById(inherited.getFileId()).bytes), "the default-encrypted file's download is not the plaintext"); + }); + } + + private static void assertSseB2(String what, B2FileSseForResponse sse) { + Support.check(sse != null, what + " carries no serverSideEncryption"); + Support.eq(what + " mode", B2ServerSideEncryptionMode.SSE_B2, sse.getMode()); + Support.eq(what + " algorithm", "AES256", sse.getAlgorithm()); + } +} diff --git a/.sdkharness/tests/conformance/EncSseC.java b/.sdkharness/tests/conformance/EncSseC.java new file mode 100644 index 000000000..e90969112 --- /dev/null +++ b/.sdkharness/tests/conformance/EncSseC.java @@ -0,0 +1,70 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.contentSources.B2ByteArrayContentSource; +import com.backblaze.b2.client.exceptions.B2Exception; +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2FileSseForRequest; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2ServerSideEncryptionMode; +import com.backblaze.b2.client.structures.B2UploadFileRequest; + +import java.security.MessageDigest; +import java.util.Arrays; +import java.util.Base64; + +/** + * enc.sse_c. Run by ../run-conformance (see ../../README.md). + * + *

b2_upload_file takes the X-Bz-Server-Side-Encryption-Customer-Algorithm (AES256), -Customer-Key and + * -Customer-Key-Md5 headers; its response serverSideEncryption carries "the mode ("SSE-B2" or "SSE-C") and + * algorithm" (https://www.backblaze.com/apidocs/b2-upload-file). The key's MD5 comes back on download: + * X-Bz-Server-Side-Encryption-Customer-Key-Md5, "if the file was encrypted with SSE-C, then the value is the MD5 + * digest of the customer-managed encryption key used for encryption" + * (https://www.backblaze.com/apidocs/b2-download-file-by-id). "Backblaze B2 manages the encryption process using the + * key that you provide when uploading or accessing each file"; the documented error is for the WRONG key, "a 403 + * error is returned" (https://www.backblaze.com/docs/cloud-storage-server-side-encryption). + * + *

A 32-byte customer key (derived from a fixed string; it protects nothing and is never printed) is used for an + * upload. Asserted: the upload response reports SSE-C / AES256; downloading with the key returns the plaintext and the + * response carries the key's MD5; downloading without the key fails with a service error (B2 documents no status for + * a missing key, so a 400, 401 or 403 counts) and does not return the plaintext; downloading with a different key + * fails with 403, the documented status for a wrong key. + */ +public class EncSseC { + public static void main(String[] args) { + Support.run("enc.sse_c", () -> { + Support.step("bucket and keys"); + final B2Bucket bucket = Support.bucket(); + final byte[] keyBytes = MessageDigest.getInstance("SHA-256").digest(Support.utf8("sdkharness-java-sse-c-key")); + final byte[] otherBytes = MessageDigest.getInstance("SHA-256").digest(Support.utf8("sdkharness-java-sse-c-other-key")); + final B2FileSseForRequest sse = B2FileSseForRequest.createSseCAes256(Base64.getEncoder().encodeToString(keyBytes)); + final B2FileSseForRequest wrong = B2FileSseForRequest.createSseCAes256(Base64.getEncoder().encodeToString(otherBytes)); + final String keyMd5 = Base64.getEncoder().encodeToString(MessageDigest.getInstance("MD5").digest(keyBytes)); + + Support.step("upload with SSE-C"); + final byte[] data = Support.payload(1024, 92); + final B2FileVersion uploaded = Support.client.uploadSmallFile( + B2UploadFileRequest.builder(bucket.getBucketId(), "st/enc/sse-c.bin", "application/octet-stream", B2ByteArrayContentSource.build(data)) + .setServerSideEncryption(sse).build()); + Support.check(uploaded.getServerSideEncryption() != null, "the upload response carries no serverSideEncryption"); + Support.eq("upload response mode", B2ServerSideEncryptionMode.SSE_C, uploaded.getServerSideEncryption().getMode()); + Support.eq("upload response algorithm", "AES256", uploaded.getServerSideEncryption().getAlgorithm()); + + Support.step("download with the key"); + final Support.Downloaded got = Support.downloadById(uploaded.getFileId(), sse); + Support.check(Arrays.equals(data, got.bytes), "the download with the key is not the plaintext"); + Support.eq("X-Bz-Server-Side-Encryption-Customer-Key-Md5 on the download", keyMd5, got.headers.getSseCustomerKeyMd5OrNull()); + Support.eq("X-Bz-Server-Side-Encryption-Customer-Algorithm on the download", "AES256", got.headers.getSseCustomerAlgorithmOrNull()); + + Support.step("download without the key"); + final B2Exception missing = Support.expectError("download of an SSE-C file without its key", () -> Support.downloadById(uploaded.getFileId())); + Support.check(Arrays.asList(400, 401, 403).contains(missing.getStatus()), + "refused with status " + missing.getStatus() + " code " + missing.getCode() + ", expected 400, 401 or 403"); + + Support.step("download with a different key"); + final B2Exception other = Support.expectError("download of an SSE-C file with the wrong key", () -> Support.downloadById(uploaded.getFileId(), wrong)); + Support.eq("status for the wrong key", 403, other.getStatus()); + }); + } +} diff --git a/.sdkharness/tests/conformance/FilesServerSideCopy.java b/.sdkharness/tests/conformance/FilesServerSideCopy.java new file mode 100644 index 000000000..8fa7fd113 --- /dev/null +++ b/.sdkharness/tests/conformance/FilesServerSideCopy.java @@ -0,0 +1,72 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2CopyFileRequest; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.util.B2ByteRange; + +import java.util.Arrays; +import java.util.Collections; + +/** + * files.server_side_copy. Run by ../run-conformance (see ../../README.md). + * + *

b2_copy_file "Copies an entire file" from a sourceFileId to a new fileName: the request carries no + * file bytes, and the response's fileId is "The unique identifier for this version of this file" while its + * contentSha1 is "The SHA1 of the bytes stored in the file", so a whole-file copy reports the source's sha1 + * under a new fileId. The check also reads the copy back, and asserts the metadata directive (COPY keeps the + * source's content type and file info, REPLACE takes the ones in the request) and a ranged copy. + * https://www.backblaze.com/apidocs/b2-copy-file + */ +public class FilesServerSideCopy { + public static void main(String[] args) { + Support.run("files.server_side_copy", true, () -> { + Support.step("bucket"); + final B2Bucket bucket = Support.bucket(); + + Support.step("source"); + final byte[] data = Support.payload(512, 11); + final B2FileVersion source = Support.upload(bucket, "st/copy/source.bin", data, "application/x-sdkharness", + Collections.singletonMap("origin", "source")); + + Support.step("copy"); + Support.wire.clear(); + final B2FileVersion copy = Support.client.copySmallFile( + B2CopyFileRequest.builder(source.getFileId(), "st/copy/destination.bin").build()); + Support.eq("copy fileName", "st/copy/destination.bin", copy.getFileName()); + Support.check(!source.getFileId().equals(copy.getFileId()), "the copy reused the source's fileId"); + Support.eq("copy contentSha1", source.getContentSha1(), copy.getContentSha1()); + Support.eq("copy contentLength", (long) data.length, copy.getContentLength()); + Support.eq("COPY keeps the content type", "application/x-sdkharness", copy.getContentType()); + Support.eq("COPY keeps the file info", "source", copy.getFileInfo().get("origin")); + + Support.step("no bytes were uploaded"); + Support.eq("b2_copy_file requests", 1, Support.wireCount("POST", "b2_copy_file")); + Support.eq("b2_upload_file requests during the copy", 0, Support.wireCount("POST", "b2_upload_file")); + + Support.step("read back"); + Support.check(Arrays.equals(data, Support.downloadById(copy.getFileId()).bytes), "the copy's bytes differ from the source's"); + Support.check(Arrays.equals(data, Support.downloadById(source.getFileId()).bytes), "the source changed"); + + Support.step("metadata directive REPLACE"); + final B2FileVersion replaced = Support.client.copySmallFile( + B2CopyFileRequest.builder(source.getFileId(), "st/copy/replaced.bin") + .setMetadataDirective(B2CopyFileRequest.REPLACE_METADATA_DIRECTIVE) + .setContentType("text/x-replaced") + .setFileInfo(Collections.singletonMap("origin", "replaced")) + .build()); + Support.eq("REPLACE content type", "text/x-replaced", replaced.getContentType()); + Support.eq("REPLACE file info", "replaced", replaced.getFileInfo().get("origin")); + Support.eq("REPLACE contentSha1", source.getContentSha1(), replaced.getContentSha1()); + + Support.step("a range copy"); + final B2FileVersion ranged = Support.client.copySmallFile( + B2CopyFileRequest.builder(source.getFileId(), "st/copy/ranged.bin") + .setRange(B2ByteRange.between(100, 199)).build()); + Support.eq("range copy length", 100L, ranged.getContentLength()); + Support.check(Arrays.equals(Arrays.copyOfRange(data, 100, 200), Support.downloadById(ranged.getFileId()).bytes), + "the range copy's bytes are not bytes 100..199 of the source"); + }); + } +} diff --git a/.sdkharness/tests/conformance/GuardSelfTest.java b/.sdkharness/tests/conformance/GuardSelfTest.java index 01e2a644f..55bd47ce5 100644 --- a/.sdkharness/tests/conformance/GuardSelfTest.java +++ b/.sdkharness/tests/conformance/GuardSelfTest.java @@ -17,7 +17,9 @@ * *

It checks that a request to a loopback server passes, that an absolute request URI to a * non-loopback host (a documentation address, never contacted) is refused before it is sent, and that - * a loopback server redirecting to a non-loopback host is refused on the redirect hop. + * a loopback server redirecting to a non-loopback host is refused on the redirect hop. It also checks the one + * request path that does not go through the SDK's client, {@code Support.rawGet} (a share-link fetch, used by + * {@code urls.native_download}): loopback passes, a non-loopback host is refused before anything is sent. */ public final class GuardSelfTest { private GuardSelfTest() { @@ -58,6 +60,15 @@ public static void main(String[] args) throws Exception { problem = mustBeBlocked(client, new HttpGet("http://localhost:" + server.getAddress().getPort() + "/ok"), "a loopback name that is not 127.0.0.1", "localhost"); } + if (problem == null) { + problem = rawGetMustBeBlocked("http://203.0.113.7:9/share-link", "a share-link fetch to a non-loopback host", "203.0.113.7"); + } + if (problem == null) { + final Support.Raw ok = Support.rawGet(base + "/ok", null); + if (ok.status != 200) { + problem = "a share-link fetch from loopback did not reach the server"; + } + } } catch (Exception e) { problem = "unexpected " + Support.describe(e); } finally { @@ -72,6 +83,20 @@ public static void main(String[] args) throws Exception { } } + /** {@code Support.rawGet} is the one request path that does not go through the SDK's client; it carries its own host check. */ + private static String rawGetMustBeBlocked(String url, String what, String host) { + final int before = Support.blocked.size(); + try { + Support.rawGet(url, null); + return what + " was sent"; + } catch (Exception e) { + if (Support.blocked.size() != before + 1 || !host.equals(Support.blocked.get(before))) { + return what + " failed without being blocked by the guard: " + Support.describe(e); + } + return null; + } + } + private static String mustBeBlocked(CloseableHttpClient client, HttpGet get, String what, String host) { final int before = Support.blocked.size(); try (CloseableHttpResponse r = client.execute(get)) { diff --git a/.sdkharness/tests/conformance/KeysCrud.java b/.sdkharness/tests/conformance/KeysCrud.java new file mode 100644 index 000000000..24e4a07f8 --- /dev/null +++ b/.sdkharness/tests/conformance/KeysCrud.java @@ -0,0 +1,96 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.B2StorageClient; +import com.backblaze.b2.client.structures.B2ApplicationKey; +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2CreateKeyRequest; +import com.backblaze.b2.client.structures.B2CreatedApplicationKey; +import com.backblaze.b2.client.structures.B2DeleteKeyRequest; +import com.backblaze.b2.client.structures.B2ListBucketsRequest; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * keys.crud. Run by ../run-conformance (see ../../README.md). + * + *

b2_create_key's applicationKey is "The secret part of the key. This is the only time it will be returned, so + * you need to keep it. This is not returned when you list the keys in your account."; b2_list_keys' key objects carry + * no applicationKey field; b2_delete_key "Deletes the application key specified" + * (https://www.backblaze.com/apidocs/b2-create-key, https://www.backblaze.com/apidocs/b2-list-keys, + * https://www.backblaze.com/apidocs/b2-delete-key). + * + *

A key restricted to one bucket and to three capabilities is created, listed and deleted. Asserted: create returns + * the key id and a secret and echoes the name, capabilities and bucket; the listing finds the key with the same name, + * capabilities and bucket, and nothing the SDK returns for a listed key carries the secret (no accessor of + * {@code B2ApplicationKey} returns it, nor does its text form); the secret works (a second client authorized with it + * reports the restricted bucket as its allowed bucket and lists that bucket by name); after the delete the key is no longer listed and its secret no longer + * authorizes. The key is restricted to one bucket because this SDK pins API version 2, which cannot represent a key + * scoped to several (card: keys.multi_bucket is "no"). + */ +public class KeysCrud { + public static void main(String[] args) { + Support.run("keys.crud", () -> { + Support.step("bucket"); + final B2Bucket bucket = Support.bucket(); + final Set capabilities = new HashSet<>(Arrays.asList("listBuckets", "listFiles", "readFiles")); + final String name = "sdkharness-" + bucket.getBucketName().substring(bucket.getBucketName().length() - 8); + + Support.step("create"); + final B2CreatedApplicationKey created = Support.client.createKey(B2CreateKeyRequest.builder(capabilities, name).setBucketId(bucket.getBucketId()).build()); + Support.trackKey(created.getApplicationKeyId()); + final String secret = created.getApplicationKey(); + Support.check(created.getApplicationKeyId() != null && !created.getApplicationKeyId().isEmpty(), "create returned no key id"); + Support.check(secret != null && !secret.isEmpty(), "create returned no secret"); + Support.eq("created key name", name, created.getKeyName()); + Support.eq("created key capabilities", capabilities, new HashSet<>(created.getCapabilities())); + Support.eq("created key bucket", bucket.getBucketId(), created.getBucketId()); + + Support.step("list"); + B2ApplicationKey listed = find(created.getApplicationKeyId()); + Support.check(listed != null, "the new key is not in the listing"); + Support.eq("listed key name", name, listed.getKeyName()); + Support.eq("listed key capabilities", capabilities, new HashSet<>(listed.getCapabilities())); + Support.eq("listed key bucket", bucket.getBucketId(), listed.getBucketId()); + + Support.step("the secret is returned only on create"); + Support.check(!listed.toString().contains(secret), "the listed key's text form carries the secret"); + for (Method method : B2ApplicationKey.class.getMethods()) { + if (method.getParameterCount() == 0 && method.getReturnType() == String.class) { + Support.check(!secret.equals(method.invoke(listed)), "B2ApplicationKey." + method.getName() + "() returns the secret"); + } + } + + Support.step("the secret works"); + final B2StorageClient restricted = Support.clientFor(created.getApplicationKeyId(), secret); + // A key restricted to a bucket must name that bucket when it lists buckets (b2_list_buckets), so it does. + final List visible = new ArrayList<>(); + for (B2Bucket b : restricted.listBuckets(B2ListBucketsRequest.builder(restricted.getAccountId()).setBucketName(bucket.getBucketName()).build()).getBuckets()) { + visible.add(b.getBucketName()); + } + Support.eq("buckets the restricted key lists by name", Arrays.asList(bucket.getBucketName()), visible); + Support.eq("authorization bucket of the restricted key", bucket.getBucketId(), restricted.getAccountAuthorization().getAllowed().getBucketId()); + + Support.step("delete"); + Support.client.deleteKey(B2DeleteKeyRequest.builder(created.getApplicationKeyId()).build()); + Support.untrackKey(created.getApplicationKeyId()); + Support.check(find(created.getApplicationKeyId()) == null, "the deleted key is still listed"); + Support.eq("status of authorizing with a deleted key", 401, Support.expectError("authorizing with a deleted key", + () -> Support.clientFor(created.getApplicationKeyId(), secret).buckets()).getStatus()); + }); + } + + private static B2ApplicationKey find(String keyId) throws Exception { + for (B2ApplicationKey key : Support.client.applicationKeys()) { + if (keyId.equals(key.getApplicationKeyId())) { + return key; + } + } + return null; + } +} diff --git a/.sdkharness/tests/conformance/LargeComposeMixed.java b/.sdkharness/tests/conformance/LargeComposeMixed.java new file mode 100644 index 000000000..3737b0641 --- /dev/null +++ b/.sdkharness/tests/conformance/LargeComposeMixed.java @@ -0,0 +1,76 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.B2CopyingPartStorer; +import com.backblaze.b2.client.B2PartStorer; +import com.backblaze.b2.client.B2UploadingPartStorer; +import com.backblaze.b2.client.contentSources.B2ByteArrayContentSource; +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2StartLargeFileRequest; +import com.backblaze.b2.util.B2ByteRange; + +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * large.compose_mixed. Run by ../run-conformance (see ../../README.md). + * + *

"a large file can be assembled from a mix of uploaded and copied parts" + * (https://www.backblaze.com/docs/cloud-storage-large-files). b2_copy_part "Copies from an existing B2 file, + * storing it as a part of a large file which has already been started with b2_start_large_file"; its range is + * "The range of bytes to copy. If not provided, the whole source file will be copied." + * (https://www.backblaze.com/apidocs/b2-copy-part). Every part but the last is at least the realm's absolute + * minimum part size ("the minimum part size is 5 MB except for the last part in a file which has a minimum size of + * 1 byte", https://www.backblaze.com/docs/cloud-storage-create-large-files-with-the-native-api; the size is read + * from the realm, never hard-coded). + * + *

One large file is composed with {@code storeLargeFile} from three pieces: bytes held locally (an uploading + * part), a byte range of an existing file (a copying part with a range) and a whole existing file (a copying part + * without one). Asserted: the SDK sent one upload part and two copy parts and no other part, and the finished + * file's bytes equal the ordered concatenation of the three. + */ +public class LargeComposeMixed { + public static void main(String[] args) { + Support.run("large.compose_mixed", true, () -> { + Support.step("bucket"); + final B2Bucket bucket = Support.bucket(); + final int piece = (int) Support.partSize(); + + Support.step("sources"); + final byte[] local = Support.payload(piece, 51); + final byte[] rangedSource = Support.payload(3 * piece, 52); + final byte[] wholeSource = Support.payload(piece + 7, 53); + final B2FileVersion ranged = Support.upload(bucket, "st/compose/ranged-source.bin", rangedSource, "application/octet-stream", null); + final B2FileVersion whole = Support.upload(bucket, "st/compose/whole-source.bin", wholeSource, "application/octet-stream", null); + final byte[] rangePiece = Arrays.copyOfRange(rangedSource, piece, 2 * piece); + + Support.step("compose"); + final B2FileVersion started = Support.client.startLargeFile( + B2StartLargeFileRequest.builder(bucket.getBucketId(), "st/compose/composed.bin", "application/octet-stream").build()); + final List parts = new ArrayList<>(); + parts.add(new B2UploadingPartStorer(1, B2ByteArrayContentSource.build(local))); + parts.add(new B2CopyingPartStorer(2, ranged.getFileId(), B2ByteRange.between(piece, 2L * piece - 1))); + parts.add(new B2CopyingPartStorer(3, whole.getFileId())); + Support.wire.clear(); + final B2FileVersion composed = Support.client.storeLargeFile(started, parts, null, Support.executor(2)); + + Support.step("the SDK used a mix of parts"); + Support.eq("b2_upload_part requests", 1, Support.wireCount("POST", "b2_upload_part")); + Support.eq("b2_copy_part requests", 2, Support.wireCount("POST", "b2_copy_part")); + Support.eq("b2_finish_large_file requests", 1, Support.wireCount("POST", "b2_finish_large_file")); + Support.eq("composed fileId", started.getFileId(), composed.getFileId()); + + Support.step("bytes"); + final ByteArrayOutputStream expected = new ByteArrayOutputStream(); + expected.write(local); + expected.write(rangePiece); + expected.write(wholeSource); + Support.eq("composed contentLength", (long) expected.size(), Support.client.getFileInfo(composed.getFileId()).getContentLength()); + Support.check(Arrays.equals(expected.toByteArray(), Support.downloadById(composed.getFileId()).bytes), + "the composed file is not the ordered concatenation of the local bytes, the copied range and the whole copied file"); + }); + } +} diff --git a/.sdkharness/tests/conformance/LargeConcurrentParts.java b/.sdkharness/tests/conformance/LargeConcurrentParts.java new file mode 100644 index 000000000..0900a08e5 --- /dev/null +++ b/.sdkharness/tests/conformance/LargeConcurrentParts.java @@ -0,0 +1,132 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.contentSources.B2ContentSource; +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2UploadFileRequest; + +import java.io.ByteArrayInputStream; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.concurrent.ExecutorService; + +/** + * large.concurrent_parts. Run by ../run-conformance (see ../../README.md). + * + *

B2 permits uploading the parts of one large file in parallel: "Parts of a large file can be uploaded and + * copied in parallel" (https://www.backblaze.com/docs/cloud-storage-large-files) and "Each thread that uploads + * should get its own URL" (https://www.backblaze.com/docs/cloud-storage-create-large-files-with-the-native-api). + * No B2 document requires a client to do it, so the expected value comes from the SDK's own contract: the + * executor passed to {@code uploadLargeFile} sets how many parts are sent at once. + * + *

KNOWN WEAK by design, and stated here rather than hidden: the check does not trust wall time. The same + * eight-part file is uploaded twice, once on a one-thread executor and once on a four-thread executor, and the + * HTTP client the check hands the SDK counts how many {@code b2_upload_part} requests are in flight at the same + * instant (from the SDK starting to send to the response arriving). Each part's body is read slowly, so + * simultaneous requests overlap on a loopback server. Asserted: with one worker never more than one is in + * flight (this proves the instrument can tell the difference), with four workers more than one is. It is an + * in-process observation at the SDK's own HTTP client, not a packet capture; the wall times are only a NOTE. + */ +public class LargeConcurrentParts { + private static final int PARTS = 8; + private static final long PAUSE_MILLIS = 150; + + public static void main(String[] args) { + Support.run("large.concurrent_parts", true, () -> { + Support.step("bucket"); + final B2Bucket bucket = Support.bucket(); + final long partSize = Support.partSize(); + final byte[] data = Support.payload((int) (PARTS * partSize + 3), 31); + final String sha1 = Support.sha1(data); + Support.inFlightMarker = "b2_upload_part"; + + Support.step("one worker"); + final long oneMillis = upload(bucket, "st/large/serial.bin", data, sha1, Support.executor(1)); + final int serialMax = Support.maxInFlight.get(); + Support.eq("b2_upload_part requests in flight at once with one worker", 1, serialMax); + + Support.step("four workers"); + Support.maxInFlight.set(0); + final long fourMillis = upload(bucket, "st/large/parallel.bin", data, sha1, Support.executor(4)); + final int parallelMax = Support.maxInFlight.get(); + Support.check(parallelMax > 1, "with four workers at most " + parallelMax + " b2_upload_part request(s) were ever in flight at once"); + + Support.step("bytes"); + for (String name : new String[] {"st/large/serial.bin", "st/large/parallel.bin"}) { + final B2FileVersion stored = Support.versions(bucket.getBucketId(), name).get(0); + Support.check(Arrays.equals(data, Support.downloadById(stored.getFileId()).bytes), name + ": the downloaded bytes differ"); + } + Support.note("evidence is in-process: peak b2_upload_part requests in flight " + serialMax + " with 1 worker and " + parallelMax + + " with 4; wall time " + oneMillis + " ms and " + fourMillis + " ms (a corroboration, never the verdict)"); + }); + } + + private static long upload(B2Bucket bucket, String name, byte[] data, String sha1, ExecutorService executor) throws Exception { + final long start = System.nanoTime(); + final B2FileVersion file = Support.client.uploadLargeFile( + B2UploadFileRequest.builder(bucket.getBucketId(), name, "application/octet-stream", new SlowSource(data, sha1)).build(), + executor); + Support.eq(name + " contentLength", (long) data.length, file.getContentLength()); + return (System.nanoTime() - start) / 1_000_000; + } + + /** The bytes, read slowly: the first read of every stream waits, so uploads that run together overlap. */ + private static final class SlowSource implements B2ContentSource { + private final byte[] data; + private final String sha1; + + SlowSource(byte[] data, String sha1) { + this.data = data; + this.sha1 = sha1; + } + + @Override + public long getContentLength() { + return data.length; + } + + @Override + public String getSha1OrNull() { + return sha1; + } + + @Override + public Long getSrcLastModifiedMillisOrNull() { + return null; + } + + @Override + public InputStream createInputStream() { + return new FilterInputStream(new ByteArrayInputStream(data)) { + private boolean first = true; + + private void pause() throws IOException { + if (first) { + first = false; + try { + Thread.sleep(PAUSE_MILLIS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("interrupted"); + } + } + } + + @Override + public int read() throws IOException { + pause(); + return super.read(); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + pause(); + return super.read(b, off, len); + } + }; + } + } +} diff --git a/.sdkharness/tests/conformance/LargeMultipart.java b/.sdkharness/tests/conformance/LargeMultipart.java new file mode 100644 index 000000000..5bd52cc3f --- /dev/null +++ b/.sdkharness/tests/conformance/LargeMultipart.java @@ -0,0 +1,63 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.contentSources.B2ByteArrayContentSource; +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2UploadFileRequest; + +import java.util.Arrays; +import java.util.concurrent.ExecutorService; + +/** + * large.multipart. Run by ../run-conformance (see ../../README.md). + * + *

A payload of three parts' worth of bytes (the part size is the larger of the realm's recommended and + * absolute minimum part size, never a hard-coded 5 MB) is uploaded with {@code uploadLargeFile}. Asserted: + * the SDK started one large file, sent more than one part and finished it; the file's contentSha1 is "none" + * ("Large files do not have SHA1 checksums, and the value is none", + * https://www.backblaze.com/apidocs/b2-finish-large-file) and the sha1 the caller gave travels as + * {@code large_file_sha1} ("If the caller knows the SHA1 of the entire large file being uploaded, Backblaze + * recommends using large_file_sha1 as the name", https://www.backblaze.com/apidocs/b2-start-large-file); the + * downloaded bytes equal what was uploaded. A large file "must consist of at least two parts" + * (https://www.backblaze.com/docs/cloud-storage-large-files). + * + *

NEEDS REVIEW (as in the Python check): the docs recommend large_file_sha1, they do not require it. + */ +public class LargeMultipart { + public static void main(String[] args) { + Support.run("large.multipart", true, () -> { + Support.step("bucket"); + final B2Bucket bucket = Support.bucket(); + + Support.step("upload"); + final long partSize = Support.partSize(); + final byte[] data = Support.payload((int) (3 * partSize + 17), 21); + final String sha1 = Support.sha1(data); + final ExecutorService executor = Support.executor(2); + Support.wire.clear(); + final B2FileVersion large = Support.client.uploadLargeFile( + B2UploadFileRequest.builder(bucket.getBucketId(), "st/large/multipart.bin", "application/octet-stream", + B2ByteArrayContentSource.builder(data).setSha1OrNull(sha1).build()).build(), + executor); + + Support.step("the SDK used the large-file calls"); + Support.eq("b2_start_large_file requests", 1, Support.wireCount("POST", "b2_start_large_file")); + Support.eq("b2_finish_large_file requests", 1, Support.wireCount("POST", "b2_finish_large_file")); + final int parts = Support.wireCount("POST", "b2_upload_part"); + Support.check(parts > 1, "expected more than one b2_upload_part request, saw " + parts); + Support.eq("b2_upload_file requests (a large file is not a small upload)", 0, Support.wireCount("POST", "b2_upload_file")); + + Support.step("the finished file"); + Support.eq("uploaded contentLength", (long) data.length, large.getContentLength()); + final B2FileVersion stored = Support.client.getFileInfo(large.getFileId()); + Support.eq("contentSha1 of a large file", "none", stored.getContentSha1()); + Support.eq("large_file_sha1", sha1, stored.getFileInfo().get("large_file_sha1")); + Support.eq("stored contentLength", (long) data.length, stored.getContentLength()); + + Support.step("bytes"); + Support.check(Arrays.equals(data, Support.downloadById(large.getFileId()).bytes), + "the downloaded bytes differ from the uploaded ones"); + }); + } +} diff --git a/.sdkharness/tests/conformance/LargeResume.java b/.sdkharness/tests/conformance/LargeResume.java new file mode 100644 index 000000000..5c7f02920 --- /dev/null +++ b/.sdkharness/tests/conformance/LargeResume.java @@ -0,0 +1,201 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.contentSources.B2ContentSource; +import com.backblaze.b2.client.exceptions.B2Exception; +import com.backblaze.b2.client.exceptions.B2LocalException; +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2Part; +import com.backblaze.b2.client.structures.B2ListUnfinishedLargeFilesRequest; +import com.backblaze.b2.client.structures.B2UploadFileRequest; + +import java.io.ByteArrayInputStream; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * large.resume. Run by ../run-conformance (see ../../README.md). + * + *

"Any number of large files can be in progress at the same time. You can use b2_list_unfinished_large_files + * to get a list of them. For any one unfinished large file, you can use b2_list_parts to get a list of the parts + * that were uploaded so far." (https://www.backblaze.com/docs/cloud-storage-create-large-files-with-the-native-api; + * https://www.backblaze.com/apidocs/b2-list-parts). B2 makes reusing the unfinished fileId possible; that the SDK + * does it is the capability under test, not a B2 promise. + * + *

The first upload is interrupted from inside the content source: the bytes of the third part cannot be read + * (an unchecked exception, not an IOException: the SDK retries an IOException for about two minutes), so with one + * worker the first two parts are stored and the call fails before the file is finished. The large file is then + * unfinished (a "start" version in the unfinished listing, parts in b2_list_parts). The SDK's resume call + * {@code finishUploadingLargeFile} is given that file and the same content. Asserted: the rerun sends no + * b2_start_large_file, uploads only the parts that were missing (fewer than the total, and exactly the missing + * ones), finishes the same fileId, and the stored bytes equal the content. + * + *

Two legs, because the SDK treats the two content shapes differently. Leg "no sha1": the content source does + * not know its sha1, so the large file carries no large_file_sha1 and the rerun is the same request as the first + * run. Leg "sha1": the content source knows its sha1 (the SDK records it as the large_file_sha1 file info); the + * same request is first tried as-is, and, if the SDK refuses it with a "mismatch" error, the check records that on + * a NOTE line and resumes with the large_file_sha1 custom field added to the request (which the SDK then accepts). + * The NOTE is a finding about the SDK, not about B2: B2LargeFileUploader removes the key + * {@code X-Bz-Info-large_file_sha1} (B2Headers.LARGE_FILE_SHA1) from the file info before comparing it with the + * request's, but the info map is keyed {@code large_file_sha1} (B2Headers.LARGE_FILE_SHA1_INFO_NAME). + */ +public class LargeResume { + private static final int PARTS = 4; + + public static void main(String[] args) { + Support.run("large.resume", true, () -> { + Support.step("bucket"); + final B2Bucket bucket = Support.bucket(); + final long partSize = Support.partSize(); + final byte[] data = Support.payload((int) (PARTS * partSize + 3), 41); + + Support.step("leg no sha1"); + resume(bucket, "st/large/resume-no-sha1.bin", data, null); + + Support.step("leg sha1"); + resume(bucket, "st/large/resume-sha1.bin", data, Support.sha1(data)); + }); + } + + private static void resume(B2Bucket bucket, String name, byte[] data, String sha1) throws Exception { + final long partSize = Support.partSize(); + final String leg = sha1 == null ? "no sha1: " : "sha1: "; + + Support.wire.clear(); + try { + Support.client.uploadLargeFile(request(bucket, name, new Source(data, sha1, 2 * partSize), null), Support.executor(1)); + Support.fail(leg + "the interrupted upload finished instead of failing"); + } catch (B2Exception expected) { + Support.note(leg + "the first upload stopped as intended: " + Support.describe(expected)); + } + Support.eq(leg + "b2_start_large_file requests in the first run", 1, Support.wireCount("POST", "b2_start_large_file")); + Support.eq(leg + "b2_finish_large_file requests in the first run", 0, Support.wireCount("POST", "b2_finish_large_file")); + + final List unfinished = unfinished(bucket, name); + Support.eq(leg + "unfinished large files named " + name, 1, unfinished.size()); + final B2FileVersion pending = unfinished.get(0); + Support.eq(leg + "unfinished file name", name, pending.getFileName()); + int storedBefore = 0; + for (B2Part part : Support.client.parts(pending.getFileId())) { + storedBefore++; + } + Support.check(storedBefore >= 1, leg + "no part was stored before the interruption"); + Support.check(storedBefore < PARTS, leg + "all " + PARTS + " parts were stored; nothing is left to resume"); + + Support.wire.clear(); + B2FileVersion finished = null; + try { + finished = Support.client.finishUploadingLargeFile(pending, request(bucket, name, new Source(data, sha1, -1), null), Support.executor(2)); + } catch (B2LocalException e) { + if (sha1 == null || !"mismatch".equals(e.getCode())) { + throw e; + } + Support.note("FINDING b2-sdk-java: finishUploadingLargeFile refused the same request that started the large file: " + + e.getMessage() + ". It resumes only when the request carries the large_file_sha1 custom field itself."); + } + if (finished == null) { + Support.eq(leg + "b2_upload_part requests after the refused resume", 0, Support.wireCount("POST", "b2_upload_part")); + finished = Support.client.finishUploadingLargeFile(pending, + request(bucket, name, new Source(data, sha1, -1), sha1), Support.executor(2)); + } + Support.eq(leg + "b2_start_large_file requests in the rerun", 0, Support.wireCount("POST", "b2_start_large_file")); + Support.eq(leg + "finished fileId is the unfinished one", pending.getFileId(), finished.getFileId()); + Support.eq(leg + "parts uploaded in the rerun", PARTS - storedBefore, Support.wireCount("POST", "b2_upload_part")); + Support.eq(leg + "b2_finish_large_file requests in the rerun", 1, Support.wireCount("POST", "b2_finish_large_file")); + + Support.eq(leg + "large files still unfinished after the rerun", 0, unfinished(bucket, name).size()); + Support.eq(leg + "stored contentLength", (long) data.length, Support.client.getFileInfo(finished.getFileId()).getContentLength()); + Support.check(Arrays.equals(data, Support.downloadById(finished.getFileId()).bytes), leg + "the downloaded bytes differ from the content"); + } + + private static List unfinished(B2Bucket bucket, String name) throws B2Exception { + final List all = new ArrayList<>(); + for (B2FileVersion v : Support.client.unfinishedLargeFiles( + B2ListUnfinishedLargeFilesRequest.builder(bucket.getBucketId()).setNamePrefix(name).build())) { + all.add(v); + } + return all; + } + + /** {@code sha1Field}, when not null, is added to the request as the large_file_sha1 custom field. */ + private static B2UploadFileRequest request(B2Bucket bucket, String name, B2ContentSource source, String sha1Field) { + final B2UploadFileRequest.Builder builder = B2UploadFileRequest.builder(bucket.getBucketId(), name, "application/octet-stream", source); + if (sha1Field != null) { + builder.setCustomField("large_file_sha1", sha1Field); + } + return builder.build(); + } + + /** The content; with {@code failAt >= 0}, reading past that byte offset fails the way an interrupted run would. */ + private static final class Source implements B2ContentSource { + private final byte[] data; + private final String sha1; + private final long failAt; + + Source(byte[] data, String sha1, long failAt) { + this.data = data; + this.sha1 = sha1; + this.failAt = failAt; + } + + @Override + public long getContentLength() { + return data.length; + } + + @Override + public String getSha1OrNull() { + return sha1; + } + + @Override + public Long getSrcLastModifiedMillisOrNull() { + return null; + } + + @Override + public InputStream createInputStream() { + return new FilterInputStream(new ByteArrayInputStream(data)) { + private long pos; + + private void guard() { + if (failAt >= 0 && pos >= failAt) { + throw new IllegalStateException("interrupted by the check at byte " + pos); + } + } + + @Override + public int read() throws IOException { + guard(); + final int b = super.read(); + if (b >= 0) { + pos++; + } + return b; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + guard(); + final int n = super.read(b, off, failAt >= 0 ? (int) Math.min(len, failAt - pos) : len); + if (n > 0) { + pos += n; + } + return n; + } + + @Override + public long skip(long n) throws IOException { + final long skipped = super.skip(n); + pos += skipped; + return skipped; + } + }; + } + } +} diff --git a/.sdkharness/tests/conformance/LockBucketDefault.java b/.sdkharness/tests/conformance/LockBucketDefault.java new file mode 100644 index 000000000..2f3d5da49 --- /dev/null +++ b/.sdkharness/tests/conformance/LockBucketDefault.java @@ -0,0 +1,59 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2BucketDefaultRetention; +import com.backblaze.b2.client.structures.B2FileRetention; +import com.backblaze.b2.client.structures.B2FileRetentionMode; +import com.backblaze.b2.client.structures.B2FileRetentionPeriodUnit; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2UpdateBucketRequest; + +/** + * lock.bucket_default. Run by ../run-conformance (see ../../README.md). + * + *

Object Lock: a bucket's default retention means "All of the files that are uploaded or copied to that bucket, + * from that time on, are protected by those Object Lock retention settings by default." + * (https://www.backblaze.com/docs/cloud-storage-object-lock). The read-back shape, defaultRetention {mode, period + * {duration, unit}}, is the native API's + * (https://www.backblaze.com/docs/cloud-storage-enable-object-lock-with-the-native-api). + * + *

A file-lock-enabled bucket gets a default retention of governance, 1 day. Asserted: the update's response and + * a fresh read of the bucket report the mode and the period, and a file uploaded afterwards inherits a governance + * retention that ends about one day from now (within 30 minutes). + */ +public class LockBucketDefault { + private static final long DAY_MILLIS = 24L * 60 * 60 * 1000; + private static final long TOLERANCE_MILLIS = 30L * 60 * 1000; + + public static void main(String[] args) { + Support.run("lock.bucket_default", () -> { + Support.step("bucket"); + final B2Bucket bucket = Support.lockedBucket(); + Support.check(bucket.getFileLockConfiguration().isFileLockEnabled(), "the new bucket does not report file lock as enabled"); + + Support.step("set the default retention"); + final B2BucketDefaultRetention wanted = new B2BucketDefaultRetention(B2FileRetentionMode.GOVERNANCE, 1, B2FileRetentionPeriodUnit.DAYS); + final B2Bucket updated = Support.client.updateBucket(B2UpdateBucketRequest.builder(bucket).setDefaultRetention(wanted).build()); + Support.eq("default retention in the update response", wanted, updated.getFileLockConfiguration().getDefaultRetention()); + + Support.step("read back"); + final B2Bucket reread = Support.client.getBucketOrNullByName(bucket.getBucketName()); + Support.check(reread != null, "the bucket is not listed"); + Support.eq("default retention mode", B2FileRetentionMode.GOVERNANCE, reread.getFileLockConfiguration().getDefaultRetentionMode()); + Support.eq("default retention duration", 1, reread.getFileLockConfiguration().getDefaultRetentionPeriod().getDuration()); + Support.eq("default retention unit", B2FileRetentionPeriodUnit.DAYS, reread.getFileLockConfiguration().getDefaultRetentionPeriod().getUnit()); + Support.check(reread.getFileLockConfiguration().isFileLockEnabled(), "file lock is no longer enabled after the update"); + + Support.step("an uploaded file inherits it"); + final long before = System.currentTimeMillis(); + final B2FileVersion uploaded = Support.upload(reread, "st/lock/default.txt", Support.payload(256, 81), "text/plain", null); + final long after = System.currentTimeMillis(); + final B2FileRetention retention = Support.client.getFileInfo(uploaded.getFileId()).getFileRetention(); + Support.eq("inherited retention mode", B2FileRetentionMode.GOVERNANCE, retention.getMode()); + final long until = retention.getRetainUntilTimestamp(); + Support.check(until >= before + DAY_MILLIS - TOLERANCE_MILLIS && until <= after + DAY_MILLIS + TOLERANCE_MILLIS, + "inherited retention ends " + (until - before) / 60000 + " minutes from now, expected about " + DAY_MILLIS / 60000); + }); + } +} diff --git a/.sdkharness/tests/conformance/LockBypassGovernance.java b/.sdkharness/tests/conformance/LockBypassGovernance.java new file mode 100644 index 000000000..82aec91f2 --- /dev/null +++ b/.sdkharness/tests/conformance/LockBypassGovernance.java @@ -0,0 +1,55 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.exceptions.B2Exception; +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2DeleteFileVersionRequest; +import com.backblaze.b2.client.structures.B2FileRetention; +import com.backblaze.b2.client.structures.B2FileRetentionMode; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2UpdateFileRetentionRequest; + +import java.util.Arrays; + +/** + * lock.bypass_governance. Run by ../run-conformance (see ../../README.md). + * + *

b2_delete_file_version's bypassGovernance "Must be specified and set to true if deleting a file version + * protected by Object Lock governance mode retention settings. Setting the value requires the bypassGovernance + * application key capability." Without it the delete is refused: 401 access_denied, "The delete was not allowed + * because Object Lock is enabled on the file." (https://www.backblaze.com/apidocs/b2-delete-file-version). The + * account key the simulator hands every check holds the capability. + * + *

An object under governance retention (now + 2 days) is deleted without bypass, then with + * {@code setBypassGovernance(true)}. Asserted: the first is refused with 401 access_denied and leaves the version + * in place; the second succeeds and the version is gone from the listing and from download by id. The SDK treats a + * 401 on a call other than an upload as a reason to reauthorize and retry; the retries end in the same refusal, + * which is what the check asserts, so the extra requests are not a failure. + */ +public class LockBypassGovernance { + public static void main(String[] args) { + Support.run("lock.bypass_governance", () -> { + Support.step("bucket and object under governance"); + final B2Bucket bucket = Support.lockedBucket(); + final byte[] data = Support.payload(256, 84); + final B2FileVersion file = Support.upload(bucket, "st/lock/bypass.txt", data, "text/plain", null); + Support.client.updateFileRetention(B2UpdateFileRetentionRequest.builder(file.getFileName(), file.getFileId(), + new B2FileRetention(B2FileRetentionMode.GOVERNANCE, System.currentTimeMillis() + 2L * 24 * 60 * 60 * 1000)).build()); + + Support.step("delete without bypass is refused"); + final B2Exception refused = Support.expectError("delete of a governance-locked version without bypass", + () -> Support.client.deleteFileVersion(B2DeleteFileVersionRequest.builder(file.getFileName(), file.getFileId()).build())); + Support.eq("refusal status", 401, refused.getStatus()); + Support.eq("refusal code", "access_denied", refused.getCode()); + Support.check(Arrays.equals(data, Support.downloadById(file.getFileId()).bytes), "the refused delete changed the file"); + + Support.step("delete with bypass"); + Support.client.deleteFileVersion(B2DeleteFileVersionRequest.builder(file.getFileName(), file.getFileId()).setBypassGovernance(true).build()); + + Support.step("the version is gone"); + Support.eq("versions left after the bypassing delete", 0, Support.versions(bucket.getBucketId(), file.getFileName()).size()); + Support.eq("download by id of the deleted version", 404, Support.expectError("download of a deleted version", + () -> Support.downloadById(file.getFileId())).getStatus()); + }); + } +} diff --git a/.sdkharness/tests/conformance/LockLegalHold.java b/.sdkharness/tests/conformance/LockLegalHold.java new file mode 100644 index 000000000..5d8e272a4 --- /dev/null +++ b/.sdkharness/tests/conformance/LockLegalHold.java @@ -0,0 +1,56 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.exceptions.B2Exception; +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2LegalHold; +import com.backblaze.b2.client.structures.B2UpdateFileLegalHoldRequest; + +import java.util.Arrays; + +/** + * lock.legal_hold. Run by ../run-conformance (see ../../README.md). + * + *

b2_update_file_legal_hold takes legalHold "on" or "off" and is "Used to enable legal hold for a file in an + * Object Lock-enabled bucket, preventing it from being deleted, or to disable legal hold protections" + * (https://www.backblaze.com/apidocs/b2-update-file-legal-hold). "If either legal hold or retention settings, or even + * both, are active for a given file, then that file is protected from being deleted" + * (https://www.backblaze.com/docs/cloud-storage-object-lock). + * + *

An object in a file-lock-enabled bucket: the hold is set on, read back, and set off. Asserted: while on, the + * update's response and the metadata say on, deleting the version is refused (B2 documents the protection, not the + * status, so a 400, 401, 403 or 409 service error counts) and the file is still there; once off, the metadata says + * off and deleting the version is permitted and removes it. + */ +public class LockLegalHold { + public static void main(String[] args) { + Support.run("lock.legal_hold", () -> { + Support.step("bucket and object"); + final B2Bucket bucket = Support.lockedBucket(); + final byte[] data = Support.payload(256, 83); + final B2FileVersion file = Support.upload(bucket, "st/lock/hold.txt", data, "text/plain", null); + + Support.step("hold on"); + Support.eq("legal hold in the update response", B2LegalHold.ON, Support.client.updateFileLegalHold( + B2UpdateFileLegalHoldRequest.builder(file.getFileName(), file.getFileId(), B2LegalHold.ON).build()).getLegalHold()); + Support.eq("legal hold in the metadata", B2LegalHold.ON, Support.client.getFileInfo(file.getFileId()).getLegalHold()); + + Support.step("delete while held is refused"); + final B2Exception refused = Support.expectError("delete of a version under legal hold", + () -> Support.client.deleteFileVersion(file.getFileName(), file.getFileId())); + Support.check(Arrays.asList(400, 401, 403, 409).contains(refused.getStatus()), + "refused with status " + refused.getStatus() + " code " + refused.getCode() + ", expected 400, 401, 403 or 409"); + Support.check(Arrays.equals(data, Support.downloadById(file.getFileId()).bytes), "the refused delete changed the file"); + + Support.step("hold off"); + Support.eq("legal hold in the update response", B2LegalHold.OFF, Support.client.updateFileLegalHold( + B2UpdateFileLegalHoldRequest.builder(file.getFileName(), file.getFileId(), B2LegalHold.OFF).build()).getLegalHold()); + Support.eq("legal hold in the metadata", B2LegalHold.OFF, Support.client.getFileInfo(file.getFileId()).getLegalHold()); + + Support.step("delete once released"); + Support.client.deleteFileVersion(file.getFileName(), file.getFileId()); + Support.eq("versions left after the delete", 0, Support.versions(bucket.getBucketId(), file.getFileName()).size()); + }); + } +} diff --git a/.sdkharness/tests/conformance/LockPerFileRetention.java b/.sdkharness/tests/conformance/LockPerFileRetention.java new file mode 100644 index 000000000..2c1b58811 --- /dev/null +++ b/.sdkharness/tests/conformance/LockPerFileRetention.java @@ -0,0 +1,59 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.exceptions.B2Exception; +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2FileRetention; +import com.backblaze.b2.client.structures.B2FileRetentionMode; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2UpdateFileRetentionRequest; + +import java.util.Arrays; + +/** + * lock.per_file_retention. Run by ../run-conformance (see ../../README.md). + * + *

b2_update_file_retention sets fileRetention {mode "governance" or "compliance", retainUntilTimestamp} and + * returns it (https://www.backblaze.com/apidocs/b2-update-file-retention). "If either legal hold or retention + * settings, or even both, are active for a given file, then that file is protected from being deleted" + * (https://www.backblaze.com/docs/cloud-storage-object-lock), and b2_delete_file_version's bypassGovernance "Must be + * specified and set to true if deleting a file version protected by Object Lock governance mode retention settings" + * (https://www.backblaze.com/apidocs/b2-delete-file-version). + * + *

An object in a file-lock-enabled bucket gets a governance retention of now + 2 days. Asserted: the update's + * response and the file's metadata report that mode and timestamp, and deleting that version without bypass is + * refused with 401 access_denied (the answer B2's own contract test records for this call, and the simulator's) and + * leaves the version in place. Compliance mode is never used: nothing could lift it, and the check must clean up. + */ +public class LockPerFileRetention { + private static final long TWO_DAYS_MILLIS = 2L * 24 * 60 * 60 * 1000; + + public static void main(String[] args) { + Support.run("lock.per_file_retention", () -> { + Support.step("bucket and object"); + final B2Bucket bucket = Support.lockedBucket(); + final byte[] data = Support.payload(256, 82); + final B2FileVersion file = Support.upload(bucket, "st/lock/retention.txt", data, "text/plain", null); + + Support.step("set the retention"); + final long until = System.currentTimeMillis() + TWO_DAYS_MILLIS; + final B2FileRetention wanted = new B2FileRetention(B2FileRetentionMode.GOVERNANCE, until); + Support.eq("retention in the update response", wanted, + Support.client.updateFileRetention(B2UpdateFileRetentionRequest.builder(file.getFileName(), file.getFileId(), wanted).build()).getFileRetention()); + + Support.step("read back"); + final B2FileRetention reported = Support.client.getFileInfo(file.getFileId()).getFileRetention(); + Support.eq("file retention mode", B2FileRetentionMode.GOVERNANCE, reported.getMode()); + Support.eq("file retention retainUntilTimestamp", until, reported.getRetainUntilTimestamp()); + + Support.step("delete without bypass is refused"); + final B2Exception refused = Support.expectError("delete of a governance-locked version", + () -> Support.client.deleteFileVersion(file.getFileName(), file.getFileId())); + Support.eq("refusal status", 401, refused.getStatus()); + Support.eq("refusal code", "access_denied", refused.getCode()); + + Support.step("the version is still there"); + Support.check(Arrays.equals(data, Support.downloadById(file.getFileId()).bytes), "the refused delete changed the file"); + }); + } +} diff --git a/.sdkharness/tests/conformance/Support.java b/.sdkharness/tests/conformance/Support.java index 6b6b8f973..c6ddedc4f 100644 --- a/.sdkharness/tests/conformance/Support.java +++ b/.sdkharness/tests/conformance/Support.java @@ -7,8 +7,12 @@ import com.backblaze.b2.client.contentSources.B2Headers; import com.backblaze.b2.client.exceptions.B2Exception; import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2CreateBucketRequest; +import com.backblaze.b2.client.structures.B2DeleteFileVersionRequest; import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2LegalHold; import com.backblaze.b2.client.structures.B2ListFileVersionsRequest; +import com.backblaze.b2.client.structures.B2UpdateFileLegalHoldRequest; import com.backblaze.b2.client.structures.B2UploadFileRequest; import com.backblaze.b2.client.webApiHttpClient.B2StorageHttpClientBuilder; import com.backblaze.b2.client.webApiHttpClient.HttpClientFactory; @@ -17,6 +21,7 @@ import org.apache.http.HttpHost; import org.apache.http.HttpRequest; import org.apache.http.HttpRequestInterceptor; +import org.apache.http.HttpResponseInterceptor; import org.apache.http.client.config.RequestConfig; import org.apache.http.config.Registry; import org.apache.http.config.RegistryBuilder; @@ -36,6 +41,10 @@ import java.util.Map; import java.util.Random; import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; /** * Shared plumbing for the sdkharness conformance checks. Compiled together with one check class @@ -85,9 +94,20 @@ static final class Downloaded { /** "METHOD uri" of every HTTP request the SDK sent, when the check asked for the recording client. */ static final List wire = Collections.synchronizedList(new ArrayList<>()); + /** + * With the recording client, requests whose URI contains {@link #inFlightMarker} are counted while they are in + * flight (from the moment the SDK starts sending to the moment the response arrives); {@link #maxInFlight} is the + * most that overlapped. + */ + static volatile String inFlightMarker; + static final AtomicInteger inFlight = new AtomicInteger(); + static final AtomicInteger maxInFlight = new AtomicInteger(); + + private static final List executors = new ArrayList<>(); private static String capability = "unknown"; private static String step = "configuration"; private static final List created = new ArrayList<>(); + private static final java.util.Set lockEnabled = new java.util.HashSet<>(); private static final String LOOPBACK_HOST = "127.0.0.1"; /** Hosts of requests the guard refused (the SDK may swallow the exception, so the verdict reads this). */ static final List blocked = Collections.synchronizedList(new ArrayList<>()); @@ -133,8 +153,24 @@ static void run(String cap, boolean recordWire, Body body) { } } + /** + * A second client, authorized with a key the check itself created (keys.crud), through the same guarded factory. + * It is closed with the check. The credential is the caller's; nothing here prints it. + */ + static B2StorageClient clientFor(String keyId, String key) { + final B2StorageClient other = newClient(System.getenv("SDKHARNESS_SIMULATOR_URL"), keyId, key, false); + extraClients.add(other); + return other; + } + + private static final List extraClients = new ArrayList<>(); + private static B2StorageClient newClient(String masterUrl, boolean recordWire) { - final B2ClientConfig config = B2ClientConfig.builder(KEY_ID, KEY, "b2-sdk-java-sdkharness-conformance") + return newClient(masterUrl, KEY_ID, KEY, recordWire); + } + + private static B2StorageClient newClient(String masterUrl, String keyId, String key, boolean recordWire) { + final B2ClientConfig config = B2ClientConfig.builder(keyId, key, "b2-sdk-java-sdkharness-conformance") .setMasterUrl(masterUrl) .build(); // The SDK's own request settings (timeouts, no URI normalization); the guarded factory below @@ -169,6 +205,16 @@ public CloseableHttpClient create() { requireLoopback(request, (HttpHost) context.getAttribute(HttpCoreContext.HTTP_TARGET_HOST)); if (recordWire) { wire.add(request.getRequestLine().getMethod() + " " + request.getRequestLine().getUri()); + if (matchesInFlight(request)) { + final int now = inFlight.incrementAndGet(); + maxInFlight.accumulateAndGet(now, Math::max); + } + } + }) + .addInterceptorLast((HttpResponseInterceptor) (response, context) -> { + final HttpRequest request = (HttpRequest) context.getAttribute(HttpCoreContext.HTTP_REQUEST); + if (recordWire && request != null && matchesInFlight(request)) { + inFlight.decrementAndGet(); } }) .build(); @@ -181,6 +227,11 @@ public void close() { }; } + private static boolean matchesInFlight(HttpRequest request) { + final String marker = inFlightMarker; + return marker != null && request.getRequestLine().getUri().contains(marker); + } + /** * Throws, and records the host, unless the request's target is 127.0.0.1. The target host is the * one the client is about to connect to (set on the context before interceptors run, also on each @@ -210,9 +261,43 @@ private static void cleanup() { if (client == null) { return; } + for (ExecutorService executor : executors) { + executor.shutdownNow(); + } + for (B2StorageClient other : extraClients) { + try { + other.close(); + } catch (RuntimeException e) { + // nothing left to release + } + } + for (String keyId : createdKeys) { + try { + client.deleteKey(com.backblaze.b2.client.structures.B2DeleteKeyRequest.builder(keyId).build()); + } catch (B2Exception | RuntimeException e) { + cleanupErrors.add("key " + keyId + ": " + describe(e)); + } + } for (B2Bucket bucket : created) { try { - client.deleteAllFilesInBucket(bucket.getBucketId()); + // A large file left unfinished would keep the bucket from being deleted. + for (B2FileVersion unfinished : client.unfinishedLargeFiles(bucket.getBucketId())) { + client.cancelLargeFile(unfinished.getFileId()); + } + if (lockEnabled.contains(bucket.getBucketId())) { + // Files under a lock: a legal hold is released and governance retention is bypassed (the simulator's + // key holds that capability). Compliance retention is never set by a check, since nothing can lift it. + for (B2FileVersion version : client.fileVersions(bucket.getBucketId())) { + try { + client.updateFileLegalHold(B2UpdateFileLegalHoldRequest.builder(version.getFileName(), version.getFileId(), B2LegalHold.OFF).build()); + } catch (B2Exception e) { + // a hide marker has no legal hold to release + } + client.deleteFileVersion(B2DeleteFileVersionRequest.builder(version.getFileName(), version.getFileId()).setBypassGovernance(true).build()); + } + } else { + client.deleteAllFilesInBucket(bucket.getBucketId()); + } client.deleteBucket(bucket.getBucketId()); } catch (B2Exception | RuntimeException e) { cleanupErrors.add(bucket.getBucketName() + ": " + describe(e)); @@ -291,10 +376,44 @@ static B2Bucket bucket() throws B2Exception { return bucket; } + /** A fresh private bucket with Object Lock enabled; cleanup releases legal holds and bypasses governance retention. */ + static B2Bucket lockedBucket() throws B2Exception { + return bucket(true, null); + } + + /** A fresh private bucket made from a request the caller may customize; {@code lock} also enables Object Lock. */ + static B2Bucket bucket(boolean lock, java.util.function.Consumer customize) throws B2Exception { + final B2CreateBucketRequest.Builder builder = B2CreateBucketRequest.builder(newBucketName(), "allPrivate"); + if (lock) { + builder.setFileLockEnabled(true); + } + if (customize != null) { + customize.accept(builder); + } + final B2Bucket bucket = client.createBucket(builder.build()); + created.add(bucket); + if (lock) { + lockEnabled.add(bucket.getBucketId()); + } + return bucket; + } + static String newBucketName() { return "sdkharness-conf-" + UUID.randomUUID().toString().replace("-", "").substring(0, 12); } + private static final List createdKeys = new ArrayList<>(); + + /** Records an application key the check created, so cleanup deletes it if the check does not. */ + static void trackKey(String keyId) { + createdKeys.add(keyId); + } + + /** The check deleted this key itself. */ + static void untrackKey(String keyId) { + createdKeys.remove(keyId); + } + /** Records a bucket the check created itself, so cleanup deletes it if the check does not. */ static void track(B2Bucket bucket) { created.add(bucket); @@ -305,6 +424,37 @@ static void untrack(B2Bucket bucket) { created.removeIf(b -> b.getBucketId().equals(bucket.getBucketId())); } + /** A thread pool the SDK's large-file calls run on; shut down when the check ends. */ + static ExecutorService executor(int threads) { + final ExecutorService executor = Executors.newFixedThreadPool(threads); + executors.add(executor); + return executor; + } + + /** The part size to use: the larger of the account's recommended and absolute minimum part size, as the realm reports them. */ + static long partSize() { + try { + final long recommended = client.getAccountAuthorization().getRecommendedPartSize(); + final long minimum = client.getAccountAuthorization().getAbsoluteMinimumPartSize(); + return Math.max(recommended, minimum); + } catch (B2Exception e) { + throw new Failure("could not read the account's part sizes: " + describe(e)); + } + } + + /** How many recorded requests were "METHOD uri" with the given method and a URI containing {@code part}. */ + static int wireCount(String method, String part) { + int n = 0; + synchronized (wire) { + for (String line : wire) { + if (line.startsWith(method + " ") && line.contains(part)) { + n++; + } + } + } + return n; + } + static B2FileVersion upload(B2Bucket bucket, String name, byte[] data, String contentType, Map fileInfo) throws B2Exception { final B2UploadFileRequest.Builder request = B2UploadFileRequest @@ -346,6 +496,17 @@ static Downloaded downloadById(String fileId) throws B2Exception { return got; } + /** Download by id with server-side-encryption request headers (an SSE-C key); {@code sse} may be null. */ + static Downloaded downloadById(String fileId, com.backblaze.b2.client.structures.B2FileSseForRequest sse) throws B2Exception { + final Downloaded got = new Downloaded(); + client.downloadById(com.backblaze.b2.client.structures.B2DownloadByIdRequest.builder(fileId).setServerSideEncryption(sse).build(), + (headers, in) -> { + got.headers = headers; + got.bytes = in.readAllBytes(); + }); + return got; + } + static Downloaded downloadByName(String bucketName, String fileName) throws B2Exception { final Downloaded got = new Downloaded(); client.downloadByName(bucketName, fileName, (headers, in) -> { @@ -372,6 +533,42 @@ static List names(Iterable versions) { return names; } + /** What a plain HTTP GET answered. */ + static final class Raw { + int status; + byte[] bytes; + } + + /** + * A GET made by the check itself, not by the SDK (a share link is used by someone who has no SDK): refused + * unless its host is 127.0.0.1, recorded in {@link #blocked} like the SDK's own requests. {@code authorization} + * is sent as the Authorization header when not null. + */ + static Raw rawGet(String url, String authorization) throws java.io.IOException { + final String host = URI.create(url).getHost(); + if (!LOOPBACK_HOST.equals(host)) { + blocked.add(String.valueOf(host)); + throw new Failure("blocked non-loopback host: " + host); + } + final java.net.HttpURLConnection connection = (java.net.HttpURLConnection) URI.create(url).toURL().openConnection(); + try { + connection.setInstanceFollowRedirects(false); + connection.setConnectTimeout(10_000); + connection.setReadTimeout(30_000); + if (authorization != null) { + connection.setRequestProperty("Authorization", authorization); + } + final Raw raw = new Raw(); + raw.status = connection.getResponseCode(); + try (java.io.InputStream in = raw.status < 400 ? connection.getInputStream() : connection.getErrorStream()) { + raw.bytes = in == null ? new byte[0] : in.readAllBytes(); + } + return raw; + } finally { + connection.disconnect(); + } + } + static byte[] utf8(String s) { return s.getBytes(StandardCharsets.UTF_8); } diff --git a/.sdkharness/tests/conformance/UrlsNativeDownload.java b/.sdkharness/tests/conformance/UrlsNativeDownload.java new file mode 100644 index 000000000..bc1520640 --- /dev/null +++ b/.sdkharness/tests/conformance/UrlsNativeDownload.java @@ -0,0 +1,71 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2DownloadAuthorization; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2GetDownloadAuthorizationRequest; + +import java.util.Arrays; + +/** + * urls.native_download. Run by ../run-conformance (see ../../README.md). + * + *

The share link is B2's native one, never an S3 presigned URL. b2_get_download_authorization "Returns an + * authorization token that can be passed to ******* in the Authorization header or as an + * Authorization parameter", limited to a fileNamePrefix and valid for validDurationInSeconds + * (https://www.backblaze.com/apidocs/b2-get-download-authorization). ******* addresses + * {@code /file//}, answers a private file without a token with 401, and an expired token + * with 401 expired_auth_token (https://www.backblaze.com/apidocs/b2-download-file-by-name). + * + *

Asserted, over plain HTTP made by the check itself (the holder of a share link has no SDK): the SDK's + * {@code getDownloadByNameUrl} is a /file// URL with no X-Amz-* material; the file is 401 without a + * token; with the token (Authorization header) it is 200 and the bytes are right; a file outside the token's + * prefix is 401; after the validity has passed the same token is 401. + */ +public class UrlsNativeDownload { + private static final int VALID_SECONDS = 2; + + public static void main(String[] args) { + Support.run("urls.native_download", () -> { + Support.step("bucket"); + final B2Bucket bucket = Support.bucket(); + + Support.step("files"); + final byte[] data = Support.payload(300, 61); + final B2FileVersion shared = Support.upload(bucket, "st/share/file.bin", data, "application/octet-stream", null); + Support.upload(bucket, "st/other/file.bin", data, "application/octet-stream", null); + + Support.step("native url"); + final String url = Support.client.getDownloadByNameUrl(bucket.getBucketName(), shared.getFileName()); + Support.check(url.contains("/file/" + bucket.getBucketName() + "/st/share/file.bin"), "not a native /file// URL: " + url.replaceAll("\\?.*", "?...")); + Support.check(!url.contains("X-Amz-"), "the URL carries S3 presigning material"); + + Support.step("no token"); + Support.eq("a private file without a token", 401, Support.rawGet(url, null).status); + + Support.step("authorization"); + final B2DownloadAuthorization authorization = Support.client.getDownloadAuthorization( + B2GetDownloadAuthorizationRequest.builder(bucket.getBucketId(), "st/share/", VALID_SECONDS).build()); + Support.eq("authorization prefix", "st/share/", authorization.getFileNamePrefix()); + Support.eq("authorization bucket", bucket.getBucketId(), authorization.getBucketId()); + final long issued = System.nanoTime(); + + Support.step("token works within its validity"); + final Support.Raw ok = Support.rawGet(url, authorization.getAuthorizationToken()); + Support.eq("status with the token", 200, ok.status); + Support.check(Arrays.equals(data, ok.bytes), "the bytes fetched with the token differ from the upload"); + + Support.step("token is limited to its prefix"); + final String otherUrl = Support.client.getDownloadByNameUrl(bucket.getBucketName(), "st/other/file.bin"); + Support.eq("a file outside the token's prefix", 401, Support.rawGet(otherUrl, authorization.getAuthorizationToken()).status); + + Support.step("token expires"); + final long wait = VALID_SECONDS * 1000L + 1500 - (System.nanoTime() - issued) / 1_000_000; + if (wait > 0) { + Thread.sleep(wait); + } + Support.eq("status after the validity has passed", 401, Support.rawGet(url, authorization.getAuthorizationToken()).status); + }); + } +} diff --git a/.sdkharness/tests/run-conformance b/.sdkharness/tests/run-conformance index 38fdb2093..849dd5578 100755 --- a/.sdkharness/tests/run-conformance +++ b/.sdkharness/tests/run-conformance @@ -29,6 +29,25 @@ case "$SCENARIO" in files.hide) class=FilesHide ;; files.metadata) class=FilesMetadata ;; bucket.crud) class=BucketCrud ;; + files.server_side_copy) class=FilesServerSideCopy ;; + large.multipart) class=LargeMultipart ;; + large.concurrent_parts) class=LargeConcurrentParts ;; + large.resume) class=LargeResume ;; + large.compose_mixed) class=LargeComposeMixed ;; + urls.native_download) class=UrlsNativeDownload ;; + client.progress) class=ClientProgress ;; + lock.bucket_default) class=LockBucketDefault ;; + lock.per_file_retention) class=LockPerFileRetention ;; + keys.crud) class=KeysCrud ;; + bucket.cors) class=BucketCors ;; + bucket.lifecycle) class=BucketLifecycle ;; + bucket.notification_rules) class=BucketNotificationRules ;; + bucket.inbound_webhook) class=BucketInboundWebhook ;; + bucket.replication_config) class=BucketReplicationConfig ;; + enc.sse_b2) class=EncSseB2 ;; + enc.sse_c) class=EncSseC ;; + lock.legal_hold) class=LockLegalHold ;; + lock.bypass_governance) class=LockBypassGovernance ;; *) fail 'configuration: no check for this scenario' ;; esac [[ "${SDKHARNESS_SIMULATOR_URL:-}" =~ ^http://127\.0\.0\.1:[0-9]{1,5}/?$ ]] \ @@ -44,7 +63,7 @@ for name in $(compgen -e); do case "$name" in B2_*) unset "$name" ;; esac; done work="$(mktemp -d "${TMPDIR:-/tmp}/sdkharness-java-conformance.XXXXXX")" || fail 'setup: could not create a scratch directory' trap 'rm -rf "$work"' EXIT -if ! javac --release 11 -nowarn -d "$work/classes" -cp "$SDKHARNESS_JAVA_CLASSPATH" \ +if ! javac --release 11 -encoding UTF-8 -nowarn -d "$work/classes" -cp "$SDKHARNESS_JAVA_CLASSPATH" \ "$HERE/conformance/Support.java" "$HERE/conformance/$class.java" >"$work/javac.log" 2>&1; then fail "build: the check did not compile against this checkout: $(tidy "$(head -c 400 "$work/javac.log")")" fi diff --git a/.sdkharness/tests/run-resilience b/.sdkharness/tests/run-resilience index 328424697..66a8a7805 100755 --- a/.sdkharness/tests/run-resilience +++ b/.sdkharness/tests/run-resilience @@ -66,7 +66,7 @@ cleanup() { } trap cleanup EXIT -if ! javac --release 11 -nowarn -d "$work/classes" -cp "$SDKHARNESS_JAVA_CLASSPATH" \ +if ! javac --release 11 -encoding UTF-8 -nowarn -d "$work/classes" -cp "$SDKHARNESS_JAVA_CLASSPATH" \ "$HERE/resilience/Support.java" "$HERE/resilience/$class.java" >"$work/javac.log" 2>&1; then fail "build: the check did not compile against this checkout: $(tidy "$(head -c 400 "$work/javac.log")")" fi From 12af861eefa57088c598694987a83c33429a6e76 Mon Sep 17 00:00:00 2001 From: scarreras Date: Tue, 20 Oct 2026 13:00:19 -0600 Subject: [PATCH 2/2] sdkharness: address review on conformance checks - large.resume: drop the retry that re-supplied sha1 as the custom field; the original request is kept and the scenario fails with the FINDING as its reason when the SDK refuses it with "mismatch". - keys.crud: assert revocation through a fresh authorization (getAccountAuthorization) with the deleted key; the earlier authorization of the live restricted key is the control. - Support.rawGet: parse the URL once and validate and connect on the same URI. - large.concurrent_parts: reset inFlight beside maxInFlight. Co-Authored-By: Claude Sonnet 5.5 --- .sdkharness/README.md | 6 ++-- .sdkharness/tests/conformance/KeysCrud.java | 4 ++- .../conformance/LargeConcurrentParts.java | 1 + .../tests/conformance/LargeResume.java | 31 +++++++------------ .sdkharness/tests/conformance/Support.java | 5 +-- 5 files changed, 21 insertions(+), 26 deletions(-) diff --git a/.sdkharness/README.md b/.sdkharness/README.md index 16a9a08bc..2ade014f6 100644 --- a/.sdkharness/README.md +++ b/.sdkharness/README.md @@ -55,13 +55,13 @@ deletion, and a deletion that fails turns the verdict into a `FAIL` (`cleanup -- | `bucket.inbound_webhook` | offline: a payload signed in the check with the documented HMAC is accepted and parsed by `B2EventNotification.parse`; a wrong secret, a changed payload and a malformed signature are refused; a test event parses | | `bucket.replication_config` | a source rule (destination bucket, prefix, priority, flags) with a replication key is set and read back field for field | -`large.resume` records one SDK behavior on a `NOTE ... FINDING` line. When the content source knows its sha1, the +`large.resume` reports one SDK behavior as a `FAIL` whose reason starts with `FINDING`. When the content source knows its sha1, the SDK stores it as the file info `large_file_sha1`; `finishUploadingLargeFile` then compares the request's file info with the stored one after removing the key `X-Bz-Info-large_file_sha1` (`B2Headers.LARGE_FILE_SHA1`, `B2LargeFileUploader.throwIfLargeFileVersionDoesntSeemToMatchRequest`), but the stored key is `large_file_sha1` (`B2Headers.LARGE_FILE_SHA1_INFO_NAME`), so the same request that started the file is refused with a -`mismatch` error. The check then resumes with `large_file_sha1` added to the request's custom fields, which the SDK -accepts, and asserts the rest of the scenario; if the SDK is fixed the first attempt succeeds and the note disappears. +`mismatch` error. The check does not work around that refusal: the scenario fails with the finding as its reason (the +harness detects, it does not fix the SDK) and passes once the SDK accepts the same request. Every check that locks files releases them on exit (legal hold off, governance retention bypassed); none uses compliance mode, which nothing could lift. diff --git a/.sdkharness/tests/conformance/KeysCrud.java b/.sdkharness/tests/conformance/KeysCrud.java index 24e4a07f8..a7ba22bc5 100644 --- a/.sdkharness/tests/conformance/KeysCrud.java +++ b/.sdkharness/tests/conformance/KeysCrud.java @@ -80,8 +80,10 @@ public static void main(String[] args) { Support.client.deleteKey(B2DeleteKeyRequest.builder(created.getApplicationKeyId()).build()); Support.untrackKey(created.getApplicationKeyId()); Support.check(find(created.getApplicationKeyId()) == null, "the deleted key is still listed"); + // Revocation is asserted through authorization itself (a fresh b2_authorize_account). A bucket-restricted key + // calling buckets() gets 401 even when valid, so that call cannot tell a revoked key from a restricted one. Support.eq("status of authorizing with a deleted key", 401, Support.expectError("authorizing with a deleted key", - () -> Support.clientFor(created.getApplicationKeyId(), secret).buckets()).getStatus()); + () -> Support.clientFor(created.getApplicationKeyId(), secret).getAccountAuthorization()).getStatus()); }); } diff --git a/.sdkharness/tests/conformance/LargeConcurrentParts.java b/.sdkharness/tests/conformance/LargeConcurrentParts.java index 0900a08e5..57723760f 100644 --- a/.sdkharness/tests/conformance/LargeConcurrentParts.java +++ b/.sdkharness/tests/conformance/LargeConcurrentParts.java @@ -50,6 +50,7 @@ public static void main(String[] args) { Support.step("four workers"); Support.maxInFlight.set(0); + Support.inFlight.set(0); final long fourMillis = upload(bucket, "st/large/parallel.bin", data, sha1, Support.executor(4)); final int parallelMax = Support.maxInFlight.get(); Support.check(parallelMax > 1, "with four workers at most " + parallelMax + " b2_upload_part request(s) were ever in flight at once"); diff --git a/.sdkharness/tests/conformance/LargeResume.java b/.sdkharness/tests/conformance/LargeResume.java index 5c7f02920..53be57139 100644 --- a/.sdkharness/tests/conformance/LargeResume.java +++ b/.sdkharness/tests/conformance/LargeResume.java @@ -38,9 +38,9 @@ *

Two legs, because the SDK treats the two content shapes differently. Leg "no sha1": the content source does * not know its sha1, so the large file carries no large_file_sha1 and the rerun is the same request as the first * run. Leg "sha1": the content source knows its sha1 (the SDK records it as the large_file_sha1 file info); the - * same request is first tried as-is, and, if the SDK refuses it with a "mismatch" error, the check records that on - * a NOTE line and resumes with the large_file_sha1 custom field added to the request (which the SDK then accepts). - * The NOTE is a finding about the SDK, not about B2: B2LargeFileUploader removes the key + * rerun is again the same request as the first run. If the SDK refuses it with a "mismatch" error, the scenario + * FAILS with a FINDING as the reason: the check detects, it does not work around the refusal. The finding is about + * the SDK, not about B2: B2LargeFileUploader removes the key * {@code X-Bz-Info-large_file_sha1} (B2Headers.LARGE_FILE_SHA1) from the file info before comparing it with the * request's, but the info map is keyed {@code large_file_sha1} (B2Headers.LARGE_FILE_SHA1_INFO_NAME). */ @@ -68,7 +68,7 @@ private static void resume(B2Bucket bucket, String name, byte[] data, String sha Support.wire.clear(); try { - Support.client.uploadLargeFile(request(bucket, name, new Source(data, sha1, 2 * partSize), null), Support.executor(1)); + Support.client.uploadLargeFile(request(bucket, name, new Source(data, sha1, 2 * partSize)), Support.executor(1)); Support.fail(leg + "the interrupted upload finished instead of failing"); } catch (B2Exception expected) { Support.note(leg + "the first upload stopped as intended: " + Support.describe(expected)); @@ -88,20 +88,16 @@ private static void resume(B2Bucket bucket, String name, byte[] data, String sha Support.check(storedBefore < PARTS, leg + "all " + PARTS + " parts were stored; nothing is left to resume"); Support.wire.clear(); - B2FileVersion finished = null; + final B2FileVersion finished; try { - finished = Support.client.finishUploadingLargeFile(pending, request(bucket, name, new Source(data, sha1, -1), null), Support.executor(2)); + finished = Support.client.finishUploadingLargeFile(pending, request(bucket, name, new Source(data, sha1, -1)), Support.executor(2)); } catch (B2LocalException e) { if (sha1 == null || !"mismatch".equals(e.getCode())) { throw e; } - Support.note("FINDING b2-sdk-java: finishUploadingLargeFile refused the same request that started the large file: " - + e.getMessage() + ". It resumes only when the request carries the large_file_sha1 custom field itself."); - } - if (finished == null) { - Support.eq(leg + "b2_upload_part requests after the refused resume", 0, Support.wireCount("POST", "b2_upload_part")); - finished = Support.client.finishUploadingLargeFile(pending, - request(bucket, name, new Source(data, sha1, -1), sha1), Support.executor(2)); + Support.fail("FINDING b2-sdk-java: finishUploadingLargeFile refused the same request that started the large file: " + + e.getMessage()); + return; } Support.eq(leg + "b2_start_large_file requests in the rerun", 0, Support.wireCount("POST", "b2_start_large_file")); Support.eq(leg + "finished fileId is the unfinished one", pending.getFileId(), finished.getFileId()); @@ -122,13 +118,8 @@ private static List unfinished(B2Bucket bucket, String name) thro return all; } - /** {@code sha1Field}, when not null, is added to the request as the large_file_sha1 custom field. */ - private static B2UploadFileRequest request(B2Bucket bucket, String name, B2ContentSource source, String sha1Field) { - final B2UploadFileRequest.Builder builder = B2UploadFileRequest.builder(bucket.getBucketId(), name, "application/octet-stream", source); - if (sha1Field != null) { - builder.setCustomField("large_file_sha1", sha1Field); - } - return builder.build(); + private static B2UploadFileRequest request(B2Bucket bucket, String name, B2ContentSource source) { + return B2UploadFileRequest.builder(bucket.getBucketId(), name, "application/octet-stream", source).build(); } /** The content; with {@code failAt >= 0}, reading past that byte offset fails the way an interrupted run would. */ diff --git a/.sdkharness/tests/conformance/Support.java b/.sdkharness/tests/conformance/Support.java index c6ddedc4f..b496e7e2d 100644 --- a/.sdkharness/tests/conformance/Support.java +++ b/.sdkharness/tests/conformance/Support.java @@ -545,12 +545,13 @@ static final class Raw { * is sent as the Authorization header when not null. */ static Raw rawGet(String url, String authorization) throws java.io.IOException { - final String host = URI.create(url).getHost(); + final URI uri = URI.create(url); + final String host = uri.getHost(); if (!LOOPBACK_HOST.equals(host)) { blocked.add(String.valueOf(host)); throw new Failure("blocked non-loopback host: " + host); } - final java.net.HttpURLConnection connection = (java.net.HttpURLConnection) URI.create(url).toURL().openConnection(); + final java.net.HttpURLConnection connection = (java.net.HttpURLConnection) uri.toURL().openConnection(); try { connection.setInstanceFollowRedirects(false); connection.setConnectTimeout(10_000);