diff --git a/.sdkharness/README.md b/.sdkharness/README.md index fe6b78d92..2ade014f6 100644 --- a/.sdkharness/README.md +++ b/.sdkharness/README.md @@ -5,7 +5,7 @@ assertions in `tests/`; the harness builds this checkout, starts a fresh local B2 simulator, and retains and reports the evidence. -Scenarios so far: `health/golden-path` and eight `conformance` and sixteen `resilience` scenarios (below). `tests/run-health` runs +Scenarios so far: `health/golden-path` and 27 `conformance` (one per capability the SDK claims) and sixteen `resilience` scenarios (below). `tests/run-health` runs `tests/health/GoldenPath.java` as a single-file Java program that authorizes, uploads a small file, downloads it and compares the bytes, lists it, deletes it and confirms it is gone, then prints one result line: @@ -35,6 +35,36 @@ deletion, and a deletion that fails turns the verdict into a `FAIL` (`cleanup -- | `files.hide` | hide marker is a new version; not in the default listing, visible in the version listing; 404 by name; the original still downloads by id | | `bucket.crud` | create, duplicate name refused, list, update type and info (revision grows), delete refused while non-empty, delete | | `files.metadata` | by id (`b2_get_file_info`) and by name (an HTTP HEAD of the download URL: the cell is `partial`); the check records the SDK's own requests and asserts that boundary | +| `files.server_side_copy` | `b2_copy_file` sends no bytes (no `b2_upload_file`), same sha1 under a new fileId, the COPY and REPLACE metadata directives, a ranged copy | +| `large.multipart` | `uploadLargeFile` of three parts' worth: one start, more than one `b2_upload_part`, one finish, contentSha1 `none`, `large_file_sha1` carried, bytes intact | +| `large.concurrent_parts` | the same eight-part file on a one-thread and a four-thread executor: the SDK's own HTTP client counts `b2_upload_part` requests in flight (1 with one worker, more than 1 with four; wall time is only a `NOTE`) | +| `large.resume` | a first upload interrupted after two parts, then `finishUploadingLargeFile` on the unfinished file: no new `b2_start_large_file`, only the missing parts, same fileId, bytes intact; run with and without a known sha1 (see the note below) | +| `large.compose_mixed` | one large file from local bytes, a copied byte range and a whole copied file (`storeLargeFile` with `B2UploadingPartStorer` and `B2CopyingPartStorer`), equal to their ordered concatenation | +| `urls.native_download` | `getDownloadByNameUrl` is a native `/file//` URL (no `X-Amz-*`); 401 without a token, 200 with a download authorization, 401 outside its prefix and after it expires | +| `client.progress` | a `B2UploadListener` on a large upload (one worker): it fires, every part's bytes and the running total never go down, every part ends SUCCEEDED, the total equals the byte count | +| `lock.bucket_default` | a default retention of governance, 1 day on a lock-enabled bucket: read back, and an uploaded file inherits a retention of about now + 1 day | +| `lock.per_file_retention` | governance retention now + 2 days on a file: read back, and a delete without bypass is refused (401 `access_denied`) and leaves the file | +| `lock.legal_hold` | hold on: read back and the delete is refused; hold off: read back and the delete succeeds | +| `lock.bypass_governance` | under governance retention a delete is refused (401 `access_denied`), `setBypassGovernance(true)` deletes the version | +| `enc.sse_b2` | an SSE-B2 upload reports SSE-B2 / AES256 and downloads as plaintext; a bucket default of SSE-B2 applies to a plain upload | +| `enc.sse_c` | an SSE-C upload reports SSE-C / AES256; download with the key returns the plaintext and the key's MD5; without it and with another key it is refused (403 for a wrong key) | +| `keys.crud` | a bucket-restricted key is created (secret returned), listed (no secret anywhere in what the SDK returns), used by a second client, deleted (gone, and its secret no longer authorizes) | +| `bucket.cors` | typed `B2CorsRule`: set at create, replaced by update, cleared; read back field for field | +| `bucket.lifecycle` | `daysFromHidingToDeleting` 1 round trips with no other field populated; a replacement list replaces; an empty list clears | +| `bucket.notification_rules` | a webhook rule with a signing secret is set, read back (the secret unchanged) and deleted | +| `bucket.inbound_webhook` | offline: a payload signed in the check with the documented HMAC is accepted and parsed by `B2EventNotification.parse`; a wrong secret, a changed payload and a malformed signature are refused; a test event parses | +| `bucket.replication_config` | a source rule (destination bucket, prefix, priority, flags) with a replication key is set and read back field for field | + +`large.resume` reports one SDK behavior as a `FAIL` whose reason starts with `FINDING`. When the content source knows its sha1, the +SDK stores it as the file info `large_file_sha1`; `finishUploadingLargeFile` then compares the request's file info +with the stored one after removing the key `X-Bz-Info-large_file_sha1` (`B2Headers.LARGE_FILE_SHA1`, +`B2LargeFileUploader.throwIfLargeFileVersionDoesntSeemToMatchRequest`), but the stored key is +`large_file_sha1` (`B2Headers.LARGE_FILE_SHA1_INFO_NAME`), so the same request that started the file is refused with a +`mismatch` error. The check does not work around that refusal: the scenario fails with the finding as its reason (the +harness detects, it does not fix the SDK) and passes once the SDK accepts the same request. + +Every check that locks files releases them on exit (legal hold off, governance retention bypassed); none uses +compliance mode, which nothing could lift. A failing scenario prints one `SDKHARNESS_RESULT ... FAIL` line with the step and the reason. Details are formatted to one capped printable line from exception class names, status codes and the check's own diff --git a/.sdkharness/tests.tsv b/.sdkharness/tests.tsv index 5dc227f1b..87e929c20 100644 --- a/.sdkharness/tests.tsv +++ b/.sdkharness/tests.tsv @@ -8,6 +8,25 @@ conformance files.delete_version simulator ./.sdkharness/tests/run-conformance conformance files.hide simulator ./.sdkharness/tests/run-conformance conformance bucket.crud simulator ./.sdkharness/tests/run-conformance conformance files.metadata simulator ./.sdkharness/tests/run-conformance +conformance files.server_side_copy simulator ./.sdkharness/tests/run-conformance +conformance large.multipart simulator ./.sdkharness/tests/run-conformance +conformance large.concurrent_parts simulator ./.sdkharness/tests/run-conformance +conformance large.resume simulator ./.sdkharness/tests/run-conformance +conformance large.compose_mixed simulator ./.sdkharness/tests/run-conformance +conformance urls.native_download simulator ./.sdkharness/tests/run-conformance +conformance client.progress simulator ./.sdkharness/tests/run-conformance +conformance lock.bucket_default simulator ./.sdkharness/tests/run-conformance +conformance lock.per_file_retention simulator ./.sdkharness/tests/run-conformance +conformance lock.legal_hold simulator ./.sdkharness/tests/run-conformance +conformance lock.bypass_governance simulator ./.sdkharness/tests/run-conformance +conformance enc.sse_b2 simulator ./.sdkharness/tests/run-conformance +conformance enc.sse_c simulator ./.sdkharness/tests/run-conformance +conformance keys.crud simulator ./.sdkharness/tests/run-conformance +conformance bucket.cors simulator ./.sdkharness/tests/run-conformance +conformance bucket.lifecycle simulator ./.sdkharness/tests/run-conformance +conformance bucket.notification_rules simulator ./.sdkharness/tests/run-conformance +conformance bucket.inbound_webhook simulator ./.sdkharness/tests/run-conformance +conformance bucket.replication_config simulator ./.sdkharness/tests/run-conformance resilience upload.retry_503 simulator ./.sdkharness/tests/run-resilience resilience upload.retry_500 simulator ./.sdkharness/tests/run-resilience resilience upload.retry_408 simulator ./.sdkharness/tests/run-resilience diff --git a/.sdkharness/tests/conformance/BucketCors.java b/.sdkharness/tests/conformance/BucketCors.java new file mode 100644 index 000000000..ad262a1f9 --- /dev/null +++ b/.sdkharness/tests/conformance/BucketCors.java @@ -0,0 +1,82 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2CorsRule; +import com.backblaze.b2.client.structures.B2UpdateBucketRequest; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +/** + * bucket.cors. Run by ../run-conformance (see ../../README.md). + * + *

Every field sent is a documented rule field (corsRuleName, allowedOrigins, allowedOperations, allowedHeaders, + * exposeHeaders, maxAgeSeconds), and b2_download_file_by_name and b2_download_file_by_id are documented + * allowedOperations values. Rules can be "set when you create the bucket with b2_create_bucket, or updated on an + * existing bucket using b2_update_bucket", and b2_list_buckets returns corsRules + * (https://www.backblaze.com/docs/cloud-storage-cross-origin-resource-sharing-rules, + * https://www.backblaze.com/apidocs/b2-update-bucket, https://www.backblaze.com/apidocs/b2-list-buckets). + * + *

This SDK models a rule as the typed {@code B2CorsRule} (the card's cell is "yes", not "partial"). Asserted: a + * bucket created with one rule reports it, field for field, when listed; updating the bucket to a different pair of + * rules replaces the set (the read-back is exactly the new pair); updating to an empty list removes every rule. + */ +public class BucketCors { + public static void main(String[] args) { + Support.run("bucket.cors", () -> { + final B2CorsRule first = B2CorsRule.builder("sdkharness-first", Arrays.asList("https://example.com", "https://app.example.com"), + new HashSet<>(Arrays.asList("b2_download_file_by_name", "b2_download_file_by_id")), 3600) + .setAllowedHeaders(Arrays.asList("range", "authorization")) + .setExposeHeaders(Arrays.asList("x-bz-content-sha1")) + .build(); + + Support.step("create with a rule"); + final B2Bucket bucket = Support.bucket(false, b -> b.setCorsRules(Collections.singletonList(first))); + Support.step("read back the created rule"); + assertRules("rules on create", Collections.singletonList(first), bucket.getCorsRules()); + assertRules("rules when listed", Collections.singletonList(first), listed(bucket).getCorsRules()); + + Support.step("replace the rules"); + final B2CorsRule second = B2CorsRule.builder("sdkharness-second", Collections.singletonList("*"), + new HashSet<>(Collections.singletonList("b2_download_file_by_name")), 60).build(); + final B2CorsRule third = B2CorsRule.builder("sdkharness-third", Collections.singletonList("https://other.example.com"), + new HashSet<>(Collections.singletonList("b2_download_file_by_id")), 120) + .setAllowedHeaders(Collections.singletonList("*")).build(); + final B2Bucket replaced = Support.client.updateBucket(B2UpdateBucketRequest.builder(bucket).setCorsRules(Arrays.asList(second, third)).build()); + assertRules("rules after the update", Arrays.asList(second, third), listed(bucket).getCorsRules()); + + Support.step("remove the rules"); + // updateBucket is optimistic: each update names the revision it was made from, so it starts from the last response. + Support.client.updateBucket(B2UpdateBucketRequest.builder(replaced).setCorsRules(Collections.emptyList()).build()); + Support.eq("rules after clearing", 0, listed(bucket).getCorsRules().size()); + }); + } + + private static B2Bucket listed(B2Bucket bucket) throws Exception { + final B2Bucket found = Support.client.getBucketOrNullByName(bucket.getBucketName()); + Support.check(found != null, "the bucket is not listed"); + return found; + } + + /** Field for field, in order. */ + private static void assertRules(String what, List expected, List actual) { + Support.eq(what + ": number of rules", expected.size(), actual.size()); + for (int i = 0; i < expected.size(); i++) { + final B2CorsRule e = expected.get(i); + final B2CorsRule a = actual.get(i); + Support.eq(what + ": corsRuleName", e.getCorsRuleName(), a.getCorsRuleName()); + Support.eq(what + ": allowedOrigins of " + e.getCorsRuleName(), e.getAllowedOrigins(), a.getAllowedOrigins()); + Support.eq(what + ": allowedOperations of " + e.getCorsRuleName(), new HashSet<>(e.getAllowedOperations()), new HashSet<>(a.getAllowedOperations())); + Support.eq(what + ": allowedHeaders of " + e.getCorsRuleName(), nullToEmpty(e.getAllowedHeaders()), nullToEmpty(a.getAllowedHeaders())); + Support.eq(what + ": exposeHeaders of " + e.getCorsRuleName(), nullToEmpty(e.getExposeHeaders()), nullToEmpty(a.getExposeHeaders())); + Support.eq(what + ": maxAgeSeconds of " + e.getCorsRuleName(), e.getMaxAgeSeconds(), a.getMaxAgeSeconds()); + } + } + + private static List nullToEmpty(List list) { + return list == null ? Collections.emptyList() : list; + } +} diff --git a/.sdkharness/tests/conformance/BucketInboundWebhook.java b/.sdkharness/tests/conformance/BucketInboundWebhook.java new file mode 100644 index 000000000..c51064f40 --- /dev/null +++ b/.sdkharness/tests/conformance/BucketInboundWebhook.java @@ -0,0 +1,105 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.exceptions.B2SignatureVerificationException; +import com.backblaze.b2.client.structures.B2EventNotification; +import com.backblaze.b2.client.structures.B2EventNotificationEvent; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import java.nio.charset.StandardCharsets; + +/** + * bucket.inbound_webhook. Run by ../run-conformance (see ../../README.md). + * + *

The X-Bz-Event-Notification-Signature value "comprises the signature version, always v1, followed by an equals + * sign =, followed by the lowercase hex-encoded signature value", and to verify it "create an HMAC SHA-256 digest + * using the hmacSha256SigningSecret ... as the key, with the request payload as the data. The resulting hex-encoded + * HMAC digest value must match the signature in the header." The payload's event fields are the documented ones + * (https://www.backblaze.com/docs/cloud-storage-event-notifications-reference-guide). + * + *

The SDK's part of this capability is parsing and verifying what a service receives, so nothing here needs a + * server round trip: a payload shaped like the documented example is signed in this check with its own HMAC code (not + * the SDK's), then given to {@code B2EventNotification.parse}. Asserted: with the right secret the payload is accepted + * and every documented event field is read back; with another secret, with the payload changed after signing, and + * with a signature that is not v1 hex, it is refused with {@code B2SignatureVerificationException}; a test event (no + * object fields) parses. + */ +public class BucketInboundWebhook { + private static final String PAYLOAD = "{\"events\":[{" + + "\"accountId\":\"sdkharness-account\",\"bucketId\":\"sdkharness-bucket-id\",\"bucketName\":\"sdkharness-conf-bucket\"," + + "\"eventId\":\"sdkharness-event-1\",\"eventTimestamp\":1761000000000,\"eventType\":\"b2:ObjectCreated:Upload\"," + + "\"eventVersion\":1,\"matchedRuleName\":\"sdkharness-rule\",\"objectName\":\"st/new.txt\",\"objectSize\":1234," + + "\"objectVersionId\":\"sdkharness-version-1\"}]}"; + private static final String TEST_PAYLOAD = "{\"events\":[{" + + "\"accountId\":\"sdkharness-account\",\"bucketId\":\"sdkharness-bucket-id\",\"bucketName\":\"sdkharness-conf-bucket\"," + + "\"eventId\":\"sdkharness-event-2\",\"eventTimestamp\":1761000000001,\"eventType\":\"b2:TestEvent\"," + + "\"eventVersion\":1,\"matchedRuleName\":\"sdkharness-rule\"}]}"; + private static final String SECRET = "*******"; + + public static void main(String[] args) { + Support.run("bucket.inbound_webhook", () -> { + Support.step("accept a correctly signed payload"); + final byte[] body = PAYLOAD.getBytes(StandardCharsets.UTF_8); + final B2EventNotification accepted = B2EventNotification.parse(body, sign(body, SECRET), SECRET); + Support.eq("number of events", 1, accepted.getEvents().size()); + final B2EventNotificationEvent event = accepted.getEvents().get(0); + Support.eq("accountId", "sdkharness-account", event.getAccountId()); + Support.eq("bucketId", "sdkharness-bucket-id", event.getBucketId()); + Support.eq("bucketName", "sdkharness-conf-bucket", event.getBucketName()); + Support.eq("eventId", "sdkharness-event-1", event.getEventId()); + Support.eq("eventTimestamp", 1761000000000L, event.getEventTimestamp()); + Support.eq("eventType", "b2:ObjectCreated:Upload", event.getEventType()); + Support.eq("eventVersion", 1, event.getEventVersion()); + Support.eq("matchedRuleName", "sdkharness-rule", event.getMatchedRuleName()); + Support.eq("objectName", "st/new.txt", event.getObjectName()); + Support.eq("objectSize", 1234L, event.getObjectSize()); + Support.eq("objectVersionId", "sdkharness-version-1", event.getObjectVersionId()); + + Support.step("the string overload agrees"); + Support.eq("event id through the string overload", "sdkharness-event-1", + B2EventNotification.parse(PAYLOAD, sign(body, SECRET), SECRET).getEvents().get(0).getEventId()); + + Support.step("refuse a wrong secret"); + refused("a payload verified with another secret", () -> B2EventNotification.parse(body, sign(body, SECRET), "another-secret")); + + Support.step("refuse a changed payload"); + final byte[] changed = PAYLOAD.replace("st/new.txt", "st/evil.txt").getBytes(StandardCharsets.UTF_8); + refused("a payload changed after signing", () -> B2EventNotification.parse(changed, sign(body, SECRET), SECRET)); + + Support.step("refuse a signature that is not v1 hex"); + refused("a signature without the v1= prefix", () -> B2EventNotification.parse(body, sign(body, SECRET).substring(3), SECRET)); + refused("an empty signature", () -> B2EventNotification.parse(body, "", SECRET)); + + Support.step("a test event"); + final byte[] test = TEST_PAYLOAD.getBytes(StandardCharsets.UTF_8); + final B2EventNotificationEvent testEvent = B2EventNotification.parse(test, sign(test, SECRET), SECRET).getEvents().get(0); + Support.eq("test event type", "b2:TestEvent", testEvent.getEventType()); + Support.eq("test event object name", null, testEvent.getObjectName()); + }); + } + + private interface Parse { + void run() throws Exception; + } + + private static void refused(String what, Parse parse) throws Exception { + try { + parse.run(); + } catch (B2SignatureVerificationException expected) { + return; + } + Support.fail(what + " was accepted"); + } + + /** The documented signature: "v1=" and the lowercase hex of HMAC-SHA256(secret, payload). Not the SDK's code. */ + private static String sign(byte[] payload, String secret) throws Exception { + final Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(secret.getBytes(StandardCharsets.UTF_8), "HmacSHA256")); + final StringBuilder hex = new StringBuilder("v1="); + for (byte b : mac.doFinal(payload)) { + hex.append(String.format("%02x", b)); + } + return hex.toString(); + } +} diff --git a/.sdkharness/tests/conformance/BucketLifecycle.java b/.sdkharness/tests/conformance/BucketLifecycle.java new file mode 100644 index 000000000..fa52fa82f --- /dev/null +++ b/.sdkharness/tests/conformance/BucketLifecycle.java @@ -0,0 +1,74 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2LifecycleRule; +import com.backblaze.b2.client.structures.B2UpdateBucketRequest; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * bucket.lifecycle. Run by ../run-conformance (see ../../README.md). + * + *

daysFromHidingToDeleting and fileNamePrefix are documented rule fields; B2's own example rule is + * daysFromHidingToDeleting 1 with daysFromUploadingToHiding null. "Unspecified properties are treated as having a + * null value", which is the expected value of every field not set. b2_update_bucket lifecycleRules: "If specified, + * the existing lifecycle rules will be replaced with this new list." + * (https://www.backblaze.com/docs/cloud-storage-lifecycle-rules, https://www.backblaze.com/apidocs/b2-update-bucket). + * + *

Asserted: a rule {prefix st/, daysFromHidingToDeleting 1} set on an existing bucket is read back with the same + * values and no other field populated; setting a different list replaces the first (the read-back is exactly the new + * list, all three day fields included); an empty list removes every rule. + */ +public class BucketLifecycle { + public static void main(String[] args) { + Support.run("bucket.lifecycle", () -> { + Support.step("bucket"); + final B2Bucket bucket = Support.bucket(); + Support.eq("rules on a new bucket", 0, bucket.getLifecycleRules().size()); + + Support.step("set one rule"); + final B2LifecycleRule rule = B2LifecycleRule.builder("st/").setDaysFromHidingToDeleting(1).build(); + final B2Bucket withRule = Support.client.updateBucket(B2UpdateBucketRequest.builder(bucket).setLifecycleRules(Collections.singletonList(rule)).build()); + final List read = listed(bucket).getLifecycleRules(); + Support.eq("number of rules", 1, read.size()); + Support.eq("fileNamePrefix", "st/", read.get(0).getFileNamePrefix()); + Support.eq("daysFromHidingToDeleting", 1, read.get(0).getDaysFromHidingToDeleting()); + Support.eq("daysFromUploadingToHiding (not set)", null, read.get(0).getDaysFromUploadingToHiding()); + Support.eq("daysFromStartingToCancelingUnfinishedLargeFiles (not set)", null, read.get(0).getDaysFromStartingToCancelingUnfinishedLargeFiles()); + + Support.step("replace the rules"); + final B2LifecycleRule hide = B2LifecycleRule.builder("logs/").setDaysFromUploadingToHiding(30).setDaysFromHidingToDeleting(7).build(); + final B2LifecycleRule cancel = B2LifecycleRule.builder("big/").setDaysFromStartingToCancelingUnfinishedLargeFiles(2).build(); + // updateBucket is optimistic: each update names the revision it was made from, so it starts from the last response. + final B2Bucket replacedBucket = Support.client.updateBucket(B2UpdateBucketRequest.builder(withRule).setLifecycleRules(Arrays.asList(hide, cancel)).build()); + final List replaced = listed(bucket).getLifecycleRules(); + Support.eq("number of rules after the replacement", 2, replaced.size()); + for (B2LifecycleRule expected : Arrays.asList(hide, cancel)) { + B2LifecycleRule actual = null; + for (B2LifecycleRule candidate : replaced) { + if (expected.getFileNamePrefix().equals(candidate.getFileNamePrefix())) { + actual = candidate; + } + } + Support.check(actual != null, "no rule for prefix " + expected.getFileNamePrefix() + " after the replacement"); + Support.eq(expected.getFileNamePrefix() + " daysFromUploadingToHiding", expected.getDaysFromUploadingToHiding(), actual.getDaysFromUploadingToHiding()); + Support.eq(expected.getFileNamePrefix() + " daysFromHidingToDeleting", expected.getDaysFromHidingToDeleting(), actual.getDaysFromHidingToDeleting()); + Support.eq(expected.getFileNamePrefix() + " daysFromStartingToCancelingUnfinishedLargeFiles", + expected.getDaysFromStartingToCancelingUnfinishedLargeFiles(), actual.getDaysFromStartingToCancelingUnfinishedLargeFiles()); + } + + Support.step("remove the rules"); + Support.client.updateBucket(B2UpdateBucketRequest.builder(replacedBucket).setLifecycleRules(Collections.emptyList()).build()); + Support.eq("rules after clearing", 0, listed(bucket).getLifecycleRules().size()); + }); + } + + private static B2Bucket listed(B2Bucket bucket) throws Exception { + final B2Bucket found = Support.client.getBucketOrNullByName(bucket.getBucketName()); + Support.check(found != null, "the bucket is not listed"); + return found; + } +} diff --git a/.sdkharness/tests/conformance/BucketNotificationRules.java b/.sdkharness/tests/conformance/BucketNotificationRules.java new file mode 100644 index 000000000..ad42d111d --- /dev/null +++ b/.sdkharness/tests/conformance/BucketNotificationRules.java @@ -0,0 +1,67 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2EventNotificationRule; +import com.backblaze.b2.client.structures.B2SetBucketNotificationRulesRequest; +import com.backblaze.b2.client.structures.B2WebhookConfiguration; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.TreeSet; + +/** + * bucket.notification_rules. Run by ../run-conformance (see ../../README.md). + * + *

b2_get_bucket_notification_rules lists hmacSha256SigningSecret as a response field and its example response + * carries it unredacted, so the secret is expected back unchanged + * (https://www.backblaze.com/apidocs/b2-get-bucket-notification-rules); b2_set_bucket_notification_rules replaces the + * whole rule set (https://www.backblaze.com/apidocs/b2-set-bucket-notification-rules). The webhook URL is never + * called: the rule only has to be stored and read back. + * + *

Asserted: a rule with a webhook target and a signing secret is set; the SDK's read-back (and the set call's own + * response) carries the same name, event types, object name prefix, enabled flag, target URL and signing secret; + * setting an empty list deletes it. The secret is generated here, compared and never printed. + */ +public class BucketNotificationRules { + private static final String WEBHOOK_URL = "https://example.com/sdkharness-conf"; + + public static void main(String[] args) { + Support.run("bucket.notification_rules", () -> { + Support.step("bucket"); + final B2Bucket bucket = Support.bucket(); + Support.eq("rules on a new bucket", 0, Support.client.getBucketNotificationRules(bucket.getBucketId()).getEventNotificationRules().size()); + + Support.step("set a rule"); + // 32 alphanumeric characters, the documented shape of a signing secret. + final String secret = "*******" + bucket.getBucketName().substring(bucket.getBucketName().length() - 12) + "0123456789abcdef"; + final B2EventNotificationRule rule = new B2EventNotificationRule("sdkharness-conf", + new TreeSet<>(Arrays.asList("b2:ObjectCreated:*", "b2:ObjectDeleted:*")), "st/", + new B2WebhookConfiguration(WEBHOOK_URL, secret), true); + final List set = Support.client.setBucketNotificationRules( + B2SetBucketNotificationRulesRequest.builder(bucket.getBucketId(), Collections.singletonList(rule)).build()).getEventNotificationRules(); + assertRule("set response", rule, set, secret); + + Support.step("read back"); + assertRule("read-back", rule, Support.client.getBucketNotificationRules(bucket.getBucketId()).getEventNotificationRules(), secret); + + Support.step("delete"); + Support.client.setBucketNotificationRules(B2SetBucketNotificationRulesRequest.builder(bucket.getBucketId(), Collections.emptyList()).build()); + Support.eq("rules after deleting", 0, Support.client.getBucketNotificationRules(bucket.getBucketId()).getEventNotificationRules().size()); + }); + } + + private static void assertRule(String what, B2EventNotificationRule expected, List actual, String secret) { + Support.eq(what + ": number of rules", 1, actual.size()); + final B2EventNotificationRule rule = actual.get(0); + Support.eq(what + ": name", expected.getName(), rule.getName()); + Support.eq(what + ": eventTypes", expected.getEventTypes(), rule.getEventTypes()); + Support.eq(what + ": objectNamePrefix", expected.getObjectNamePrefix(), rule.getObjectNamePrefix()); + Support.eq(what + ": isEnabled", true, rule.isEnabled()); + Support.check(rule.getTargetConfiguration() instanceof B2WebhookConfiguration, what + ": the target is not a webhook"); + final B2WebhookConfiguration target = (B2WebhookConfiguration) rule.getTargetConfiguration(); + Support.eq(what + ": webhook url", WEBHOOK_URL, target.getUrl()); + Support.check(secret.equals(target.getHmacSha256SigningSecret()), what + ": the signing secret did not come back unchanged"); + } +} diff --git a/.sdkharness/tests/conformance/BucketReplicationConfig.java b/.sdkharness/tests/conformance/BucketReplicationConfig.java new file mode 100644 index 000000000..11e0e319d --- /dev/null +++ b/.sdkharness/tests/conformance/BucketReplicationConfig.java @@ -0,0 +1,75 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2BucketReplicationConfiguration; +import com.backblaze.b2.client.structures.B2CreateKeyRequest; +import com.backblaze.b2.client.structures.B2CreatedApplicationKey; +import com.backblaze.b2.client.structures.B2ReplicationRule; +import com.backblaze.b2.client.structures.B2UpdateBucketRequest; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +/** + * bucket.replication_config. Run by ../run-conformance (see ../../README.md). + * + *

b2_update_bucket takes replicationConfiguration; "At least one of the asReplicationSource or + * asReplicationDestination parameters is required". Every rule field set here (replicationRuleName, + * destinationBucketId, fileNamePrefix, includeExistingFiles, isEnabled, priority) plus sourceApplicationKeyId is a + * documented field, and "Call b2_list_buckets to retrieve all of the replication rules associated with one or more + * buckets." (https://www.backblaze.com/apidocs/b2-update-bucket, + * https://www.backblaze.com/docs/cloud-storage-create-a-cloud-replication-rule-with-the-native-api). + * + *

Two buckets and a replication-capable key (the capabilities the documentation lists for a source) are created + * and one source rule is set on the first bucket, naming the second as the destination. Asserted: the read-back of + * the source bucket carries the source key id and the rule, field for field, and no destination side; the + * destination bucket reports no source configuration. Nothing is replicated: the check only stores and reads back + * the configuration, which is the whole of this capability (the SDK offers the structures, not a helper that runs + * replication). + */ +public class BucketReplicationConfig { + public static void main(String[] args) { + Support.run("bucket.replication_config", () -> { + Support.step("buckets and a key"); + final B2Bucket source = Support.bucket(); + final B2Bucket destination = Support.bucket(); + final B2CreatedApplicationKey key = Support.client.createKey(B2CreateKeyRequest.builder(new HashSet<>(Arrays.asList( + "listBuckets", "listFiles", "readFiles", "shareFiles", "writeFiles", "deleteFiles", + "readFileLegalHolds", "readFileRetentions", "readBucketReplications", "writeBucketReplications")), + "sdkharness-" + source.getBucketName().substring(source.getBucketName().length() - 8)).build()); + Support.trackKey(key.getApplicationKeyId()); + + Support.step("set a source rule"); + final B2ReplicationRule rule = new B2ReplicationRule("sdkharness-conf", destination.getBucketId(), 1, "st/", true, false); + Support.client.updateBucket(B2UpdateBucketRequest.builder(source) + .setReplicationConfiguration(B2BucketReplicationConfiguration.createForSource(key.getApplicationKeyId(), Collections.singletonList(rule))).build()); + + Support.step("read back"); + final B2BucketReplicationConfiguration read = listed(source).getReplicationConfiguration(); + Support.eq("source application key id", key.getApplicationKeyId(), read.getSourceApplicationKeyIdOrNull()); + final List rules = read.getReplicationRulesOrNull(); + Support.check(rules != null && rules.size() == 1, "expected one replication rule, got " + (rules == null ? "none" : rules.size())); + final B2ReplicationRule actual = rules.get(0); + Support.eq("replicationRuleName", rule.getReplicationRuleName(), actual.getReplicationRuleName()); + Support.eq("destinationBucketId", rule.getDestinationBucketId(), actual.getDestinationBucketId()); + Support.eq("priority", rule.getPriority(), actual.getPriority()); + Support.eq("fileNamePrefix", rule.getFileNamePrefix(), actual.getFileNamePrefix()); + Support.eq("isEnabled", rule.isEnabled(), actual.isEnabled()); + Support.eq("includeExistingFiles", rule.includeExistingFiles(), actual.includeExistingFiles()); + Support.eq("a source-only bucket has no destination mapping", null, read.getSourceToDestinationKeyMappingOrNull()); + + Support.step("the destination bucket is not a source"); + final B2BucketReplicationConfiguration other = listed(destination).getReplicationConfiguration(); + Support.check(other == null || other.getReplicationRulesOrNull() == null, "the destination bucket reports replication rules it was never given"); + }); + } + + private static B2Bucket listed(B2Bucket bucket) throws Exception { + final B2Bucket found = Support.client.getBucketOrNullByName(bucket.getBucketName()); + Support.check(found != null, "the bucket is not listed"); + return found; + } +} diff --git a/.sdkharness/tests/conformance/ClientProgress.java b/.sdkharness/tests/conformance/ClientProgress.java new file mode 100644 index 000000000..f9777b27c --- /dev/null +++ b/.sdkharness/tests/conformance/ClientProgress.java @@ -0,0 +1,94 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.contentSources.B2ByteArrayContentSource; +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2UploadFileRequest; +import com.backblaze.b2.client.structures.B2UploadListener; +import com.backblaze.b2.client.structures.B2UploadProgress; +import com.backblaze.b2.client.structures.B2UploadState; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/** + * client.progress. Run by ../run-conformance (see ../../README.md). + * + *

NO DOCUMENTED CONTRACT: progress reporting is the SDK's own ergonomics (B2UploadListener); the B2 API has no + * progress surface, so the expected values come from the scenario, not from a Backblaze page. Scenario: a large + * object is uploaded with a listener attached; the callbacks fire, are monotonically non-decreasing, and the final + * value equals the total byte count. + * + *

One worker, deliberately: with several, the per-part reports of concurrent parts legitimately interleave, and + * "non-decreasing" would test the thread pool, not the listener. The SDK reports per part (part index, bytes so far, + * state), so the check sums the latest report of every part into one running total. Asserted: callbacks fire; each + * part's own bytes-so-far never goes down; the running total never goes down; every part ends SUCCEEDED with + * bytes-so-far equal to its length; the final total equals the byte count of the object and the part lengths add up + * to it. + */ +public class ClientProgress { + public static void main(String[] args) { + Support.run("client.progress", () -> { + Support.step("bucket"); + final B2Bucket bucket = Support.bucket(); + final byte[] data = Support.payload((int) (4 * Support.partSize() + 5), 71); + + Support.step("upload with a listener"); + final List reports = Collections.synchronizedList(new ArrayList<>()); + final B2UploadListener listener = reports::add; + final B2FileVersion file = Support.client.uploadLargeFile( + B2UploadFileRequest.builder(bucket.getBucketId(), "st/progress/large.bin", "application/octet-stream", + B2ByteArrayContentSource.build(data)).setListener(listener).build(), + Support.executor(1)); + Support.eq("uploaded contentLength", (long) data.length, file.getContentLength()); + + Support.step("callbacks fired"); + Support.check(!reports.isEmpty(), "the listener was never called"); + + Support.step("monotonic"); + final Map latest = new TreeMap<>(); + final Map lengths = new TreeMap<>(); + long previousTotal = 0; + synchronized (reports) { + for (B2UploadProgress report : reports) { + final Long before = latest.get(report.getPartIndex()); + Support.check(before == null || report.getBytesSoFar() >= before, + "part " + report.getPartIndex() + " went backwards from " + before + " to " + report.getBytesSoFar() + " (" + report.getState() + ")"); + latest.put(report.getPartIndex(), report.getBytesSoFar()); + lengths.put(report.getPartIndex(), report.getLength()); + long total = 0; + for (long bytes : latest.values()) { + total += bytes; + } + Support.check(total >= previousTotal, "the running total went backwards from " + previousTotal + " to " + total); + previousTotal = total; + } + } + + Support.step("final value"); + Support.check(latest.size() > 1, "expected progress for more than one part, saw " + latest.size()); + Support.eq("final running total", (long) data.length, previousTotal); + long lengthSum = 0; + for (long length : lengths.values()) { + lengthSum += length; + } + Support.eq("sum of the parts' lengths", (long) data.length, lengthSum); + for (int part : latest.keySet()) { + B2UploadProgress last = null; + synchronized (reports) { + for (B2UploadProgress report : reports) { + if (report.getPartIndex() == part) { + last = report; + } + } + } + Support.eq("part " + part + " final state", B2UploadState.SUCCEEDED, last.getState()); + Support.eq("part " + part + " final bytes", last.getLength(), last.getBytesSoFar()); + } + }); + } +} diff --git a/.sdkharness/tests/conformance/EncSseB2.java b/.sdkharness/tests/conformance/EncSseB2.java new file mode 100644 index 000000000..e89b13ff7 --- /dev/null +++ b/.sdkharness/tests/conformance/EncSseB2.java @@ -0,0 +1,67 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.contentSources.B2ByteArrayContentSource; +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2BucketServerSideEncryption; +import com.backblaze.b2.client.structures.B2FileSseForRequest; +import com.backblaze.b2.client.structures.B2FileSseForResponse; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2ServerSideEncryptionMode; +import com.backblaze.b2.client.structures.B2UploadFileRequest; + +import java.util.Arrays; + +/** + * enc.sse_b2. Run by ../run-conformance (see ../../README.md). + * + *

b2_upload_file's X-Bz-Server-Side-Encryption: "B2 will encrypt the uploaded data before storing the file using + * ... (SSE-B2) with the algorithm specified in this header. Currently, the only supported value for this header is + * AES256." Its response serverSideEncryption carries "the mode ("SSE-B2" or "SSE-C") and algorithm" + * (https://www.backblaze.com/apidocs/b2-upload-file). A bucket default: "All uploads to that bucket, from the time + * default encryption is enabled onward, will then be encrypted with SSE-B2 by default unless you explicitly specify + * SSE-C encryption for a given file at upload time." (https://www.backblaze.com/docs/cloud-storage-server-side-encryption); + * defaultServerSideEncryption is "The default server-side encryption settings for new files uploaded to this + * bucket", example {"mode": "SSE-B2", "algorithm": "AES256"} (https://www.backblaze.com/apidocs/b2-create-bucket). + * + *

Asserted: a file uploaded requesting SSE-B2 reports SSE-B2 / AES256 in the upload response and in its metadata, + * the download returns the plaintext and its response says AES256; a bucket created with SSE-B2 as its default + * reports that default, and a file uploaded to it without asking reports SSE-B2 too. + */ +public class EncSseB2 { + public static void main(String[] args) { + Support.run("enc.sse_b2", () -> { + Support.step("bucket"); + final B2Bucket bucket = Support.bucket(); + + Support.step("upload requesting SSE-B2"); + final byte[] data = Support.payload(1024, 91); + final B2FileVersion uploaded = Support.client.uploadSmallFile( + B2UploadFileRequest.builder(bucket.getBucketId(), "st/enc/sse-b2.bin", "application/octet-stream", B2ByteArrayContentSource.build(data)) + .setServerSideEncryption(B2FileSseForRequest.createSseB2Aes256()).build()); + assertSseB2("upload response", uploaded.getServerSideEncryption()); + + Support.step("metadata"); + assertSseB2("file metadata", Support.client.getFileInfo(uploaded.getFileId()).getServerSideEncryption()); + + Support.step("download"); + final Support.Downloaded got = Support.downloadById(uploaded.getFileId()); + Support.check(Arrays.equals(data, got.bytes), "the download is not the plaintext"); + Support.eq("X-Bz-Server-Side-Encryption on the download", "AES256", got.headers.getServerSideEncryptionOrNull()); + + Support.step("bucket default"); + final B2Bucket defaultBucket = Support.bucket(false, b -> b.setDefaultServerSideEncryption(B2BucketServerSideEncryption.createSseB2Aes256())); + Support.eq("bucket default mode", B2ServerSideEncryptionMode.SSE_B2, defaultBucket.getDefaultServerSideEncryption().getMode()); + Support.eq("bucket default algorithm", "AES256", defaultBucket.getDefaultServerSideEncryption().getAlgorithm()); + final B2FileVersion inherited = Support.upload(defaultBucket, "st/enc/default.bin", data, "application/octet-stream", null); + assertSseB2("a file uploaded to a bucket with default encryption", inherited.getServerSideEncryption()); + Support.check(Arrays.equals(data, Support.downloadById(inherited.getFileId()).bytes), "the default-encrypted file's download is not the plaintext"); + }); + } + + private static void assertSseB2(String what, B2FileSseForResponse sse) { + Support.check(sse != null, what + " carries no serverSideEncryption"); + Support.eq(what + " mode", B2ServerSideEncryptionMode.SSE_B2, sse.getMode()); + Support.eq(what + " algorithm", "AES256", sse.getAlgorithm()); + } +} diff --git a/.sdkharness/tests/conformance/EncSseC.java b/.sdkharness/tests/conformance/EncSseC.java new file mode 100644 index 000000000..e90969112 --- /dev/null +++ b/.sdkharness/tests/conformance/EncSseC.java @@ -0,0 +1,70 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.contentSources.B2ByteArrayContentSource; +import com.backblaze.b2.client.exceptions.B2Exception; +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2FileSseForRequest; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2ServerSideEncryptionMode; +import com.backblaze.b2.client.structures.B2UploadFileRequest; + +import java.security.MessageDigest; +import java.util.Arrays; +import java.util.Base64; + +/** + * enc.sse_c. Run by ../run-conformance (see ../../README.md). + * + *

b2_upload_file takes the X-Bz-Server-Side-Encryption-Customer-Algorithm (AES256), -Customer-Key and + * -Customer-Key-Md5 headers; its response serverSideEncryption carries "the mode ("SSE-B2" or "SSE-C") and + * algorithm" (https://www.backblaze.com/apidocs/b2-upload-file). The key's MD5 comes back on download: + * X-Bz-Server-Side-Encryption-Customer-Key-Md5, "if the file was encrypted with SSE-C, then the value is the MD5 + * digest of the customer-managed encryption key used for encryption" + * (https://www.backblaze.com/apidocs/b2-download-file-by-id). "Backblaze B2 manages the encryption process using the + * key that you provide when uploading or accessing each file"; the documented error is for the WRONG key, "a 403 + * error is returned" (https://www.backblaze.com/docs/cloud-storage-server-side-encryption). + * + *

A 32-byte customer key (derived from a fixed string; it protects nothing and is never printed) is used for an + * upload. Asserted: the upload response reports SSE-C / AES256; downloading with the key returns the plaintext and the + * response carries the key's MD5; downloading without the key fails with a service error (B2 documents no status for + * a missing key, so a 400, 401 or 403 counts) and does not return the plaintext; downloading with a different key + * fails with 403, the documented status for a wrong key. + */ +public class EncSseC { + public static void main(String[] args) { + Support.run("enc.sse_c", () -> { + Support.step("bucket and keys"); + final B2Bucket bucket = Support.bucket(); + final byte[] keyBytes = MessageDigest.getInstance("SHA-256").digest(Support.utf8("sdkharness-java-sse-c-key")); + final byte[] otherBytes = MessageDigest.getInstance("SHA-256").digest(Support.utf8("sdkharness-java-sse-c-other-key")); + final B2FileSseForRequest sse = B2FileSseForRequest.createSseCAes256(Base64.getEncoder().encodeToString(keyBytes)); + final B2FileSseForRequest wrong = B2FileSseForRequest.createSseCAes256(Base64.getEncoder().encodeToString(otherBytes)); + final String keyMd5 = Base64.getEncoder().encodeToString(MessageDigest.getInstance("MD5").digest(keyBytes)); + + Support.step("upload with SSE-C"); + final byte[] data = Support.payload(1024, 92); + final B2FileVersion uploaded = Support.client.uploadSmallFile( + B2UploadFileRequest.builder(bucket.getBucketId(), "st/enc/sse-c.bin", "application/octet-stream", B2ByteArrayContentSource.build(data)) + .setServerSideEncryption(sse).build()); + Support.check(uploaded.getServerSideEncryption() != null, "the upload response carries no serverSideEncryption"); + Support.eq("upload response mode", B2ServerSideEncryptionMode.SSE_C, uploaded.getServerSideEncryption().getMode()); + Support.eq("upload response algorithm", "AES256", uploaded.getServerSideEncryption().getAlgorithm()); + + Support.step("download with the key"); + final Support.Downloaded got = Support.downloadById(uploaded.getFileId(), sse); + Support.check(Arrays.equals(data, got.bytes), "the download with the key is not the plaintext"); + Support.eq("X-Bz-Server-Side-Encryption-Customer-Key-Md5 on the download", keyMd5, got.headers.getSseCustomerKeyMd5OrNull()); + Support.eq("X-Bz-Server-Side-Encryption-Customer-Algorithm on the download", "AES256", got.headers.getSseCustomerAlgorithmOrNull()); + + Support.step("download without the key"); + final B2Exception missing = Support.expectError("download of an SSE-C file without its key", () -> Support.downloadById(uploaded.getFileId())); + Support.check(Arrays.asList(400, 401, 403).contains(missing.getStatus()), + "refused with status " + missing.getStatus() + " code " + missing.getCode() + ", expected 400, 401 or 403"); + + Support.step("download with a different key"); + final B2Exception other = Support.expectError("download of an SSE-C file with the wrong key", () -> Support.downloadById(uploaded.getFileId(), wrong)); + Support.eq("status for the wrong key", 403, other.getStatus()); + }); + } +} diff --git a/.sdkharness/tests/conformance/FilesServerSideCopy.java b/.sdkharness/tests/conformance/FilesServerSideCopy.java new file mode 100644 index 000000000..8fa7fd113 --- /dev/null +++ b/.sdkharness/tests/conformance/FilesServerSideCopy.java @@ -0,0 +1,72 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2CopyFileRequest; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.util.B2ByteRange; + +import java.util.Arrays; +import java.util.Collections; + +/** + * files.server_side_copy. Run by ../run-conformance (see ../../README.md). + * + *

b2_copy_file "Copies an entire file" from a sourceFileId to a new fileName: the request carries no + * file bytes, and the response's fileId is "The unique identifier for this version of this file" while its + * contentSha1 is "The SHA1 of the bytes stored in the file", so a whole-file copy reports the source's sha1 + * under a new fileId. The check also reads the copy back, and asserts the metadata directive (COPY keeps the + * source's content type and file info, REPLACE takes the ones in the request) and a ranged copy. + * https://www.backblaze.com/apidocs/b2-copy-file + */ +public class FilesServerSideCopy { + public static void main(String[] args) { + Support.run("files.server_side_copy", true, () -> { + Support.step("bucket"); + final B2Bucket bucket = Support.bucket(); + + Support.step("source"); + final byte[] data = Support.payload(512, 11); + final B2FileVersion source = Support.upload(bucket, "st/copy/source.bin", data, "application/x-sdkharness", + Collections.singletonMap("origin", "source")); + + Support.step("copy"); + Support.wire.clear(); + final B2FileVersion copy = Support.client.copySmallFile( + B2CopyFileRequest.builder(source.getFileId(), "st/copy/destination.bin").build()); + Support.eq("copy fileName", "st/copy/destination.bin", copy.getFileName()); + Support.check(!source.getFileId().equals(copy.getFileId()), "the copy reused the source's fileId"); + Support.eq("copy contentSha1", source.getContentSha1(), copy.getContentSha1()); + Support.eq("copy contentLength", (long) data.length, copy.getContentLength()); + Support.eq("COPY keeps the content type", "application/x-sdkharness", copy.getContentType()); + Support.eq("COPY keeps the file info", "source", copy.getFileInfo().get("origin")); + + Support.step("no bytes were uploaded"); + Support.eq("b2_copy_file requests", 1, Support.wireCount("POST", "b2_copy_file")); + Support.eq("b2_upload_file requests during the copy", 0, Support.wireCount("POST", "b2_upload_file")); + + Support.step("read back"); + Support.check(Arrays.equals(data, Support.downloadById(copy.getFileId()).bytes), "the copy's bytes differ from the source's"); + Support.check(Arrays.equals(data, Support.downloadById(source.getFileId()).bytes), "the source changed"); + + Support.step("metadata directive REPLACE"); + final B2FileVersion replaced = Support.client.copySmallFile( + B2CopyFileRequest.builder(source.getFileId(), "st/copy/replaced.bin") + .setMetadataDirective(B2CopyFileRequest.REPLACE_METADATA_DIRECTIVE) + .setContentType("text/x-replaced") + .setFileInfo(Collections.singletonMap("origin", "replaced")) + .build()); + Support.eq("REPLACE content type", "text/x-replaced", replaced.getContentType()); + Support.eq("REPLACE file info", "replaced", replaced.getFileInfo().get("origin")); + Support.eq("REPLACE contentSha1", source.getContentSha1(), replaced.getContentSha1()); + + Support.step("a range copy"); + final B2FileVersion ranged = Support.client.copySmallFile( + B2CopyFileRequest.builder(source.getFileId(), "st/copy/ranged.bin") + .setRange(B2ByteRange.between(100, 199)).build()); + Support.eq("range copy length", 100L, ranged.getContentLength()); + Support.check(Arrays.equals(Arrays.copyOfRange(data, 100, 200), Support.downloadById(ranged.getFileId()).bytes), + "the range copy's bytes are not bytes 100..199 of the source"); + }); + } +} diff --git a/.sdkharness/tests/conformance/GuardSelfTest.java b/.sdkharness/tests/conformance/GuardSelfTest.java index 01e2a644f..55bd47ce5 100644 --- a/.sdkharness/tests/conformance/GuardSelfTest.java +++ b/.sdkharness/tests/conformance/GuardSelfTest.java @@ -17,7 +17,9 @@ * *

It checks that a request to a loopback server passes, that an absolute request URI to a * non-loopback host (a documentation address, never contacted) is refused before it is sent, and that - * a loopback server redirecting to a non-loopback host is refused on the redirect hop. + * a loopback server redirecting to a non-loopback host is refused on the redirect hop. It also checks the one + * request path that does not go through the SDK's client, {@code Support.rawGet} (a share-link fetch, used by + * {@code urls.native_download}): loopback passes, a non-loopback host is refused before anything is sent. */ public final class GuardSelfTest { private GuardSelfTest() { @@ -58,6 +60,15 @@ public static void main(String[] args) throws Exception { problem = mustBeBlocked(client, new HttpGet("http://localhost:" + server.getAddress().getPort() + "/ok"), "a loopback name that is not 127.0.0.1", "localhost"); } + if (problem == null) { + problem = rawGetMustBeBlocked("http://203.0.113.7:9/share-link", "a share-link fetch to a non-loopback host", "203.0.113.7"); + } + if (problem == null) { + final Support.Raw ok = Support.rawGet(base + "/ok", null); + if (ok.status != 200) { + problem = "a share-link fetch from loopback did not reach the server"; + } + } } catch (Exception e) { problem = "unexpected " + Support.describe(e); } finally { @@ -72,6 +83,20 @@ public static void main(String[] args) throws Exception { } } + /** {@code Support.rawGet} is the one request path that does not go through the SDK's client; it carries its own host check. */ + private static String rawGetMustBeBlocked(String url, String what, String host) { + final int before = Support.blocked.size(); + try { + Support.rawGet(url, null); + return what + " was sent"; + } catch (Exception e) { + if (Support.blocked.size() != before + 1 || !host.equals(Support.blocked.get(before))) { + return what + " failed without being blocked by the guard: " + Support.describe(e); + } + return null; + } + } + private static String mustBeBlocked(CloseableHttpClient client, HttpGet get, String what, String host) { final int before = Support.blocked.size(); try (CloseableHttpResponse r = client.execute(get)) { diff --git a/.sdkharness/tests/conformance/KeysCrud.java b/.sdkharness/tests/conformance/KeysCrud.java new file mode 100644 index 000000000..a7ba22bc5 --- /dev/null +++ b/.sdkharness/tests/conformance/KeysCrud.java @@ -0,0 +1,98 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.B2StorageClient; +import com.backblaze.b2.client.structures.B2ApplicationKey; +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2CreateKeyRequest; +import com.backblaze.b2.client.structures.B2CreatedApplicationKey; +import com.backblaze.b2.client.structures.B2DeleteKeyRequest; +import com.backblaze.b2.client.structures.B2ListBucketsRequest; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * keys.crud. Run by ../run-conformance (see ../../README.md). + * + *

b2_create_key's applicationKey is "The secret part of the key. This is the only time it will be returned, so + * you need to keep it. This is not returned when you list the keys in your account."; b2_list_keys' key objects carry + * no applicationKey field; b2_delete_key "Deletes the application key specified" + * (https://www.backblaze.com/apidocs/b2-create-key, https://www.backblaze.com/apidocs/b2-list-keys, + * https://www.backblaze.com/apidocs/b2-delete-key). + * + *

A key restricted to one bucket and to three capabilities is created, listed and deleted. Asserted: create returns + * the key id and a secret and echoes the name, capabilities and bucket; the listing finds the key with the same name, + * capabilities and bucket, and nothing the SDK returns for a listed key carries the secret (no accessor of + * {@code B2ApplicationKey} returns it, nor does its text form); the secret works (a second client authorized with it + * reports the restricted bucket as its allowed bucket and lists that bucket by name); after the delete the key is no longer listed and its secret no longer + * authorizes. The key is restricted to one bucket because this SDK pins API version 2, which cannot represent a key + * scoped to several (card: keys.multi_bucket is "no"). + */ +public class KeysCrud { + public static void main(String[] args) { + Support.run("keys.crud", () -> { + Support.step("bucket"); + final B2Bucket bucket = Support.bucket(); + final Set capabilities = new HashSet<>(Arrays.asList("listBuckets", "listFiles", "readFiles")); + final String name = "sdkharness-" + bucket.getBucketName().substring(bucket.getBucketName().length() - 8); + + Support.step("create"); + final B2CreatedApplicationKey created = Support.client.createKey(B2CreateKeyRequest.builder(capabilities, name).setBucketId(bucket.getBucketId()).build()); + Support.trackKey(created.getApplicationKeyId()); + final String secret = created.getApplicationKey(); + Support.check(created.getApplicationKeyId() != null && !created.getApplicationKeyId().isEmpty(), "create returned no key id"); + Support.check(secret != null && !secret.isEmpty(), "create returned no secret"); + Support.eq("created key name", name, created.getKeyName()); + Support.eq("created key capabilities", capabilities, new HashSet<>(created.getCapabilities())); + Support.eq("created key bucket", bucket.getBucketId(), created.getBucketId()); + + Support.step("list"); + B2ApplicationKey listed = find(created.getApplicationKeyId()); + Support.check(listed != null, "the new key is not in the listing"); + Support.eq("listed key name", name, listed.getKeyName()); + Support.eq("listed key capabilities", capabilities, new HashSet<>(listed.getCapabilities())); + Support.eq("listed key bucket", bucket.getBucketId(), listed.getBucketId()); + + Support.step("the secret is returned only on create"); + Support.check(!listed.toString().contains(secret), "the listed key's text form carries the secret"); + for (Method method : B2ApplicationKey.class.getMethods()) { + if (method.getParameterCount() == 0 && method.getReturnType() == String.class) { + Support.check(!secret.equals(method.invoke(listed)), "B2ApplicationKey." + method.getName() + "() returns the secret"); + } + } + + Support.step("the secret works"); + final B2StorageClient restricted = Support.clientFor(created.getApplicationKeyId(), secret); + // A key restricted to a bucket must name that bucket when it lists buckets (b2_list_buckets), so it does. + final List visible = new ArrayList<>(); + for (B2Bucket b : restricted.listBuckets(B2ListBucketsRequest.builder(restricted.getAccountId()).setBucketName(bucket.getBucketName()).build()).getBuckets()) { + visible.add(b.getBucketName()); + } + Support.eq("buckets the restricted key lists by name", Arrays.asList(bucket.getBucketName()), visible); + Support.eq("authorization bucket of the restricted key", bucket.getBucketId(), restricted.getAccountAuthorization().getAllowed().getBucketId()); + + Support.step("delete"); + Support.client.deleteKey(B2DeleteKeyRequest.builder(created.getApplicationKeyId()).build()); + Support.untrackKey(created.getApplicationKeyId()); + Support.check(find(created.getApplicationKeyId()) == null, "the deleted key is still listed"); + // Revocation is asserted through authorization itself (a fresh b2_authorize_account). A bucket-restricted key + // calling buckets() gets 401 even when valid, so that call cannot tell a revoked key from a restricted one. + Support.eq("status of authorizing with a deleted key", 401, Support.expectError("authorizing with a deleted key", + () -> Support.clientFor(created.getApplicationKeyId(), secret).getAccountAuthorization()).getStatus()); + }); + } + + private static B2ApplicationKey find(String keyId) throws Exception { + for (B2ApplicationKey key : Support.client.applicationKeys()) { + if (keyId.equals(key.getApplicationKeyId())) { + return key; + } + } + return null; + } +} diff --git a/.sdkharness/tests/conformance/LargeComposeMixed.java b/.sdkharness/tests/conformance/LargeComposeMixed.java new file mode 100644 index 000000000..3737b0641 --- /dev/null +++ b/.sdkharness/tests/conformance/LargeComposeMixed.java @@ -0,0 +1,76 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.B2CopyingPartStorer; +import com.backblaze.b2.client.B2PartStorer; +import com.backblaze.b2.client.B2UploadingPartStorer; +import com.backblaze.b2.client.contentSources.B2ByteArrayContentSource; +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2StartLargeFileRequest; +import com.backblaze.b2.util.B2ByteRange; + +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * large.compose_mixed. Run by ../run-conformance (see ../../README.md). + * + *

"a large file can be assembled from a mix of uploaded and copied parts" + * (https://www.backblaze.com/docs/cloud-storage-large-files). b2_copy_part "Copies from an existing B2 file, + * storing it as a part of a large file which has already been started with b2_start_large_file"; its range is + * "The range of bytes to copy. If not provided, the whole source file will be copied." + * (https://www.backblaze.com/apidocs/b2-copy-part). Every part but the last is at least the realm's absolute + * minimum part size ("the minimum part size is 5 MB except for the last part in a file which has a minimum size of + * 1 byte", https://www.backblaze.com/docs/cloud-storage-create-large-files-with-the-native-api; the size is read + * from the realm, never hard-coded). + * + *

One large file is composed with {@code storeLargeFile} from three pieces: bytes held locally (an uploading + * part), a byte range of an existing file (a copying part with a range) and a whole existing file (a copying part + * without one). Asserted: the SDK sent one upload part and two copy parts and no other part, and the finished + * file's bytes equal the ordered concatenation of the three. + */ +public class LargeComposeMixed { + public static void main(String[] args) { + Support.run("large.compose_mixed", true, () -> { + Support.step("bucket"); + final B2Bucket bucket = Support.bucket(); + final int piece = (int) Support.partSize(); + + Support.step("sources"); + final byte[] local = Support.payload(piece, 51); + final byte[] rangedSource = Support.payload(3 * piece, 52); + final byte[] wholeSource = Support.payload(piece + 7, 53); + final B2FileVersion ranged = Support.upload(bucket, "st/compose/ranged-source.bin", rangedSource, "application/octet-stream", null); + final B2FileVersion whole = Support.upload(bucket, "st/compose/whole-source.bin", wholeSource, "application/octet-stream", null); + final byte[] rangePiece = Arrays.copyOfRange(rangedSource, piece, 2 * piece); + + Support.step("compose"); + final B2FileVersion started = Support.client.startLargeFile( + B2StartLargeFileRequest.builder(bucket.getBucketId(), "st/compose/composed.bin", "application/octet-stream").build()); + final List parts = new ArrayList<>(); + parts.add(new B2UploadingPartStorer(1, B2ByteArrayContentSource.build(local))); + parts.add(new B2CopyingPartStorer(2, ranged.getFileId(), B2ByteRange.between(piece, 2L * piece - 1))); + parts.add(new B2CopyingPartStorer(3, whole.getFileId())); + Support.wire.clear(); + final B2FileVersion composed = Support.client.storeLargeFile(started, parts, null, Support.executor(2)); + + Support.step("the SDK used a mix of parts"); + Support.eq("b2_upload_part requests", 1, Support.wireCount("POST", "b2_upload_part")); + Support.eq("b2_copy_part requests", 2, Support.wireCount("POST", "b2_copy_part")); + Support.eq("b2_finish_large_file requests", 1, Support.wireCount("POST", "b2_finish_large_file")); + Support.eq("composed fileId", started.getFileId(), composed.getFileId()); + + Support.step("bytes"); + final ByteArrayOutputStream expected = new ByteArrayOutputStream(); + expected.write(local); + expected.write(rangePiece); + expected.write(wholeSource); + Support.eq("composed contentLength", (long) expected.size(), Support.client.getFileInfo(composed.getFileId()).getContentLength()); + Support.check(Arrays.equals(expected.toByteArray(), Support.downloadById(composed.getFileId()).bytes), + "the composed file is not the ordered concatenation of the local bytes, the copied range and the whole copied file"); + }); + } +} diff --git a/.sdkharness/tests/conformance/LargeConcurrentParts.java b/.sdkharness/tests/conformance/LargeConcurrentParts.java new file mode 100644 index 000000000..57723760f --- /dev/null +++ b/.sdkharness/tests/conformance/LargeConcurrentParts.java @@ -0,0 +1,133 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.contentSources.B2ContentSource; +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2UploadFileRequest; + +import java.io.ByteArrayInputStream; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.concurrent.ExecutorService; + +/** + * large.concurrent_parts. Run by ../run-conformance (see ../../README.md). + * + *

B2 permits uploading the parts of one large file in parallel: "Parts of a large file can be uploaded and + * copied in parallel" (https://www.backblaze.com/docs/cloud-storage-large-files) and "Each thread that uploads + * should get its own URL" (https://www.backblaze.com/docs/cloud-storage-create-large-files-with-the-native-api). + * No B2 document requires a client to do it, so the expected value comes from the SDK's own contract: the + * executor passed to {@code uploadLargeFile} sets how many parts are sent at once. + * + *

KNOWN WEAK by design, and stated here rather than hidden: the check does not trust wall time. The same + * eight-part file is uploaded twice, once on a one-thread executor and once on a four-thread executor, and the + * HTTP client the check hands the SDK counts how many {@code b2_upload_part} requests are in flight at the same + * instant (from the SDK starting to send to the response arriving). Each part's body is read slowly, so + * simultaneous requests overlap on a loopback server. Asserted: with one worker never more than one is in + * flight (this proves the instrument can tell the difference), with four workers more than one is. It is an + * in-process observation at the SDK's own HTTP client, not a packet capture; the wall times are only a NOTE. + */ +public class LargeConcurrentParts { + private static final int PARTS = 8; + private static final long PAUSE_MILLIS = 150; + + public static void main(String[] args) { + Support.run("large.concurrent_parts", true, () -> { + Support.step("bucket"); + final B2Bucket bucket = Support.bucket(); + final long partSize = Support.partSize(); + final byte[] data = Support.payload((int) (PARTS * partSize + 3), 31); + final String sha1 = Support.sha1(data); + Support.inFlightMarker = "b2_upload_part"; + + Support.step("one worker"); + final long oneMillis = upload(bucket, "st/large/serial.bin", data, sha1, Support.executor(1)); + final int serialMax = Support.maxInFlight.get(); + Support.eq("b2_upload_part requests in flight at once with one worker", 1, serialMax); + + Support.step("four workers"); + Support.maxInFlight.set(0); + Support.inFlight.set(0); + final long fourMillis = upload(bucket, "st/large/parallel.bin", data, sha1, Support.executor(4)); + final int parallelMax = Support.maxInFlight.get(); + Support.check(parallelMax > 1, "with four workers at most " + parallelMax + " b2_upload_part request(s) were ever in flight at once"); + + Support.step("bytes"); + for (String name : new String[] {"st/large/serial.bin", "st/large/parallel.bin"}) { + final B2FileVersion stored = Support.versions(bucket.getBucketId(), name).get(0); + Support.check(Arrays.equals(data, Support.downloadById(stored.getFileId()).bytes), name + ": the downloaded bytes differ"); + } + Support.note("evidence is in-process: peak b2_upload_part requests in flight " + serialMax + " with 1 worker and " + parallelMax + + " with 4; wall time " + oneMillis + " ms and " + fourMillis + " ms (a corroboration, never the verdict)"); + }); + } + + private static long upload(B2Bucket bucket, String name, byte[] data, String sha1, ExecutorService executor) throws Exception { + final long start = System.nanoTime(); + final B2FileVersion file = Support.client.uploadLargeFile( + B2UploadFileRequest.builder(bucket.getBucketId(), name, "application/octet-stream", new SlowSource(data, sha1)).build(), + executor); + Support.eq(name + " contentLength", (long) data.length, file.getContentLength()); + return (System.nanoTime() - start) / 1_000_000; + } + + /** The bytes, read slowly: the first read of every stream waits, so uploads that run together overlap. */ + private static final class SlowSource implements B2ContentSource { + private final byte[] data; + private final String sha1; + + SlowSource(byte[] data, String sha1) { + this.data = data; + this.sha1 = sha1; + } + + @Override + public long getContentLength() { + return data.length; + } + + @Override + public String getSha1OrNull() { + return sha1; + } + + @Override + public Long getSrcLastModifiedMillisOrNull() { + return null; + } + + @Override + public InputStream createInputStream() { + return new FilterInputStream(new ByteArrayInputStream(data)) { + private boolean first = true; + + private void pause() throws IOException { + if (first) { + first = false; + try { + Thread.sleep(PAUSE_MILLIS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("interrupted"); + } + } + } + + @Override + public int read() throws IOException { + pause(); + return super.read(); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + pause(); + return super.read(b, off, len); + } + }; + } + } +} diff --git a/.sdkharness/tests/conformance/LargeMultipart.java b/.sdkharness/tests/conformance/LargeMultipart.java new file mode 100644 index 000000000..5bd52cc3f --- /dev/null +++ b/.sdkharness/tests/conformance/LargeMultipart.java @@ -0,0 +1,63 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.contentSources.B2ByteArrayContentSource; +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2UploadFileRequest; + +import java.util.Arrays; +import java.util.concurrent.ExecutorService; + +/** + * large.multipart. Run by ../run-conformance (see ../../README.md). + * + *

A payload of three parts' worth of bytes (the part size is the larger of the realm's recommended and + * absolute minimum part size, never a hard-coded 5 MB) is uploaded with {@code uploadLargeFile}. Asserted: + * the SDK started one large file, sent more than one part and finished it; the file's contentSha1 is "none" + * ("Large files do not have SHA1 checksums, and the value is none", + * https://www.backblaze.com/apidocs/b2-finish-large-file) and the sha1 the caller gave travels as + * {@code large_file_sha1} ("If the caller knows the SHA1 of the entire large file being uploaded, Backblaze + * recommends using large_file_sha1 as the name", https://www.backblaze.com/apidocs/b2-start-large-file); the + * downloaded bytes equal what was uploaded. A large file "must consist of at least two parts" + * (https://www.backblaze.com/docs/cloud-storage-large-files). + * + *

NEEDS REVIEW (as in the Python check): the docs recommend large_file_sha1, they do not require it. + */ +public class LargeMultipart { + public static void main(String[] args) { + Support.run("large.multipart", true, () -> { + Support.step("bucket"); + final B2Bucket bucket = Support.bucket(); + + Support.step("upload"); + final long partSize = Support.partSize(); + final byte[] data = Support.payload((int) (3 * partSize + 17), 21); + final String sha1 = Support.sha1(data); + final ExecutorService executor = Support.executor(2); + Support.wire.clear(); + final B2FileVersion large = Support.client.uploadLargeFile( + B2UploadFileRequest.builder(bucket.getBucketId(), "st/large/multipart.bin", "application/octet-stream", + B2ByteArrayContentSource.builder(data).setSha1OrNull(sha1).build()).build(), + executor); + + Support.step("the SDK used the large-file calls"); + Support.eq("b2_start_large_file requests", 1, Support.wireCount("POST", "b2_start_large_file")); + Support.eq("b2_finish_large_file requests", 1, Support.wireCount("POST", "b2_finish_large_file")); + final int parts = Support.wireCount("POST", "b2_upload_part"); + Support.check(parts > 1, "expected more than one b2_upload_part request, saw " + parts); + Support.eq("b2_upload_file requests (a large file is not a small upload)", 0, Support.wireCount("POST", "b2_upload_file")); + + Support.step("the finished file"); + Support.eq("uploaded contentLength", (long) data.length, large.getContentLength()); + final B2FileVersion stored = Support.client.getFileInfo(large.getFileId()); + Support.eq("contentSha1 of a large file", "none", stored.getContentSha1()); + Support.eq("large_file_sha1", sha1, stored.getFileInfo().get("large_file_sha1")); + Support.eq("stored contentLength", (long) data.length, stored.getContentLength()); + + Support.step("bytes"); + Support.check(Arrays.equals(data, Support.downloadById(large.getFileId()).bytes), + "the downloaded bytes differ from the uploaded ones"); + }); + } +} diff --git a/.sdkharness/tests/conformance/LargeResume.java b/.sdkharness/tests/conformance/LargeResume.java new file mode 100644 index 000000000..53be57139 --- /dev/null +++ b/.sdkharness/tests/conformance/LargeResume.java @@ -0,0 +1,192 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.contentSources.B2ContentSource; +import com.backblaze.b2.client.exceptions.B2Exception; +import com.backblaze.b2.client.exceptions.B2LocalException; +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2Part; +import com.backblaze.b2.client.structures.B2ListUnfinishedLargeFilesRequest; +import com.backblaze.b2.client.structures.B2UploadFileRequest; + +import java.io.ByteArrayInputStream; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * large.resume. Run by ../run-conformance (see ../../README.md). + * + *

"Any number of large files can be in progress at the same time. You can use b2_list_unfinished_large_files + * to get a list of them. For any one unfinished large file, you can use b2_list_parts to get a list of the parts + * that were uploaded so far." (https://www.backblaze.com/docs/cloud-storage-create-large-files-with-the-native-api; + * https://www.backblaze.com/apidocs/b2-list-parts). B2 makes reusing the unfinished fileId possible; that the SDK + * does it is the capability under test, not a B2 promise. + * + *

The first upload is interrupted from inside the content source: the bytes of the third part cannot be read + * (an unchecked exception, not an IOException: the SDK retries an IOException for about two minutes), so with one + * worker the first two parts are stored and the call fails before the file is finished. The large file is then + * unfinished (a "start" version in the unfinished listing, parts in b2_list_parts). The SDK's resume call + * {@code finishUploadingLargeFile} is given that file and the same content. Asserted: the rerun sends no + * b2_start_large_file, uploads only the parts that were missing (fewer than the total, and exactly the missing + * ones), finishes the same fileId, and the stored bytes equal the content. + * + *

Two legs, because the SDK treats the two content shapes differently. Leg "no sha1": the content source does + * not know its sha1, so the large file carries no large_file_sha1 and the rerun is the same request as the first + * run. Leg "sha1": the content source knows its sha1 (the SDK records it as the large_file_sha1 file info); the + * rerun is again the same request as the first run. If the SDK refuses it with a "mismatch" error, the scenario + * FAILS with a FINDING as the reason: the check detects, it does not work around the refusal. The finding is about + * the SDK, not about B2: B2LargeFileUploader removes the key + * {@code X-Bz-Info-large_file_sha1} (B2Headers.LARGE_FILE_SHA1) from the file info before comparing it with the + * request's, but the info map is keyed {@code large_file_sha1} (B2Headers.LARGE_FILE_SHA1_INFO_NAME). + */ +public class LargeResume { + private static final int PARTS = 4; + + public static void main(String[] args) { + Support.run("large.resume", true, () -> { + Support.step("bucket"); + final B2Bucket bucket = Support.bucket(); + final long partSize = Support.partSize(); + final byte[] data = Support.payload((int) (PARTS * partSize + 3), 41); + + Support.step("leg no sha1"); + resume(bucket, "st/large/resume-no-sha1.bin", data, null); + + Support.step("leg sha1"); + resume(bucket, "st/large/resume-sha1.bin", data, Support.sha1(data)); + }); + } + + private static void resume(B2Bucket bucket, String name, byte[] data, String sha1) throws Exception { + final long partSize = Support.partSize(); + final String leg = sha1 == null ? "no sha1: " : "sha1: "; + + Support.wire.clear(); + try { + Support.client.uploadLargeFile(request(bucket, name, new Source(data, sha1, 2 * partSize)), Support.executor(1)); + Support.fail(leg + "the interrupted upload finished instead of failing"); + } catch (B2Exception expected) { + Support.note(leg + "the first upload stopped as intended: " + Support.describe(expected)); + } + Support.eq(leg + "b2_start_large_file requests in the first run", 1, Support.wireCount("POST", "b2_start_large_file")); + Support.eq(leg + "b2_finish_large_file requests in the first run", 0, Support.wireCount("POST", "b2_finish_large_file")); + + final List unfinished = unfinished(bucket, name); + Support.eq(leg + "unfinished large files named " + name, 1, unfinished.size()); + final B2FileVersion pending = unfinished.get(0); + Support.eq(leg + "unfinished file name", name, pending.getFileName()); + int storedBefore = 0; + for (B2Part part : Support.client.parts(pending.getFileId())) { + storedBefore++; + } + Support.check(storedBefore >= 1, leg + "no part was stored before the interruption"); + Support.check(storedBefore < PARTS, leg + "all " + PARTS + " parts were stored; nothing is left to resume"); + + Support.wire.clear(); + final B2FileVersion finished; + try { + finished = Support.client.finishUploadingLargeFile(pending, request(bucket, name, new Source(data, sha1, -1)), Support.executor(2)); + } catch (B2LocalException e) { + if (sha1 == null || !"mismatch".equals(e.getCode())) { + throw e; + } + Support.fail("FINDING b2-sdk-java: finishUploadingLargeFile refused the same request that started the large file: " + + e.getMessage()); + return; + } + Support.eq(leg + "b2_start_large_file requests in the rerun", 0, Support.wireCount("POST", "b2_start_large_file")); + Support.eq(leg + "finished fileId is the unfinished one", pending.getFileId(), finished.getFileId()); + Support.eq(leg + "parts uploaded in the rerun", PARTS - storedBefore, Support.wireCount("POST", "b2_upload_part")); + Support.eq(leg + "b2_finish_large_file requests in the rerun", 1, Support.wireCount("POST", "b2_finish_large_file")); + + Support.eq(leg + "large files still unfinished after the rerun", 0, unfinished(bucket, name).size()); + Support.eq(leg + "stored contentLength", (long) data.length, Support.client.getFileInfo(finished.getFileId()).getContentLength()); + Support.check(Arrays.equals(data, Support.downloadById(finished.getFileId()).bytes), leg + "the downloaded bytes differ from the content"); + } + + private static List unfinished(B2Bucket bucket, String name) throws B2Exception { + final List all = new ArrayList<>(); + for (B2FileVersion v : Support.client.unfinishedLargeFiles( + B2ListUnfinishedLargeFilesRequest.builder(bucket.getBucketId()).setNamePrefix(name).build())) { + all.add(v); + } + return all; + } + + private static B2UploadFileRequest request(B2Bucket bucket, String name, B2ContentSource source) { + return B2UploadFileRequest.builder(bucket.getBucketId(), name, "application/octet-stream", source).build(); + } + + /** The content; with {@code failAt >= 0}, reading past that byte offset fails the way an interrupted run would. */ + private static final class Source implements B2ContentSource { + private final byte[] data; + private final String sha1; + private final long failAt; + + Source(byte[] data, String sha1, long failAt) { + this.data = data; + this.sha1 = sha1; + this.failAt = failAt; + } + + @Override + public long getContentLength() { + return data.length; + } + + @Override + public String getSha1OrNull() { + return sha1; + } + + @Override + public Long getSrcLastModifiedMillisOrNull() { + return null; + } + + @Override + public InputStream createInputStream() { + return new FilterInputStream(new ByteArrayInputStream(data)) { + private long pos; + + private void guard() { + if (failAt >= 0 && pos >= failAt) { + throw new IllegalStateException("interrupted by the check at byte " + pos); + } + } + + @Override + public int read() throws IOException { + guard(); + final int b = super.read(); + if (b >= 0) { + pos++; + } + return b; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + guard(); + final int n = super.read(b, off, failAt >= 0 ? (int) Math.min(len, failAt - pos) : len); + if (n > 0) { + pos += n; + } + return n; + } + + @Override + public long skip(long n) throws IOException { + final long skipped = super.skip(n); + pos += skipped; + return skipped; + } + }; + } + } +} diff --git a/.sdkharness/tests/conformance/LockBucketDefault.java b/.sdkharness/tests/conformance/LockBucketDefault.java new file mode 100644 index 000000000..2f3d5da49 --- /dev/null +++ b/.sdkharness/tests/conformance/LockBucketDefault.java @@ -0,0 +1,59 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2BucketDefaultRetention; +import com.backblaze.b2.client.structures.B2FileRetention; +import com.backblaze.b2.client.structures.B2FileRetentionMode; +import com.backblaze.b2.client.structures.B2FileRetentionPeriodUnit; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2UpdateBucketRequest; + +/** + * lock.bucket_default. Run by ../run-conformance (see ../../README.md). + * + *

Object Lock: a bucket's default retention means "All of the files that are uploaded or copied to that bucket, + * from that time on, are protected by those Object Lock retention settings by default." + * (https://www.backblaze.com/docs/cloud-storage-object-lock). The read-back shape, defaultRetention {mode, period + * {duration, unit}}, is the native API's + * (https://www.backblaze.com/docs/cloud-storage-enable-object-lock-with-the-native-api). + * + *

A file-lock-enabled bucket gets a default retention of governance, 1 day. Asserted: the update's response and + * a fresh read of the bucket report the mode and the period, and a file uploaded afterwards inherits a governance + * retention that ends about one day from now (within 30 minutes). + */ +public class LockBucketDefault { + private static final long DAY_MILLIS = 24L * 60 * 60 * 1000; + private static final long TOLERANCE_MILLIS = 30L * 60 * 1000; + + public static void main(String[] args) { + Support.run("lock.bucket_default", () -> { + Support.step("bucket"); + final B2Bucket bucket = Support.lockedBucket(); + Support.check(bucket.getFileLockConfiguration().isFileLockEnabled(), "the new bucket does not report file lock as enabled"); + + Support.step("set the default retention"); + final B2BucketDefaultRetention wanted = new B2BucketDefaultRetention(B2FileRetentionMode.GOVERNANCE, 1, B2FileRetentionPeriodUnit.DAYS); + final B2Bucket updated = Support.client.updateBucket(B2UpdateBucketRequest.builder(bucket).setDefaultRetention(wanted).build()); + Support.eq("default retention in the update response", wanted, updated.getFileLockConfiguration().getDefaultRetention()); + + Support.step("read back"); + final B2Bucket reread = Support.client.getBucketOrNullByName(bucket.getBucketName()); + Support.check(reread != null, "the bucket is not listed"); + Support.eq("default retention mode", B2FileRetentionMode.GOVERNANCE, reread.getFileLockConfiguration().getDefaultRetentionMode()); + Support.eq("default retention duration", 1, reread.getFileLockConfiguration().getDefaultRetentionPeriod().getDuration()); + Support.eq("default retention unit", B2FileRetentionPeriodUnit.DAYS, reread.getFileLockConfiguration().getDefaultRetentionPeriod().getUnit()); + Support.check(reread.getFileLockConfiguration().isFileLockEnabled(), "file lock is no longer enabled after the update"); + + Support.step("an uploaded file inherits it"); + final long before = System.currentTimeMillis(); + final B2FileVersion uploaded = Support.upload(reread, "st/lock/default.txt", Support.payload(256, 81), "text/plain", null); + final long after = System.currentTimeMillis(); + final B2FileRetention retention = Support.client.getFileInfo(uploaded.getFileId()).getFileRetention(); + Support.eq("inherited retention mode", B2FileRetentionMode.GOVERNANCE, retention.getMode()); + final long until = retention.getRetainUntilTimestamp(); + Support.check(until >= before + DAY_MILLIS - TOLERANCE_MILLIS && until <= after + DAY_MILLIS + TOLERANCE_MILLIS, + "inherited retention ends " + (until - before) / 60000 + " minutes from now, expected about " + DAY_MILLIS / 60000); + }); + } +} diff --git a/.sdkharness/tests/conformance/LockBypassGovernance.java b/.sdkharness/tests/conformance/LockBypassGovernance.java new file mode 100644 index 000000000..82aec91f2 --- /dev/null +++ b/.sdkharness/tests/conformance/LockBypassGovernance.java @@ -0,0 +1,55 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.exceptions.B2Exception; +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2DeleteFileVersionRequest; +import com.backblaze.b2.client.structures.B2FileRetention; +import com.backblaze.b2.client.structures.B2FileRetentionMode; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2UpdateFileRetentionRequest; + +import java.util.Arrays; + +/** + * lock.bypass_governance. Run by ../run-conformance (see ../../README.md). + * + *

b2_delete_file_version's bypassGovernance "Must be specified and set to true if deleting a file version + * protected by Object Lock governance mode retention settings. Setting the value requires the bypassGovernance + * application key capability." Without it the delete is refused: 401 access_denied, "The delete was not allowed + * because Object Lock is enabled on the file." (https://www.backblaze.com/apidocs/b2-delete-file-version). The + * account key the simulator hands every check holds the capability. + * + *

An object under governance retention (now + 2 days) is deleted without bypass, then with + * {@code setBypassGovernance(true)}. Asserted: the first is refused with 401 access_denied and leaves the version + * in place; the second succeeds and the version is gone from the listing and from download by id. The SDK treats a + * 401 on a call other than an upload as a reason to reauthorize and retry; the retries end in the same refusal, + * which is what the check asserts, so the extra requests are not a failure. + */ +public class LockBypassGovernance { + public static void main(String[] args) { + Support.run("lock.bypass_governance", () -> { + Support.step("bucket and object under governance"); + final B2Bucket bucket = Support.lockedBucket(); + final byte[] data = Support.payload(256, 84); + final B2FileVersion file = Support.upload(bucket, "st/lock/bypass.txt", data, "text/plain", null); + Support.client.updateFileRetention(B2UpdateFileRetentionRequest.builder(file.getFileName(), file.getFileId(), + new B2FileRetention(B2FileRetentionMode.GOVERNANCE, System.currentTimeMillis() + 2L * 24 * 60 * 60 * 1000)).build()); + + Support.step("delete without bypass is refused"); + final B2Exception refused = Support.expectError("delete of a governance-locked version without bypass", + () -> Support.client.deleteFileVersion(B2DeleteFileVersionRequest.builder(file.getFileName(), file.getFileId()).build())); + Support.eq("refusal status", 401, refused.getStatus()); + Support.eq("refusal code", "access_denied", refused.getCode()); + Support.check(Arrays.equals(data, Support.downloadById(file.getFileId()).bytes), "the refused delete changed the file"); + + Support.step("delete with bypass"); + Support.client.deleteFileVersion(B2DeleteFileVersionRequest.builder(file.getFileName(), file.getFileId()).setBypassGovernance(true).build()); + + Support.step("the version is gone"); + Support.eq("versions left after the bypassing delete", 0, Support.versions(bucket.getBucketId(), file.getFileName()).size()); + Support.eq("download by id of the deleted version", 404, Support.expectError("download of a deleted version", + () -> Support.downloadById(file.getFileId())).getStatus()); + }); + } +} diff --git a/.sdkharness/tests/conformance/LockLegalHold.java b/.sdkharness/tests/conformance/LockLegalHold.java new file mode 100644 index 000000000..5d8e272a4 --- /dev/null +++ b/.sdkharness/tests/conformance/LockLegalHold.java @@ -0,0 +1,56 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.exceptions.B2Exception; +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2LegalHold; +import com.backblaze.b2.client.structures.B2UpdateFileLegalHoldRequest; + +import java.util.Arrays; + +/** + * lock.legal_hold. Run by ../run-conformance (see ../../README.md). + * + *

b2_update_file_legal_hold takes legalHold "on" or "off" and is "Used to enable legal hold for a file in an + * Object Lock-enabled bucket, preventing it from being deleted, or to disable legal hold protections" + * (https://www.backblaze.com/apidocs/b2-update-file-legal-hold). "If either legal hold or retention settings, or even + * both, are active for a given file, then that file is protected from being deleted" + * (https://www.backblaze.com/docs/cloud-storage-object-lock). + * + *

An object in a file-lock-enabled bucket: the hold is set on, read back, and set off. Asserted: while on, the + * update's response and the metadata say on, deleting the version is refused (B2 documents the protection, not the + * status, so a 400, 401, 403 or 409 service error counts) and the file is still there; once off, the metadata says + * off and deleting the version is permitted and removes it. + */ +public class LockLegalHold { + public static void main(String[] args) { + Support.run("lock.legal_hold", () -> { + Support.step("bucket and object"); + final B2Bucket bucket = Support.lockedBucket(); + final byte[] data = Support.payload(256, 83); + final B2FileVersion file = Support.upload(bucket, "st/lock/hold.txt", data, "text/plain", null); + + Support.step("hold on"); + Support.eq("legal hold in the update response", B2LegalHold.ON, Support.client.updateFileLegalHold( + B2UpdateFileLegalHoldRequest.builder(file.getFileName(), file.getFileId(), B2LegalHold.ON).build()).getLegalHold()); + Support.eq("legal hold in the metadata", B2LegalHold.ON, Support.client.getFileInfo(file.getFileId()).getLegalHold()); + + Support.step("delete while held is refused"); + final B2Exception refused = Support.expectError("delete of a version under legal hold", + () -> Support.client.deleteFileVersion(file.getFileName(), file.getFileId())); + Support.check(Arrays.asList(400, 401, 403, 409).contains(refused.getStatus()), + "refused with status " + refused.getStatus() + " code " + refused.getCode() + ", expected 400, 401, 403 or 409"); + Support.check(Arrays.equals(data, Support.downloadById(file.getFileId()).bytes), "the refused delete changed the file"); + + Support.step("hold off"); + Support.eq("legal hold in the update response", B2LegalHold.OFF, Support.client.updateFileLegalHold( + B2UpdateFileLegalHoldRequest.builder(file.getFileName(), file.getFileId(), B2LegalHold.OFF).build()).getLegalHold()); + Support.eq("legal hold in the metadata", B2LegalHold.OFF, Support.client.getFileInfo(file.getFileId()).getLegalHold()); + + Support.step("delete once released"); + Support.client.deleteFileVersion(file.getFileName(), file.getFileId()); + Support.eq("versions left after the delete", 0, Support.versions(bucket.getBucketId(), file.getFileName()).size()); + }); + } +} diff --git a/.sdkharness/tests/conformance/LockPerFileRetention.java b/.sdkharness/tests/conformance/LockPerFileRetention.java new file mode 100644 index 000000000..2c1b58811 --- /dev/null +++ b/.sdkharness/tests/conformance/LockPerFileRetention.java @@ -0,0 +1,59 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.exceptions.B2Exception; +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2FileRetention; +import com.backblaze.b2.client.structures.B2FileRetentionMode; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2UpdateFileRetentionRequest; + +import java.util.Arrays; + +/** + * lock.per_file_retention. Run by ../run-conformance (see ../../README.md). + * + *

b2_update_file_retention sets fileRetention {mode "governance" or "compliance", retainUntilTimestamp} and + * returns it (https://www.backblaze.com/apidocs/b2-update-file-retention). "If either legal hold or retention + * settings, or even both, are active for a given file, then that file is protected from being deleted" + * (https://www.backblaze.com/docs/cloud-storage-object-lock), and b2_delete_file_version's bypassGovernance "Must be + * specified and set to true if deleting a file version protected by Object Lock governance mode retention settings" + * (https://www.backblaze.com/apidocs/b2-delete-file-version). + * + *

An object in a file-lock-enabled bucket gets a governance retention of now + 2 days. Asserted: the update's + * response and the file's metadata report that mode and timestamp, and deleting that version without bypass is + * refused with 401 access_denied (the answer B2's own contract test records for this call, and the simulator's) and + * leaves the version in place. Compliance mode is never used: nothing could lift it, and the check must clean up. + */ +public class LockPerFileRetention { + private static final long TWO_DAYS_MILLIS = 2L * 24 * 60 * 60 * 1000; + + public static void main(String[] args) { + Support.run("lock.per_file_retention", () -> { + Support.step("bucket and object"); + final B2Bucket bucket = Support.lockedBucket(); + final byte[] data = Support.payload(256, 82); + final B2FileVersion file = Support.upload(bucket, "st/lock/retention.txt", data, "text/plain", null); + + Support.step("set the retention"); + final long until = System.currentTimeMillis() + TWO_DAYS_MILLIS; + final B2FileRetention wanted = new B2FileRetention(B2FileRetentionMode.GOVERNANCE, until); + Support.eq("retention in the update response", wanted, + Support.client.updateFileRetention(B2UpdateFileRetentionRequest.builder(file.getFileName(), file.getFileId(), wanted).build()).getFileRetention()); + + Support.step("read back"); + final B2FileRetention reported = Support.client.getFileInfo(file.getFileId()).getFileRetention(); + Support.eq("file retention mode", B2FileRetentionMode.GOVERNANCE, reported.getMode()); + Support.eq("file retention retainUntilTimestamp", until, reported.getRetainUntilTimestamp()); + + Support.step("delete without bypass is refused"); + final B2Exception refused = Support.expectError("delete of a governance-locked version", + () -> Support.client.deleteFileVersion(file.getFileName(), file.getFileId())); + Support.eq("refusal status", 401, refused.getStatus()); + Support.eq("refusal code", "access_denied", refused.getCode()); + + Support.step("the version is still there"); + Support.check(Arrays.equals(data, Support.downloadById(file.getFileId()).bytes), "the refused delete changed the file"); + }); + } +} diff --git a/.sdkharness/tests/conformance/Support.java b/.sdkharness/tests/conformance/Support.java index 6b6b8f973..b496e7e2d 100644 --- a/.sdkharness/tests/conformance/Support.java +++ b/.sdkharness/tests/conformance/Support.java @@ -7,8 +7,12 @@ import com.backblaze.b2.client.contentSources.B2Headers; import com.backblaze.b2.client.exceptions.B2Exception; import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2CreateBucketRequest; +import com.backblaze.b2.client.structures.B2DeleteFileVersionRequest; import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2LegalHold; import com.backblaze.b2.client.structures.B2ListFileVersionsRequest; +import com.backblaze.b2.client.structures.B2UpdateFileLegalHoldRequest; import com.backblaze.b2.client.structures.B2UploadFileRequest; import com.backblaze.b2.client.webApiHttpClient.B2StorageHttpClientBuilder; import com.backblaze.b2.client.webApiHttpClient.HttpClientFactory; @@ -17,6 +21,7 @@ import org.apache.http.HttpHost; import org.apache.http.HttpRequest; import org.apache.http.HttpRequestInterceptor; +import org.apache.http.HttpResponseInterceptor; import org.apache.http.client.config.RequestConfig; import org.apache.http.config.Registry; import org.apache.http.config.RegistryBuilder; @@ -36,6 +41,10 @@ import java.util.Map; import java.util.Random; import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; /** * Shared plumbing for the sdkharness conformance checks. Compiled together with one check class @@ -85,9 +94,20 @@ static final class Downloaded { /** "METHOD uri" of every HTTP request the SDK sent, when the check asked for the recording client. */ static final List wire = Collections.synchronizedList(new ArrayList<>()); + /** + * With the recording client, requests whose URI contains {@link #inFlightMarker} are counted while they are in + * flight (from the moment the SDK starts sending to the moment the response arrives); {@link #maxInFlight} is the + * most that overlapped. + */ + static volatile String inFlightMarker; + static final AtomicInteger inFlight = new AtomicInteger(); + static final AtomicInteger maxInFlight = new AtomicInteger(); + + private static final List executors = new ArrayList<>(); private static String capability = "unknown"; private static String step = "configuration"; private static final List created = new ArrayList<>(); + private static final java.util.Set lockEnabled = new java.util.HashSet<>(); private static final String LOOPBACK_HOST = "127.0.0.1"; /** Hosts of requests the guard refused (the SDK may swallow the exception, so the verdict reads this). */ static final List blocked = Collections.synchronizedList(new ArrayList<>()); @@ -133,8 +153,24 @@ static void run(String cap, boolean recordWire, Body body) { } } + /** + * A second client, authorized with a key the check itself created (keys.crud), through the same guarded factory. + * It is closed with the check. The credential is the caller's; nothing here prints it. + */ + static B2StorageClient clientFor(String keyId, String key) { + final B2StorageClient other = newClient(System.getenv("SDKHARNESS_SIMULATOR_URL"), keyId, key, false); + extraClients.add(other); + return other; + } + + private static final List extraClients = new ArrayList<>(); + private static B2StorageClient newClient(String masterUrl, boolean recordWire) { - final B2ClientConfig config = B2ClientConfig.builder(KEY_ID, KEY, "b2-sdk-java-sdkharness-conformance") + return newClient(masterUrl, KEY_ID, KEY, recordWire); + } + + private static B2StorageClient newClient(String masterUrl, String keyId, String key, boolean recordWire) { + final B2ClientConfig config = B2ClientConfig.builder(keyId, key, "b2-sdk-java-sdkharness-conformance") .setMasterUrl(masterUrl) .build(); // The SDK's own request settings (timeouts, no URI normalization); the guarded factory below @@ -169,6 +205,16 @@ public CloseableHttpClient create() { requireLoopback(request, (HttpHost) context.getAttribute(HttpCoreContext.HTTP_TARGET_HOST)); if (recordWire) { wire.add(request.getRequestLine().getMethod() + " " + request.getRequestLine().getUri()); + if (matchesInFlight(request)) { + final int now = inFlight.incrementAndGet(); + maxInFlight.accumulateAndGet(now, Math::max); + } + } + }) + .addInterceptorLast((HttpResponseInterceptor) (response, context) -> { + final HttpRequest request = (HttpRequest) context.getAttribute(HttpCoreContext.HTTP_REQUEST); + if (recordWire && request != null && matchesInFlight(request)) { + inFlight.decrementAndGet(); } }) .build(); @@ -181,6 +227,11 @@ public void close() { }; } + private static boolean matchesInFlight(HttpRequest request) { + final String marker = inFlightMarker; + return marker != null && request.getRequestLine().getUri().contains(marker); + } + /** * Throws, and records the host, unless the request's target is 127.0.0.1. The target host is the * one the client is about to connect to (set on the context before interceptors run, also on each @@ -210,9 +261,43 @@ private static void cleanup() { if (client == null) { return; } + for (ExecutorService executor : executors) { + executor.shutdownNow(); + } + for (B2StorageClient other : extraClients) { + try { + other.close(); + } catch (RuntimeException e) { + // nothing left to release + } + } + for (String keyId : createdKeys) { + try { + client.deleteKey(com.backblaze.b2.client.structures.B2DeleteKeyRequest.builder(keyId).build()); + } catch (B2Exception | RuntimeException e) { + cleanupErrors.add("key " + keyId + ": " + describe(e)); + } + } for (B2Bucket bucket : created) { try { - client.deleteAllFilesInBucket(bucket.getBucketId()); + // A large file left unfinished would keep the bucket from being deleted. + for (B2FileVersion unfinished : client.unfinishedLargeFiles(bucket.getBucketId())) { + client.cancelLargeFile(unfinished.getFileId()); + } + if (lockEnabled.contains(bucket.getBucketId())) { + // Files under a lock: a legal hold is released and governance retention is bypassed (the simulator's + // key holds that capability). Compliance retention is never set by a check, since nothing can lift it. + for (B2FileVersion version : client.fileVersions(bucket.getBucketId())) { + try { + client.updateFileLegalHold(B2UpdateFileLegalHoldRequest.builder(version.getFileName(), version.getFileId(), B2LegalHold.OFF).build()); + } catch (B2Exception e) { + // a hide marker has no legal hold to release + } + client.deleteFileVersion(B2DeleteFileVersionRequest.builder(version.getFileName(), version.getFileId()).setBypassGovernance(true).build()); + } + } else { + client.deleteAllFilesInBucket(bucket.getBucketId()); + } client.deleteBucket(bucket.getBucketId()); } catch (B2Exception | RuntimeException e) { cleanupErrors.add(bucket.getBucketName() + ": " + describe(e)); @@ -291,10 +376,44 @@ static B2Bucket bucket() throws B2Exception { return bucket; } + /** A fresh private bucket with Object Lock enabled; cleanup releases legal holds and bypasses governance retention. */ + static B2Bucket lockedBucket() throws B2Exception { + return bucket(true, null); + } + + /** A fresh private bucket made from a request the caller may customize; {@code lock} also enables Object Lock. */ + static B2Bucket bucket(boolean lock, java.util.function.Consumer customize) throws B2Exception { + final B2CreateBucketRequest.Builder builder = B2CreateBucketRequest.builder(newBucketName(), "allPrivate"); + if (lock) { + builder.setFileLockEnabled(true); + } + if (customize != null) { + customize.accept(builder); + } + final B2Bucket bucket = client.createBucket(builder.build()); + created.add(bucket); + if (lock) { + lockEnabled.add(bucket.getBucketId()); + } + return bucket; + } + static String newBucketName() { return "sdkharness-conf-" + UUID.randomUUID().toString().replace("-", "").substring(0, 12); } + private static final List createdKeys = new ArrayList<>(); + + /** Records an application key the check created, so cleanup deletes it if the check does not. */ + static void trackKey(String keyId) { + createdKeys.add(keyId); + } + + /** The check deleted this key itself. */ + static void untrackKey(String keyId) { + createdKeys.remove(keyId); + } + /** Records a bucket the check created itself, so cleanup deletes it if the check does not. */ static void track(B2Bucket bucket) { created.add(bucket); @@ -305,6 +424,37 @@ static void untrack(B2Bucket bucket) { created.removeIf(b -> b.getBucketId().equals(bucket.getBucketId())); } + /** A thread pool the SDK's large-file calls run on; shut down when the check ends. */ + static ExecutorService executor(int threads) { + final ExecutorService executor = Executors.newFixedThreadPool(threads); + executors.add(executor); + return executor; + } + + /** The part size to use: the larger of the account's recommended and absolute minimum part size, as the realm reports them. */ + static long partSize() { + try { + final long recommended = client.getAccountAuthorization().getRecommendedPartSize(); + final long minimum = client.getAccountAuthorization().getAbsoluteMinimumPartSize(); + return Math.max(recommended, minimum); + } catch (B2Exception e) { + throw new Failure("could not read the account's part sizes: " + describe(e)); + } + } + + /** How many recorded requests were "METHOD uri" with the given method and a URI containing {@code part}. */ + static int wireCount(String method, String part) { + int n = 0; + synchronized (wire) { + for (String line : wire) { + if (line.startsWith(method + " ") && line.contains(part)) { + n++; + } + } + } + return n; + } + static B2FileVersion upload(B2Bucket bucket, String name, byte[] data, String contentType, Map fileInfo) throws B2Exception { final B2UploadFileRequest.Builder request = B2UploadFileRequest @@ -346,6 +496,17 @@ static Downloaded downloadById(String fileId) throws B2Exception { return got; } + /** Download by id with server-side-encryption request headers (an SSE-C key); {@code sse} may be null. */ + static Downloaded downloadById(String fileId, com.backblaze.b2.client.structures.B2FileSseForRequest sse) throws B2Exception { + final Downloaded got = new Downloaded(); + client.downloadById(com.backblaze.b2.client.structures.B2DownloadByIdRequest.builder(fileId).setServerSideEncryption(sse).build(), + (headers, in) -> { + got.headers = headers; + got.bytes = in.readAllBytes(); + }); + return got; + } + static Downloaded downloadByName(String bucketName, String fileName) throws B2Exception { final Downloaded got = new Downloaded(); client.downloadByName(bucketName, fileName, (headers, in) -> { @@ -372,6 +533,43 @@ static List names(Iterable versions) { return names; } + /** What a plain HTTP GET answered. */ + static final class Raw { + int status; + byte[] bytes; + } + + /** + * A GET made by the check itself, not by the SDK (a share link is used by someone who has no SDK): refused + * unless its host is 127.0.0.1, recorded in {@link #blocked} like the SDK's own requests. {@code authorization} + * is sent as the Authorization header when not null. + */ + static Raw rawGet(String url, String authorization) throws java.io.IOException { + final URI uri = URI.create(url); + final String host = uri.getHost(); + if (!LOOPBACK_HOST.equals(host)) { + blocked.add(String.valueOf(host)); + throw new Failure("blocked non-loopback host: " + host); + } + final java.net.HttpURLConnection connection = (java.net.HttpURLConnection) uri.toURL().openConnection(); + try { + connection.setInstanceFollowRedirects(false); + connection.setConnectTimeout(10_000); + connection.setReadTimeout(30_000); + if (authorization != null) { + connection.setRequestProperty("Authorization", authorization); + } + final Raw raw = new Raw(); + raw.status = connection.getResponseCode(); + try (java.io.InputStream in = raw.status < 400 ? connection.getInputStream() : connection.getErrorStream()) { + raw.bytes = in == null ? new byte[0] : in.readAllBytes(); + } + return raw; + } finally { + connection.disconnect(); + } + } + static byte[] utf8(String s) { return s.getBytes(StandardCharsets.UTF_8); } diff --git a/.sdkharness/tests/conformance/UrlsNativeDownload.java b/.sdkharness/tests/conformance/UrlsNativeDownload.java new file mode 100644 index 000000000..bc1520640 --- /dev/null +++ b/.sdkharness/tests/conformance/UrlsNativeDownload.java @@ -0,0 +1,71 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2DownloadAuthorization; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2GetDownloadAuthorizationRequest; + +import java.util.Arrays; + +/** + * urls.native_download. Run by ../run-conformance (see ../../README.md). + * + *

The share link is B2's native one, never an S3 presigned URL. b2_get_download_authorization "Returns an + * authorization token that can be passed to ******* in the Authorization header or as an + * Authorization parameter", limited to a fileNamePrefix and valid for validDurationInSeconds + * (https://www.backblaze.com/apidocs/b2-get-download-authorization). ******* addresses + * {@code /file//}, answers a private file without a token with 401, and an expired token + * with 401 expired_auth_token (https://www.backblaze.com/apidocs/b2-download-file-by-name). + * + *

Asserted, over plain HTTP made by the check itself (the holder of a share link has no SDK): the SDK's + * {@code getDownloadByNameUrl} is a /file// URL with no X-Amz-* material; the file is 401 without a + * token; with the token (Authorization header) it is 200 and the bytes are right; a file outside the token's + * prefix is 401; after the validity has passed the same token is 401. + */ +public class UrlsNativeDownload { + private static final int VALID_SECONDS = 2; + + public static void main(String[] args) { + Support.run("urls.native_download", () -> { + Support.step("bucket"); + final B2Bucket bucket = Support.bucket(); + + Support.step("files"); + final byte[] data = Support.payload(300, 61); + final B2FileVersion shared = Support.upload(bucket, "st/share/file.bin", data, "application/octet-stream", null); + Support.upload(bucket, "st/other/file.bin", data, "application/octet-stream", null); + + Support.step("native url"); + final String url = Support.client.getDownloadByNameUrl(bucket.getBucketName(), shared.getFileName()); + Support.check(url.contains("/file/" + bucket.getBucketName() + "/st/share/file.bin"), "not a native /file// URL: " + url.replaceAll("\\?.*", "?...")); + Support.check(!url.contains("X-Amz-"), "the URL carries S3 presigning material"); + + Support.step("no token"); + Support.eq("a private file without a token", 401, Support.rawGet(url, null).status); + + Support.step("authorization"); + final B2DownloadAuthorization authorization = Support.client.getDownloadAuthorization( + B2GetDownloadAuthorizationRequest.builder(bucket.getBucketId(), "st/share/", VALID_SECONDS).build()); + Support.eq("authorization prefix", "st/share/", authorization.getFileNamePrefix()); + Support.eq("authorization bucket", bucket.getBucketId(), authorization.getBucketId()); + final long issued = System.nanoTime(); + + Support.step("token works within its validity"); + final Support.Raw ok = Support.rawGet(url, authorization.getAuthorizationToken()); + Support.eq("status with the token", 200, ok.status); + Support.check(Arrays.equals(data, ok.bytes), "the bytes fetched with the token differ from the upload"); + + Support.step("token is limited to its prefix"); + final String otherUrl = Support.client.getDownloadByNameUrl(bucket.getBucketName(), "st/other/file.bin"); + Support.eq("a file outside the token's prefix", 401, Support.rawGet(otherUrl, authorization.getAuthorizationToken()).status); + + Support.step("token expires"); + final long wait = VALID_SECONDS * 1000L + 1500 - (System.nanoTime() - issued) / 1_000_000; + if (wait > 0) { + Thread.sleep(wait); + } + Support.eq("status after the validity has passed", 401, Support.rawGet(url, authorization.getAuthorizationToken()).status); + }); + } +} diff --git a/.sdkharness/tests/run-conformance b/.sdkharness/tests/run-conformance index 38fdb2093..849dd5578 100755 --- a/.sdkharness/tests/run-conformance +++ b/.sdkharness/tests/run-conformance @@ -29,6 +29,25 @@ case "$SCENARIO" in files.hide) class=FilesHide ;; files.metadata) class=FilesMetadata ;; bucket.crud) class=BucketCrud ;; + files.server_side_copy) class=FilesServerSideCopy ;; + large.multipart) class=LargeMultipart ;; + large.concurrent_parts) class=LargeConcurrentParts ;; + large.resume) class=LargeResume ;; + large.compose_mixed) class=LargeComposeMixed ;; + urls.native_download) class=UrlsNativeDownload ;; + client.progress) class=ClientProgress ;; + lock.bucket_default) class=LockBucketDefault ;; + lock.per_file_retention) class=LockPerFileRetention ;; + keys.crud) class=KeysCrud ;; + bucket.cors) class=BucketCors ;; + bucket.lifecycle) class=BucketLifecycle ;; + bucket.notification_rules) class=BucketNotificationRules ;; + bucket.inbound_webhook) class=BucketInboundWebhook ;; + bucket.replication_config) class=BucketReplicationConfig ;; + enc.sse_b2) class=EncSseB2 ;; + enc.sse_c) class=EncSseC ;; + lock.legal_hold) class=LockLegalHold ;; + lock.bypass_governance) class=LockBypassGovernance ;; *) fail 'configuration: no check for this scenario' ;; esac [[ "${SDKHARNESS_SIMULATOR_URL:-}" =~ ^http://127\.0\.0\.1:[0-9]{1,5}/?$ ]] \ @@ -44,7 +63,7 @@ for name in $(compgen -e); do case "$name" in B2_*) unset "$name" ;; esac; done work="$(mktemp -d "${TMPDIR:-/tmp}/sdkharness-java-conformance.XXXXXX")" || fail 'setup: could not create a scratch directory' trap 'rm -rf "$work"' EXIT -if ! javac --release 11 -nowarn -d "$work/classes" -cp "$SDKHARNESS_JAVA_CLASSPATH" \ +if ! javac --release 11 -encoding UTF-8 -nowarn -d "$work/classes" -cp "$SDKHARNESS_JAVA_CLASSPATH" \ "$HERE/conformance/Support.java" "$HERE/conformance/$class.java" >"$work/javac.log" 2>&1; then fail "build: the check did not compile against this checkout: $(tidy "$(head -c 400 "$work/javac.log")")" fi diff --git a/.sdkharness/tests/run-resilience b/.sdkharness/tests/run-resilience index 328424697..66a8a7805 100755 --- a/.sdkharness/tests/run-resilience +++ b/.sdkharness/tests/run-resilience @@ -66,7 +66,7 @@ cleanup() { } trap cleanup EXIT -if ! javac --release 11 -nowarn -d "$work/classes" -cp "$SDKHARNESS_JAVA_CLASSPATH" \ +if ! javac --release 11 -encoding UTF-8 -nowarn -d "$work/classes" -cp "$SDKHARNESS_JAVA_CLASSPATH" \ "$HERE/resilience/Support.java" "$HERE/resilience/$class.java" >"$work/javac.log" 2>&1; then fail "build: the check did not compile against this checkout: $(tidy "$(head -c 400 "$work/javac.log")")" fi