Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package aws-c-s3 for openSUSE:Factory checked in at 2026-08-14 22:06:13 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/aws-c-s3 (Old) and /work/SRC/openSUSE:Factory/.aws-c-s3.new.1258 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "aws-c-s3" Fri Aug 14 22:06:13 2026 rev:46 rq:1370960 version:0.13.5 Changes: -------- --- /work/SRC/openSUSE:Factory/aws-c-s3/aws-c-s3.changes 2026-08-06 16:25:58.672350364 +0200 +++ /work/SRC/openSUSE:Factory/.aws-c-s3.new.1258/aws-c-s3.changes 2026-08-14 22:06:30.093976011 +0200 @@ -1,0 +2,8 @@ +Mon Aug 10 12:15:02 UTC 2026 - John Paul Adrian Glaubitz <[email protected]> + +- Update to version + * support opaque etags by @TingDaoK in (#664) + * Combine per-part CRCs for whole-object download checksum + validation by @TingDaoK in (#663) + +------------------------------------------------------------------- Old: ---- v0.13.4.tar.gz New: ---- v0.13.5.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ aws-c-s3.spec ++++++ --- /var/tmp/diff_new_pack.9aiKZm/_old 2026-08-14 22:06:30.949006530 +0200 +++ /var/tmp/diff_new_pack.9aiKZm/_new 2026-08-14 22:06:30.952006637 +0200 @@ -19,7 +19,7 @@ %define library_version 1.0.0 %define library_soversion 0unstable Name: aws-c-s3 -Version: 0.13.4 +Version: 0.13.5 Release: 0 Summary: AWS Cross-Platform, C99 wrapper for cryptography primitives License: Apache-2.0 ++++++ v0.13.4.tar.gz -> v0.13.5.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/aws-c-s3-0.13.4/.github/workflows/ci.yml new/aws-c-s3-0.13.5/.github/workflows/ci.yml --- old/aws-c-s3-0.13.4/.github/workflows/ci.yml 2026-07-31 02:02:00.000000000 +0200 +++ new/aws-c-s3-0.13.5/.github/workflows/ci.yml 2026-08-06 19:29:07.000000000 +0200 @@ -69,25 +69,23 @@ - name: gcc-7 - name: gcc-8 - name: gcc-11 + steps: + - uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=${{ matrix.compiler.name }} --cmake-extra=-DASSERT_LOCK_HELD=ON --cmake-extra=-DAWS_ENABLE_S3_ENDPOINT_RESOLVER=ON + + linux-newer-compiler-compat: + runs-on: ubuntu-24.04 # latest + strategy: + fail-fast: false + matrix: + compiler: - name: gcc-13 - # See Issue: https://github.com/llvm/llvm-project/issues/59007. Although this issue - # has been fixed in LLVM, the fix will probably not propagate to older versions of Ubuntu and GCC 13.1. - # - # Starting with GLIBC version 2.34, the `dn_expand` function, previously found in `libresolv.so`, was moved to `libc.so`. This - # function is used internally by the `getaddrinfo()` system call. - # - # In our setup (As of December 2024), we are using an Ubuntu 18 Docker image on a newer Ubuntu host. - # However, due to compatibility issues between newer libasan.so in GCC 13.1 - # and the older Ubuntu image, the linker does not link with `libresolv.so`. - # This results in crashes in `getaddrinfo()` since Ubuntu-18 GLIBC is 2.31. - # - # This problem does not occur on Ubuntu 22 and newer because GLIBC versions 2.34 - # and above include `dn_expand` in `libc.so`, eliminating the dependency on - # `libresolv.so`. - # - # We can bypass this problem by linking with "resolv" manually until we bump - # our base Linux image to Ubuntu 22. - extra-build-flag: --cmake-extra=-DCMAKE_EXE_LINKER_FLAGS="-lresolv" steps: - uses: aws-actions/configure-aws-credentials@v6 with: @@ -96,7 +94,7 @@ - name: Build ${{ env.PACKAGE_NAME }} run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=${{ matrix.compiler.name }} ${{ matrix.compiler.extra-build-flag }} --cmake-extra=-DASSERT_LOCK_HELD=ON --cmake-extra=-DAWS_ENABLE_S3_ENDPOINT_RESOLVER=ON + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-ubuntu-22-x64 build -p ${{ env.PACKAGE_NAME }} --compiler=${{ matrix.compiler.name }} --cmake-extra=-DASSERT_LOCK_HELD=ON --cmake-extra=-DAWS_ENABLE_S3_ENDPOINT_RESOLVER=ON clang-sanitizers: runs-on: ubuntu-24.04 # latest diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/aws-c-s3-0.13.4/include/aws/s3/private/s3_checksums.h new/aws-c-s3-0.13.5/include/aws/s3/private/s3_checksums.h --- old/aws-c-s3-0.13.4/include/aws/s3/private/s3_checksums.h 2026-07-31 02:02:00.000000000 +0200 +++ new/aws-c-s3-0.13.5/include/aws/s3/private/s3_checksums.h 2026-08-06 19:29:07.000000000 +0200 @@ -30,6 +30,8 @@ void (*destroy)(struct aws_s3_checksum *checksum); int (*update)(struct aws_s3_checksum *checksum, const struct aws_byte_cursor *buf); int (*finalize)(struct aws_s3_checksum *checksum, struct aws_byte_buf *out); + /* Optional. NULL for algorithms that cannot be combined. See aws_checksum_combine_digest. */ + int (*combine)(struct aws_s3_checksum *head, uint64_t tail_value, uint64_t tail_length); }; struct aws_s3_checksum { @@ -222,6 +224,38 @@ AWS_S3_API int aws_checksum_finalize(struct aws_s3_checksum *checksum, struct aws_byte_buf *output); +/** + * True if checksums of the algorithm can be combined via aws_checksum_combine_digest. + * Only the CRC algorithms (CRC32, CRC32C, CRC64NVME) can be. + */ +AWS_S3_API +bool aws_checksum_algorithm_is_combinable(enum aws_s3_checksum_algorithm algorithm); + +/* Largest digest produced by an algorithm that satisfies aws_checksum_algorithm_is_combinable. + * The combinable algorithms are all CRCs, so sizeof(uint64_t) covers CRC64NVME and CRC32s. */ +#define AWS_S3_COMBINABLE_DIGEST_MAX_LEN sizeof(uint64_t) + +/** + * Folds the digest of one data block into `head`, so that `head` becomes the checksum of its own data + * followed by that block, without re-scanning either: + * + * head = checksum(block_head) + * tail_digest = checksum_finalize(block_tail) + * aws_checksum_combine_digest(head, tail_digest, block_tail_length) + * -> head == checksum(block_head || block_tail) + * + * Taking a digest rather than a live checksum lets the caller fold in a block long after the + * checksum that produced it is gone. `tail_length` is the length in bytes of the data that produced + * `tail_digest`, not the digest size. + * + * `head`'s algorithm must satisfy aws_checksum_algorithm_is_combinable. + * AWS_ERROR_UNSUPPORTED_OPERATION for a non-combinable algorithm, + * AWS_ERROR_INVALID_STATE if `head` is already finalized, + * AWS_ERROR_INVALID_ARGUMENT if `tail_digest` is not exactly the algorithm's digest size. + */ +AWS_S3_API +int aws_checksum_combine_digest(struct aws_s3_checksum *head, struct aws_byte_cursor tail_digest, uint64_t tail_length); + AWS_S3_API int aws_s3_meta_request_checksum_config_storage_init( struct aws_allocator *allocator, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/aws-c-s3-0.13.4/include/aws/s3/private/s3_meta_request_impl.h new/aws-c-s3-0.13.5/include/aws/s3/private/s3_meta_request_impl.h --- old/aws-c-s3-0.13.4/include/aws/s3/private/s3_meta_request_impl.h 2026-07-31 02:02:00.000000000 +0200 +++ new/aws-c-s3-0.13.5/include/aws/s3/private/s3_meta_request_impl.h 2026-08-06 19:29:07.000000000 +0200 @@ -54,6 +54,22 @@ void *user_data; }; +/* One part's contribution to the meta request's whole-object checksum. Lives in + * aws_s3_meta_request.combine_slots at index (part_number - 1). + * + * Deliberately plain data: it owns nothing and points at nothing, so its lifetime is the meta + * request's and is independent of the request. That is what lets a request tear down its own running + * checksums at stream completion while its contribution to the whole-object checksum outlives it. */ +struct aws_s3_combine_slot { + /* Length in bytes of the part body the digest covers. */ + uint64_t length; + /* Raw (not base64) digest of the part body, `digest_len` bytes. */ + uint8_t digest[AWS_S3_COMBINABLE_DIGEST_MAX_LEN]; + /* Zero until this part records its digest. A slot still zero when the meta request finishes means + * the part never completed, so the whole-object checksum cannot be assembled. */ + size_t digest_len; +}; + /* An event to be delivered on the meta-request's io_event_loop thread. */ struct aws_s3_meta_request_event { enum aws_s3_meta_request_event_type { @@ -352,6 +368,23 @@ /* running checksum of all the parts of a default get, or ranged get meta request*/ struct aws_s3_checksum *meta_request_level_running_response_sum; + /* True when meta_request_level_running_response_sum uses an algorithm that aws_checksum_combine supports + * (the CRCs). In that case each part computes a digest of its own body on its connection's thread and + * records it in combine_slots, and the whole-object sum is assembled from those digests with an O(1) + * combine per part when the meta request finishes, so no thread re-reads the body. Otherwise the delivery + * thread feeds bytes into the running sum directly, which requires delivery to be in object order. */ + bool meta_request_level_checksum_combinable; + + /* Per-part digests will be folded into meta_request_level_running_response_sum, indexed by + * (part_number - 1). NULL unless meta_request_level_checksum_combinable is true. + * + * Not protected by the synced data lock, and does not need to be: the array is allocated once, before any + * part is dispatched, and never resized, so slot addresses are stable; each part writes only its own slot + * and touches no shared bookkeeping. */ + struct aws_s3_combine_slot *combine_slots; + /* Number of entries in combine_slots. Zero when combine_slots is NULL. */ + uint32_t combine_slot_count; + /* The receiving file handler */ FILE *recv_file; struct aws_string *recv_filepath; @@ -482,6 +515,26 @@ struct aws_s3_meta_request *meta_request, struct aws_s3_request *request); +/* Decides how the whole-object response checksum will be built, once the discovery response has told us which + * algorithm the object uses and how many parts it has. Call after aws_s3_check_headers_for_checksum() has run + * at the meta request level, before any part is dispatched, with the meta request's synced data lock HELD. + * + * For the CRCs, each part checksums its own body on its own connection's thread and records the digest in + * meta_request->combine_slots, which this function allocates. The digests are folded together with an O(1) + * combine per part when the meta request finishes, so no thread ever re-reads the object. Everything else + * falls back to feeding the running sum from the delivery thread. + * + * `discovery_request` is the request whose response headers were just inspected. If it carried body bytes of + * its own (a partNumber=1 GET rather than a HEAD), its digest is computed here, since its body may arrive before + * the algorithm was known. + * + * Only worth calling for multipart downloads; with a single request there are no parts to combine. */ +AWS_S3_API +int aws_s3_meta_request_setup_checksum_combine_synced( + struct aws_s3_meta_request *meta_request, + struct aws_s3_request *discovery_request, + uint32_t total_num_parts); + /* Add an event for delivery on the meta-request's io_event_loop thread. * These events usually correspond to callbacks that must fire sequentially and non-overlapping, * such as delivery of a part's response body. */ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/aws-c-s3-0.13.4/include/aws/s3/private/s3_request.h new/aws-c-s3-0.13.5/include/aws/s3/private/s3_request.h --- old/aws-c-s3-0.13.4/include/aws/s3/private/s3_request.h 2026-07-31 02:02:00.000000000 +0200 +++ new/aws-c-s3-0.13.5/include/aws/s3/private/s3_request.h 2026-08-06 19:29:07.000000000 +0200 @@ -264,8 +264,20 @@ /* checksum found in the header of an individual get part http request */ struct aws_byte_buf request_level_response_header_checksum; - /* running checksum of the response to an individual get part http request */ + /* Running checksum used to validate this part against its own checksum header, which + * request_level_response_header_checksum holds the expected value for. NULL when the response carried no + * checksum header of its own. Its algorithm is whichever one that header named. */ struct aws_s3_checksum *request_level_running_response_sum; + + /* Running checksum used only to produce this part's digest for the meta request's whole-object combine. + * NULL when this part does not contribute to a combine. + * + * Kept separate from request_level_running_response_sum because the two answer to different algorithms: + * this one always uses the whole-object algorithm from the discovery response, while the validation sum + * uses whatever algorithm this part's own checksum header named. Nothing requires those to agree — an + * object can carry a whole-object CRC64NVME while its parts carry per-part CRC32 — and folding a digest + * of the wrong algorithm would silently corrupt the whole-object sum. */ + struct aws_s3_checksum *request_level_combine_sum; /* The algorithm used to validate the checksum */ enum aws_s3_checksum_algorithm validation_algorithm; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/aws-c-s3-0.13.4/source/s3_auto_ranged_get.c new/aws-c-s3-0.13.5/source/s3_auto_ranged_get.c --- old/aws-c-s3-0.13.4/source/s3_auto_ranged_get.c 2026-07-31 02:02:00.000000000 +0200 +++ new/aws-c-s3-0.13.5/source/s3_auto_ranged_get.c 2026-08-06 19:29:07.000000000 +0200 @@ -835,6 +835,8 @@ num_parts, auto_ranged_get->estimated_object_stored_part_size); } else { + /* avoid leaking the error code */ + aws_reset_error(); num_parts = 1; /* Failed to parse ETags */ AWS_LOGF_WARN( @@ -842,7 +844,6 @@ "id=%p Failed to parse ETags, fallback to default part size.", (void *)meta_request); auto_ranged_get->estimated_object_stored_part_size = g_default_part_size_fallback; - goto update_synced_data; } auto_ranged_get->num_stored_parts = num_parts; } @@ -1014,6 +1015,15 @@ object_range_start, object_range_end); } + + /* Only now is the part count known, which is what sizes the per-part checksum slots. Deciding + * here also means every part dispatched afterwards sees the decision already made. */ + if (meta_request->checksum_config.validate_response_checksum && error_code == AWS_ERROR_SUCCESS) { + if (aws_s3_meta_request_setup_checksum_combine_synced( + meta_request, request, auto_ranged_get->synced_data.total_num_parts) != AWS_OP_SUCCESS) { + error_code = aws_last_error_or_unknown(); + } + } } switch (request->request_tag) { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/aws-c-s3-0.13.4/source/s3_checksums.c new/aws-c-s3-0.13.5/source/s3_checksums.c --- old/aws-c-s3-0.13.4/source/s3_checksums.c 2026-07-31 02:02:00.000000000 +0200 +++ new/aws-c-s3-0.13.5/source/s3_checksums.c 2026-08-06 19:29:07.000000000 +0200 @@ -253,6 +253,22 @@ aws_mem_release(checksum->allocator, checksum); } +static int s_crc32_combine(struct aws_s3_checksum *head, uint64_t tail_value, uint64_t tail_length) { + head->impl.crc_val_32bit = aws_checksums_crc32_combine(head->impl.crc_val_32bit, (uint32_t)tail_value, tail_length); + return AWS_OP_SUCCESS; +} + +static int s_crc32c_combine(struct aws_s3_checksum *head, uint64_t tail_value, uint64_t tail_length) { + head->impl.crc_val_32bit = + aws_checksums_crc32c_combine(head->impl.crc_val_32bit, (uint32_t)tail_value, tail_length); + return AWS_OP_SUCCESS; +} + +static int s_crc64nvme_combine(struct aws_s3_checksum *head, uint64_t tail_value, uint64_t tail_length) { + head->impl.crc_val_64bit = aws_checksums_crc64nvme_combine(head->impl.crc_val_64bit, tail_value, tail_length); + return AWS_OP_SUCCESS; +} + static struct aws_checksum_vtable hash_vtable = { .update = s_hash_update, .finalize = s_hash_finalize, @@ -269,16 +285,19 @@ .update = s_crc32_checksum_update, .finalize = s_crc32_finalize, .destroy = s_crc_destroy, + .combine = s_crc32_combine, }; static struct aws_checksum_vtable crc32c_vtable = { .update = s_crc32c_checksum_update, .finalize = s_crc32_finalize, .destroy = s_crc_destroy, + .combine = s_crc32c_combine, }; static struct aws_checksum_vtable crc64nvme_vtable = { .update = s_crc64nvme_checksum_update, .finalize = s_crc64_finalize, .destroy = s_crc_destroy, + .combine = s_crc64nvme_combine, }; struct aws_s3_checksum *aws_hash_new(struct aws_allocator *allocator, aws_hash_new_fn hash_fn) { @@ -405,6 +424,49 @@ return checksum->vtable->finalize(checksum, output); } +bool aws_checksum_algorithm_is_combinable(enum aws_s3_checksum_algorithm algorithm) { + switch (algorithm) { + case AWS_SCA_CRC32: + case AWS_SCA_CRC32C: + case AWS_SCA_CRC64NVME: + return true; + default: + return false; + } +} + +int aws_checksum_combine_digest( + struct aws_s3_checksum *head, + struct aws_byte_cursor tail_digest, + uint64_t tail_length) { + + if (head == NULL || head->vtable == NULL || head->vtable->combine == NULL) { + return aws_raise_error(AWS_ERROR_UNSUPPORTED_OPERATION); + } + if (!head->good) { + return aws_raise_error(AWS_ERROR_INVALID_STATE); + } + if (tail_digest.len != head->digest_size) { + return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); + } + + /* The CRC finalizers write digests big-endian, so read them back the same way. */ + uint64_t tail_value = 0; + if (head->digest_size == AWS_CRC32_LEN) { + uint32_t value_32 = 0; + if (!aws_byte_cursor_read_be32(&tail_digest, &value_32)) { + return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); + } + tail_value = value_32; + } else { + if (!aws_byte_cursor_read_be64(&tail_digest, &tail_value)) { + return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); + } + } + + return head->vtable->combine(head, tail_value, tail_length); +} + static int s_checksum_compute_fn( struct aws_allocator *allocator, const struct aws_byte_cursor *input, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/aws-c-s3-0.13.4/source/s3_meta_request.c new/aws-c-s3-0.13.5/source/s3_meta_request.c --- old/aws-c-s3-0.13.4/source/s3_meta_request.c 2026-07-31 02:02:00.000000000 +0200 +++ new/aws-c-s3-0.13.5/source/s3_meta_request.c 2026-08-06 19:29:07.000000000 +0200 @@ -38,6 +38,7 @@ static int s_s3_request_priority_queue_pred(const void *a, const void *b); static int s_s3_pending_prepare_entry_pred(const void *a, const void *b); +static bool s_s3_meta_request_fold_combine_slots(struct aws_s3_meta_request *meta_request); static void s_s3_meta_request_destroy(void *user_data); static void s_s3_meta_request_init_signing_date_time( @@ -100,7 +101,7 @@ aws_mutex_unlock(&meta_request->synced_data.lock); } -/* True if the checksum validated and matched, false otherwise. */ +/* True if the checksum validated and matched, false otherwise. Finalizes checksum_to_validate. */ static bool s_validate_checksum( struct aws_s3_checksum *checksum_to_validate, struct aws_byte_buf *expected_encoded_checksum) { @@ -139,17 +140,33 @@ struct aws_s3_meta_request *meta_request, struct aws_s3_meta_request_result *meta_request_result) { + /* No lock. meta_request_level_running_response_sum has a single owner at any point in its life: the + * connection thread that creates it at discovery, then either the delivery loop (byte-wise) or this + * function (folding recorded part digests), never both. Each handoff sits behind a lock that the path + * already takes, so nothing here races. See the combine_slots declaration for the ordering argument. */ if (meta_request_result->error_code == AWS_OP_SUCCESS && meta_request->meta_request_level_running_response_sum) { - meta_request_result->did_validate = true; - meta_request_result->validation_algorithm = meta_request->meta_request_level_running_response_sum->algorithm; - if (!s_validate_checksum( - meta_request->meta_request_level_running_response_sum, - &meta_request->meta_request_level_response_header_checksum)) { - meta_request_result->error_code = AWS_ERROR_S3_RESPONSE_CHECKSUM_MISMATCH; - AWS_LOGF_ERROR(AWS_LS_S3_META_REQUEST, "id=%p Checksum mismatch!", (void *)meta_request); + /* A part that never completed leaves the whole-object sum covering only a subset of the object, so + * validating it would be meaningless. Report it as unvalidated rather than as a failure: the caller + * asked for the object, and every part that arrived was still checked against whatever checksum it + * carried of its own. */ + bool complete = + !meta_request->meta_request_level_checksum_combinable || s_s3_meta_request_fold_combine_slots(meta_request); + + if (complete) { + meta_request_result->did_validate = true; + meta_request_result->validation_algorithm = + meta_request->meta_request_level_running_response_sum->algorithm; + + if (!s_validate_checksum( + meta_request->meta_request_level_running_response_sum, + &meta_request->meta_request_level_response_header_checksum)) { + meta_request_result->error_code = AWS_ERROR_S3_RESPONSE_CHECKSUM_MISMATCH; + AWS_LOGF_ERROR(AWS_LS_S3_META_REQUEST, "id=%p Checksum mismatch!", (void *)meta_request); + } } } aws_checksum_destroy(meta_request->meta_request_level_running_response_sum); + meta_request->meta_request_level_running_response_sum = NULL; aws_byte_buf_clean_up(&meta_request->meta_request_level_response_header_checksum); } @@ -715,6 +732,11 @@ AWS_ASSERT(aws_priority_queue_size(&meta_request->synced_data.pending_body_streaming_requests) == 0); aws_priority_queue_clean_up(&meta_request->synced_data.pending_body_streaming_requests); + /* Slots own nothing, so a cancelled or failed meta request can just drop them. */ + aws_mem_release(meta_request->allocator, meta_request->combine_slots); + meta_request->combine_slots = NULL; + meta_request->combine_slot_count = 0; + AWS_ASSERT(aws_array_list_length(&meta_request->synced_data.event_delivery_array) == 0); aws_array_list_clean_up(&meta_request->synced_data.event_delivery_array); @@ -755,6 +777,147 @@ return (*request_a)->part_number > (*request_b)->part_number; } +/* Records one finished part's digest so it can be folded into the whole-object checksum at finish. + * + * No lock: the slot array was sized before any part was dispatched and is never resized, and this part owns + * its slot exclusively. See the combine_slots declaration for why the write is visible to the fold. */ +static void s_s3_meta_request_record_part_digest( + struct aws_s3_meta_request *meta_request, + uint32_t part_number, + const struct aws_byte_buf *digest, + uint64_t length) { + + AWS_PRECONDITION(meta_request); + AWS_PRECONDITION(digest); + AWS_PRECONDITION(part_number > 0); + AWS_FATAL_ASSERT(digest->len > 0 && digest->len <= AWS_S3_COMBINABLE_DIGEST_MAX_LEN); + + if (part_number > meta_request->combine_slot_count) { + /* More parts turned up than the discovery response accounted for. Leave the slots alone; the fold at + * finish will report the whole-object checksum as unvalidated rather than assembling a partial sum. */ + AWS_LOGF_WARN( + AWS_LS_S3_META_REQUEST, + "id=%p Part %" PRIu32 " is beyond the %" PRIu32 " parts discovered, so its checksum cannot " + "contribute to the whole-object checksum.", + (void *)meta_request, + part_number, + meta_request->combine_slot_count); + return; + } + + struct aws_s3_combine_slot *slot = &meta_request->combine_slots[part_number - 1]; + slot->length = length; + memcpy(slot->digest, digest->buffer, digest->len); + slot->digest_len = digest->len; +} + +/* Assembles the whole-object checksum from the recorded part digests. Returns false, leaving the running sum + * untouched, if any part never recorded one. + * + * CRCs compose: folding part digests left to right in object order yields exactly the checksum of the + * concatenated object, so no thread re-reads the body. Slots are already in object order, so this is a single + * forward pass. + * + * No lock, and none needed: by the time the meta request finishes, update() has confirmed every dispatched + * part completed, so all slot writes are done and published. */ +static bool s_s3_meta_request_fold_combine_slots(struct aws_s3_meta_request *meta_request) { + AWS_PRECONDITION(meta_request); + + struct aws_s3_checksum *running_sum = meta_request->meta_request_level_running_response_sum; + AWS_FATAL_ASSERT(running_sum != NULL); + + if (meta_request->combine_slots == NULL || meta_request->combine_slot_count == 0) { + AWS_LOGF_WARN( + AWS_LS_S3_META_REQUEST, + "id=%p No part checksums were recorded, so the whole-object checksum cannot be validated.", + (void *)meta_request); + return false; + } + + for (uint32_t i = 0; i < meta_request->combine_slot_count; ++i) { + const struct aws_s3_combine_slot *slot = &meta_request->combine_slots[i]; + if (slot->digest_len == 0) { + AWS_LOGF_WARN( + AWS_LS_S3_META_REQUEST, + "id=%p Part %" PRIu32 " never recorded a checksum, so the whole-object checksum cannot be " + "validated.", + (void *)meta_request, + i + 1); + return false; + } + if (aws_checksum_combine_digest( + running_sum, aws_byte_cursor_from_array(slot->digest, slot->digest_len), slot->length)) { + AWS_LOGF_ERROR( + AWS_LS_S3_META_REQUEST, + "id=%p Failed to combine checksum for part %" PRIu32 ". last error:%s", + (void *)meta_request, + i + 1, + aws_error_name(aws_last_error_or_unknown())); + return false; + } + } + + return true; +} + +int aws_s3_meta_request_setup_checksum_combine_synced( + struct aws_s3_meta_request *meta_request, + struct aws_s3_request *discovery_request, + uint32_t total_num_parts) { + + AWS_ERROR_PRECONDITION(meta_request); + AWS_ERROR_PRECONDITION(discovery_request); + ASSERT_SYNCED_DATA_LOCK_HELD(meta_request); + + meta_request->meta_request_level_checksum_combinable = false; + + if (meta_request->meta_request_level_running_response_sum == NULL) { + /* Nothing to validate at the meta request level. */ + return AWS_OP_SUCCESS; + } + + enum aws_s3_checksum_algorithm algorithm = meta_request->meta_request_level_running_response_sum->algorithm; + if (!aws_checksum_algorithm_is_combinable(algorithm)) { + /* Leave it to the delivery loop, which sees the body in object order. */ + return AWS_OP_SUCCESS; + } + if (total_num_parts == 0) { + /* An empty object, so there is nothing to combine. */ + return AWS_OP_SUCCESS; + } + + AWS_FATAL_ASSERT(meta_request->combine_slots == NULL); + meta_request->combine_slots = + aws_mem_calloc(meta_request->allocator, total_num_parts, sizeof(struct aws_s3_combine_slot)); + meta_request->combine_slot_count = total_num_parts; + meta_request->meta_request_level_checksum_combinable = true; + + /* From here on, each part checksums itself as it streams. The discovery request is the exception: if it + * carried body bytes, they arrived before we knew which algorithm to use, so checksum that one part from + * its buffer now. A HEAD discovery has no body and needs nothing. */ + if (discovery_request->part_number == 0 || discovery_request->send_data.response_body.len == 0) { + return AWS_OP_SUCCESS; + } + + struct aws_s3_checksum *part_sum = aws_checksum_new(meta_request->allocator, algorithm); + if (part_sum == NULL) { + return AWS_OP_ERR; + } + + struct aws_byte_cursor body = aws_byte_cursor_from_buf(&discovery_request->send_data.response_body); + uint8_t digest_storage[AWS_S3_COMBINABLE_DIGEST_MAX_LEN]; + struct aws_byte_buf part_digest = aws_byte_buf_from_empty_array(digest_storage, sizeof(digest_storage)); + + if (aws_checksum_update(part_sum, &body) || aws_checksum_finalize(part_sum, &part_digest)) { + aws_checksum_destroy(part_sum); + return AWS_OP_ERR; + } + aws_checksum_destroy(part_sum); + + s_s3_meta_request_record_part_digest(meta_request, discovery_request->part_number, &part_digest, body.len); + return AWS_OP_SUCCESS; +} + static int s_s3_pending_prepare_entry_pred(const void *a, const void *b) { const struct aws_s3_pending_prepare_entry *entry_a = a; const struct aws_s3_pending_prepare_entry *entry_b = b; @@ -1466,6 +1629,38 @@ return false; } +/* Check to see if we need to create a request_level_combine_sum for combine the checksum for the full object */ +static int s_ensure_part_combine_sum(struct aws_s3_meta_request *meta_request, struct aws_s3_request *request) { + AWS_ERROR_PRECONDITION(meta_request); + AWS_ERROR_PRECONDITION(request); + + if (request->request_level_combine_sum != NULL) { + /* Already running, skipping. */ + return AWS_OP_SUCCESS; + } + if (request->part_number == 0) { + /* Not a part, so it has no place in the object's byte order. */ + return AWS_OP_SUCCESS; + } + if (!meta_request->meta_request_level_checksum_combinable || + meta_request->meta_request_level_running_response_sum == NULL) { + /* No track on the meta request level for combine the checksum */ + return AWS_OP_SUCCESS; + } + + request->request_level_combine_sum = + aws_checksum_new(meta_request->allocator, meta_request->meta_request_level_running_response_sum->algorithm); + if (request->request_level_combine_sum == NULL) { + AWS_LOGF_ERROR( + AWS_LS_S3_META_REQUEST, + "id=%p Could not create part checksum for request %p", + (void *)meta_request, + (void *)request); + return AWS_OP_ERR; + } + return AWS_OP_SUCCESS; +} + static int s_s3_meta_request_incoming_headers( struct aws_http_stream *stream, enum aws_http_header_block header_block, @@ -1628,6 +1823,18 @@ * because when object_size_hint is provided the buffer is sized to min(hint, part_size), which may be * smaller than part_size. Using part_size here would miss the case where hint < content_length <= part_size. */ + struct aws_s3_meta_request *meta_request = request->meta_request; + AWS_PRECONDITION(meta_request); + + /* Check if we need to create a `request_level_combine_sum` or not */ + if (meta_request->checksum_config.validate_response_checksum && + request->request_type == AWS_S3_REQUEST_TYPE_GET_OBJECT && + s_s3_meta_request_error_code_from_response_status(request->send_data.response_status) == AWS_ERROR_SUCCESS) { + if (s_ensure_part_combine_sum(meta_request, request)) { + return AWS_OP_ERR; + } + } + if (request->request_type == AWS_S3_REQUEST_TYPE_GET_OBJECT && request->request_tag == AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_GET_OBJECT_WITH_PART_NUMBER_1) { uint64_t content_length; @@ -1679,9 +1886,26 @@ AWS_LOGF_TRACE(AWS_LS_S3_META_REQUEST, "response body: \n" PRInSTR "\n", AWS_BYTE_CURSOR_PRI(*data)); } - if (meta_request->checksum_config.validate_response_checksum && request->request_level_running_response_sum) { - /* Update the request level checksum. */ - aws_checksum_update(request->request_level_running_response_sum, data); + if (meta_request->checksum_config.validate_response_checksum) { + /* Update whichever of this part's running checksums are active, while the data is still hot from the + * socket read on this connection's thread. The two are independent: validation_sum is present only + * when this part's response carried a checksum header of its own, combine_sum only when this part + * feeds the whole-object checksum. They are separate objects because those two checksums may not use + * the same algorithm. */ + struct aws_s3_checksum *validation_sum = request->request_level_running_response_sum; + struct aws_s3_checksum *combine_sum = request->request_level_combine_sum; + + if ((validation_sum != NULL && aws_checksum_update(validation_sum, data)) || + (combine_sum != NULL && aws_checksum_update(combine_sum, data))) { + AWS_LOGF_ERROR( + AWS_LS_S3_META_REQUEST, + "id=%p: Request %p could not update checksum due to error %d (%s).", + (void *)meta_request, + (void *)request, + aws_last_error_or_unknown(), + aws_error_str(aws_last_error_or_unknown())); + return AWS_OP_ERR; + } } if (request->send_data.response_body.capacity == 0) { @@ -1761,12 +1985,15 @@ struct aws_s3_meta_request *meta_request = request->meta_request; if (meta_request->checksum_config.validate_response_checksum) { - /* finish the request level checksum validation. */ - if (error_code == AWS_OP_SUCCESS && request->request_level_running_response_sum) { + /* Validate this part against its own checksum header, if it had one. The presence of an expected + * value is what decides, not the presence of a running sum: a part that only contributes to the + * whole-object checksum has a combine sum but nothing to compare anything against on its own. */ + if (error_code == AWS_OP_SUCCESS && request->request_level_response_header_checksum.len > 0) { + struct aws_s3_checksum *part_sum = request->request_level_running_response_sum; + AWS_FATAL_ASSERT(part_sum != NULL); request->did_validate = true; - request->validation_algorithm = request->request_level_running_response_sum->algorithm; - request->checksum_match = s_validate_checksum( - request->request_level_running_response_sum, &request->request_level_response_header_checksum); + request->validation_algorithm = part_sum->algorithm; + request->checksum_match = s_validate_checksum(part_sum, &request->request_level_response_header_checksum); if (!request->checksum_match) { AWS_LOGF_ERROR( AWS_LS_S3_META_REQUEST, @@ -1779,10 +2006,29 @@ } else { request->did_validate = false; } + + /* Record this part's digest for the whole-object combine, which happens once the meta request + * finishes. It is copied into a slot the meta request owns, so the running checksum below can be torn + * down here exactly as it always was, and nothing of this request outlives it. */ + if (error_code == AWS_OP_SUCCESS && request->request_level_combine_sum != NULL) { + uint8_t digest_storage[AWS_S3_COMBINABLE_DIGEST_MAX_LEN] = {0}; + struct aws_byte_buf part_digest = aws_byte_buf_from_empty_array(digest_storage, sizeof(digest_storage)); + + if (aws_checksum_finalize(request->request_level_combine_sum, &part_digest)) { + error_code = aws_last_error_or_unknown(); + } else { + s_s3_meta_request_record_part_digest( + meta_request, request->part_number, &part_digest, request->send_data.response_body.len); + } + } + aws_checksum_destroy(request->request_level_running_response_sum); - aws_byte_buf_clean_up(&request->request_level_response_header_checksum); request->request_level_running_response_sum = NULL; + aws_checksum_destroy(request->request_level_combine_sum); + request->request_level_combine_sum = NULL; + aws_byte_buf_clean_up(&request->request_level_response_header_checksum); } + /* BEGIN CRITICAL SECTION */ { aws_s3_meta_request_lock_synced_data(meta_request); @@ -2373,7 +2619,12 @@ meta_request->io_threaded_data.next_deliver_range_start += response_body.len; /* 4. Checksum update */ - if (error_code == AWS_ERROR_SUCCESS && meta_request->meta_request_level_running_response_sum) { + if (error_code == AWS_ERROR_SUCCESS && meta_request->meta_request_level_running_response_sum && + !meta_request->meta_request_level_checksum_combinable) { + /* Non-combinable algorithm, so the whole-object checksum can only be built by feeding it + * bytes in object order, which is what this delivery loop guarantees. Combinable + * algorithms fold in each part's own checksum when the part completes instead, so there + * is nothing to do here. */ if (aws_checksum_update(meta_request->meta_request_level_running_response_sum, &response_body)) { error_code = aws_last_error(); AWS_LOGF_ERROR( diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/aws-c-s3-0.13.4/tests/CMakeLists.txt new/aws-c-s3-0.13.5/tests/CMakeLists.txt --- old/aws-c-s3-0.13.4/tests/CMakeLists.txt 2026-07-31 02:02:00.000000000 +0200 +++ new/aws-c-s3-0.13.5/tests/CMakeLists.txt 2026-08-06 19:29:07.000000000 +0200 @@ -194,6 +194,9 @@ add_net_test_case(test_s3_round_trip) add_net_test_case(test_s3_round_trip_default_get) add_net_test_case(test_s3_round_trip_multipart_get_fc) +add_net_test_case(test_s3_multipart_get_whole_object_checksum) +add_net_test_case(test_s3_multipart_get_full_object_checksum_header) +add_net_test_case(test_s3_multipart_get_full_object_checksum_callback) add_net_test_case(test_s3_round_trip_default_get_fc) add_net_test_case(test_s3_round_trip_empty_fc) add_net_test_case(test_s3_round_trip_mpu_multipart_get_fc) @@ -326,6 +329,15 @@ add_test_case(crc64nvme_test_invalid_buffer) add_test_case(crc64nvme_test_invalid_state) +add_test_case(checksum_combine_crc32) +add_test_case(checksum_combine_crc32c) +add_test_case(checksum_combine_crc64nvme) +add_test_case(checksum_combine_many_blocks) +add_test_case(checksum_combine_identity) +add_test_case(checksum_combine_empty_tail) +add_test_case(checksum_combine_unsupported_algorithms) +add_test_case(checksum_combine_invalid_state) + add_test_case(crc32_nist_test_case_1) add_test_case(crc32_nist_test_case_2) add_test_case(crc32_nist_test_case_3) @@ -400,8 +412,12 @@ add_net_test_case(async_internal_error_from_complete_multipart_mock_server) add_net_test_case(async_access_denied_from_complete_multipart_mock_server) add_net_test_case(get_object_modified_mock_server) + add_net_test_case(get_object_opaque_etag_mock_server) add_net_test_case(get_object_invalid_responses_mock_server) add_net_test_case(get_object_mismatch_checksum_responses_mock_server) + add_net_test_case(multipart_download_checksum_combine_mock_server) + add_net_test_case(multipart_download_checksum_combine_out_of_order_mock_server) + add_net_test_case(download_checksum_single_part_with_part_header_mock_server) add_net_test_case(get_object_throughput_failure_mock_server) add_net_test_case(get_object_long_error_mock_server) add_net_test_case(upload_part_invalid_response_mock_server) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/aws-c-s3-0.13.4/tests/mock_s3_server/GetObject/get_object_checksum_combine_head.json new/aws-c-s3-0.13.5/tests/mock_s3_server/GetObject/get_object_checksum_combine_head.json --- old/aws-c-s3-0.13.4/tests/mock_s3_server/GetObject/get_object_checksum_combine_head.json 1970-01-01 01:00:00.000000000 +0100 +++ new/aws-c-s3-0.13.5/tests/mock_s3_server/GetObject/get_object_checksum_combine_head.json 2026-08-06 19:29:07.000000000 +0200 @@ -0,0 +1,15 @@ +{ + "status": 200, + "headers": { + "ETag": "checksumcombinemocketag", + "Date": "Thu, 12 Jan 2023 00:04:21 GMT", + "Last-Modified": "Tue, 10 Jan 2023 23:39:32 GMT", + "Accept-Ranges": "bytes", + "Content-Length": "262144", + "Content-Type": "binary/octet-stream", + "x-amz-checksum-crc32": "uo2NxA==" + }, + "body": [ + "" + ] +} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/aws-c-s3-0.13.4/tests/mock_s3_server/GetObject/get_object_checksum_combine_part.json new/aws-c-s3-0.13.5/tests/mock_s3_server/GetObject/get_object_checksum_combine_part.json --- old/aws-c-s3-0.13.4/tests/mock_s3_server/GetObject/get_object_checksum_combine_part.json 1970-01-01 01:00:00.000000000 +0100 +++ new/aws-c-s3-0.13.5/tests/mock_s3_server/GetObject/get_object_checksum_combine_part.json 2026-08-06 19:29:07.000000000 +0200 @@ -0,0 +1,14 @@ +{ + "status": 206, + "headers": { + "ETag": "checksumcombinemocketag", + "Date": "Thu, 12 Jan 2023 00:04:21 GMT", + "Last-Modified": "Tue, 10 Jan 2023 23:39:32 GMT", + "Accept-Ranges": "bytes", + "Content-Range": "<request-range>/262144", + "Content-Type": "binary/octet-stream" + }, + "body": [ + "<data-to-send>" + ] +} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/aws-c-s3-0.13.4/tests/mock_s3_server/GetObject/get_object_checksum_combine_slow_part.json new/aws-c-s3-0.13.5/tests/mock_s3_server/GetObject/get_object_checksum_combine_slow_part.json --- old/aws-c-s3-0.13.4/tests/mock_s3_server/GetObject/get_object_checksum_combine_slow_part.json 1970-01-01 01:00:00.000000000 +0100 +++ new/aws-c-s3-0.13.5/tests/mock_s3_server/GetObject/get_object_checksum_combine_slow_part.json 2026-08-06 19:29:07.000000000 +0200 @@ -0,0 +1,15 @@ +{ + "delay": 1, + "status": 206, + "headers": { + "ETag": "checksumcombinemocketag", + "Date": "Thu, 12 Jan 2023 00:04:21 GMT", + "Last-Modified": "Tue, 10 Jan 2023 23:39:32 GMT", + "Accept-Ranges": "bytes", + "Content-Range": "<request-range>/262144", + "Content-Type": "binary/octet-stream" + }, + "body": [ + "<data-to-send>" + ] +} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/aws-c-s3-0.13.4/tests/mock_s3_server/GetObject/get_object_checksum_single_part.json new/aws-c-s3-0.13.5/tests/mock_s3_server/GetObject/get_object_checksum_single_part.json --- old/aws-c-s3-0.13.4/tests/mock_s3_server/GetObject/get_object_checksum_single_part.json 1970-01-01 01:00:00.000000000 +0100 +++ new/aws-c-s3-0.13.5/tests/mock_s3_server/GetObject/get_object_checksum_single_part.json 2026-08-06 19:29:07.000000000 +0200 @@ -0,0 +1,15 @@ +{ + "status": 206, + "headers": { + "ETag": "checksumsinglepartmocketag", + "Date": "Thu, 12 Jan 2023 00:04:21 GMT", + "Last-Modified": "Tue, 10 Jan 2023 23:39:32 GMT", + "Accept-Ranges": "bytes", + "Content-Range": "<request-range>/65536", + "Content-Type": "binary/octet-stream", + "x-amz-checksum-crc32": "wyCR/w==" + }, + "body": [ + "<data-to-send>" + ] +} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/aws-c-s3-0.13.4/tests/mock_s3_server/GetObject/get_object_checksum_single_part_head.json new/aws-c-s3-0.13.5/tests/mock_s3_server/GetObject/get_object_checksum_single_part_head.json --- old/aws-c-s3-0.13.4/tests/mock_s3_server/GetObject/get_object_checksum_single_part_head.json 1970-01-01 01:00:00.000000000 +0100 +++ new/aws-c-s3-0.13.5/tests/mock_s3_server/GetObject/get_object_checksum_single_part_head.json 2026-08-06 19:29:07.000000000 +0200 @@ -0,0 +1,15 @@ +{ + "status": 200, + "headers": { + "ETag": "checksumsinglepartmocketag", + "Date": "Thu, 12 Jan 2023 00:04:21 GMT", + "Last-Modified": "Tue, 10 Jan 2023 23:39:32 GMT", + "Accept-Ranges": "bytes", + "Content-Length": "65536", + "Content-Type": "binary/octet-stream", + "x-amz-checksum-crc32": "wyCR/w==" + }, + "body": [ + "" + ] +} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/aws-c-s3-0.13.4/tests/mock_s3_server/GetObject/get_object_opaque_etag.json new/aws-c-s3-0.13.5/tests/mock_s3_server/GetObject/get_object_opaque_etag.json --- old/aws-c-s3-0.13.4/tests/mock_s3_server/GetObject/get_object_opaque_etag.json 1970-01-01 01:00:00.000000000 +0100 +++ new/aws-c-s3-0.13.5/tests/mock_s3_server/GetObject/get_object_opaque_etag.json 2026-08-06 19:29:07.000000000 +0200 @@ -0,0 +1,14 @@ +{ + "status": 206, + "headers": { + "ETag": "IAMRANDOMETAG", + "Date": "Thu, 12 Jan 2023 00:04:21 GMT", + "Last-Modified": "Tue, 10 Jan 2023 23:39:32 GMT", + "Accept-Ranges": "bytes", + "Content-Range": "<request-range>/262144", + "Content-Type": "binary/octet-stream" + }, + "body": [ + "<data-to-send>" + ] +} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/aws-c-s3-0.13.4/tests/mock_s3_server/mock_s3_server.py new/aws-c-s3-0.13.5/tests/mock_s3_server/mock_s3_server.py --- old/aws-c-s3-0.13.4/tests/mock_s3_server/mock_s3_server.py 2026-07-31 02:02:00.000000000 +0200 +++ new/aws-c-s3-0.13.5/tests/mock_s3_server/mock_s3_server.py 2026-08-06 19:29:07.000000000 +0200 @@ -442,6 +442,31 @@ if parsed_path.path == "/get_object_modified": return handle_get_object_modified(start_range, end_range, request) + if parsed_path.path == "/get_object_checksum_single_part": + # 64 KiB object that downloads as a single part, where the part response carries the + # whole-object CRC32 as its own checksum header (correct value, so per-part validation + # passes). Exercises a part being both validated against its own header and folded into + # the whole-object checksum. + if head_request: + return ResponseConfig("/get_object_checksum_single_part_head", request=request) + response_config = ResponseConfig("/get_object_checksum_single_part", request=request) + response_config.generate_body_size = data_length + return response_config + + if parsed_path.path in ("/get_object_checksum_combine", "/get_object_checksum_combine_out_of_order"): + # 256 KiB object of repeated 'a'. The whole-object CRC32 is advertised on the HEAD response only, + # the way real S3 does for a single-part upload: ranged part responses carry no checksum header. + # Validation can therefore only succeed by combining the per-part CRCs. + if head_request: + return ResponseConfig("/get_object_checksum_combine_head", request=request) + if parsed_path.path == "/get_object_checksum_combine_out_of_order" and start_range == 65536: + # Delay part 2 so parts 3 and 4 complete first and have to wait their turn to combine. + response_config = ResponseConfig("/get_object_checksum_combine_slow_part", request=request) + else: + response_config = ResponseConfig("/get_object_checksum_combine_part", request=request) + response_config.generate_body_size = data_length + return response_config + if parsed_path.path == "/get_object_pause_delay_part": # 256 KiB object served in 64 KiB parts (4 parts). The part at offset 65536 # (part 2) is delayed so the test can pause while parts 1, 3, 4 have completed. @@ -472,6 +497,14 @@ response_config.generate_body_size = data_length return response_config + if parsed_path.path == "/get_object_opaque_etag": + # 256 KiB object whose ETag is an opaque identifier that does not follow the + # S3 "<md5>-<num_parts>" format. Used to verify auto-ranged GET still works + # when the part count cannot be parsed. + response_config = ResponseConfig(parsed_path.path, request=request) + response_config.generate_body_size = data_length + return response_config + if parsed_path.path == "/get_object_error_part_3": # Same 256 KiB object, but part 3 (offset 131072) fails with 403 after a short # delay (long enough for parts 1 and 2 to be delivered first). diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/aws-c-s3-0.13.4/tests/s3_checksums_combine_tests.c new/aws-c-s3-0.13.5/tests/s3_checksums_combine_tests.c --- old/aws-c-s3-0.13.4/tests/s3_checksums_combine_tests.c 1970-01-01 01:00:00.000000000 +0100 +++ new/aws-c-s3-0.13.5/tests/s3_checksums_combine_tests.c 2026-08-06 19:29:07.000000000 +0200 @@ -0,0 +1,351 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include "aws/s3/private/s3_checksums.h" +#include <aws/common/byte_buf.h> +#include <aws/testing/aws_test_harness.h> + +/* Computes checksum(input) in one shot, for comparison against a combined result. */ +static int s_compute_whole( + struct aws_allocator *allocator, + enum aws_s3_checksum_algorithm algorithm, + struct aws_byte_cursor input, + struct aws_byte_buf *out) { + + aws_byte_buf_init(out, allocator, aws_get_digest_size_from_checksum_algorithm(algorithm)); + return aws_checksum_compute(allocator, algorithm, &input, out); +} + +/* Folds `tail` into `head` the way the download path does: finalize the tail to get its digest, then + * fold that digest in. Consumes `tail`. */ +static int s_combine_checksums(struct aws_s3_checksum *head, struct aws_s3_checksum *tail, uint64_t tail_length) { + uint8_t digest_storage[AWS_S3_COMBINABLE_DIGEST_MAX_LEN] = {0}; + struct aws_byte_buf tail_digest = aws_byte_buf_from_empty_array(digest_storage, sizeof(digest_storage)); + if (aws_checksum_finalize(tail, &tail_digest)) { + return AWS_OP_ERR; + } + return aws_checksum_combine_digest(head, aws_byte_cursor_from_buf(&tail_digest), tail_length); +} + +/* Splits input at `split`, checksums each half independently, combines them, and asserts the + * result equals the checksum of the whole input. */ +static int s_verify_combine_at_split( + struct aws_allocator *allocator, + enum aws_s3_checksum_algorithm algorithm, + struct aws_byte_cursor input, + size_t split) { + + AWS_FATAL_ASSERT(split <= input.len); + + struct aws_byte_cursor head_bytes = aws_byte_cursor_from_array(input.ptr, split); + struct aws_byte_cursor tail_bytes = aws_byte_cursor_from_array(input.ptr + split, input.len - split); + + struct aws_s3_checksum *head = aws_checksum_new(allocator, algorithm); + struct aws_s3_checksum *tail = aws_checksum_new(allocator, algorithm); + ASSERT_NOT_NULL(head); + ASSERT_NOT_NULL(tail); + + ASSERT_SUCCESS(aws_checksum_update(head, &head_bytes)); + ASSERT_SUCCESS(aws_checksum_update(tail, &tail_bytes)); + ASSERT_SUCCESS(s_combine_checksums(head, tail, tail_bytes.len)); + + struct aws_byte_buf combined; + aws_byte_buf_init(&combined, allocator, aws_get_digest_size_from_checksum_algorithm(algorithm)); + ASSERT_SUCCESS(aws_checksum_finalize(head, &combined)); + + struct aws_byte_buf expected; + ASSERT_SUCCESS(s_compute_whole(allocator, algorithm, input, &expected)); + + ASSERT_BIN_ARRAYS_EQUALS(expected.buffer, expected.len, combined.buffer, combined.len); + + aws_byte_buf_clean_up(&expected); + aws_byte_buf_clean_up(&combined); + aws_checksum_destroy(tail); + aws_checksum_destroy(head); + + return AWS_OP_SUCCESS; +} + +/* Exercises every split offset, including the degenerate 0 and input.len cases. */ +static int s_verify_combine_all_splits( + struct aws_allocator *allocator, + enum aws_s3_checksum_algorithm algorithm, + struct aws_byte_cursor input) { + + for (size_t split = 0; split <= input.len; ++split) { + if (s_verify_combine_at_split(allocator, algorithm, input, split)) { + return AWS_OP_ERR; + } + } + return AWS_OP_SUCCESS; +} + +static struct aws_byte_cursor s_combine_test_input(void) { + return aws_byte_cursor_from_c_str("abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmn" + "hijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"); +} + +static int s_checksum_combine_crc32_fn(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + aws_s3_library_init(allocator); + ASSERT_SUCCESS(s_verify_combine_all_splits(allocator, AWS_SCA_CRC32, s_combine_test_input())); + aws_s3_library_clean_up(); + return AWS_OP_SUCCESS; +} +AWS_TEST_CASE(checksum_combine_crc32, s_checksum_combine_crc32_fn) + +static int s_checksum_combine_crc32c_fn(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + aws_s3_library_init(allocator); + ASSERT_SUCCESS(s_verify_combine_all_splits(allocator, AWS_SCA_CRC32C, s_combine_test_input())); + aws_s3_library_clean_up(); + return AWS_OP_SUCCESS; +} +AWS_TEST_CASE(checksum_combine_crc32c, s_checksum_combine_crc32c_fn) + +static int s_checksum_combine_crc64nvme_fn(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + aws_s3_library_init(allocator); + ASSERT_SUCCESS(s_verify_combine_all_splits(allocator, AWS_SCA_CRC64NVME, s_combine_test_input())); + aws_s3_library_clean_up(); + return AWS_OP_SUCCESS; +} +AWS_TEST_CASE(checksum_combine_crc64nvme, s_checksum_combine_crc64nvme_fn) + +/* Combining N blocks left-to-right must match the whole-buffer checksum. This is the shape the + * download path uses: one accumulator folded forward part by part. */ +static int s_checksum_combine_many_blocks_fn(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + aws_s3_library_init(allocator); + + const enum aws_s3_checksum_algorithm algorithms[] = {AWS_SCA_CRC32, AWS_SCA_CRC32C, AWS_SCA_CRC64NVME}; + /* Deliberately uneven block sizes, mirroring a final short part. */ + const size_t block_sizes[] = {1, 7, 16, 3, 64, 21}; + + uint8_t input_bytes[112]; + for (size_t i = 0; i < sizeof(input_bytes); ++i) { + input_bytes[i] = (uint8_t)(i * 31 + 7); + } + struct aws_byte_cursor input = aws_byte_cursor_from_array(input_bytes, sizeof(input_bytes)); + + for (size_t algo_i = 0; algo_i < AWS_ARRAY_SIZE(algorithms); ++algo_i) { + enum aws_s3_checksum_algorithm algorithm = algorithms[algo_i]; + size_t digest_size = aws_get_digest_size_from_checksum_algorithm(algorithm); + + struct aws_s3_checksum *accumulator = aws_checksum_new(allocator, algorithm); + ASSERT_NOT_NULL(accumulator); + + struct aws_byte_cursor remaining = input; + for (size_t block_i = 0; block_i < AWS_ARRAY_SIZE(block_sizes) && remaining.len > 0; ++block_i) { + size_t block_len = aws_min_size(block_sizes[block_i], remaining.len); + struct aws_byte_cursor block = aws_byte_cursor_from_array(remaining.ptr, block_len); + + struct aws_s3_checksum *block_sum = aws_checksum_new(allocator, algorithm); + ASSERT_NOT_NULL(block_sum); + ASSERT_SUCCESS(aws_checksum_update(block_sum, &block)); + ASSERT_SUCCESS(s_combine_checksums(accumulator, block_sum, block_len)); + aws_checksum_destroy(block_sum); + + aws_byte_cursor_advance(&remaining, block_len); + } + /* Whatever the block sizes did not cover, fold in as one final block. */ + if (remaining.len > 0) { + struct aws_s3_checksum *block_sum = aws_checksum_new(allocator, algorithm); + ASSERT_NOT_NULL(block_sum); + ASSERT_SUCCESS(aws_checksum_update(block_sum, &remaining)); + ASSERT_SUCCESS(s_combine_checksums(accumulator, block_sum, remaining.len)); + aws_checksum_destroy(block_sum); + } + + struct aws_byte_buf combined; + aws_byte_buf_init(&combined, allocator, digest_size); + ASSERT_SUCCESS(aws_checksum_finalize(accumulator, &combined)); + + struct aws_byte_buf expected; + ASSERT_SUCCESS(s_compute_whole(allocator, algorithm, input, &expected)); + ASSERT_BIN_ARRAYS_EQUALS(expected.buffer, expected.len, combined.buffer, combined.len); + + aws_byte_buf_clean_up(&expected); + aws_byte_buf_clean_up(&combined); + aws_checksum_destroy(accumulator); + } + + aws_s3_library_clean_up(); + return AWS_OP_SUCCESS; +} +AWS_TEST_CASE(checksum_combine_many_blocks, s_checksum_combine_many_blocks_fn) + +/* A fresh accumulator is the identity element, so folding a block into it must yield that + * block's own checksum. The download path relies on this for the first part. */ +static int s_checksum_combine_identity_fn(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + aws_s3_library_init(allocator); + + struct aws_byte_cursor input = s_combine_test_input(); + const enum aws_s3_checksum_algorithm algorithms[] = {AWS_SCA_CRC32, AWS_SCA_CRC32C, AWS_SCA_CRC64NVME}; + + for (size_t i = 0; i < AWS_ARRAY_SIZE(algorithms); ++i) { + enum aws_s3_checksum_algorithm algorithm = algorithms[i]; + + struct aws_s3_checksum *accumulator = aws_checksum_new(allocator, algorithm); + struct aws_s3_checksum *block_sum = aws_checksum_new(allocator, algorithm); + ASSERT_NOT_NULL(accumulator); + ASSERT_NOT_NULL(block_sum); + + ASSERT_SUCCESS(aws_checksum_update(block_sum, &input)); + ASSERT_SUCCESS(s_combine_checksums(accumulator, block_sum, input.len)); + + struct aws_byte_buf combined; + aws_byte_buf_init(&combined, allocator, aws_get_digest_size_from_checksum_algorithm(algorithm)); + ASSERT_SUCCESS(aws_checksum_finalize(accumulator, &combined)); + + struct aws_byte_buf expected; + ASSERT_SUCCESS(s_compute_whole(allocator, algorithm, input, &expected)); + ASSERT_BIN_ARRAYS_EQUALS(expected.buffer, expected.len, combined.buffer, combined.len); + + aws_byte_buf_clean_up(&expected); + aws_byte_buf_clean_up(&combined); + aws_checksum_destroy(block_sum); + aws_checksum_destroy(accumulator); + } + + aws_s3_library_clean_up(); + return AWS_OP_SUCCESS; +} +AWS_TEST_CASE(checksum_combine_identity, s_checksum_combine_identity_fn) + +/* Combining a zero-length block must leave the accumulator untouched. Empty part responses take + * this path. */ +static int s_checksum_combine_empty_tail_fn(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + aws_s3_library_init(allocator); + + struct aws_byte_cursor input = s_combine_test_input(); + const enum aws_s3_checksum_algorithm algorithms[] = {AWS_SCA_CRC32, AWS_SCA_CRC32C, AWS_SCA_CRC64NVME}; + + for (size_t i = 0; i < AWS_ARRAY_SIZE(algorithms); ++i) { + enum aws_s3_checksum_algorithm algorithm = algorithms[i]; + + struct aws_s3_checksum *accumulator = aws_checksum_new(allocator, algorithm); + struct aws_s3_checksum *empty = aws_checksum_new(allocator, algorithm); + ASSERT_NOT_NULL(accumulator); + ASSERT_NOT_NULL(empty); + + ASSERT_SUCCESS(aws_checksum_update(accumulator, &input)); + ASSERT_SUCCESS(s_combine_checksums(accumulator, empty, 0)); + + struct aws_byte_buf combined; + aws_byte_buf_init(&combined, allocator, aws_get_digest_size_from_checksum_algorithm(algorithm)); + ASSERT_SUCCESS(aws_checksum_finalize(accumulator, &combined)); + + struct aws_byte_buf expected; + ASSERT_SUCCESS(s_compute_whole(allocator, algorithm, input, &expected)); + ASSERT_BIN_ARRAYS_EQUALS(expected.buffer, expected.len, combined.buffer, combined.len); + + aws_byte_buf_clean_up(&expected); + aws_byte_buf_clean_up(&combined); + aws_checksum_destroy(empty); + aws_checksum_destroy(accumulator); + } + + aws_s3_library_clean_up(); + return AWS_OP_SUCCESS; +} +AWS_TEST_CASE(checksum_combine_empty_tail, s_checksum_combine_empty_tail_fn) + +/* Non-CRC algorithms have no combine identity, so the API must reject them rather than produce a + * silently wrong digest. */ +static int s_checksum_combine_unsupported_algorithms_fn(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + aws_s3_library_init(allocator); + + const enum aws_s3_checksum_algorithm combinable[] = {AWS_SCA_CRC32, AWS_SCA_CRC32C, AWS_SCA_CRC64NVME}; + const enum aws_s3_checksum_algorithm not_combinable[] = { + AWS_SCA_SHA1, + AWS_SCA_SHA256, + AWS_SCA_SHA512, + AWS_SCA_XXHASH64, + AWS_SCA_XXHASH3_64, + AWS_SCA_XXHASH3_128, + }; + + for (size_t i = 0; i < AWS_ARRAY_SIZE(combinable); ++i) { + ASSERT_TRUE(aws_checksum_algorithm_is_combinable(combinable[i])); + } + + struct aws_byte_cursor input = s_combine_test_input(); + for (size_t i = 0; i < AWS_ARRAY_SIZE(not_combinable); ++i) { + if (i <= AWS_SCA_SHA512) { +#ifdef BYO_CRYPTO + /* Skip SHA based algo for BYO_CRYPTO since they are libcrypto based. */ + continue; +#endif + } + enum aws_s3_checksum_algorithm algorithm = not_combinable[i]; + ASSERT_FALSE(aws_checksum_algorithm_is_combinable(algorithm)); + + struct aws_s3_checksum *head = aws_checksum_new(allocator, algorithm); + struct aws_s3_checksum *tail = aws_checksum_new(allocator, algorithm); + ASSERT_NOT_NULL(head); + ASSERT_NOT_NULL(tail); + ASSERT_SUCCESS(aws_checksum_update(head, &input)); + ASSERT_SUCCESS(aws_checksum_update(tail, &input)); + uint8_t unused_digest[AWS_S3_COMBINABLE_DIGEST_MAX_LEN] = {0}; + ASSERT_ERROR( + AWS_ERROR_UNSUPPORTED_OPERATION, + aws_checksum_combine_digest( + head, aws_byte_cursor_from_array(unused_digest, sizeof(unused_digest)), input.len)); + aws_checksum_destroy(tail); + aws_checksum_destroy(head); + } + + aws_s3_library_clean_up(); + return AWS_OP_SUCCESS; +} +AWS_TEST_CASE(checksum_combine_unsupported_algorithms, s_checksum_combine_unsupported_algorithms_fn) + +/* Once finalized, a checksum is spent. Combining into or from one must fail loudly. */ +static int s_checksum_combine_invalid_state_fn(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + aws_s3_library_init(allocator); + + struct aws_byte_cursor input = s_combine_test_input(); + size_t digest_size = aws_get_digest_size_from_checksum_algorithm(AWS_SCA_CRC32); + + /* Finalized head. */ + { + struct aws_s3_checksum *head = aws_checksum_new(allocator, AWS_SCA_CRC32); + struct aws_s3_checksum *tail = aws_checksum_new(allocator, AWS_SCA_CRC32); + struct aws_byte_buf digest; + aws_byte_buf_init(&digest, allocator, digest_size); + + ASSERT_SUCCESS(aws_checksum_update(head, &input)); + ASSERT_SUCCESS(aws_checksum_finalize(head, &digest)); + ASSERT_ERROR(AWS_ERROR_INVALID_STATE, s_combine_checksums(head, tail, 0)); + + aws_byte_buf_clean_up(&digest); + aws_checksum_destroy(tail); + aws_checksum_destroy(head); + } + + /* Finalized tail. */ + { + struct aws_s3_checksum *head = aws_checksum_new(allocator, AWS_SCA_CRC32); + struct aws_s3_checksum *tail = aws_checksum_new(allocator, AWS_SCA_CRC32); + struct aws_byte_buf digest; + aws_byte_buf_init(&digest, allocator, digest_size); + + ASSERT_SUCCESS(aws_checksum_update(tail, &input)); + ASSERT_SUCCESS(aws_checksum_finalize(tail, &digest)); + ASSERT_ERROR(AWS_ERROR_INVALID_STATE, s_combine_checksums(head, tail, input.len)); + + aws_byte_buf_clean_up(&digest); + aws_checksum_destroy(tail); + aws_checksum_destroy(head); + } + + aws_s3_library_clean_up(); + return AWS_OP_SUCCESS; +} +AWS_TEST_CASE(checksum_combine_invalid_state, s_checksum_combine_invalid_state_fn) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/aws-c-s3-0.13.4/tests/s3_data_plane_tests.c new/aws-c-s3-0.13.5/tests/s3_data_plane_tests.c --- old/aws-c-s3-0.13.4/tests/s3_data_plane_tests.c 2026-07-31 02:02:00.000000000 +0200 +++ new/aws-c-s3-0.13.5/tests/s3_data_plane_tests.c 2026-08-06 19:29:07.000000000 +0200 @@ -4994,6 +4994,146 @@ return s_test_s3_round_trip_multipart_get_fc_helper(allocator, ctx, true); } +/* Uploads an object that S3 stores a whole-object checksum for, then downloads it split into many parts, for + * every algorithm in the priority list. + * + * Two part sizes, because the upload and download have to be split differently. The upload needs S3 to end + * up holding a whole-object checksum rather than a composite one, while the download needs to be split into + * parts for those parts to be recombined into that checksum. A per-meta-request part size gives each one + * what it needs from a single client, and on a PUT it doubles as the multipart threshold. + * + * With validate_response_checksum set, the download discovers the object with a HEAD, which never carries + * x-amz-mp-parts-count, so the whole-object checksum is taken at the meta request level (a composite + * checksum is still rejected there, since its trailing "-N" makes the value the wrong length). Combinable + * algorithms then have each part digest its own body and fold the digests together when the meta request + * finishes; the rest fall back to feeding the running sum from the delivery thread in object order. Looping + * the whole priority list covers both branches. + * + * s_s3_test_validate_checksum asserts the whole-object checksum was actually validated, so a part that + * failed to contribute its digest fails the test rather than silently downgrading to unvalidated. */ +static int s_test_s3_multipart_get_full_object_checksum_helper( + struct aws_allocator *allocator, + void *ctx, + enum aws_s3_tester_full_object_checksum full_object_checksum) { + (void)ctx; + + struct aws_s3_tester tester; + ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester)); + + /* Without an explicit full-object checksum, the object has to stay under the upload's multipart threshold + * so the upload is a single PutObject, which is what makes S3 store a whole-object checksum for every + * algorithm. With one, the upload can be a real multipart upload that declares the checksum itself. */ + bool explicit_full_object_checksum = full_object_checksum != AWS_TEST_FOC_NONE; + uint32_t object_size_mb = explicit_full_object_checksum ? 10 : 1; + size_t upload_part_size = MB_TO_BYTES(5); + size_t download_part_size = explicit_full_object_checksum ? MB_TO_BYTES(1) : 64 * 1024; + + struct aws_s3_tester_client_options client_options = { + .part_size = upload_part_size, + }; + struct aws_s3_client *client = NULL; + ASSERT_SUCCESS(aws_s3_tester_client_new(&tester, &client_options, &client)); + + for (size_t i = 0; i < AWS_ARRAY_SIZE(s_checksum_algo_priority_list); i++) { + enum aws_s3_checksum_algorithm algorithm = s_checksum_algo_priority_list[i]; + + if (explicit_full_object_checksum && !aws_checksum_algorithm_is_combinable(algorithm)) { + /* S3 only offers a full object checksum for a multipart upload with the CRCs. The very property + * that makes them combinable here is what lets S3 assemble one from the parts on its side. */ + continue; + } + + struct aws_byte_buf path_buf; + AWS_ZERO_STRUCT(path_buf); + char object_path_sprintf_buffer[128] = ""; + snprintf( + object_path_sprintf_buffer, + sizeof(object_path_sprintf_buffer), + "/prefix/round_trip/test_multipart_get_foc_%d_%d.txt", + algorithm, + full_object_checksum); + ASSERT_SUCCESS(aws_s3_tester_upload_file_path_init( + allocator, &path_buf, aws_byte_cursor_from_c_str(object_path_sprintf_buffer))); + struct aws_byte_cursor object_path = aws_byte_cursor_from_buf(&path_buf); + + /*** PUT FILE ***/ + + struct aws_s3_tester_meta_request_options put_options = { + .allocator = allocator, + .meta_request_type = AWS_S3_META_REQUEST_TYPE_PUT_OBJECT, + .client = client, + .part_size = upload_part_size, + .checksum_algorithm = algorithm, + .validate_get_response_checksum = false, + .put_options = + { + .object_size_mb = object_size_mb, + .object_path_override = object_path, + .full_object_checksum = full_object_checksum, + }, + }; + + ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(&tester, &put_options, NULL)); + + /*** GET FILE ***/ + + struct aws_s3_tester_meta_request_options get_options = { + .allocator = allocator, + .meta_request_type = AWS_S3_META_REQUEST_TYPE_GET_OBJECT, + .validate_type = AWS_S3_TESTER_VALIDATE_TYPE_EXPECT_SUCCESS, + .client = client, + .part_size = download_part_size, + .expected_validate_checksum_alg = algorithm, + .validate_get_response_checksum = true, + .get_options = + { + .object_path = object_path, + }, + .finish_callback = s_s3_test_validate_checksum, + }; + + struct aws_s3_meta_request_test_results out_results; + aws_s3_meta_request_test_results_init(&out_results, allocator); + ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(&tester, &get_options, &out_results)); + + ASSERT_UINT_EQUALS(AWS_ERROR_SUCCESS, out_results.finished_error_code); + ASSERT_TRUE(out_results.did_validate); + ASSERT_UINT_EQUALS(algorithm, out_results.validation_algorithm); + /* Every byte must have been delivered, otherwise a passing checksum would be meaningless. */ + ASSERT_UINT_EQUALS(MB_TO_BYTES((uint64_t)object_size_mb), out_results.received_body_size); + + aws_s3_meta_request_test_results_clean_up(&out_results); + aws_byte_buf_clean_up(&path_buf); + } + + aws_s3_client_release(client); + aws_s3_tester_clean_up(&tester); + + return 0; +} + +/* Whole-object checksum from a single PutObject, downloaded in 16 parts. Covers all algorithms. */ +AWS_TEST_CASE(test_s3_multipart_get_whole_object_checksum, s_test_s3_multipart_get_whole_object_checksum) +static int s_test_s3_multipart_get_whole_object_checksum(struct aws_allocator *allocator, void *ctx) { + return s_test_s3_multipart_get_full_object_checksum_helper(allocator, ctx, AWS_TEST_FOC_NONE); +} + +/* Full-object checksum declared on a multipart upload, downloaded in 10 parts. S3 only supports this for the + * CRCs, which are exactly the combinable algorithms, so this variant only exercises the combine path. The + * non-combinable fallback is covered by test_s3_multipart_get_whole_object_checksum above, where a single + * PutObject gives S3 a whole-object checksum for any algorithm. */ +AWS_TEST_CASE(test_s3_multipart_get_full_object_checksum_header, s_test_s3_multipart_get_full_object_checksum_header) +static int s_test_s3_multipart_get_full_object_checksum_header(struct aws_allocator *allocator, void *ctx) { + return s_test_s3_multipart_get_full_object_checksum_helper(allocator, ctx, AWS_TEST_FOC_HEADER); +} + +AWS_TEST_CASE( + test_s3_multipart_get_full_object_checksum_callback, + s_test_s3_multipart_get_full_object_checksum_callback) +static int s_test_s3_multipart_get_full_object_checksum_callback(struct aws_allocator *allocator, void *ctx) { + return s_test_s3_multipart_get_full_object_checksum_helper(allocator, ctx, AWS_TEST_FOC_CALLBACK); +} + /* Test the multipart uploaded object was downloaded with same part size, which will download the object matches all the * parts and validate the parts checksum. */ static int s_test_s3_round_trip_mpu_multipart_get_fc_helper( diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/aws-c-s3-0.13.4/tests/s3_mock_server_tests.c new/aws-c-s3-0.13.5/tests/s3_mock_server_tests.c --- old/aws-c-s3-0.13.4/tests/s3_mock_server_tests.c 2026-07-31 02:02:00.000000000 +0200 +++ new/aws-c-s3-0.13.5/tests/s3_mock_server_tests.c 2026-08-06 19:29:07.000000000 +0200 @@ -1353,6 +1353,48 @@ return AWS_OP_SUCCESS; } +TEST_CASE(get_object_opaque_etag_mock_server) { + (void)ctx; + + struct aws_s3_tester tester; + ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester)); + struct aws_s3_tester_client_options client_options = { + .part_size = 64 * 1024, + .tls_usage = AWS_S3_TLS_DISABLED, + }; + + struct aws_s3_client *client = NULL; + ASSERT_SUCCESS(aws_s3_tester_client_new(&tester, &client_options, &client)); + + /* Mock server returns a 256 KiB object whose ETag is opaque and cannot be + * parsed into a part count. The download must still succeed using the fallback part size. */ + struct aws_byte_cursor object_path = aws_byte_cursor_from_c_str("/get_object_opaque_etag"); + + struct aws_s3_tester_meta_request_options get_options = { + .allocator = allocator, + .meta_request_type = AWS_S3_META_REQUEST_TYPE_GET_OBJECT, + .client = client, + .get_options = + { + .object_path = object_path, + }, + .mock_server = true, + .validate_type = AWS_S3_TESTER_VALIDATE_TYPE_EXPECT_SUCCESS, + }; + struct aws_s3_meta_request_test_results out_results; + aws_s3_meta_request_test_results_init(&out_results, allocator); + + ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(&tester, &get_options, &out_results)); + ASSERT_UINT_EQUALS(AWS_ERROR_SUCCESS, out_results.finished_error_code); + ASSERT_UINT_EQUALS(262144, out_results.received_body_size); + + aws_s3_meta_request_test_results_clean_up(&out_results); + aws_s3_client_release(client); + aws_s3_tester_clean_up(&tester); + + return AWS_OP_SUCCESS; +} + TEST_CASE(get_object_invalid_responses_mock_server) { (void)ctx; @@ -1464,6 +1506,114 @@ aws_s3_meta_request_test_results_clean_up(&out_results); aws_s3_client_release(client); + aws_s3_tester_clean_up(&tester); + + return AWS_OP_SUCCESS; +} + +/* Downloads a 256 KiB object in four 64 KiB parts, where the whole-object CRC32 is only advertised on + * the HEAD response. Each part's CRC is computed on its own connection and the four are combined, so a + * correct result proves the combined checksum equals the checksum of the concatenated object. + * + * `object_path` selects whether parts complete in order or with part 2 delayed. */ +static int s_test_multipart_download_checksum_combine( + struct aws_allocator *allocator, + struct aws_byte_cursor object_path) { + + struct aws_s3_tester tester; + ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester)); + struct aws_s3_tester_client_options client_options = { + .part_size = 64 * 1024, + .tls_usage = AWS_S3_TLS_DISABLED, + }; + + struct aws_s3_client *client = NULL; + ASSERT_SUCCESS(aws_s3_tester_client_new(&tester, &client_options, &client)); + + struct aws_s3_tester_meta_request_options get_options = { + .allocator = allocator, + .meta_request_type = AWS_S3_META_REQUEST_TYPE_GET_OBJECT, + .client = client, + .expected_validate_checksum_alg = AWS_SCA_CRC32, + .validate_get_response_checksum = true, + .get_options = + { + .object_path = object_path, + }, + .mock_server = true, + .validate_type = AWS_S3_TESTER_VALIDATE_TYPE_EXPECT_SUCCESS, + }; + struct aws_s3_meta_request_test_results out_results; + aws_s3_meta_request_test_results_init(&out_results, allocator); + + ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(&tester, &get_options, &out_results)); + + ASSERT_UINT_EQUALS(AWS_ERROR_SUCCESS, out_results.finished_error_code); + ASSERT_UINT_EQUALS(AWS_SCA_CRC32, out_results.algorithm); + /* All 262144 bytes must have been delivered, otherwise a passing checksum would be meaningless. */ + ASSERT_UINT_EQUALS(262144, out_results.received_body_size); + + aws_s3_meta_request_test_results_clean_up(&out_results); + aws_s3_client_release(client); + aws_s3_tester_clean_up(&tester); + + return AWS_OP_SUCCESS; +} + +TEST_CASE(multipart_download_checksum_combine_mock_server) { + (void)ctx; + return s_test_multipart_download_checksum_combine( + allocator, aws_byte_cursor_from_c_str("/get_object_checksum_combine")); +} + +TEST_CASE(multipart_download_checksum_combine_out_of_order_mock_server) { + (void)ctx; + /* Part 2 is served slowly, so parts 3 and 4 finish first and must wait in the combine queue. + * Combining out of arrival order has to produce the same digest as combining in order. */ + return s_test_multipart_download_checksum_combine( + allocator, aws_byte_cursor_from_c_str("/get_object_checksum_combine_out_of_order")); +} + +/* A 64 KiB object that downloads as a single part, where the part response carries the whole-object + * CRC32 as its own checksum header with the correct value. That part is therefore both validated + * against its own header and folded into the whole-object checksum, and both must succeed. */ +TEST_CASE(download_checksum_single_part_with_part_header_mock_server) { + (void)ctx; + + struct aws_s3_tester tester; + ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester)); + struct aws_s3_tester_client_options client_options = { + .part_size = 64 * 1024, + .tls_usage = AWS_S3_TLS_DISABLED, + }; + + struct aws_s3_client *client = NULL; + ASSERT_SUCCESS(aws_s3_tester_client_new(&tester, &client_options, &client)); + + struct aws_s3_tester_meta_request_options get_options = { + .allocator = allocator, + .meta_request_type = AWS_S3_META_REQUEST_TYPE_GET_OBJECT, + .client = client, + .expected_validate_checksum_alg = AWS_SCA_CRC32, + .validate_get_response_checksum = true, + .get_options = + { + .object_path = aws_byte_cursor_from_c_str("/get_object_checksum_single_part"), + }, + .mock_server = true, + .validate_type = AWS_S3_TESTER_VALIDATE_TYPE_EXPECT_SUCCESS, + }; + struct aws_s3_meta_request_test_results out_results; + aws_s3_meta_request_test_results_init(&out_results, allocator); + + ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(&tester, &get_options, &out_results)); + + ASSERT_UINT_EQUALS(AWS_ERROR_SUCCESS, out_results.finished_error_code); + ASSERT_UINT_EQUALS(AWS_SCA_CRC32, out_results.algorithm); + ASSERT_UINT_EQUALS(65536, out_results.received_body_size); + + aws_s3_meta_request_test_results_clean_up(&out_results); + aws_s3_client_release(client); aws_s3_tester_clean_up(&tester); return AWS_OP_SUCCESS;
