This is an automated email from the ASF dual-hosted git repository. PsiACE pushed a commit to branch s3-crc-fast in repository https://gitbox.apache.org/repos/asf/opendal.git
commit 0f4c58d0e3e786600828ee1b5e8c648079620f7a Author: Chojan Shang <[email protected]> AuthorDate: Mon Jun 1 17:55:40 2026 +0800 refactor: switch S3 crc32c checksum to crc-fast --- core/Cargo.lock | 11 +---------- core/services/s3/Cargo.toml | 2 +- core/services/s3/src/core.rs | 17 +++++++++++------ 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/core/Cargo.lock b/core/Cargo.lock index f2d0b3de1..f958ff7da 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -2049,15 +2049,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index"; checksum = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff" -[[package]] -name = "crc32c" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index"; -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" -dependencies = [ - "rustc_version", -] - [[package]] name = "crc32fast" version = "1.5.0" @@ -7336,7 +7327,7 @@ version = "0.57.0" dependencies = [ "base64 0.22.1", "bytes", - "crc32c", + "crc-fast", "http 1.4.0", "log", "md-5 0.11.0", diff --git a/core/services/s3/Cargo.toml b/core/services/s3/Cargo.toml index 7e9c0926f..52b4343cf 100644 --- a/core/services/s3/Cargo.toml +++ b/core/services/s3/Cargo.toml @@ -33,7 +33,7 @@ all-features = true [dependencies] base64 = { workspace = true } bytes = { workspace = true } -crc32c = "0.6.6" +crc-fast = "1.6.0" http = { workspace = true } log = { workspace = true } md-5 = "0.11.0" diff --git a/core/services/s3/src/core.rs b/core/services/s3/src/core.rs index 944a1d32c..831ff7c24 100644 --- a/core/services/s3/src/core.rs +++ b/core/services/s3/src/core.rs @@ -24,6 +24,8 @@ use base64::Engine; use base64::prelude::BASE64_STANDARD; use bytes::Bytes; use constants::X_AMZ_META_PREFIX; +use crc_fast::CrcAlgorithm; +use crc_fast::Digest as CrcDigest; use http::HeaderValue; use http::Request; use http::Response; @@ -114,6 +116,14 @@ pub(crate) struct S3UploadPartCopyRequest<'a> { pub(crate) range: BytesRange, } +fn format_crc32c_iter(body: Buffer) -> String { + let mut digest = CrcDigest::new(CrcAlgorithm::Crc32Iscsi); + body.for_each(|b| digest.update(&b)); + + let crc = digest.finalize() as u32; + BASE64_STANDARD.encode(crc.to_be_bytes()) +} + impl Debug for S3Core { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("S3Core") @@ -264,12 +274,7 @@ impl S3Core { pub fn calculate_checksum(&self, body: &Buffer) -> Option<String> { match self.checksum_algorithm { None => None, - Some(ChecksumAlgorithm::Crc32c) => { - let mut crc = 0u32; - body.clone() - .for_each(|b| crc = crc32c::crc32c_append(crc, &b)); - Some(BASE64_STANDARD.encode(crc.to_be_bytes())) - } + Some(ChecksumAlgorithm::Crc32c) => Some(format_crc32c_iter(body.clone())), Some(ChecksumAlgorithm::Md5) => Some(format_content_md5_iter(body.clone())), } }
