plusplusjiajia commented on code in PR #3082: URL: https://github.com/apache/iceberg-rust/pull/3082#discussion_r4165795574
########## crates/catalog/rest/src/auth/sigv4.rs: ########## @@ -0,0 +1,1269 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! AWS SigV4 request signing for the REST catalog. + +use chrono::{DateTime, Utc}; +#[cfg(test)] +use hmac::{Hmac, Mac}; +use iceberg::{Error, ErrorKind, Result}; +use sha2::{Digest, Sha256}; + +/// Hex SHA-256 of the empty string. +const EMPTY_BODY_HEX_SHA256: &str = + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + +/// How the payload hash is encoded in the `x-amz-content-sha256` header. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PayloadHashMode { Review Comment: Good call, added. ########## crates/catalog/rest/src/auth/sigv4.rs: ########## @@ -0,0 +1,1269 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! AWS SigV4 request signing for the REST catalog. + +use chrono::{DateTime, Utc}; +#[cfg(test)] +use hmac::{Hmac, Mac}; +use iceberg::{Error, ErrorKind, Result}; +use sha2::{Digest, Sha256}; + +/// Hex SHA-256 of the empty string. +const EMPTY_BODY_HEX_SHA256: &str = + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + +/// How the payload hash is encoded in the `x-amz-content-sha256` header. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PayloadHashMode { + /// Iceberg Java's RESTSigV4 style: base64 header when there is a body, hex + /// when there is none; the canonical request always uses hex. A caller-set Review Comment: Added a note on where the base64 comes from. I kept it neutral on which mode to pick, since #3092 defaults to `IcebergRest` to match Java clients. ########## crates/catalog/rest/src/auth/sigv4.rs: ########## @@ -0,0 +1,1269 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! AWS SigV4 request signing for the REST catalog. + +use chrono::{DateTime, Utc}; +#[cfg(test)] +use hmac::{Hmac, Mac}; +use iceberg::{Error, ErrorKind, Result}; +use sha2::{Digest, Sha256}; + +/// Hex SHA-256 of the empty string. +const EMPTY_BODY_HEX_SHA256: &str = + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + +/// How the payload hash is encoded in the `x-amz-content-sha256` header. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PayloadHashMode { + /// Iceberg Java's RESTSigV4 style: base64 header when there is a body, hex + /// when there is none; the canonical request always uses hex. A caller-set + /// header is replaced, and moved to `Original-x-amz-content-sha256` when it + /// differed. Java replaces it too for a bodiless request, but signs the + /// caller's value when a body is present. + IcebergRest, + /// Standard AWS SigV4 style: hex everywhere (e.g. AWS Glue). + StandardAws, +} + +/// Derives the AWS SigV4 signing key. +#[cfg(test)] +fn hmac_sha256(key: &[u8], data: &[u8]) -> Vec<u8> { + let mut mac = <Hmac<Sha256> as Mac>::new_from_slice(key).expect("HMAC takes a key of any size"); + mac.update(data); + mac.finalize().into_bytes().to_vec() +} + +fn hex_sha256(data: &[u8]) -> String { + encode_hex(&Sha256::digest(data)) +} + +#[cfg(test)] +fn hex_hmac_sha256(key: &[u8], data: &[u8]) -> String { + encode_hex(&hmac_sha256(key, data)) +} + +fn encode_hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn base64_encode(bytes: &[u8]) -> String { + base64::engine::Engine::encode(&base64::engine::general_purpose::STANDARD, bytes) +} + +#[cfg(test)] +fn signing_key(secret: &str, date: &str, region: &str, service: &str) -> Vec<u8> { + let k_date = hmac_sha256(format!("AWS4{secret}").as_bytes(), date.as_bytes()); + let k_region = hmac_sha256(&k_date, region.as_bytes()); + let k_service = hmac_sha256(&k_region, service.as_bytes()); + hmac_sha256(&k_service, b"aws4_request") +} + +/// The `x-amz-content-sha256` value. `None` means no body at all, which the +/// two modes encode differently. +fn content_sha256_header(body: Option<&[u8]>, mode: PayloadHashMode) -> String { + match mode { + PayloadHashMode::StandardAws => hex_sha256(body.unwrap_or_default()), + PayloadHashMode::IcebergRest => match body { + None => EMPTY_BODY_HEX_SHA256.to_string(), + Some(body) => base64_encode(&Sha256::digest(body)), + }, + } +} + +/// Signs REST catalog requests the way Iceberg Java's `RESTSigV4AuthSession` +/// does. Carries no credentials, so one signer serves every session. +#[derive(Clone)] +pub struct SigV4Signer { + region: String, + service: String, + mode: PayloadHashMode, +} + +impl SigV4Signer { + /// Creates a new SigV4 signer. + pub fn new(region: String, service: String, mode: PayloadHashMode) -> Self { Review Comment: Switched to `impl Into<String>`, thanks. The struct's fields are private, so adding one later isn't breaking. ########## crates/catalog/rest/src/auth/sigv4.rs: ########## @@ -0,0 +1,1269 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! AWS SigV4 request signing for the REST catalog. + +use chrono::{DateTime, Utc}; +#[cfg(test)] +use hmac::{Hmac, Mac}; +use iceberg::{Error, ErrorKind, Result}; +use sha2::{Digest, Sha256}; + +/// Hex SHA-256 of the empty string. +const EMPTY_BODY_HEX_SHA256: &str = + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + +/// How the payload hash is encoded in the `x-amz-content-sha256` header. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PayloadHashMode { + /// Iceberg Java's RESTSigV4 style: base64 header when there is a body, hex + /// when there is none; the canonical request always uses hex. A caller-set + /// header is replaced, and moved to `Original-x-amz-content-sha256` when it + /// differed. Java replaces it too for a bodiless request, but signs the + /// caller's value when a body is present. + IcebergRest, + /// Standard AWS SigV4 style: hex everywhere (e.g. AWS Glue). + StandardAws, +} + +/// Derives the AWS SigV4 signing key. +#[cfg(test)] +fn hmac_sha256(key: &[u8], data: &[u8]) -> Vec<u8> { + let mut mac = <Hmac<Sha256> as Mac>::new_from_slice(key).expect("HMAC takes a key of any size"); + mac.update(data); + mac.finalize().into_bytes().to_vec() +} + +fn hex_sha256(data: &[u8]) -> String { + encode_hex(&Sha256::digest(data)) +} + +#[cfg(test)] +fn hex_hmac_sha256(key: &[u8], data: &[u8]) -> String { + encode_hex(&hmac_sha256(key, data)) +} + +fn encode_hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn base64_encode(bytes: &[u8]) -> String { + base64::engine::Engine::encode(&base64::engine::general_purpose::STANDARD, bytes) +} + +#[cfg(test)] +fn signing_key(secret: &str, date: &str, region: &str, service: &str) -> Vec<u8> { + let k_date = hmac_sha256(format!("AWS4{secret}").as_bytes(), date.as_bytes()); + let k_region = hmac_sha256(&k_date, region.as_bytes()); + let k_service = hmac_sha256(&k_region, service.as_bytes()); + hmac_sha256(&k_service, b"aws4_request") +} + +/// The `x-amz-content-sha256` value. `None` means no body at all, which the +/// two modes encode differently. +fn content_sha256_header(body: Option<&[u8]>, mode: PayloadHashMode) -> String { + match mode { + PayloadHashMode::StandardAws => hex_sha256(body.unwrap_or_default()), + PayloadHashMode::IcebergRest => match body { + None => EMPTY_BODY_HEX_SHA256.to_string(), + Some(body) => base64_encode(&Sha256::digest(body)), + }, + } +} + +/// Signs REST catalog requests the way Iceberg Java's `RESTSigV4AuthSession` +/// does. Carries no credentials, so one signer serves every session. +#[derive(Clone)] +pub struct SigV4Signer { + region: String, + service: String, + mode: PayloadHashMode, +} + +impl SigV4Signer { + /// Creates a new SigV4 signer. + pub fn new(region: String, service: String, mode: PayloadHashMode) -> Self { + Self { + region, + service, + mode, + } + } + + /// Signs `request` in place, rewriting it as signing requires: an existing + /// `Authorization` becomes `Original-Authorization`, userinfo leaves the + /// URL, and a `+` in the query becomes `%20`. + /// + /// A `+` is therefore taken to be an encoded space; write a literal plus as + /// `%2B`. + /// + /// Fails rather than sign a streaming body or a non-UTF-8 header, neither + /// of which canonicalizes faithfully. + /// + /// Send the result through a client that does not follow redirects: a + /// redirect replays a signature made for another URL, and across hosts + /// reqwest drops `Authorization` but keeps `Original-Authorization`. + /// + /// `aws_sigv4` traces the headers it is given, and its redaction list does + /// not cover the `Original-` copy. An installed `tracing` subscriber is + /// muted for the call; with no subscriber, or with `tracing`'s `log-always` + /// feature, its `log` bridge still forwards those events, so keep + /// `aws_sigv4` below trace level there. + pub fn sign( Review Comment: Agreed, added to the `sign()` doc. The #3092 session resolves credentials per request, like Java. ########## crates/catalog/rest/src/auth/sigv4.rs: ########## @@ -0,0 +1,1269 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! AWS SigV4 request signing for the REST catalog. + +use chrono::{DateTime, Utc}; +#[cfg(test)] +use hmac::{Hmac, Mac}; +use iceberg::{Error, ErrorKind, Result}; +use sha2::{Digest, Sha256}; + +/// Hex SHA-256 of the empty string. +const EMPTY_BODY_HEX_SHA256: &str = + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + +/// How the payload hash is encoded in the `x-amz-content-sha256` header. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PayloadHashMode { + /// Iceberg Java's RESTSigV4 style: base64 header when there is a body, hex + /// when there is none; the canonical request always uses hex. A caller-set + /// header is replaced, and moved to `Original-x-amz-content-sha256` when it + /// differed. Java replaces it too for a bodiless request, but signs the + /// caller's value when a body is present. + IcebergRest, + /// Standard AWS SigV4 style: hex everywhere (e.g. AWS Glue). + StandardAws, +} + +/// Derives the AWS SigV4 signing key. +#[cfg(test)] +fn hmac_sha256(key: &[u8], data: &[u8]) -> Vec<u8> { + let mut mac = <Hmac<Sha256> as Mac>::new_from_slice(key).expect("HMAC takes a key of any size"); + mac.update(data); + mac.finalize().into_bytes().to_vec() +} + +fn hex_sha256(data: &[u8]) -> String { + encode_hex(&Sha256::digest(data)) +} + +#[cfg(test)] +fn hex_hmac_sha256(key: &[u8], data: &[u8]) -> String { + encode_hex(&hmac_sha256(key, data)) +} + +fn encode_hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn base64_encode(bytes: &[u8]) -> String { + base64::engine::Engine::encode(&base64::engine::general_purpose::STANDARD, bytes) +} + +#[cfg(test)] +fn signing_key(secret: &str, date: &str, region: &str, service: &str) -> Vec<u8> { + let k_date = hmac_sha256(format!("AWS4{secret}").as_bytes(), date.as_bytes()); + let k_region = hmac_sha256(&k_date, region.as_bytes()); + let k_service = hmac_sha256(&k_region, service.as_bytes()); + hmac_sha256(&k_service, b"aws4_request") +} + +/// The `x-amz-content-sha256` value. `None` means no body at all, which the +/// two modes encode differently. +fn content_sha256_header(body: Option<&[u8]>, mode: PayloadHashMode) -> String { + match mode { + PayloadHashMode::StandardAws => hex_sha256(body.unwrap_or_default()), + PayloadHashMode::IcebergRest => match body { + None => EMPTY_BODY_HEX_SHA256.to_string(), + Some(body) => base64_encode(&Sha256::digest(body)), + }, + } +} + +/// Signs REST catalog requests the way Iceberg Java's `RESTSigV4AuthSession` +/// does. Carries no credentials, so one signer serves every session. +#[derive(Clone)] +pub struct SigV4Signer { + region: String, + service: String, + mode: PayloadHashMode, +} + +impl SigV4Signer { + /// Creates a new SigV4 signer. + pub fn new(region: String, service: String, mode: PayloadHashMode) -> Self { + Self { + region, + service, + mode, + } + } + + /// Signs `request` in place, rewriting it as signing requires: an existing + /// `Authorization` becomes `Original-Authorization`, userinfo leaves the + /// URL, and a `+` in the query becomes `%20`. + /// + /// A `+` is therefore taken to be an encoded space; write a literal plus as + /// `%2B`. + /// + /// Fails rather than sign a streaming body or a non-UTF-8 header, neither + /// of which canonicalizes faithfully. + /// + /// Send the result through a client that does not follow redirects: a + /// redirect replays a signature made for another URL, and across hosts + /// reqwest drops `Authorization` but keeps `Original-Authorization`. + /// + /// `aws_sigv4` traces the headers it is given, and its redaction list does + /// not cover the `Original-` copy. An installed `tracing` subscriber is + /// muted for the call; with no subscriber, or with `tracing`'s `log-always` + /// feature, its `log` bridge still forwards those events, so keep + /// `aws_sigv4` below trace level there. + pub fn sign( + &self, + request: &mut crate::HttpRequest, + credentials: &aws_credential_types::Credentials, + ) -> Result<()> { + self.sign_at(request, credentials, Utc::now()) + } + + fn sign_at( + &self, + request: &mut crate::HttpRequest, + credentials: &aws_credential_types::Credentials, + now: DateTime<Utc>, + ) -> Result<()> { + use aws_sigv4::http_request::{SignableBody, SignableRequest, sign}; + use aws_sigv4::sign::v4; + use tracing::subscriber::NoSubscriber; + + let body = signable_body(request)?; + let content_header = content_sha256_header(body.as_deref(), self.mode); + + convert_headers(request); + + // Relocated after signing, so the `Original-` copy is not signed. + let displaced_content_hash: Vec<_> = request + .headers() + .get_all(CONTENT_SHA256) + .iter() + .filter(|v| v.as_bytes() != content_header.as_bytes()) + .cloned() + .collect(); + request + .headers_mut() + .insert(CONTENT_SHA256, content_header.parse().unwrap()); Review Comment: Makes sense, it returns an error now. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
