This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new bd4b26bc0 feat(services/swift): add TempURL presigned URL support
(#7214)
bd4b26bc0 is described below
commit bd4b26bc0e367a92333ff1a9728b4d1e0fb1bf1c
Author: Ben Roeder <[email protected]>
AuthorDate: Sun Feb 22 21:54:35 2026 -0800
feat(services/swift): add TempURL presigned URL support (#7214)
Add HMAC-signed TempURL support for presign operations (stat, read,
write). Uses the prefixed base64 signature format (e.g. sha256:<base64>)
which works universally across Swift deployments.
- Supports SHA1, SHA256 (default), and SHA512 algorithms via
temp_url_hash_algorithm config
- Requires temp_url_key matching X-Account-Meta-Temp-URL-Key or
X-Container-Meta-Temp-URL-Key on the Swift account/container
- Configure TempURL key on SAIO for CI presign test coverage
- Add multi-arch Dockerfile for native arm64 local testing
---
.github/services/swift/swift/action.yml | 3 +
core/Cargo.lock | 5 +
core/services/swift/Cargo.toml | 5 +
core/services/swift/src/backend.rs | 64 +++++++++
core/services/swift/src/config.rs | 12 ++
core/services/swift/src/core.rs | 194 ++++++++++++++++++++++++++
fixtures/swift/Dockerfile | 87 ++++++++++++
fixtures/swift/docker-compose-swift-local.yml | 40 ++++++
8 files changed, 410 insertions(+)
diff --git a/.github/services/swift/swift/action.yml
b/.github/services/swift/swift/action.yml
index 7dc7e4ff1..3eff4fd32 100644
--- a/.github/services/swift/swift/action.yml
+++ b/.github/services/swift/swift/action.yml
@@ -34,6 +34,8 @@ runs:
token=$(echo "$response" | grep X-Auth-Token | head -n1 | awk '{print
$2}' | tr -d '[:space:]')
endpoint=$(echo "$response" | grep X-Storage-Url | head -n1 | awk
'{print $2}' | tr -d '[:space:]')
curl --location --request PUT "${endpoint}/testing" --header
"X-Auth-Token: $token"
+ # Configure TempURL key on the account for presigned URL tests
+ curl -s -X POST -H "X-Auth-Token: $token" -H
"X-Account-Meta-Temp-URL-Key: opendal-swift-test-key" "${endpoint}"
echo "OPENDAL_SWIFT_TOKEN=${token}" >> $GITHUB_ENV
echo "OPENDAL_SWIFT_ENDPOINT=${endpoint}" >> $GITHUB_ENV
@@ -42,5 +44,6 @@ runs:
run: |
cat << EOF >> $GITHUB_ENV
OPENDAL_SWIFT_CONTAINER=testing
+ OPENDAL_SWIFT_TEMP_URL_KEY=opendal-swift-test-key
OPENDAL_SWIFT_ROOT=/
EOF
diff --git a/core/Cargo.lock b/core/Cargo.lock
index 2c47306c7..eda8e61f5 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -7043,13 +7043,18 @@ dependencies = [
name = "opendal-service-swift"
version = "0.55.0"
dependencies = [
+ "base64 0.22.1",
"bytes",
+ "hmac",
"http 1.4.0",
"log",
"opendal-core",
+ "percent-encoding",
"quick-xml",
"serde",
"serde_json",
+ "sha1",
+ "sha2",
"tokio",
"uuid",
]
diff --git a/core/services/swift/Cargo.toml b/core/services/swift/Cargo.toml
index 0d5b9b02c..fab50090c 100644
--- a/core/services/swift/Cargo.toml
+++ b/core/services/swift/Cargo.toml
@@ -31,13 +31,18 @@ version = { workspace = true }
all-features = true
[dependencies]
+base64 = { workspace = true }
bytes = { workspace = true }
+hmac = "0.12.1"
http = { workspace = true }
log = { workspace = true }
opendal-core = { path = "../../core", version = "0.55.0", default-features =
false }
+percent-encoding = "2"
quick-xml = { workspace = true, features = ["serialize", "overlapped-lists"] }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
+sha1 = "0.10.6"
+sha2 = { workspace = true }
uuid = { workspace = true, features = ["v4"] }
[dev-dependencies]
diff --git a/core/services/swift/src/backend.rs
b/core/services/swift/src/backend.rs
index 1a6ad309b..594e65f46 100644
--- a/core/services/swift/src/backend.rs
+++ b/core/services/swift/src/backend.rs
@@ -95,6 +95,30 @@ impl SwiftBuilder {
}
self
}
+
+ /// Set the TempURL key for generating presigned URLs.
+ ///
+ /// This should match the `X-Account-Meta-Temp-URL-Key` or
+ /// `X-Container-Meta-Temp-URL-Key` value configured on the Swift
+ /// account or container.
+ pub fn temp_url_key(mut self, key: &str) -> Self {
+ if !key.is_empty() {
+ self.config.temp_url_key = Some(key.to_string());
+ }
+ self
+ }
+
+ /// Set the hash algorithm for TempURL signing.
+ ///
+ /// Supported values: `sha1`, `sha256`, `sha512`. Defaults to `sha256`.
+ /// The cluster must have the chosen algorithm in its
+ /// `tempurl.allowed_digests` (check `GET /info`).
+ pub fn temp_url_hash_algorithm(mut self, algo: &str) -> Self {
+ if !algo.is_empty() {
+ self.config.temp_url_hash_algorithm = Some(algo.to_string());
+ }
+ self
+ }
}
impl Builder for SwiftBuilder {
@@ -135,6 +159,12 @@ impl Builder for SwiftBuilder {
};
let token = self.config.token.unwrap_or_default();
+ let temp_url_key = self.config.temp_url_key.unwrap_or_default();
+ let has_temp_url_key = !temp_url_key.is_empty();
+ let temp_url_hash_algorithm = match
&self.config.temp_url_hash_algorithm {
+ Some(algo) => TempUrlHashAlgorithm::from_str_opt(algo)?,
+ None => TempUrlHashAlgorithm::Sha256,
+ };
Ok(SwiftBackend {
core: Arc::new(SwiftCore {
@@ -178,6 +208,11 @@ impl Builder for SwiftBuilder {
list: true,
list_with_recursive: true,
+ presign: has_temp_url_key,
+ presign_stat: has_temp_url_key,
+ presign_read: has_temp_url_key,
+ presign_write: has_temp_url_key,
+
shared: true,
..Default::default()
@@ -188,6 +223,8 @@ impl Builder for SwiftBuilder {
endpoint,
container,
token,
+ temp_url_key,
+ temp_url_hash_algorithm,
}),
})
}
@@ -271,6 +308,33 @@ impl Access for SwiftBackend {
Ok((RpList::default(), oio::PageLister::new(l)))
}
+ async fn presign(&self, path: &str, args: OpPresign) -> Result<RpPresign> {
+ let (expire, op) = args.into_parts();
+
+ let method = match &op {
+ PresignOperation::Stat(_) => http::Method::HEAD,
+ PresignOperation::Read(_) => http::Method::GET,
+ PresignOperation::Write(_) => http::Method::PUT,
+ _ => {
+ return Err(Error::new(
+ ErrorKind::Unsupported,
+ "presign operation is not supported",
+ ));
+ }
+ };
+
+ let url = self.core.swift_temp_url(&method, path, expire)?;
+ let uri: http::Uri = url.parse().map_err(|e| {
+ Error::new(ErrorKind::Unexpected, "failed to parse presigned
URL").set_source(e)
+ })?;
+
+ Ok(RpPresign::new(PresignedRequest::new(
+ method,
+ uri,
+ http::HeaderMap::new(),
+ )))
+ }
+
async fn copy(&self, from: &str, to: &str, _args: OpCopy) ->
Result<RpCopy> {
// cannot copy objects larger than 5 GB.
// Reference:
https://docs.openstack.org/api-ref/object-store/#copy-object
diff --git a/core/services/swift/src/config.rs
b/core/services/swift/src/config.rs
index 63e928aac..efaf46653 100644
--- a/core/services/swift/src/config.rs
+++ b/core/services/swift/src/config.rs
@@ -36,6 +36,18 @@ pub struct SwiftConfig {
pub root: Option<String>,
/// The token for Swift.
pub token: Option<String>,
+ /// The TempURL key for generating presigned URLs.
+ ///
+ /// This corresponds to the `X-Account-Meta-Temp-URL-Key` or
+ /// `X-Container-Meta-Temp-URL-Key` header value configured on the
+ /// Swift account or container.
+ pub temp_url_key: Option<String>,
+ /// The hash algorithm for TempURL signing.
+ ///
+ /// Supported values: `sha1`, `sha256`, `sha512`. Defaults to `sha256`.
+ /// The cluster must have the chosen algorithm in its
+ /// `tempurl.allowed_digests` (check `GET /info`).
+ pub temp_url_hash_algorithm: Option<String>,
}
impl Debug for SwiftConfig {
diff --git a/core/services/swift/src/core.rs b/core/services/swift/src/core.rs
index ccd5d34c5..310e248fc 100644
--- a/core/services/swift/src/core.rs
+++ b/core/services/swift/src/core.rs
@@ -17,7 +17,12 @@
use std::fmt::Debug;
use std::sync::Arc;
+use std::time::Duration;
+use std::time::SystemTime;
+use hmac::Hmac;
+use hmac::Mac;
+use http::Method;
use http::Request;
use http::Response;
use http::header;
@@ -27,16 +32,85 @@ use http::header::IF_NONE_MATCH;
use http::header::IF_UNMODIFIED_SINCE;
use serde::Deserialize;
use serde::Serialize;
+use sha1::Sha1;
+use sha2::Sha256;
+use sha2::Sha512;
use opendal_core::raw::*;
use opendal_core::*;
+/// The HMAC hash algorithm used for TempURL signing.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TempUrlHashAlgorithm {
+ Sha1,
+ Sha256,
+ Sha512,
+}
+
+impl TempUrlHashAlgorithm {
+ pub fn from_str_opt(s: &str) -> Result<Self> {
+ match s.to_lowercase().as_str() {
+ "sha1" => Ok(Self::Sha1),
+ "sha256" => Ok(Self::Sha256),
+ "sha512" => Ok(Self::Sha512),
+ _ => Err(Error::new(
+ ErrorKind::ConfigInvalid,
+ format!(
+ "unsupported temp_url_hash_algorithm: {s}. Expected: sha1,
sha256, or sha512"
+ ),
+ )),
+ }
+ }
+
+ /// Compute HMAC and return the signature in `algo:base64` format.
+ ///
+ /// Swift's TempURL middleware supports two signature formats
+ /// (see `extract_digest_and_algorithm` in swift/common/digest.py):
+ /// - Plain hex with length-based algorithm detection
+ /// (40 chars = SHA1, 64 = SHA256, 128 = SHA512)
+ /// - Prefixed base64: `sha1:<base64>`, `sha256:<base64>`,
`sha512:<base64>`
+ ///
+ /// We use the prefixed base64 format as it explicitly declares the
+ /// algorithm and avoids ambiguity.
+ ///
+ /// References:
+ /// -
<https://docs.openstack.org/swift/latest/api/temporary_url_middleware.html>
+ /// -
<https://github.com/openstack/swift/blob/master/swift/common/digest.py>
+ fn hmac_sign(&self, key: &[u8], data: &[u8]) -> String {
+ use base64::Engine;
+ let engine = base64::engine::general_purpose::STANDARD;
+
+ match self {
+ Self::Sha1 => {
+ let mut mac =
+ Hmac::<Sha1>::new_from_slice(key).expect("HMAC can take
key of any size");
+ mac.update(data);
+ format!("sha1:{}", engine.encode(mac.finalize().into_bytes()))
+ }
+ Self::Sha256 => {
+ let mut mac =
+ Hmac::<Sha256>::new_from_slice(key).expect("HMAC can take
key of any size");
+ mac.update(data);
+ format!("sha256:{}",
engine.encode(mac.finalize().into_bytes()))
+ }
+ Self::Sha512 => {
+ let mut mac =
+ Hmac::<Sha512>::new_from_slice(key).expect("HMAC can take
key of any size");
+ mac.update(data);
+ format!("sha512:{}",
engine.encode(mac.finalize().into_bytes()))
+ }
+ }
+ }
+}
+
pub struct SwiftCore {
pub info: Arc<AccessorInfo>,
pub root: String,
pub endpoint: String,
pub container: String,
pub token: String,
+ pub temp_url_key: String,
+ pub temp_url_hash_algorithm: TempUrlHashAlgorithm,
}
impl Debug for SwiftCore {
@@ -461,6 +535,70 @@ impl SwiftCore {
Ok(())
}
+
+ /// Generate a TempURL (presigned URL) for the given object.
+ ///
+ /// Uses the configured hash algorithm (default SHA256) with `algo:base64`
+ /// signature format for universal compatibility across Swift deployments.
+ ///
+ /// Reference:
<https://docs.openstack.org/swift/latest/api/temporary_url_middleware.html>
+ pub fn swift_temp_url(&self, method: &Method, path: &str, expire:
Duration) -> Result<String> {
+ if self.temp_url_key.is_empty() {
+ return Err(Error::new(
+ ErrorKind::ConfigInvalid,
+ "temp_url_key is required for presign",
+ ));
+ }
+
+ let abs = build_abs_path(&self.root, path);
+
+ // Extract the path portion from the endpoint URL for signing.
+ // The endpoint is like "https://host:port/v1/AUTH_account";.
+ // The signing path must be: /v1/AUTH_account/container/object
+ // Find the path by looking for the third '/' (after "https://host";).
+ let account_path = self
+ .endpoint
+ .find("://")
+ .and_then(|scheme_end| {
+ self.endpoint[scheme_end + 3..]
+ .find('/')
+ .map(|i| scheme_end + 3 + i)
+ })
+ .map(|path_start|
self.endpoint[path_start..].trim_end_matches('/'))
+ .unwrap_or("");
+ let signing_path = format!(
+ "{}/{}/{}",
+ account_path,
+ &self.container,
+ abs.trim_start_matches('/')
+ );
+
+ let expires = SystemTime::now()
+ .duration_since(SystemTime::UNIX_EPOCH)
+ .expect("system time before epoch")
+ .as_secs()
+ + expire.as_secs();
+
+ let sig_body = format!("{}\n{}\n{}", method.as_str(), expires,
signing_path);
+
+ let signature = self
+ .temp_url_hash_algorithm
+ .hmac_sign(self.temp_url_key.as_bytes(), sig_body.as_bytes());
+
+ // The signature is in `algo:base64` format which contains characters
+ // that need percent-encoding in query parameters (+, /, =, :).
+ let encoded_sig =
+ percent_encoding::utf8_percent_encode(&signature,
percent_encoding::NON_ALPHANUMERIC);
+
+ Ok(format!(
+ "{}/{}/{}?temp_url_sig={}&temp_url_expires={}",
+ &self.endpoint,
+ &self.container,
+ percent_encode_path(&abs),
+ &encoded_sig,
+ expires
+ ))
+ }
}
#[derive(Debug, Eq, PartialEq, Deserialize)]
@@ -620,4 +758,60 @@ mod tests {
Ok(())
}
+
+ #[test]
+ fn temp_url_sha1_signature_format() {
+ let algo = TempUrlHashAlgorithm::Sha1;
+ let sig = algo.hmac_sign(b"secret",
b"GET\n1234567890\n/v1/AUTH_test/c/obj");
+ assert!(
+ sig.starts_with("sha1:"),
+ "SHA1 signature must start with 'sha1:'"
+ );
+ // SHA1 = 20 bytes = 28 base64 chars (with padding)
+ let b64_part = &sig["sha1:".len()..];
+ assert_eq!(b64_part.len(), 28, "SHA1 base64 must be 28 chars");
+ }
+
+ #[test]
+ fn temp_url_sha256_signature_format() {
+ let algo = TempUrlHashAlgorithm::Sha256;
+ let sig = algo.hmac_sign(b"secret",
b"GET\n1234567890\n/v1/AUTH_test/c/obj");
+ assert!(
+ sig.starts_with("sha256:"),
+ "SHA256 signature must start with 'sha256:'"
+ );
+ // SHA256 = 32 bytes = 44 base64 chars (with padding)
+ let b64_part = &sig["sha256:".len()..];
+ assert_eq!(b64_part.len(), 44, "SHA256 base64 must be 44 chars");
+ }
+
+ #[test]
+ fn temp_url_sha512_signature_format() {
+ let algo = TempUrlHashAlgorithm::Sha512;
+ let sig = algo.hmac_sign(b"secret",
b"GET\n1234567890\n/v1/AUTH_test/c/obj");
+ assert!(
+ sig.starts_with("sha512:"),
+ "SHA512 signature must start with 'sha512:'"
+ );
+ // SHA512 = 64 bytes = 88 base64 chars (with padding)
+ let b64_part = &sig["sha512:".len()..];
+ assert_eq!(b64_part.len(), 88, "SHA512 base64 must be 88 chars");
+ }
+
+ #[test]
+ fn temp_url_hash_algorithm_from_str() {
+ assert_eq!(
+ TempUrlHashAlgorithm::from_str_opt("sha1").unwrap(),
+ TempUrlHashAlgorithm::Sha1
+ );
+ assert_eq!(
+ TempUrlHashAlgorithm::from_str_opt("SHA256").unwrap(),
+ TempUrlHashAlgorithm::Sha256
+ );
+ assert_eq!(
+ TempUrlHashAlgorithm::from_str_opt("Sha512").unwrap(),
+ TempUrlHashAlgorithm::Sha512
+ );
+ assert!(TempUrlHashAlgorithm::from_str_opt("md5").is_err());
+ }
}
diff --git a/fixtures/swift/Dockerfile b/fixtures/swift/Dockerfile
new file mode 100644
index 000000000..981bd2528
--- /dev/null
+++ b/fixtures/swift/Dockerfile
@@ -0,0 +1,87 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Multi-arch Swift All-In-One (SAIO) image
+#
+# Based on upstream:
https://opendev.org/openstack/swift/src/branch/master/Dockerfile
+# Modified to support both amd64 and arm64 (Apple Silicon).
+#
+# Build for current platform:
+# podman build -t opendal/swift-saio .
+#
+# Build multi-arch:
+# docker buildx build --platform linux/amd64,linux/arm64 -t
opendal/swift-saio .
+
+FROM alpine:3.16.2
+
+# TARGETARCH is automatically set by buildx/podman (amd64 or arm64).
+ARG TARGETARCH
+
+ENV S6_LOGGING=1
+ENV S6_VERSION=1.21.4.0
+ENV SOCKLOG_VERSION=3.0.1-1
+ENV BUILD_DIR="/tmp"
+ENV ENV="/etc/profile"
+
+# Map Docker arch names to s6-overlay arch names:
+# amd64 -> amd64
+# arm64 -> aarch64
+RUN if [ "$TARGETARCH" = "arm64" ]; then \
+ echo "aarch64" > /tmp/s6_arch; \
+ else \
+ echo "$TARGETARCH" > /tmp/s6_arch; \
+ fi
+
+# Clone Swift source
+RUN apk add --no-cache git && \
+ git clone --depth 1 https://opendev.org/openstack/swift.git /opt/swift && \
+ apk del git
+
+# Download s6-overlay and socklog-overlay for the target architecture
+RUN S6_ARCH=$(cat /tmp/s6_arch) && \
+ wget -q -O /tmp/s6-overlay.tar.gz \
+
"https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-${S6_ARCH}.tar.gz";
&& \
+ wget -q -O /tmp/s6-overlay.tar.gz.sig \
+
"https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-${S6_ARCH}.tar.gz.sig";
&& \
+ wget -q -O /tmp/socklog-overlay.tar.gz \
+
"https://github.com/just-containers/socklog-overlay/releases/download/v${SOCKLOG_VERSION}/socklog-overlay-${S6_ARCH}.tar.gz";
+
+RUN mkdir /etc/swift && \
+ echo "================ starting swift_needs ===================" && \
+ /opt/swift/docker/install_scripts/00_swift_needs.sh && \
+ echo "================ starting apk_install_prereqs
===================" && \
+ /opt/swift/docker/install_scripts/10_apk_install_prereqs.sh && \
+ echo "================ starting apk_install_py3 ===================" &&
\
+ /opt/swift/docker/install_scripts/21_apk_install_py3.sh && \
+ echo "================ starting swift_install ===================" && \
+ /opt/swift/docker/install_scripts/50_swift_install.sh && \
+ echo "================ installing s6-overlay ===================" && \
+ gpg --import /opt/swift/docker/s6-gpg-pub-key && \
+ gpg --verify /tmp/s6-overlay.tar.gz.sig /tmp/s6-overlay.tar.gz && \
+ gunzip -c /tmp/s6-overlay.tar.gz | tar -xf - -C / && \
+ gunzip -c /tmp/socklog-overlay.tar.gz | tar -xf - -C / && \
+ rm -rf /tmp/s6-overlay* /tmp/socklog-overlay* /tmp/s6_arch && \
+ echo "================ starting pip_uninstall_dev ==================="
&& \
+ /opt/swift/docker/install_scripts/60_pip_uninstall_dev.sh && \
+ echo "================ starting apk_uninstall_dev ==================="
&& \
+ /opt/swift/docker/install_scripts/99_apk_uninstall_dev.sh
+
+# The upstream Dockerfile uses "COPY docker/rootfs /" since it builds
+# from the Swift source tree. We cloned into /opt/swift, so copy from there.
+RUN cp -a /opt/swift/docker/rootfs/* /
+
+ENTRYPOINT ["/init"]
diff --git a/fixtures/swift/docker-compose-swift-local.yml
b/fixtures/swift/docker-compose-swift-local.yml
new file mode 100644
index 000000000..78385ddb5
--- /dev/null
+++ b/fixtures/swift/docker-compose-swift-local.yml
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Locally-built Swift SAIO for native arm64/aarch64 support.
+#
+# The upstream openstackswift/saio:py3 image is amd64-only.
+# This compose file builds from the Dockerfile which supports
+# both amd64 and arm64 natively.
+#
+# Usage:
+# docker compose -f docker-compose-swift-local.yml up -d --build --wait
+# podman compose -f docker-compose-swift-local.yml up -d --build --wait
+
+services:
+ swift:
+ build:
+ context: .
+ dockerfile: Dockerfile
+ image: opendal-swift-saio:local
+ ports:
+ - 8080:8080
+ healthcheck:
+ test: ["CMD", "curl", "-i", "-H", "X-Storage-User: test:tester", "-H",
"X-Storage-Pass: testing", "http://localhost:8080/";]
+ interval: 3s
+ timeout: 20s
+ retries: 10
