This is an automated email from the ASF dual-hosted git repository. xuanwo pushed a commit to branch xuanwo/gcp-wif-auth in repository https://gitbox.apache.org/repos/asf/opendal-reqsign.git
commit 3c1b36a1ae4c24d5c8407515110caa52b125298e Author: Xuanwo <[email protected]> AuthorDate: Fri Dec 26 20:49:57 2025 +0800 feat: Add workload identity supprot --- services/google/Cargo.toml | 1 + .../src/provide_credential/external_account.rs | 409 +++++++++++++++++++-- .../google/testdata/test_external_account.json | 6 +- services/google/tests/README.md | 12 +- services/google/tests/mocks/sts_mock_server.py | 140 +++++++ 5 files changed, 524 insertions(+), 44 deletions(-) diff --git a/services/google/Cargo.toml b/services/google/Cargo.toml index 76e1db8..6015ba3 100644 --- a/services/google/Cargo.toml +++ b/services/google/Cargo.toml @@ -28,6 +28,7 @@ rust-version.workspace = true [dependencies] async-trait = { workspace = true } +form_urlencoded = { workspace = true } http = { workspace = true } jsonwebtoken = "9.2" log = { workspace = true } diff --git a/services/google/src/provide_credential/external_account.rs b/services/google/src/provide_credential/external_account.rs index c28cca7..14c1ab4 100644 --- a/services/google/src/provide_credential/external_account.rs +++ b/services/google/src/provide_credential/external_account.rs @@ -17,6 +17,7 @@ use std::time::Duration; +use form_urlencoded::Serializer; use http::header::{ACCEPT, CONTENT_TYPE}; use log::{debug, error}; use serde::{Deserialize, Serialize}; @@ -43,18 +44,6 @@ struct ImpersonatedTokenResponse { expire_time: String, } -/// STS token exchange request. -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -struct StsTokenRequest { - grant_type: &'static str, - requested_token_type: &'static str, - audience: String, - scope: &'static str, - subject_token: String, - subject_token_type: String, -} - /// Impersonation request. #[derive(Serialize)] struct ImpersonationRequest { @@ -84,6 +73,13 @@ impl ExternalAccountCredentialProvider { self } + fn resolve_scope(&self, ctx: &Context) -> String { + self.scope + .clone() + .or_else(|| ctx.env_var(crate::constants::GOOGLE_SCOPE)) + .unwrap_or_else(|| crate::constants::DEFAULT_SCOPE.to_string()) + } + async fn load_oidc_token(&self, ctx: &Context) -> Result<String> { match &self.external_account.credential_source { external_account::Source::File(source) => { @@ -98,9 +94,19 @@ impl ExternalAccountCredentialProvider { ctx: &Context, source: &external_account::FileSource, ) -> Result<String> { - debug!("loading OIDC token from file: {}", source.file); - let content = ctx.file_read(&source.file).await?; - source.format.parse(&content) + let file = resolve_template(ctx, &source.file)?; + debug!("loading OIDC token from file: {}", file); + + let content = ctx.file_read(&file).await?; + let token = source.format.parse(&content)?; + let token = token.trim().to_string(); + if token.is_empty() { + return Err(reqsign_core::Error::credential_invalid( + "OIDC token loaded from file is empty", + )); + } + + Ok(token) } async fn load_url_sourced_token( @@ -108,13 +114,15 @@ impl ExternalAccountCredentialProvider { ctx: &Context, source: &external_account::UrlSource, ) -> Result<String> { - debug!("loading OIDC token from URL: {}", source.url); + let url = resolve_template(ctx, &source.url)?; + debug!("loading OIDC token from URL: {}", url); - let mut req = http::Request::get(&source.url); + let mut req = http::Request::get(&url); // Add custom headers if any if let Some(headers) = &source.headers { for (key, value) in headers { + let value = resolve_template(ctx, value)?; req = req.header(key, value); } } @@ -133,31 +141,46 @@ impl ExternalAccountCredentialProvider { ))); } - source.format.parse(resp.body()) + let token = source.format.parse(resp.body())?; + let token = token.trim().to_string(); + if token.is_empty() { + return Err(reqsign_core::Error::credential_invalid( + "OIDC token loaded from URL is empty", + )); + } + + Ok(token) } async fn exchange_sts_token(&self, ctx: &Context, oidc_token: &str) -> Result<Token> { debug!("exchanging OIDC token for STS access token"); - let request = StsTokenRequest { - grant_type: "urn:ietf:params:oauth:grant-type:token-exchange", - requested_token_type: "urn:ietf:params:oauth:token-type:access_token", - audience: self.external_account.audience.clone(), - scope: "https://www.googleapis.com/auth/cloud-platform";, - subject_token: oidc_token.to_string(), - subject_token_type: self.external_account.subject_token_type.clone(), - }; - - let body = serde_json::to_vec(&request).map_err(|e| { - reqsign_core::Error::unexpected("failed to serialize request").with_source(e) - })?; + let scope = self.resolve_scope(ctx); + let token_url = resolve_template(ctx, &self.external_account.token_url)?; + let audience = resolve_template(ctx, &self.external_account.audience)?; + let subject_token_type = resolve_template(ctx, &self.external_account.subject_token_type)?; + + let body = Serializer::new(String::new()) + .append_pair( + "grant_type", + "urn:ietf:params:oauth:grant-type:token-exchange", + ) + .append_pair( + "requested_token_type", + "urn:ietf:params:oauth:token-type:access_token", + ) + .append_pair("audience", &audience) + .append_pair("scope", &scope) + .append_pair("subject_token", oidc_token) + .append_pair("subject_token_type", &subject_token_type) + .finish(); let req = http::Request::builder() .method(http::Method::POST) - .uri(&self.external_account.token_url) + .uri(token_url) .header(ACCEPT, "application/json") - .header(CONTENT_TYPE, "application/json") - .body(body.into()) + .header(CONTENT_TYPE, "application/x-www-form-urlencoded") + .body(body.into_bytes().into()) .map_err(|e| { reqsign_core::Error::unexpected("failed to build HTTP request").with_source(e) })?; @@ -197,12 +220,7 @@ impl ExternalAccountCredentialProvider { debug!("impersonating service account"); - let scope = self - .scope - .clone() - .or_else(|| ctx.env_var(crate::constants::GOOGLE_SCOPE)) - .unwrap_or_else(|| crate::constants::DEFAULT_SCOPE.to_string()); - + let scope = self.resolve_scope(ctx); let lifetime = self .external_account .service_account_impersonation @@ -210,6 +228,14 @@ impl ExternalAccountCredentialProvider { .and_then(|s| s.token_lifetime_seconds) .unwrap_or(MAX_LIFETIME.as_secs() as usize); + let lifetime = if lifetime == 0 { + return Err(reqsign_core::Error::config_invalid( + "service_account_impersonation.token_lifetime_seconds must be positive", + )); + } else { + lifetime.min(MAX_LIFETIME.as_secs() as usize) + }; + let request = ImpersonationRequest { scope: vec![scope.clone()], lifetime: format!("{lifetime}s"), @@ -224,7 +250,16 @@ impl ExternalAccountCredentialProvider { .uri(url) .header(ACCEPT, "application/json") .header(CONTENT_TYPE, "application/json") - .header("Authorization", format!("Bearer {access_token}")) + .header(http::header::AUTHORIZATION, { + let mut value: http::HeaderValue = format!("Bearer {access_token}") + .parse() + .map_err(|e| { + reqsign_core::Error::unexpected("failed to parse header value") + .with_source(e) + })?; + value.set_sensitive(true); + value + }) .body(body.into()) .map_err(|e| { reqsign_core::Error::unexpected("failed to build HTTP request").with_source(e) @@ -278,3 +313,297 @@ impl ProvideCredential for ExternalAccountCredentialProvider { Ok(Some(Credential::with_token(final_token))) } } + +fn resolve_template(ctx: &Context, input: &str) -> Result<String> { + // Google external account credentials commonly contain `${VAR}` placeholders that must be + // substituted using process environment variables (e.g. GitHub Actions OIDC). + let mut out = String::with_capacity(input.len()); + let mut rest = input; + + loop { + let Some(start) = rest.find("${") else { + out.push_str(rest); + return Ok(out); + }; + + out.push_str(&rest[..start]); + rest = &rest[start + 2..]; + + let Some(end) = rest.find('}') else { + return Err(reqsign_core::Error::config_invalid(format!( + "invalid template syntax in value: {input}" + ))); + }; + + let var = &rest[..end]; + rest = &rest[end + 1..]; + + if var.is_empty() { + return Err(reqsign_core::Error::config_invalid(format!( + "empty template variable in value: {input}" + ))); + } + + let value = ctx + .env_var(var) + .filter(|v| !v.is_empty()) + .ok_or_else(|| { + reqsign_core::Error::config_invalid(format!( + "missing environment variable {var} required by template: {input}" + )) + })?; + out.push_str(&value); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use async_trait::async_trait; + use bytes::Bytes; + use http::header::{AUTHORIZATION, CONTENT_TYPE}; + use reqsign_core::{Env, FileRead, HttpSend}; + use std::collections::HashMap; + use std::path::PathBuf; + + #[derive(Debug, Default)] + struct MockEnv { + vars: HashMap<String, String>, + } + + impl MockEnv { + fn with_var(mut self, k: &str, v: &str) -> Self { + self.vars.insert(k.to_string(), v.to_string()); + self + } + } + + impl Env for MockEnv { + fn var(&self, key: &str) -> Option<String> { + self.vars.get(key).cloned() + } + + fn vars(&self) -> HashMap<String, String> { + self.vars.clone() + } + + fn home_dir(&self) -> Option<PathBuf> { + None + } + } + + #[derive(Debug, Default)] + struct MockFileRead { + files: HashMap<String, Vec<u8>>, + } + + impl MockFileRead { + fn with_file(mut self, path: &str, content: impl Into<Vec<u8>>) -> Self { + self.files.insert(path.to_string(), content.into()); + self + } + } + + #[async_trait] + impl FileRead for MockFileRead { + async fn file_read(&self, path: &str) -> Result<Vec<u8>> { + self.files.get(path).cloned().ok_or_else(|| { + reqsign_core::Error::config_invalid(format!("file not found: {path}")) + }) + } + } + + #[derive(Debug)] + struct CaptureStsHttpSend { + expected_url: String, + expected_scope: String, + expected_subject_token: String, + expected_audience: String, + expected_subject_token_type: String, + access_token: String, + } + + #[async_trait] + impl HttpSend for CaptureStsHttpSend { + async fn http_send(&self, req: http::Request<Bytes>) -> Result<http::Response<Bytes>> { + assert_eq!(req.method(), http::Method::POST); + assert_eq!(req.uri().to_string(), self.expected_url); + assert_eq!( + req.headers() + .get(CONTENT_TYPE) + .expect("content-type must exist") + .to_str() + .expect("content-type must be valid string"), + "application/x-www-form-urlencoded" + ); + + let pairs: HashMap<String, String> = + form_urlencoded::parse(req.body().as_ref()).into_owned().collect(); + assert_eq!( + pairs.get("grant_type").map(String::as_str), + Some("urn:ietf:params:oauth:grant-type:token-exchange") + ); + assert_eq!( + pairs.get("requested_token_type").map(String::as_str), + Some("urn:ietf:params:oauth:token-type:access_token") + ); + assert_eq!( + pairs.get("audience").map(String::as_str), + Some(self.expected_audience.as_str()) + ); + assert_eq!( + pairs.get("scope").map(String::as_str), + Some(self.expected_scope.as_str()) + ); + assert_eq!( + pairs.get("subject_token").map(String::as_str), + Some(self.expected_subject_token.as_str()) + ); + assert_eq!( + pairs.get("subject_token_type").map(String::as_str), + Some(self.expected_subject_token_type.as_str()) + ); + + let body = serde_json::json!({ + "access_token": &self.access_token, + "expires_in": 3600 + }); + Ok(http::Response::builder() + .status(http::StatusCode::OK) + .body(serde_json::to_vec(&body).expect("json must encode").into()) + .expect("response must build")) + } + } + + #[derive(Debug)] + struct UrlThenStsHttpSend { + expected_get_url: String, + expected_get_auth: String, + expected_post_url: String, + expected_subject_token: String, + } + + #[async_trait] + impl HttpSend for UrlThenStsHttpSend { + async fn http_send(&self, req: http::Request<Bytes>) -> Result<http::Response<Bytes>> { + match *req.method() { + http::Method::GET => { + assert_eq!(req.uri().to_string(), self.expected_get_url); + assert_eq!( + req.headers() + .get(AUTHORIZATION) + .expect("authorization must exist") + .to_str() + .expect("authorization must be valid string"), + self.expected_get_auth + ); + Ok(http::Response::builder() + .status(http::StatusCode::OK) + .body(b"test-oidc-token".as_slice().into()) + .expect("response must build")) + } + http::Method::POST => { + assert_eq!(req.uri().to_string(), self.expected_post_url); + let pairs: HashMap<String, String> = + form_urlencoded::parse(req.body().as_ref()).into_owned().collect(); + assert_eq!( + pairs.get("subject_token").map(String::as_str), + Some(self.expected_subject_token.as_str()) + ); + Ok(http::Response::builder() + .status(http::StatusCode::OK) + .body( + br#"{"access_token":"final-token","expires_in":3600}"# + .as_slice() + .into(), + ) + .expect("response must build")) + } + _ => unreachable!("unexpected method"), + } + } + } + + #[test] + fn test_resolve_template() { + let ctx = Context::new().with_env(MockEnv::default().with_var("FOO", "bar")); + assert_eq!(resolve_template(&ctx, "a${FOO}c").unwrap(), "abarc"); + } + + #[tokio::test] + async fn test_external_account_file_source_uses_form_encoded_sts() -> Result<()> { + let external_account = ExternalAccount { + audience: "aud".to_string(), + subject_token_type: "urn:ietf:params:oauth:token-type:jwt".to_string(), + token_url: "https://sts.googleapis.com/v1/token".to_string(), + credential_source: external_account::Source::File(external_account::FileSource { + file: "/var/run/token".to_string(), + format: external_account::Format::Text, + }), + service_account_impersonation_url: None, + service_account_impersonation: None, + }; + + let http = CaptureStsHttpSend { + expected_url: "https://sts.googleapis.com/v1/token".to_string(), + expected_scope: "scope-a".to_string(), + expected_subject_token: "test-oidc".to_string(), + expected_audience: "aud".to_string(), + expected_subject_token_type: "urn:ietf:params:oauth:token-type:jwt".to_string(), + access_token: "access-token".to_string(), + }; + let fs = MockFileRead::default().with_file("/var/run/token", b" test-oidc \n"); + let ctx = Context::new().with_http_send(http).with_file_read(fs); + + let provider = ExternalAccountCredentialProvider::new(external_account).with_scope("scope-a"); + let cred = provider + .provide_credential(&ctx) + .await? + .expect("credential must exist"); + assert!(cred.has_token()); + assert!(cred.has_valid_token()); + Ok(()) + } + + #[tokio::test] + async fn test_external_account_url_source_supports_env_templates() -> Result<()> { + let external_account = ExternalAccount { + audience: "aud".to_string(), + subject_token_type: "urn:ietf:params:oauth:token-type:jwt".to_string(), + token_url: "https://sts.googleapis.com/v1/token".to_string(), + credential_source: external_account::Source::Url(external_account::UrlSource { + url: "https://example.com/${PATH}".to_string(), + format: external_account::Format::Text, + headers: Some(HashMap::from([( + "Authorization".to_string(), + "Bearer ${TOKEN}".to_string(), + )])), + }), + service_account_impersonation_url: None, + service_account_impersonation: None, + }; + + let http = UrlThenStsHttpSend { + expected_get_url: "https://example.com/oidc".to_string(), + expected_get_auth: "Bearer secret".to_string(), + expected_post_url: "https://sts.googleapis.com/v1/token".to_string(), + expected_subject_token: "test-oidc-token".to_string(), + }; + + let env = MockEnv::default() + .with_var("PATH", "oidc") + .with_var("TOKEN", "secret"); + + let ctx = Context::new().with_http_send(http).with_env(env); + + let provider = ExternalAccountCredentialProvider::new(external_account); + let cred = provider + .provide_credential(&ctx) + .await? + .expect("credential must exist"); + assert!(cred.has_token()); + assert!(cred.has_valid_token()); + Ok(()) + } +} diff --git a/services/google/testdata/test_external_account.json b/services/google/testdata/test_external_account.json index 5c47b70..119c949 100644 --- a/services/google/testdata/test_external_account.json +++ b/services/google/testdata/test_external_account.json @@ -2,10 +2,10 @@ "type": "external_account", "audience": "//iam.googleapis.com/projects/000000000000/locations/global/workloadIdentityPools/reqsign/providers/reqsign-provider", "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", - "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/[email protected]:generateAccessToken";, - "token_url": "https://sts.googleapis.com/v1/token";, + "service_account_impersonation_url": "http://127.0.0.1:5000/v1/projects/-/serviceAccounts/[email protected]:generateAccessToken";, + "token_url": "http://127.0.0.1:5000/v1/token";, "credential_source": { - "url": "http://localhost:5000/token";, + "url": "http://127.0.0.1:5000/token";, "format": { "type": "json", "subject_token_field_name": "id_token" diff --git a/services/google/tests/README.md b/services/google/tests/README.md index 614665e..bcabf08 100644 --- a/services/google/tests/README.md +++ b/services/google/tests/README.md @@ -112,4 +112,14 @@ The DefaultCredentialProvider automatically detects and handles all these types. - Impersonation tests require proper IAM permissions for the source credentials - Tests use real GCS API endpoints to verify signature validity - All tests are designed to be idempotent and safe to run repeatedly -- Some credential provider tests use test data that will fail token exchange - this is expected \ No newline at end of file +- Some credential provider tests use test data that will fail token exchange - this is expected + +## Local STS Mock + +For local development without real Google Cloud credentials, you can run a simple STS mock: + +```bash +python3 services/google/tests/mocks/sts_mock_server.py 5000 +``` + +The example credential file `services/google/testdata/test_external_account.json` is configured to use this mock. diff --git a/services/google/tests/mocks/sts_mock_server.py b/services/google/tests/mocks/sts_mock_server.py new file mode 100644 index 0000000..a54e7b3 --- /dev/null +++ b/services/google/tests/mocks/sts_mock_server.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +""" +Mock server for Google STS and external account credential sources. + +This server provides: +- GET /token : returns an OIDC subject token in JSON format +- POST /v1/token : accepts RFC 8693 token exchange via application/x-www-form-urlencoded +- POST */:generateAccessToken : simulates IAMCredentials generateAccessToken (optional) + +It is intended for local testing with services/google/testdata/test_external_account.json. +""" + +import json +import sys +import time +from datetime import datetime, timedelta, timezone +from http.server import BaseHTTPRequestHandler, HTTPServer +from urllib.parse import parse_qs + + +def _read_body(handler: BaseHTTPRequestHandler) -> bytes: + length = int(handler.headers.get("Content-Length", "0")) + if length <= 0: + return b"" + return handler.rfile.read(length) + + +class StsHandler(BaseHTTPRequestHandler): + def do_GET(self): + if self.path == "/token": + self._handle_subject_token() + return + + self.send_error(404, "Not Found") + + def do_POST(self): + if self.path == "/v1/token": + self._handle_sts_token_exchange() + return + + if self.path.endswith(":generateAccessToken"): + self._handle_generate_access_token() + return + + self.send_error(404, "Not Found") + + def _handle_subject_token(self): + token_response = { + "id_token": "mock-oidc-subject-token", + } + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps(token_response).encode()) + + def _handle_sts_token_exchange(self): + content_type = self.headers.get("Content-Type", "") + if not content_type.startswith("application/x-www-form-urlencoded"): + self.send_error( + 415, + "Content-Type must be application/x-www-form-urlencoded for STS token exchange", + ) + return + + body = _read_body(self).decode("utf-8", errors="replace") + values = {k: v[0] for k, v in parse_qs(body, keep_blank_values=True).items()} + + required = [ + "grant_type", + "requested_token_type", + "audience", + "scope", + "subject_token", + "subject_token_type", + ] + missing = [k for k in required if not values.get(k)] + if missing: + self.send_error(400, f"Missing required form fields: {', '.join(missing)}") + return + + if values["grant_type"] != "urn:ietf:params:oauth:grant-type:token-exchange": + self.send_error(400, "Invalid grant_type") + return + if values["requested_token_type"] != "urn:ietf:params:oauth:token-type:access_token": + self.send_error(400, "Invalid requested_token_type") + return + + token_response = { + "access_token": f"mock-sts-access-token-{int(time.time())}", + "expires_in": 3600, + "token_type": "Bearer", + } + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps(token_response).encode()) + + def _handle_generate_access_token(self): + auth = self.headers.get("Authorization", "") + if not auth.startswith("Bearer "): + self.send_error(401, "Authorization: Bearer <token> required") + return + + expires_in = 3600 + expire_time = datetime.now(timezone.utc) + timedelta(seconds=expires_in) + token_response = { + "accessToken": f"mock-impersonated-access-token-{int(time.time())}", + "expireTime": expire_time.isoformat().replace("+00:00", "Z"), + } + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps(token_response).encode()) + + def log_message(self, format, *args): + sys.stderr.write( + f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] {format % args}\n" + ) + + +def run_server(port: int = 5000): + server_address = ("127.0.0.1", port) + httpd = HTTPServer(server_address, StsHandler) + print(f"Mock Google STS Server running on http://127.0.0.1:{port}";) + print("Press Ctrl+C to stop") + print("") + print("Endpoints:") + print(f" GET http://127.0.0.1:{port}/token";) + print(f" POST http://127.0.0.1:{port}/v1/token";) + print( + f" POST http://127.0.0.1:{port}/v1/projects/-/serviceAccounts/[email protected]:generateAccessToken"; + ) + httpd.serve_forever() + + +if __name__ == "__main__": + port = 5000 + if len(sys.argv) > 1: + port = int(sys.argv[1]) + run_server(port)
