924060929 commented on code in PR #70681: URL: https://github.com/apache/doris/pull/70681#discussion_r4610829218
########## fe/fe-connector/fe-connector-lance/src/main/java/org/apache/doris/connector/lance/LanceS3StorageProvider.java: ########## @@ -0,0 +1,214 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.lance; + +import org.apache.doris.filesystem.properties.S3CompatibleFileSystemProperties; +import org.apache.doris.filesystem.properties.StorageProperties; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * S3-compatible storage, which Lance reaches through object_store's AWS provider. + * + * <p>The emitted spelling is the one object_store reports as canonical, because that is what + * {@code StorageOptions::with_env_s3} looks for before pulling the same option out of the process + * environment: + * + * <pre> + * // lance-io/src/object_store/providers/aws.rs + * if let Ok(config_key) = AmazonS3ConfigKey::from_str(&key.to_ascii_lowercase()) + * && !self.0.contains_key(config_key.as_ref()) // "aws_access_key_id" + * </pre> + * + * <p>Any other accepted alias leaves that check unsatisfied, so a stray {@code AWS_ACCESS_KEY_ID} + * in the FE or BE environment is inserted next to the configured value, and object_store's + * {@code as_s3_options()} then folds both onto one config key and keeps whichever its HashMap + * yields last - independently per option and per process. Lance's OpenDAL S3 backend accepts these + * same spellings as serde aliases of its own field names. + */ +final class LanceS3StorageProvider implements LanceStorageProvider { + + static final LanceS3StorageProvider INSTANCE = new LanceS3StorageProvider(); + + private static final String ACCESS_KEY_ID = "aws_access_key_id"; + private static final String SECRET_ACCESS_KEY = "aws_secret_access_key"; + private static final String SESSION_TOKEN = "aws_session_token"; + private static final String REGION = "aws_region"; + private static final String ENDPOINT = "aws_endpoint"; + private static final String S3_ENDPOINT = "aws_endpoint_url_s3"; + private static final String VIRTUAL_HOSTED_STYLE = "aws_virtual_hosted_style_request"; + /** + * Not prefixed: object_store carries this as a shared client option, and reports it as + * canonical under this name. The spelling buys nothing against the environment here, though - + * {@code StorageOptions::new} overwrites this key outright from {@code AWS_ALLOW_HTTP} before + * {@code with_env_s3} runs. + */ + private static final String ALLOW_HTTP = "allow_http"; + + /** + * Every spelling object_store accepts for the options above, mapped onto the one emitted. + * + * <p>Confined to those options on purpose. They are the only ones this class contributes, so + * they are the only ones a vended option can collide with; anything else a namespace sends is + * between the namespace and Lance. + * + * <p>{@code token} is included because this provider only ever speaks for an S3 dataset, where + * it is unambiguously the session token. It means a bearer token to object_store's Azure + * parser, which is why it can only be resolved once the provider is known. + * + * <p>{@code aws_endpoint_url_s3} remains a separate coordinate: object_store prefers it over + * the generic endpoint. After the catalog/vended merge, inference fills a missing service + * endpoint from the generic one to block an unrelated environment value. + */ + private static final Map<String, String> CANONICAL_BY_ALIAS = Map.ofEntries( + Map.entry("access_key_id", ACCESS_KEY_ID), + Map.entry("aws_access_key_id", ACCESS_KEY_ID), + Map.entry("secret_access_key", SECRET_ACCESS_KEY), + Map.entry("aws_secret_access_key", SECRET_ACCESS_KEY), + Map.entry("session_token", SESSION_TOKEN), + Map.entry("aws_session_token", SESSION_TOKEN), + Map.entry("aws_token", SESSION_TOKEN), + Map.entry("token", SESSION_TOKEN), + Map.entry("endpoint", ENDPOINT), + Map.entry("endpoint_url", ENDPOINT), + Map.entry("aws_endpoint", ENDPOINT), + Map.entry("aws_endpoint_url", ENDPOINT), + Map.entry("aws_endpoint_url_s3", S3_ENDPOINT), + Map.entry("region", REGION), + Map.entry("aws_region", REGION), + Map.entry("virtual_hosted_style_request", VIRTUAL_HOSTED_STYLE), + Map.entry("aws_virtual_hosted_style_request", VIRTUAL_HOSTED_STYLE), + // OpenDAL's own field name, of which the two above are serde aliases. All three would + // be the same field supplied more than once, which fails the operator build outright. + Map.entry("enable_virtual_host_style", VIRTUAL_HOSTED_STYLE), + Map.entry("allow_http", ALLOW_HTTP), + Map.entry("aws_allow_http", ALLOW_HTTP)); + + private LanceS3StorageProvider() { + } + + @Override + public Map<String, String> normalizeDorisStorageOptions( + String datasetUri, List<StorageProperties> storageProperties) { + Map<String, String> result = new HashMap<>(); + S3CompatibleFileSystemProperties properties = selectS3Compatible(storageProperties); + if (properties == null) { + return result; Review Comment: [P2] Reject an absent S3 binding instead of falling back to ambient credentials For `CREATE CATALOG c PROPERTIES ("type"="lance", "warehouse"="s3://warehouse/lance")`, the filesystem binder has no endpoint/region/provider hint selecting S3, so its default HDFS binding reaches this method and `selectS3Compatible()` returns null. Returning an empty map bypasses the static-credential rejection below. I compiled the current adapter sources and passed the resulting `{}` to Lance 12's `Dataset.listManifestLocations()` in a child process: it successfully sent a signed request to the endpoint from `AWS_ENDPOINT_URL_S3` using `AWS_ACCESS_KEY_ID` from the process environment. Thus the documented unsupported credential-chain mode is still enabled for this reachable configuration. FE and BE can select different ambient identities/endpoints, so metadata discovery can succeed while execution fails or reads a different store at the same URI/version. Since this adapter is selected specifically for an S3 URI, fail explicitly when no S3 binding exists, and add a test with an e mpty/non-S3-only storage list (the existing credentialless test supplies an S3 holder and misses this branch). -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
