github-actions[bot] commented on code in PR #70681: URL: https://github.com/apache/doris/pull/70681#discussion_r4598106731
########## fe/fe-connector/fe-connector-lance/src/main/java/org/apache/doris/connector/lance/LanceS3StorageProvider.java: ########## @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.lance; + +import org.apache.doris.filesystem.properties.S3CompatibleFileSystemProperties; +import org.apache.doris.filesystem.properties.StorageProperties; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * S3-compatible storage, which Lance reaches through object_store's AWS provider. + * + * <p>The emitted spelling is the one object_store reports as canonical, because that is what + * {@code StorageOptions::with_env_s3} looks for before pulling the same option out of the process + * environment: + * + * <pre> + * // lance-io/src/object_store/providers/aws.rs + * if let Ok(config_key) = AmazonS3ConfigKey::from_str(&key.to_ascii_lowercase()) + * && !self.0.contains_key(config_key.as_ref()) // "aws_access_key_id" + * </pre> + * + * <p>Any other accepted alias leaves that check unsatisfied, so a stray {@code AWS_ACCESS_KEY_ID} + * in the FE or BE environment is inserted next to the configured value, and object_store's + * {@code as_s3_options()} then folds both onto one config key and keeps whichever its HashMap + * yields last - independently per option and per process. Lance's OpenDAL S3 backend accepts these + * same spellings as serde aliases of its own field names. + */ +final class LanceS3StorageProvider implements LanceStorageProvider { + + static final LanceS3StorageProvider INSTANCE = new LanceS3StorageProvider(); + + private static final String ACCESS_KEY_ID = "aws_access_key_id"; + private static final String SECRET_ACCESS_KEY = "aws_secret_access_key"; + private static final String SESSION_TOKEN = "aws_session_token"; + private static final String REGION = "aws_region"; + private static final String ENDPOINT = "aws_endpoint"; + private static final String VIRTUAL_HOSTED_STYLE = "aws_virtual_hosted_style_request"; + /** + * Not prefixed: object_store carries this as a shared client option, and reports it as + * canonical under this name. The spelling buys nothing against the environment here, though - + * {@code StorageOptions::new} overwrites this key outright from {@code AWS_ALLOW_HTTP} before + * {@code with_env_s3} runs. + */ + private static final String ALLOW_HTTP = "allow_http"; + + /** + * Every spelling object_store accepts for the options above, mapped onto the one emitted. + * + * <p>Confined to those options on purpose. They are the only ones this class contributes, so + * they are the only ones a vended option can collide with; anything else a namespace sends is + * between the namespace and Lance. + * + * <p>{@code token} is included because this provider only ever speaks for an S3 dataset, where + * it is unambiguously the session token. It means a bearer token to object_store's Azure + * parser, which is why it can only be resolved once the provider is known. + * + * <p>{@code aws_endpoint_url_s3} is deliberately absent. object_store parses it into a config + * key of its own and prefers it over the generic endpoint, so a vended one already wins + * without being rewritten, and folding it in would replace a defined precedence with map order. + */ + private static final Map<String, String> CANONICAL_BY_ALIAS = Map.ofEntries( + Map.entry("access_key_id", ACCESS_KEY_ID), + Map.entry("aws_access_key_id", ACCESS_KEY_ID), + Map.entry("secret_access_key", SECRET_ACCESS_KEY), + Map.entry("aws_secret_access_key", SECRET_ACCESS_KEY), + Map.entry("session_token", SESSION_TOKEN), + Map.entry("aws_session_token", SESSION_TOKEN), + Map.entry("aws_token", SESSION_TOKEN), + Map.entry("token", SESSION_TOKEN), + Map.entry("endpoint", ENDPOINT), + Map.entry("endpoint_url", ENDPOINT), + Map.entry("aws_endpoint", ENDPOINT), + Map.entry("aws_endpoint_url", ENDPOINT), + Map.entry("region", REGION), + Map.entry("aws_region", REGION), + Map.entry("virtual_hosted_style_request", VIRTUAL_HOSTED_STYLE), + Map.entry("aws_virtual_hosted_style_request", VIRTUAL_HOSTED_STYLE), + // OpenDAL's own field name, of which the two above are serde aliases. All three would + // be the same field supplied more than once, which fails the operator build outright. + Map.entry("enable_virtual_host_style", VIRTUAL_HOSTED_STYLE), + Map.entry("allow_http", ALLOW_HTTP), + Map.entry("aws_allow_http", ALLOW_HTTP)); + + private LanceS3StorageProvider() { + } + + @Override + public Map<String, String> normalizeDorisStorageOptions( + List<StorageProperties> storageProperties) { + Map<String, String> result = new HashMap<>(); + S3CompatibleFileSystemProperties properties = selectS3Compatible(storageProperties); + if (properties == null) { + return result; + } + LanceStorageOptions.putIfNotEmpty(result, ACCESS_KEY_ID, properties.getAccessKey()); + LanceStorageOptions.putIfNotEmpty(result, SECRET_ACCESS_KEY, properties.getSecretKey()); + LanceStorageOptions.putIfNotEmpty(result, SESSION_TOKEN, properties.getSessionToken()); + LanceStorageOptions.putIfNotEmpty(result, ENDPOINT, properties.getEndpoint()); + LanceStorageOptions.putIfNotEmpty(result, REGION, properties.getRegion()); + + String usePathStyle = properties.getUsePathStyle(); + if (usePathStyle != null && !usePathStyle.isEmpty()) { + result.put(VIRTUAL_HOSTED_STYLE, String.valueOf(!Boolean.parseBoolean(usePathStyle))); Review Comment: [P1] Qualify the endpoint before enabling virtual-hosted S3 access A normal AWS catalog with `warehouse=s3://my-bucket/lance`, static keys and `s3.region=us-east-1` defaults to `use_path_style=false`. The shared S3 holder supplies `https://s3.us-east-1.amazonaws.com`, which this method copies together with `aws_virtual_hosted_style_request=true`. Lance 12's `object_store` builder assumes an explicit endpoint in virtual-hosted mode already includes the bucket and uses it unchanged; its list requests therefore target the service endpoint with no `my-bucket`, and object requests omit the bucket too. Default AWS catalogs cannot read their metadata. Translate the endpoint and bucket-addressing policy together (or let the native builder derive a standard AWS endpoint). The MinIO regression forces path-style access, so it misses this default. ########## fe/fe-connector/fe-connector-lance/src/main/java/org/apache/doris/connector/lance/LanceS3StorageProvider.java: ########## @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.lance; + +import org.apache.doris.filesystem.properties.S3CompatibleFileSystemProperties; +import org.apache.doris.filesystem.properties.StorageProperties; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * S3-compatible storage, which Lance reaches through object_store's AWS provider. + * + * <p>The emitted spelling is the one object_store reports as canonical, because that is what + * {@code StorageOptions::with_env_s3} looks for before pulling the same option out of the process + * environment: + * + * <pre> + * // lance-io/src/object_store/providers/aws.rs + * if let Ok(config_key) = AmazonS3ConfigKey::from_str(&key.to_ascii_lowercase()) + * && !self.0.contains_key(config_key.as_ref()) // "aws_access_key_id" + * </pre> + * + * <p>Any other accepted alias leaves that check unsatisfied, so a stray {@code AWS_ACCESS_KEY_ID} + * in the FE or BE environment is inserted next to the configured value, and object_store's + * {@code as_s3_options()} then folds both onto one config key and keeps whichever its HashMap + * yields last - independently per option and per process. Lance's OpenDAL S3 backend accepts these + * same spellings as serde aliases of its own field names. + */ +final class LanceS3StorageProvider implements LanceStorageProvider { + + static final LanceS3StorageProvider INSTANCE = new LanceS3StorageProvider(); + + private static final String ACCESS_KEY_ID = "aws_access_key_id"; + private static final String SECRET_ACCESS_KEY = "aws_secret_access_key"; + private static final String SESSION_TOKEN = "aws_session_token"; + private static final String REGION = "aws_region"; + private static final String ENDPOINT = "aws_endpoint"; + private static final String VIRTUAL_HOSTED_STYLE = "aws_virtual_hosted_style_request"; + /** + * Not prefixed: object_store carries this as a shared client option, and reports it as + * canonical under this name. The spelling buys nothing against the environment here, though - + * {@code StorageOptions::new} overwrites this key outright from {@code AWS_ALLOW_HTTP} before + * {@code with_env_s3} runs. + */ + private static final String ALLOW_HTTP = "allow_http"; + + /** + * Every spelling object_store accepts for the options above, mapped onto the one emitted. + * + * <p>Confined to those options on purpose. They are the only ones this class contributes, so + * they are the only ones a vended option can collide with; anything else a namespace sends is + * between the namespace and Lance. + * + * <p>{@code token} is included because this provider only ever speaks for an S3 dataset, where + * it is unambiguously the session token. It means a bearer token to object_store's Azure + * parser, which is why it can only be resolved once the provider is known. + * + * <p>{@code aws_endpoint_url_s3} is deliberately absent. object_store parses it into a config + * key of its own and prefers it over the generic endpoint, so a vended one already wins + * without being rewritten, and folding it in would replace a defined precedence with map order. + */ + private static final Map<String, String> CANONICAL_BY_ALIAS = Map.ofEntries( + Map.entry("access_key_id", ACCESS_KEY_ID), + Map.entry("aws_access_key_id", ACCESS_KEY_ID), + Map.entry("secret_access_key", SECRET_ACCESS_KEY), + Map.entry("aws_secret_access_key", SECRET_ACCESS_KEY), + Map.entry("session_token", SESSION_TOKEN), + Map.entry("aws_session_token", SESSION_TOKEN), + Map.entry("aws_token", SESSION_TOKEN), + Map.entry("token", SESSION_TOKEN), + Map.entry("endpoint", ENDPOINT), + Map.entry("endpoint_url", ENDPOINT), + Map.entry("aws_endpoint", ENDPOINT), + Map.entry("aws_endpoint_url", ENDPOINT), + Map.entry("region", REGION), + Map.entry("aws_region", REGION), + Map.entry("virtual_hosted_style_request", VIRTUAL_HOSTED_STYLE), + Map.entry("aws_virtual_hosted_style_request", VIRTUAL_HOSTED_STYLE), + // OpenDAL's own field name, of which the two above are serde aliases. All three would + // be the same field supplied more than once, which fails the operator build outright. + Map.entry("enable_virtual_host_style", VIRTUAL_HOSTED_STYLE), + Map.entry("allow_http", ALLOW_HTTP), + Map.entry("aws_allow_http", ALLOW_HTTP)); + + private LanceS3StorageProvider() { + } + + @Override + public Map<String, String> normalizeDorisStorageOptions( + List<StorageProperties> storageProperties) { + Map<String, String> result = new HashMap<>(); + S3CompatibleFileSystemProperties properties = selectS3Compatible(storageProperties); + if (properties == null) { + return result; + } + LanceStorageOptions.putIfNotEmpty(result, ACCESS_KEY_ID, properties.getAccessKey()); + LanceStorageOptions.putIfNotEmpty(result, SECRET_ACCESS_KEY, properties.getSecretKey()); + LanceStorageOptions.putIfNotEmpty(result, SESSION_TOKEN, properties.getSessionToken()); + LanceStorageOptions.putIfNotEmpty(result, ENDPOINT, properties.getEndpoint()); Review Comment: [P2] Normalize scheme-less S3 endpoints before exporting them The shared Doris S3 properties accept endpoints such as `s3.us-east-1.amazonaws.com` without a scheme; the existing `S3ObjStorage` client adds `https://` before using them. This adapter exports that raw host unchanged. Even with `use_path_style=true`, valid static keys and a correct bucket, Lance's native builder produces `s3.us-east-1.amazonaws.com/my-bucket` as the request endpoint, and its HTTP/signing path requires an absolute URL, so namespace listing and dataset reads fail before reaching S3. Apply the same HTTPS normalization before emitting `aws_endpoint`, preserving explicit HTTP/HTTPS values. The current storage tests use only already-qualified endpoints. This failure is independent of the bucket omission under virtual-hosted addressing. ########## fe/fe-connector/fe-connector-lance/src/main/java/org/apache/doris/connector/lance/LanceS3StorageProvider.java: ########## @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.lance; + +import org.apache.doris.filesystem.properties.S3CompatibleFileSystemProperties; +import org.apache.doris.filesystem.properties.StorageProperties; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * S3-compatible storage, which Lance reaches through object_store's AWS provider. + * + * <p>The emitted spelling is the one object_store reports as canonical, because that is what + * {@code StorageOptions::with_env_s3} looks for before pulling the same option out of the process + * environment: + * + * <pre> + * // lance-io/src/object_store/providers/aws.rs + * if let Ok(config_key) = AmazonS3ConfigKey::from_str(&key.to_ascii_lowercase()) + * && !self.0.contains_key(config_key.as_ref()) // "aws_access_key_id" + * </pre> + * + * <p>Any other accepted alias leaves that check unsatisfied, so a stray {@code AWS_ACCESS_KEY_ID} + * in the FE or BE environment is inserted next to the configured value, and object_store's + * {@code as_s3_options()} then folds both onto one config key and keeps whichever its HashMap + * yields last - independently per option and per process. Lance's OpenDAL S3 backend accepts these + * same spellings as serde aliases of its own field names. + */ +final class LanceS3StorageProvider implements LanceStorageProvider { + + static final LanceS3StorageProvider INSTANCE = new LanceS3StorageProvider(); + + private static final String ACCESS_KEY_ID = "aws_access_key_id"; + private static final String SECRET_ACCESS_KEY = "aws_secret_access_key"; + private static final String SESSION_TOKEN = "aws_session_token"; + private static final String REGION = "aws_region"; + private static final String ENDPOINT = "aws_endpoint"; + private static final String VIRTUAL_HOSTED_STYLE = "aws_virtual_hosted_style_request"; + /** + * Not prefixed: object_store carries this as a shared client option, and reports it as + * canonical under this name. The spelling buys nothing against the environment here, though - + * {@code StorageOptions::new} overwrites this key outright from {@code AWS_ALLOW_HTTP} before + * {@code with_env_s3} runs. + */ + private static final String ALLOW_HTTP = "allow_http"; + + /** + * Every spelling object_store accepts for the options above, mapped onto the one emitted. + * + * <p>Confined to those options on purpose. They are the only ones this class contributes, so + * they are the only ones a vended option can collide with; anything else a namespace sends is + * between the namespace and Lance. + * + * <p>{@code token} is included because this provider only ever speaks for an S3 dataset, where + * it is unambiguously the session token. It means a bearer token to object_store's Azure + * parser, which is why it can only be resolved once the provider is known. + * + * <p>{@code aws_endpoint_url_s3} is deliberately absent. object_store parses it into a config + * key of its own and prefers it over the generic endpoint, so a vended one already wins + * without being rewritten, and folding it in would replace a defined precedence with map order. + */ + private static final Map<String, String> CANONICAL_BY_ALIAS = Map.ofEntries( + Map.entry("access_key_id", ACCESS_KEY_ID), + Map.entry("aws_access_key_id", ACCESS_KEY_ID), + Map.entry("secret_access_key", SECRET_ACCESS_KEY), + Map.entry("aws_secret_access_key", SECRET_ACCESS_KEY), + Map.entry("session_token", SESSION_TOKEN), + Map.entry("aws_session_token", SESSION_TOKEN), + Map.entry("aws_token", SESSION_TOKEN), + Map.entry("token", SESSION_TOKEN), + Map.entry("endpoint", ENDPOINT), + Map.entry("endpoint_url", ENDPOINT), + Map.entry("aws_endpoint", ENDPOINT), + Map.entry("aws_endpoint_url", ENDPOINT), + Map.entry("region", REGION), + Map.entry("aws_region", REGION), + Map.entry("virtual_hosted_style_request", VIRTUAL_HOSTED_STYLE), + Map.entry("aws_virtual_hosted_style_request", VIRTUAL_HOSTED_STYLE), + // OpenDAL's own field name, of which the two above are serde aliases. All three would + // be the same field supplied more than once, which fails the operator build outright. + Map.entry("enable_virtual_host_style", VIRTUAL_HOSTED_STYLE), + Map.entry("allow_http", ALLOW_HTTP), + Map.entry("aws_allow_http", ALLOW_HTTP)); + + private LanceS3StorageProvider() { + } + + @Override + public Map<String, String> normalizeDorisStorageOptions( + List<StorageProperties> storageProperties) { + Map<String, String> result = new HashMap<>(); + S3CompatibleFileSystemProperties properties = selectS3Compatible(storageProperties); + if (properties == null) { + return result; + } + LanceStorageOptions.putIfNotEmpty(result, ACCESS_KEY_ID, properties.getAccessKey()); Review Comment: [P2] Preserve the explicitly requested anonymous S3 mode The shared storage binder accepts `s3.credentials_provider_type=ANONYMOUS`, but this translation drops the mode and emits no `aws_skip_signature` option. With a public bucket, `use_path_style=true` and no AWS credentials on the host, Lance consequently falls back to its AWS credential chain; `object_store` requests credentials before listing because signature skipping defaults to false, so unsigned reads fail before reaching the public data. Merely omitting AK/SK does not implement anonymous access, despite the new `anonymousAccessEmitsNoCredentials` test. Translate explicit anonymous mode into the native unsigned-access policy, or reject that mode clearly if it is deferred from this port, rather than silently substituting credential-chain authentication. ########## fe/fe-connector/fe-connector-lance/src/main/java/org/apache/doris/connector/lance/LanceS3StorageProvider.java: ########## @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.lance; + +import org.apache.doris.filesystem.properties.S3CompatibleFileSystemProperties; +import org.apache.doris.filesystem.properties.StorageProperties; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * S3-compatible storage, which Lance reaches through object_store's AWS provider. + * + * <p>The emitted spelling is the one object_store reports as canonical, because that is what + * {@code StorageOptions::with_env_s3} looks for before pulling the same option out of the process + * environment: + * + * <pre> + * // lance-io/src/object_store/providers/aws.rs + * if let Ok(config_key) = AmazonS3ConfigKey::from_str(&key.to_ascii_lowercase()) + * && !self.0.contains_key(config_key.as_ref()) // "aws_access_key_id" + * </pre> + * + * <p>Any other accepted alias leaves that check unsatisfied, so a stray {@code AWS_ACCESS_KEY_ID} + * in the FE or BE environment is inserted next to the configured value, and object_store's + * {@code as_s3_options()} then folds both onto one config key and keeps whichever its HashMap + * yields last - independently per option and per process. Lance's OpenDAL S3 backend accepts these + * same spellings as serde aliases of its own field names. + */ +final class LanceS3StorageProvider implements LanceStorageProvider { + + static final LanceS3StorageProvider INSTANCE = new LanceS3StorageProvider(); + + private static final String ACCESS_KEY_ID = "aws_access_key_id"; + private static final String SECRET_ACCESS_KEY = "aws_secret_access_key"; + private static final String SESSION_TOKEN = "aws_session_token"; + private static final String REGION = "aws_region"; + private static final String ENDPOINT = "aws_endpoint"; + private static final String VIRTUAL_HOSTED_STYLE = "aws_virtual_hosted_style_request"; + /** + * Not prefixed: object_store carries this as a shared client option, and reports it as + * canonical under this name. The spelling buys nothing against the environment here, though - + * {@code StorageOptions::new} overwrites this key outright from {@code AWS_ALLOW_HTTP} before + * {@code with_env_s3} runs. + */ + private static final String ALLOW_HTTP = "allow_http"; + + /** + * Every spelling object_store accepts for the options above, mapped onto the one emitted. + * + * <p>Confined to those options on purpose. They are the only ones this class contributes, so + * they are the only ones a vended option can collide with; anything else a namespace sends is + * between the namespace and Lance. + * + * <p>{@code token} is included because this provider only ever speaks for an S3 dataset, where + * it is unambiguously the session token. It means a bearer token to object_store's Azure + * parser, which is why it can only be resolved once the provider is known. + * + * <p>{@code aws_endpoint_url_s3} is deliberately absent. object_store parses it into a config + * key of its own and prefers it over the generic endpoint, so a vended one already wins + * without being rewritten, and folding it in would replace a defined precedence with map order. + */ + private static final Map<String, String> CANONICAL_BY_ALIAS = Map.ofEntries( + Map.entry("access_key_id", ACCESS_KEY_ID), + Map.entry("aws_access_key_id", ACCESS_KEY_ID), + Map.entry("secret_access_key", SECRET_ACCESS_KEY), + Map.entry("aws_secret_access_key", SECRET_ACCESS_KEY), + Map.entry("session_token", SESSION_TOKEN), + Map.entry("aws_session_token", SESSION_TOKEN), + Map.entry("aws_token", SESSION_TOKEN), + Map.entry("token", SESSION_TOKEN), + Map.entry("endpoint", ENDPOINT), + Map.entry("endpoint_url", ENDPOINT), + Map.entry("aws_endpoint", ENDPOINT), + Map.entry("aws_endpoint_url", ENDPOINT), + Map.entry("region", REGION), + Map.entry("aws_region", REGION), + Map.entry("virtual_hosted_style_request", VIRTUAL_HOSTED_STYLE), + Map.entry("aws_virtual_hosted_style_request", VIRTUAL_HOSTED_STYLE), + // OpenDAL's own field name, of which the two above are serde aliases. All three would + // be the same field supplied more than once, which fails the operator build outright. + Map.entry("enable_virtual_host_style", VIRTUAL_HOSTED_STYLE), + Map.entry("allow_http", ALLOW_HTTP), + Map.entry("aws_allow_http", ALLOW_HTTP)); + + private LanceS3StorageProvider() { + } + + @Override + public Map<String, String> normalizeDorisStorageOptions( + List<StorageProperties> storageProperties) { + Map<String, String> result = new HashMap<>(); + S3CompatibleFileSystemProperties properties = selectS3Compatible(storageProperties); + if (properties == null) { + return result; + } + LanceStorageOptions.putIfNotEmpty(result, ACCESS_KEY_ID, properties.getAccessKey()); + LanceStorageOptions.putIfNotEmpty(result, SECRET_ACCESS_KEY, properties.getSecretKey()); + LanceStorageOptions.putIfNotEmpty(result, SESSION_TOKEN, properties.getSessionToken()); Review Comment: [P2] Keep ambient session tokens out of explicit static credentials When the catalog sets permanent AK/SK and omits `s3.session_token`, `putIfNotEmpty` leaves `aws_session_token` absent. Lance 12's `with_env_s3()` then fills it from process `AWS_SESSION_TOKEN`, and `extract_static_s3_credentials()` combines that unrelated token with the catalog's configured keys. An FE or BE launched from an assumed-role environment therefore gets invalid credentials even though the catalog selected valid permanent keys; different FE/BE environments can also make only one side fail. The existing Doris S3 factory uses tokenless `AwsBasicCredentials` for this configuration. Preserve the complete selected static identity, including the absence of a token, so the native readers cannot fill one from another identity. ########## fe/fe-connector/fe-connector-lance/src/main/java/org/apache/doris/connector/lance/LanceConnectorMetadata.java: ########## @@ -0,0 +1,251 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.lance; + +import org.apache.doris.connector.spi.ConnectorMetadata; +import org.apache.doris.connector.spi.ConnectorSession; +import org.apache.doris.connector.spi.ConnectorTableSchema; +import org.apache.doris.connector.spi.ConnectorTableStatistics; +import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.doris.connector.spi.handle.ConnectorColumnHandle; +import org.apache.doris.connector.spi.handle.ConnectorTableHandle; +import org.apache.doris.connector.spi.handle.NamedColumnHandle; +import org.apache.doris.connector.spi.mvcc.ConnectorMvccSnapshot; +import org.apache.doris.connector.spi.mvcc.ConnectorTimeTravelSpec; +import org.apache.doris.thrift.THiveTable; +import org.apache.doris.thrift.TTableDescriptor; +import org.apache.doris.thrift.TTableType; + +import org.apache.arrow.vector.types.pojo.Field; + +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; + +/** + * Serves {@code SHOW DATABASES} / {@code SHOW TABLES} / {@code DESC}, the statement's version pin and + * time travel for a Lance catalog. + * + * <p>Created per statement by the engine's metadata funnel; it holds no cross-statement state. Reads of one + * table version share one metadata read through {@link LanceStatementScope}. + */ +final class LanceConnectorMetadata implements ConnectorMetadata { + + /** Informational format name the engine records with the schema. */ + private static final String TABLE_FORMAT_TYPE = "LANCE"; + + private final LanceCatalogClient client; + + LanceConnectorMetadata(LanceCatalogClient client) { + this.client = client; + } + + // ========= ConnectorSchemaOps ========= + + @Override + public List<String> listDatabaseNames(ConnectorSession session) { + return client.listDatabaseNames(); + } + + @Override + public boolean databaseExists(ConnectorSession session, String dbName) { + return client.listDatabaseNames().contains(dbName); + } + + // ========= ConnectorTableMetadataOps ========= + + @Override + public List<String> listTableNames(ConnectorSession session, String dbName) { + return client.listTableNames(dbName); + } + + @Override + public Optional<ConnectorTableHandle> getTableHandle( + ConnectorSession session, String dbName, String tableName) { + return client.tableExists(dbName, tableName) + ? Optional.of(new LanceTableHandle(dbName, tableName)) : Optional.empty(); + } + + /** The schema of the handle's pinned version, or of the newest version when it has no pin. */ + @Override + public ConnectorTableSchema getTableSchema(ConnectorSession session, ConnectorTableHandle handle) { + LanceTableHandle lanceHandle = (LanceTableHandle) handle; + return toTableSchema(lanceHandle, metadata(session, lanceHandle)); + } + + /** The schema of a time-travel pin: each Lance version carries its own schema. */ + @Override + public ConnectorTableSchema getTableSchema(ConnectorSession session, ConnectorTableHandle handle, + ConnectorMvccSnapshot snapshot) { + LanceTableHandle lanceHandle = pinned((LanceTableHandle) handle, snapshot); + return toTableSchema(lanceHandle, metadata(session, lanceHandle)); + } + + @Override + public Map<String, ConnectorColumnHandle> getColumnHandles( + ConnectorSession session, ConnectorTableHandle handle) { + return toColumnHandles(metadata(session, (LanceTableHandle) handle)); + } + + @Override + public Map<String, ConnectorColumnHandle> getColumnHandles(ConnectorSession session, + ConnectorTableHandle handle, ConnectorMvccSnapshot snapshot) { + return toColumnHandles(metadata(session, pinned((LanceTableHandle) handle, snapshot))); + } + + @Override + public boolean supportsColumnHandleSnapshotPin(ConnectorSession session) { + return true; + } + + /** + * Lance has no table descriptor of its own, so it borrows the Hive one, as 4.x Lance tables did and as + * the other connectors that read through the generic file-scan path do. Leaving the SPI default (null) + * is not neutral: fe-core would fall back to {@code SCHEMA_TABLE} and BE would build the wrong + * descriptor. + */ + @Override + public TTableDescriptor buildTableDescriptor(ConnectorSession session, + long tableId, String tableName, String dbName, + String remoteName, int numCols, long catalogId) { + THiveTable hiveTable = new THiveTable(dbName, tableName, new HashMap<>()); + TTableDescriptor descriptor = new TTableDescriptor( + tableId, TTableType.HIVE_TABLE, numCols, 0, tableName, dbName); + descriptor.setHiveTable(hiveTable); + return descriptor; + } + + // ========= ConnectorStatisticsOps ========= + + /** The logical row count of the newest version: the fragments' rows minus their deletions. */ + @Override + public Optional<ConnectorTableStatistics> getTableStatistics(ConnectorSession session, + ConnectorTableHandle handle) { + LanceTableHandle lanceHandle = (LanceTableHandle) handle; + LanceTableMetadata metadata = metadata(session, lanceHandle); + return Optional.of(new ConnectorTableStatistics(metadata.getRowCount(), -1)); + } + + // ========= ConnectorPushdownOps ========= + + /** Nothing is pushed down to Lance yet, so no conjunct is ever translated, with a CAST or without. */ + @Override + public boolean supportsCastPredicatePushdown(ConnectorSession session) { + return false; + } + + // ========= MVCC ========= + + /** + * Pins the newest version in storage for the statement. The version is also what MTMV freshness + * compares, and Lance versions only grow on the main chain. + */ + @Override + public Optional<ConnectorMvccSnapshot> beginQuerySnapshot(ConnectorSession session, + ConnectorTableHandle handle) { + LanceTableMetadata metadata = metadata(session, ((LanceTableHandle) handle).unpinned()); + return Optional.of(ConnectorMvccSnapshot.builder().snapshotId(metadata.getVersion()).build()); Review Comment: [P2] Retain the schema of the implicit query snapshot Returning only `snapshotId` leaves `schemaId=-1` and `retainSchema=false`, so `PluginDrivenMvccExternalTable.materializeLatest()` does not publish a pinned schema. On a cold schema cache, `getPartitionColumns()` instead reaches `PluginDrivenExternalTable.initSchema()`, which uses a fresh cross-statement session and cannot reuse this statement's metadata memo. If the query pins v1 `(a)` and an external writer commits v2 `(a,b)` before that load, `SELECT *` binds the v2 columns while scan planning still applies v1; BE is then asked to read `b` from a version that has no such column. Set a schema coordinate and opt into retained schema here so the engine binds the schema already read for the pin. The single-session connector tests bypass this cache boundary. ########## build.sh: ########## @@ -1418,6 +1418,17 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then fi unset ICEBERG_CONN_LIB + # lance-core's own linux-x86-64 JNI library needs GLIBC 2.28; swap in the glibc 2.17 build, inside + # the jar the Lance plugin just unzipped (lance-core extracts the library from that jar entry and has + # no path override). After the deploy loop, because it clears and refills the plugin's lib/. + LANCE_CONN_LIB="${DORIS_OUTPUT}/fe/plugins/connector/lance/lib" + if [[ -d "${LANCE_CONN_LIB}" ]]; then + # shellcheck source=/dev/null + . "${DORIS_HOME}/docker/thirdparties/lance-jni-helpers.sh" + lance_jni_replace "${LANCE_CONN_LIB}" "${DORIS_THIRDPARTY}" "${TARGET_SYSTEM}" "${TARGET_ARCH}" Review Comment: [P2] Include Arrow C Data JNI in the older-host compatibility fix Replacing `liblance_jni` alone still leaves schema reads unusable on a stock CentOS 7 FE using its system C++ runtime. The packaged `arrow-c-data:19.0.0` x86_64 `libarrow_cdata_jni.so` requires `CXXABI_1.3.9` and `GLIBCXX_3.4.21` (confirmed by static ELF inspection), beyond that host's runtime. Every `LanceCatalogClient.readMetadata()` calls `Dataset.getSchema()`, which imports through Arrow C Data and loads this second JNI library. Neither this deployment path nor `start_fe.sh` supplies/selects a newer C++ runtime, so the first DESC/SELECT fails with a native-library error despite replacing Lance JNI. Package a compatible Arrow JNI or its required runtime as well; the new native-test helper explicitly skips this failure. ########## fe/fe-connector/fe-connector-lance/src/main/java/org/apache/doris/connector/lance/LanceNamespaceName.java: ########## @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.lance; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Converts between a hierarchical Lance namespace and the flat database name exposed by Doris. + * + * <p>The input namespace is relative to the optional parent namespace configured on the catalog. + * An empty relative namespace maps to {@code rootDatabase}. Other namespace components are joined + * with {@code delimiter}; delimiter occurrences and backslashes inside a component are escaped so + * the conversion remains reversible. A one-level namespace equal to {@code rootDatabase} receives + * an additional leading escape to distinguish it from the empty root namespace. + * + * <p>This class only converts names. It does not access the remote Lance namespace service. A malformed + * name is an {@link IllegalArgumentException}: from property validation the engine reports it as a DDL + * error, and at query time the namespace client reports it as a connector error. + */ +final class LanceNamespaceName { + private static final char ESCAPE = '\\'; + + private LanceNamespaceName() { + } + + /** + * Converts relative Lance namespace components to one Doris database name. + * + * <p>For example, with {@code "."} as the delimiter, + * {@code [company.analytics, raw]} becomes {@code company\.analytics.raw}. + * An empty namespace becomes {@code rootDatabase}. + */ + static String namespaceToDorisDatabaseName( + List<String> relativeNamespace, String delimiter, String rootDatabase) { + if (relativeNamespace.isEmpty()) { + return rootDatabase; + } + StringBuilder result = new StringBuilder(); + for (int i = 0; i < relativeNamespace.size(); i++) { + if (i > 0) { + result.append(delimiter); + } + result.append(escapeNamespaceComponent(relativeNamespace.get(i), delimiter)); + } + // The local root database represents the empty Lance namespace. Escape + // the rare, colliding one-level namespace so the mapping stays reversible. + return rootDatabase.contentEquals(result) ? ESCAPE + result.toString() : result.toString(); + } + + /** Escapes backslashes and delimiters occurring inside one namespace component. */ + private static String escapeNamespaceComponent(String component, String delimiter) { + return component.replace("\\", "\\\\").replace(delimiter, "\\" + delimiter); + } + + /** + * Restores the relative Lance namespace represented by one Doris database name. + * + * <p>For example, with {@code "."} as the delimiter, + * {@code company\.analytics.raw} becomes {@code [company.analytics, raw]}. + * A database name equal to {@code rootDatabase} becomes an empty namespace. + */ + static List<String> dorisDatabaseNameToNamespace( + String databaseName, String delimiter, String rootDatabase) { + if (rootDatabase.equals(databaseName)) { + return Collections.emptyList(); + } + if (databaseName.length() > 1 && databaseName.charAt(0) == ESCAPE Review Comment: [P3] Keep the root marker distinct from escaped namespace names With the valid custom root alias `lance.namespace.root_database=.analytics` and a real one-level namespace named `.analytics`, listing encodes the real namespace as `\.analytics`. The encoder does not add a collision marker because that escaped value differs from the raw root alias. This decoder nevertheless sees the leading backslash and the matching substring, strips the actual delimiter escape, and parses `.analytics` as a path with an empty first component. The namespace appears in SHOW DATABASES but every table lookup through it fails. Distinguish a generated root-collision marker from normal component escaping for custom aliases; add this case to the round-trip tests. ########## fe/fe-connector/fe-connector-lance/src/main/java/org/apache/doris/connector/lance/LanceConnectorMetadata.java: ########## @@ -0,0 +1,251 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.lance; + +import org.apache.doris.connector.spi.ConnectorMetadata; +import org.apache.doris.connector.spi.ConnectorSession; +import org.apache.doris.connector.spi.ConnectorTableSchema; +import org.apache.doris.connector.spi.ConnectorTableStatistics; +import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.doris.connector.spi.handle.ConnectorColumnHandle; +import org.apache.doris.connector.spi.handle.ConnectorTableHandle; +import org.apache.doris.connector.spi.handle.NamedColumnHandle; +import org.apache.doris.connector.spi.mvcc.ConnectorMvccSnapshot; +import org.apache.doris.connector.spi.mvcc.ConnectorTimeTravelSpec; +import org.apache.doris.thrift.THiveTable; +import org.apache.doris.thrift.TTableDescriptor; +import org.apache.doris.thrift.TTableType; + +import org.apache.arrow.vector.types.pojo.Field; + +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; + +/** + * Serves {@code SHOW DATABASES} / {@code SHOW TABLES} / {@code DESC}, the statement's version pin and + * time travel for a Lance catalog. + * + * <p>Created per statement by the engine's metadata funnel; it holds no cross-statement state. Reads of one + * table version share one metadata read through {@link LanceStatementScope}. + */ +final class LanceConnectorMetadata implements ConnectorMetadata { + + /** Informational format name the engine records with the schema. */ + private static final String TABLE_FORMAT_TYPE = "LANCE"; + + private final LanceCatalogClient client; + + LanceConnectorMetadata(LanceCatalogClient client) { + this.client = client; + } + + // ========= ConnectorSchemaOps ========= + + @Override + public List<String> listDatabaseNames(ConnectorSession session) { + return client.listDatabaseNames(); + } + + @Override + public boolean databaseExists(ConnectorSession session, String dbName) { + return client.listDatabaseNames().contains(dbName); + } + + // ========= ConnectorTableMetadataOps ========= + + @Override + public List<String> listTableNames(ConnectorSession session, String dbName) { + return client.listTableNames(dbName); + } + + @Override + public Optional<ConnectorTableHandle> getTableHandle( + ConnectorSession session, String dbName, String tableName) { + return client.tableExists(dbName, tableName) + ? Optional.of(new LanceTableHandle(dbName, tableName)) : Optional.empty(); + } + + /** The schema of the handle's pinned version, or of the newest version when it has no pin. */ + @Override + public ConnectorTableSchema getTableSchema(ConnectorSession session, ConnectorTableHandle handle) { + LanceTableHandle lanceHandle = (LanceTableHandle) handle; + return toTableSchema(lanceHandle, metadata(session, lanceHandle)); + } + + /** The schema of a time-travel pin: each Lance version carries its own schema. */ + @Override + public ConnectorTableSchema getTableSchema(ConnectorSession session, ConnectorTableHandle handle, + ConnectorMvccSnapshot snapshot) { + LanceTableHandle lanceHandle = pinned((LanceTableHandle) handle, snapshot); + return toTableSchema(lanceHandle, metadata(session, lanceHandle)); + } + + @Override + public Map<String, ConnectorColumnHandle> getColumnHandles( + ConnectorSession session, ConnectorTableHandle handle) { + return toColumnHandles(metadata(session, (LanceTableHandle) handle)); + } + + @Override + public Map<String, ConnectorColumnHandle> getColumnHandles(ConnectorSession session, + ConnectorTableHandle handle, ConnectorMvccSnapshot snapshot) { + return toColumnHandles(metadata(session, pinned((LanceTableHandle) handle, snapshot))); + } + + @Override + public boolean supportsColumnHandleSnapshotPin(ConnectorSession session) { + return true; + } + + /** + * Lance has no table descriptor of its own, so it borrows the Hive one, as 4.x Lance tables did and as + * the other connectors that read through the generic file-scan path do. Leaving the SPI default (null) + * is not neutral: fe-core would fall back to {@code SCHEMA_TABLE} and BE would build the wrong + * descriptor. + */ + @Override + public TTableDescriptor buildTableDescriptor(ConnectorSession session, + long tableId, String tableName, String dbName, + String remoteName, int numCols, long catalogId) { + THiveTable hiveTable = new THiveTable(dbName, tableName, new HashMap<>()); + TTableDescriptor descriptor = new TTableDescriptor( + tableId, TTableType.HIVE_TABLE, numCols, 0, tableName, dbName); + descriptor.setHiveTable(hiveTable); + return descriptor; + } + + // ========= ConnectorStatisticsOps ========= + + /** The logical row count of the newest version: the fragments' rows minus their deletions. */ + @Override + public Optional<ConnectorTableStatistics> getTableStatistics(ConnectorSession session, + ConnectorTableHandle handle) { + LanceTableHandle lanceHandle = (LanceTableHandle) handle; + LanceTableMetadata metadata = metadata(session, lanceHandle); Review Comment: [P2] Apply the snapshot in historical table-statistics requests `PluginDrivenExternalTable.fetchRowCountAtSnapshot()` creates a fresh cross-statement session, resolves an unpinned handle, and calls the three-argument `getTableStatistics(session, handle, snapshot)`. This connector only overrides the two-argument method, so the SPI default discards the snapshot and this metadata lookup counts the latest version. A query joining `FOR VERSION AS OF 1` can therefore be optimized using billions of current rows while its scan reads a one-row historical version, producing a badly skewed join order/build-side choice. Override the snapshot-aware overload and load `pinned(handle, snapshot)`, as the schema/column-handle methods already do. The native test manually pins its handle before the two-argument call and does not cover the engine's actual request. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
