zy-kkk commented on code in PR #68453: URL: https://github.com/apache/doris/pull/68453#discussion_r4129037306
########## fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceSdkNamespace.java: ########## @@ -0,0 +1,375 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance; + +import org.apache.doris.datasource.lance.storage.LanceStorageOptions; + +import com.google.common.collect.ImmutableSet; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.lance.namespace.LanceNamespace; +import org.lance.namespace.model.DescribeTableRequest; +import org.lance.namespace.model.DescribeTableResponse; +import org.lance.namespace.model.DescribeTableVersionRequest; +import org.lance.namespace.model.DescribeTableVersionResponse; +import org.lance.namespace.model.ListTableVersionsRequest; +import org.lance.namespace.model.ListTableVersionsResponse; +import org.lance.namespace.model.TableVersion; + +import java.io.ByteArrayOutputStream; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.security.SecureRandom; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; +import java.util.regex.Pattern; + +/** + * The namespace the Lance SDK is handed to open one namespace-managed dataset: the catalog's + * namespace, with three things the SDK gets wrong on its own. + * + * <p>The SDK opens with the options it is handed plus whatever its own describe vends, spelled as + * the namespace spells them. Lance then adds the process environment for any option whose + * canonical key is missing, so a vended {@code endpoint} next to an {@code AWS_ENDPOINT} in the + * FE environment leaves the FE on whichever endpoint object_store folds last, while the BE, handed + * the canonical {@code aws_endpoint}, keeps the vended one. {@link #describeTable} therefore + * returns the vended options in the vocabulary Doris uses for everything else. + * + * <p>The SDK also caches the object store of a namespace-opened dataset in the catalog Session by + * the namespace's id and the table id alone, ignoring the options: a read overlapping another one + * that still holds a store for the same table reuses that store, whatever endpoint it was built + * for (lance-io {@code StorageOptionsAccessor::accessor_id}, {@code ObjectStoreRegistry::get_store}). + * {@link #namespaceId} therefore also identifies the table location and the options the store is + * built with, less credentials the store refreshes from the namespace anyway. + * + * <p>Lance opens a finalized manifest a namespace records wherever it is, while the BE opens a + * version by the dataset URI and number, at the canonical path. {@link #describeTableVersion} and + * {@link #listTableVersions} therefore reject a finalized manifest anywhere else. + * + * <p>A namespace Lance does not implement natively is called back through JNI, which reports an + * exception thrown by the callback only as "Java exception was thrown". The last one is kept for + * {@link #unwrapCallbackFailure}, so a missing version or branch is still reported as such. + * + * <p>One instance is created per open. The datasets checked out from it resolve versions through + * it, and the store the open builds refreshes credentials through it, also for later reads that + * share that store. + */ +final class LanceSdkNamespace implements LanceNamespace { + private static final Logger LOG = LogManager.getLogger(LanceSdkNamespace.class); + + /** What jni-rs reports for a Java exception a callback threw. */ + private static final String CALLBACK_FAILURE = "Java exception was thrown"; + + private static final String EXPIRES_AT_MILLIS = "expires_at_millis"; + + /** The values lance-core's {@code str_is_truthy} accepts, lower case. */ + private static final Set<String> TRUTHY = ImmutableSet.of("1", "true", "on", "yes", "y"); + + private static final String MANIFEST_EXTENSION = ".manifest"; + + /** A URL scheme; lance-io takes a single letter before the colon for a Windows drive instead. */ + private static final Pattern URL_SCHEME = Pattern.compile("^[A-Za-z][A-Za-z0-9+.-]+:"); + + /** + * The credentials a store takes from its credential provider rather than fixing them when it + * is built: every spelling lance-io's {@code DynamicCredentials} conversions read for AWS, + * Azure and GCS, the OSS keys its dynamic OpenDAL store re-reads, and the refresh deadline. + * The provider refreshes them from the namespace only when they carry + * {@value #EXPIRES_AT_MILLIS} and the store is not an OpenDAL one; see {@link #storeIdentity}. + */ + private static final Set<String> CREDENTIAL_OPTIONS = ImmutableSet.of( + "aws_access_key_id", "access_key_id", "aws_secret_access_key", "secret_access_key", + "aws_session_token", "aws_token", "aws_security_token", "session_token", "token", + "azure_storage_sas_token", "azure_storage_sas_key", "sas_token", "sas_key", + "azure_storage_token", "bearer_token", "azure_storage_account_key", "azure_storage_access_key", + "azure_storage_master_key", "access_key", "master_key", "account_key", + "google_storage_token", + "oss_access_key_id", "oss_secret_access_key", "oss_security_token", + EXPIRES_AT_MILLIS); + + /** Keys the store digest, so an id in a log cannot be checked against guessed credentials. */ + private static final byte[] IDENTITY_KEY = new byte[32]; + + static { + new SecureRandom().nextBytes(IDENTITY_KEY); + } + + private final LanceNamespace catalogNamespace; + private final Map<String, String> sdkStorageOptions; + private final AtomicReference<RuntimeException> callbackFailure = new AtomicReference<>(); + /** The thread that opens the dataset and issues the SDK's describe; every other call is a JNI callback. */ + private final Thread openingThread = Thread.currentThread(); + /** Set by the SDK's own describe while it opens the dataset. */ + private volatile String storeIdentity; + /** The table location the SDK's own describe returned; set with {@link #storeIdentity}. */ + private volatile String tableLocation; + + /** + * @param sdkStorageOptions the options the SDK is handed in its read options, which it opens + * with under what its describe vends + */ + LanceSdkNamespace(LanceNamespace catalogNamespace, Map<String, String> sdkStorageOptions) { + this.catalogNamespace = catalogNamespace; + this.sdkStorageOptions = sdkStorageOptions; + } + + @Override + public void initialize(Map<String, String> configProperties, BufferAllocator allocator) { + throw new UnsupportedOperationException("A Lance SDK namespace wraps an initialized catalog namespace"); + } + + /** + * Read by the SDK once, when it opens the dataset: Lance 12 describes the table in + * {@code OpenDatasetBuilder.buildFromNamespaceClient} first and reads the id when the JNI + * wraps this namespace. It keys the store cache, through the credential provider the SDK + * builds from this namespace. + */ + @Override + public String namespaceId() { + String identity = storeIdentity; + if (identity == null) { + throw new IllegalStateException("The Lance SDK read the namespace id before describing the table"); + } + return "DorisSdkNamespace[" + catalogNamespace.namespaceId() + ", store=" + identity + "]"; + } + + /** + * The catalog namespace's describe, with the vended options normalized. The first call is the + * SDK's own describe while it opens the dataset, whose options the store is built with; later + * ones refresh credentials. + */ + @Override + public DescribeTableResponse describeTable(DescribeTableRequest request) { + return record(() -> { + DescribeTableResponse response = catalogNamespace.describeTable(request); + Map<String, String> vended = LanceStorageOptions.normalizeVendedStorageOptions( + response.getLocation(), response.getStorageOptions()); + // Left null when nothing was vended: a credential refresh then keeps the options it has. + if (response.getStorageOptions() != null) { + response.setStorageOptions(vended); + } + if (storeIdentity == null) { + Map<String, String> opened = new HashMap<>(sdkStorageOptions); + opened.putAll(vended); + tableLocation = response.getLocation(); + storeIdentity = storeIdentity(tableLocation, opened); + } + return response; + }); + } + + /** + * The catalog namespace's version list. Lance takes the newest entry's manifest as the head of + * a chain, so every entry is checked with {@link #checkManifestPath}. + */ + @Override + public ListTableVersionsResponse listTableVersions(ListTableVersionsRequest request) { + return record(() -> { + ListTableVersionsResponse response = catalogNamespace.listTableVersions(request); + if (response.getVersions() != null) { + response.getVersions().forEach(version -> checkManifestPath(request.getBranch(), version)); + } + return response; + }); + } + + /** The catalog namespace's describe of one version, whose manifest Lance opens; see {@link #checkManifestPath}. */ + @Override + public DescribeTableVersionResponse describeTableVersion(DescribeTableVersionRequest request) { + return record(() -> { + DescribeTableVersionResponse response = catalogNamespace.describeTableVersion(request); + checkManifestPath(request.getBranch(), response.getVersion()); + return response; + }); + } + + /** + * Rejects a finalized manifest the BE would not open. The BE opens a version by the dataset URI + * and number, at {@code <chain>/_versions/<u64::MAX - v>.manifest} (or {@code <v>.manifest} for + * the V1 naming scheme). Lance opens a manifest path ending in {@code .manifest} as recorded, + * and copies any other (staged) one to that canonical path first + * ({@code ExternalManifestCommitHandler::resolve_version_location}), so only a finalized path + * elsewhere can leave the FE and the BE reading different manifests. + */ + private void checkManifestPath(String branch, TableVersion version) { + String path = version == null ? null : version.getManifestPath(); + // Lance parses the path first, which drops surrounding slashes. + String recorded = path == null ? "" : StringUtils.strip(path, "/"); + if (version == null || version.getVersion() == null || !recorded.endsWith(MANIFEST_EXTENSION)) { + return; + } + if (storeIdentity == null) { + throw new IllegalStateException("The Lance SDK resolved a version before describing the table"); + } + String chain = objectStorePath(tableLocation); + if (branch != null) { + chain = (chain.isEmpty() ? "" : chain + "/") + "tree/" + branch; + } + String versions = (chain.isEmpty() ? "" : chain + "/") + "_versions/"; + long number = version.getVersion(); + String canonical = versions + String.format("%020d", Review Comment: Confirmed: `String.format("%020d", ..)` uses the default FORMAT locale, and under `ar-SA-u-nu-arab` it yields Arabic-Indic digits. Fixed in c57da5d5c5d. The check now lives in `LanceManifestPaths` (the wrapper is gone, see the PR comment) and pads the number by hand with `StringUtils.leftPad`, so the name no longer depends on the locale. `LanceManifestPathsTest.testCanonicalNameDoesNotDependOnTheLocale` runs the check under that locale. ########## fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceCatalogClient.java: ########## @@ -235,68 +252,581 @@ public LanceTableMetadata loadBasicTableMetadata(String dbName, String tableName } public Schema loadTableSchema(String dbName, String tableName) { - return readTableSnapshot(dbName, tableName, Optional.empty(), + return readTableSnapshot(dbName, tableName, LanceRefSelector.latest(), (dataset, access, metrics) -> metrics.measure(Stage.SCHEMA, dataset::getSchema)); } public LanceTableMetadata loadTableMetadata(String dbName, String tableName, Optional<TableSnapshot> tableSnapshot) { - return loadQueryMetadata(dbName, tableName, tableSnapshot, LanceMetadataLoader.MetadataScope.WITH_INDEXES); + return loadTableMetadata(dbName, tableName, LanceRefSelector.snapshot(tableSnapshot)); + } + + public LanceTableMetadata loadTableMetadata(String dbName, String tableName, LanceRefSelector selector) { + return loadQueryMetadata(dbName, tableName, selector, LanceMetadataLoader.MetadataScope.WITH_INDEXES); } private LanceTableMetadata loadQueryMetadata(String dbName, String tableName, Optional<TableSnapshot> tableSnapshot, LanceMetadataLoader.MetadataScope mode) { - return readTableSnapshot(dbName, tableName, tableSnapshot, + return loadQueryMetadata(dbName, tableName, LanceRefSelector.snapshot(tableSnapshot), mode); + } + + private LanceTableMetadata loadQueryMetadata(String dbName, String tableName, + LanceRefSelector selector, LanceMetadataLoader.MetadataScope mode) { + return readTableSnapshot(dbName, tableName, selector, (dataset, access, metrics) -> LanceMetadataLoader.read(dataset, access, mode, metrics)); } - /** Pins one resource generation, resolved table access, and the Dataset version for the whole read. */ - private <T> T readTableSnapshot(String dbName, String tableName, Optional<TableSnapshot> tableSnapshot, + /** + * Pins one resource generation, resolved table access, and the Dataset version for the whole read. + * + * <p>The latest version of the main chain is opened once and every other selector is a + * checkout from that handle, so the SDK resolves the ref with the same commit handler + * (the namespace's, for a managed table). A tag is resolved first to the chain and version it + * points at, so a tag created on a branch selects that branch. The two shortcuts that skip the + * latest open are an explicit version on the main chain, and the latest version of a managed + * table. For a managed table, "latest" is always the newest version the namespace records, + * never the newest manifest in storage. + */ + private <T> T readTableSnapshot(String dbName, String tableName, LanceRefSelector selector, SnapshotReader<T> reader) { - LanceTableAccess tableAccess = null; + ReadState state = new ReadState(selector, dbName + "." + tableName); LanceMetadataMetrics metrics = LanceMetadataMetrics.startMetadataRead(); try { T result; try (BufferAllocator allocator = namespaceAllocator.newChildAllocator( "lance-metadata-read", 0, namespaceAllocator.getLimit())) { - tableAccess = metrics.measure(Stage.TABLE_ACCESS, + state.access = metrics.measure(Stage.TABLE_ACCESS, () -> namespaceClient.resolveTableAccess(dbName, tableName)); - OptionalLong version = OptionalLong.empty(); - if (tableSnapshot.isPresent()) { - TableSnapshot snapshot = tableSnapshot.get(); - if (snapshot.getType() == TableSnapshot.VersionType.VERSION) { - version = OptionalLong.of(LanceSnapshotResolver.parseVersion(snapshot.getValue())); - } else { - long timestamp = TimeUtils.timeStringToLong(snapshot.getValue(), TimeUtils.getTimeZone()); - if (timestamp < 0) { - throw new IllegalArgumentException( - "Cannot parse Lance FOR TIME AS OF value '" + snapshot.getValue() + "'"); - } - try (Dataset latest = openDataset(allocator, tableAccess, OptionalLong.empty(), metrics)) { - version = OptionalLong.of(metrics.measure(Stage.VERSION_RESOLVE, - () -> LanceSnapshotResolver.getVersionAtOrBefore(latest, timestamp))); - } + OptionalLong direct = directMainVersion(state, metrics); + if (direct.isPresent() || isLatestMain(selector)) { + state.version = direct; + try (Dataset dataset = openDataset(allocator, state, direct, isLatestMain(selector), metrics)) { + result = reader.read(dataset, state.access, metrics); + } + } else { + OptionalLong mainVersion = state.access.isManagedVersioning() + ? OptionalLong.of(recordedLatestVersion(state, Optional.empty(), metrics)) + : OptionalLong.empty(); + try (Dataset main = openDataset(allocator, state, mainVersion, true, metrics)) { + result = readFromLatest(main, state, reader, metrics); } - } - try (Dataset dataset = openDataset(allocator, tableAccess, version, metrics)) { - result = reader.read(dataset, tableAccess, metrics); } } metrics.succeeded(); return result; - } catch (Exception e) { - throw LanceErrorMessages.failure("Failed to load Lance table metadata for " + dbName + "." + tableName, e, - tableAccess == null ? null : tableAccess.getDatasetUri(), - tableAccess == null ? namespaceStorageOptions : tableAccess.getStorageOptions(), catalogSecrets); + } catch (LanceUserFacingException e) { + throw new RuntimeException(e.getMessage(), e); + } catch (Exception sdkError) { + Exception e = unwrapCallbackFailure(state, sdkError); + LanceTableAccess access = state.access; + String uri = access == null ? null : access.getDatasetUri(); + Map<String, String> options = access == null ? namespaceStorageOptions : access.getStorageOptions(); + String what = state.displayName(); + if (state.branch.isPresent() && !state.branchExists && isBranchNotFound(e, state.branch.get())) { + throw new RuntimeException("Lance branch '" + state.branch.get() + "' of " + state.tableName + + state.selector.getTag().map(tag -> " (tag '" + tag + "')").orElse("") + + " was not found" + (isNamespaceMiss(e, "table branch not found") ? " in the namespace" : ""), + sanitizedCause(e, uri, options)); + } + if (state.version.isPresent() && isVersionNotFound(e)) { + throw new RuntimeException("Lance version " + state.version.getAsLong() + " of " + what + + state.selector.getTag().map(tag -> " (tag '" + tag + "')").orElse("") + + " was not found" + (isNamespaceMiss(e, "table version not found") ? " in the namespace" : ""), + sanitizedCause(e, uri, options)); + } + String hint = access != null && access.isManagedVersioning() && isAccessDenied(e) + ? " (reading a namespace-managed Lance table may need write access to finalize a staged manifest)" + : ""; + throw LanceErrorMessages.failure("Failed to load Lance table metadata for " + what + hint, e, uri, options, + catalogSecrets); } finally { metrics.close(); } } - private Dataset openDataset(BufferAllocator allocator, LanceTableAccess access, OptionalLong version, + /** What a read has resolved so far; the catch block reports errors against it. */ + private static final class ReadState { + private final LanceRefSelector selector; + private final String tableName; + private LanceTableAccess access; + /** The namespace the SDK opened a managed table through, which keeps its callbacks' failures. */ + private LanceSdkNamespace sdkNamespace; + private Optional<String> branch; + /** + * Set once the branch is known to exist: the namespace recorded versions for it, or its + * latest version was checked out. Later failures are not reported as a missing branch. + */ + private boolean branchExists; + private OptionalLong version = OptionalLong.empty(); + /** The namespace's version list per chain ("" is main), fetched at most once per read. */ + private final Map<String, List<TableVersion>> namespaceVersions = new HashMap<>(); + + private ReadState(LanceRefSelector selector, String tableName) { + this.selector = selector; + this.tableName = tableName; + this.branch = selector.getBranch(); + } + + private String displayName() { + return tableName + branch.map(name -> "@" + name).orElse(""); + } + } + + private static boolean isLatestMain(LanceRefSelector selector) { + return !selector.getTag().isPresent() && !selector.getBranch().isPresent() + && !selector.getSnapshot().isPresent(); + } + + /** + * The main-chain version a selector names without looking at the latest manifest: an explicit + * version, or the latest version of a managed table, which the namespace records. + */ + private OptionalLong directMainVersion(ReadState state, LanceMetadataMetrics metrics) { + LanceRefSelector selector = state.selector; + if (selector.getTag().isPresent() || selector.getBranch().isPresent()) { + return OptionalLong.empty(); + } + if (!selector.getSnapshot().isPresent()) { + return state.access.isManagedVersioning() + ? OptionalLong.of(recordedLatestVersion(state, Optional.empty(), metrics)) + : OptionalLong.empty(); + } + TableSnapshot snapshot = selector.getSnapshot().get(); + return snapshot.getType() == TableSnapshot.VersionType.VERSION + ? OptionalLong.of(LanceSnapshotResolver.parseVersion(snapshot.getValue())) + : OptionalLong.empty(); + } + + /** Resolves the selector against the open latest main chain and reads the selected snapshot. */ + private <T> T readFromLatest(Dataset main, ReadState state, SnapshotReader<T> reader, LanceMetadataMetrics metrics) + throws Exception { + LanceRefSelector selector = state.selector; + if (selector.getTag().isPresent()) { + // Only this tag's file is read, however many tags the table has. The SDK checks the tag + // out on the branch of the version it points at; for a managed table that is an + // explicit version the namespace resolves, never a storage fallback. + String tag = selector.getTag().get(); + state.version = OptionalLong.of(metrics.measure(Stage.VERSION_RESOLVE, () -> tagVersion(main, tag, state))); + try (Dataset target = checkout(main, Ref.ofTag(tag), metrics)) { + state.branch = branchOf(target.uri(), state.access.getDatasetUri()); + return reader.read(target, accessOf(target, state), metrics); + } + } + if (state.branch.isPresent()) { + String branch = state.branch.get(); + // Check out the branch's latest version first even when a version is already known, so + // a missing branch and a missing version inside an existing branch are told apart. + Ref branchHead = Ref.ofBranch(branch); + if (state.access.isManagedVersioning()) { Review Comment: Right for the previous code: `main` was opened and reconciled through the SDK at A, the branch head was listed afterwards, and the SDK would copy a staged manifest to A's path. That path is gone in this revision. A managed table is read at the location of the read's own `DescribeTable`, and the branch head or version Doris gets from the namespace must be recorded at its canonical path under that location's `tree/<branch>/`, or at a staged manifest beside it. If the namespace moved the table to B in between, B's path fails that check and the read asks to retry. Nothing is copied, because Doris no longer finalizes staged manifests. `LanceManagedVersioningTest.testTableMovedAfterItWasDescribedFailsTheRead` moves the table right after the describe and checks the failure and the retried read. The recorded path is relative to its bucket, as the namespace records it, so a move to another bucket under the same path is not seen; the read then uses the version the namespace named at the old location. ########## fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceCatalogClient.java: ########## @@ -235,68 +254,538 @@ public LanceTableMetadata loadBasicTableMetadata(String dbName, String tableName } public Schema loadTableSchema(String dbName, String tableName) { - return readTableSnapshot(dbName, tableName, Optional.empty(), + return readTableSnapshot(dbName, tableName, LanceRefSelector.latest(), (dataset, access, metrics) -> metrics.measure(Stage.SCHEMA, dataset::getSchema)); } public LanceTableMetadata loadTableMetadata(String dbName, String tableName, Optional<TableSnapshot> tableSnapshot) { - return loadQueryMetadata(dbName, tableName, tableSnapshot, LanceMetadataLoader.MetadataScope.WITH_INDEXES); + return loadTableMetadata(dbName, tableName, LanceRefSelector.snapshot(tableSnapshot)); + } + + public LanceTableMetadata loadTableMetadata(String dbName, String tableName, LanceRefSelector selector) { + return loadQueryMetadata(dbName, tableName, selector, LanceMetadataLoader.MetadataScope.WITH_INDEXES); } private LanceTableMetadata loadQueryMetadata(String dbName, String tableName, Optional<TableSnapshot> tableSnapshot, LanceMetadataLoader.MetadataScope mode) { - return readTableSnapshot(dbName, tableName, tableSnapshot, + return loadQueryMetadata(dbName, tableName, LanceRefSelector.snapshot(tableSnapshot), mode); + } + + private LanceTableMetadata loadQueryMetadata(String dbName, String tableName, + LanceRefSelector selector, LanceMetadataLoader.MetadataScope mode) { + return readTableSnapshot(dbName, tableName, selector, (dataset, access, metrics) -> LanceMetadataLoader.read(dataset, access, mode, metrics)); } - /** Pins one resource generation, resolved table access, and the Dataset version for the whole read. */ - private <T> T readTableSnapshot(String dbName, String tableName, Optional<TableSnapshot> tableSnapshot, + /** + * Pins one resource generation, resolved table access, and the Dataset version for the whole read. + * + * <p>The latest version of the main chain is opened once and every other selector is a + * checkout from that handle, so the SDK resolves the ref with the same commit handler + * (the namespace's, for a managed table). A tag is resolved first to the chain and version it + * points at, so a tag created on a branch selects that branch. The two shortcuts that skip the + * latest open are an explicit version on the main chain, and the latest version of a managed + * table. For a managed table, "latest" is always the newest version the namespace records, + * never the newest manifest in storage. + */ + private <T> T readTableSnapshot(String dbName, String tableName, LanceRefSelector selector, SnapshotReader<T> reader) { - LanceTableAccess tableAccess = null; + try { + return readTableSnapshotOnce(dbName, tableName, selector, reader); + } catch (StaleTableAccessException e) { + // The cached access predates a location change the SDK has already seen. Read once + // more with a fresh access, so the FE plans and the BE reads the same location. + namespaceClient.invalidateTableAccess(dbName, tableName); + try { + return readTableSnapshotOnce(dbName, tableName, selector, reader); + } catch (StaleTableAccessException again) { + throw new RuntimeException("Lance namespace reported a different location for " + dbName + "." + + tableName + " while it was being opened; retry the query"); + } + } + } + + private <T> T readTableSnapshotOnce(String dbName, String tableName, LanceRefSelector selector, + SnapshotReader<T> reader) { + ReadState state = new ReadState(selector, dbName + "." + tableName); LanceMetadataMetrics metrics = LanceMetadataMetrics.startMetadataRead(); try { T result; try (BufferAllocator allocator = namespaceAllocator.newChildAllocator( "lance-metadata-read", 0, namespaceAllocator.getLimit())) { - tableAccess = metrics.measure(Stage.TABLE_ACCESS, + state.access = metrics.measure(Stage.TABLE_ACCESS, () -> namespaceClient.resolveTableAccess(dbName, tableName)); - OptionalLong version = OptionalLong.empty(); - if (tableSnapshot.isPresent()) { - TableSnapshot snapshot = tableSnapshot.get(); - if (snapshot.getType() == TableSnapshot.VersionType.VERSION) { - version = OptionalLong.of(LanceSnapshotResolver.parseVersion(snapshot.getValue())); - } else { - long timestamp = TimeUtils.timeStringToLong(snapshot.getValue(), TimeUtils.getTimeZone()); - if (timestamp < 0) { - throw new IllegalArgumentException( - "Cannot parse Lance FOR TIME AS OF value '" + snapshot.getValue() + "'"); - } - try (Dataset latest = openDataset(allocator, tableAccess, OptionalLong.empty(), metrics)) { - version = OptionalLong.of(metrics.measure(Stage.VERSION_RESOLVE, - () -> LanceSnapshotResolver.getVersionAtOrBefore(latest, timestamp))); - } + OptionalLong direct = directMainVersion(state, metrics); + if (direct.isPresent() || isLatestMain(selector)) { + state.version = direct; + try (Dataset dataset = openDataset(allocator, state.access, direct, metrics)) { + result = reader.read(dataset, state.access, metrics); + } + } else { + OptionalLong mainVersion = state.access.isManagedVersioning() + ? OptionalLong.of(recordedLatestVersion(state, Optional.empty(), metrics)) + : OptionalLong.empty(); + try (Dataset main = openDataset(allocator, state.access, mainVersion, metrics)) { + result = readFromLatest(main, state, reader, metrics); } - } - try (Dataset dataset = openDataset(allocator, tableAccess, version, metrics)) { - result = reader.read(dataset, tableAccess, metrics); } } metrics.succeeded(); return result; + } catch (LanceUserFacingException e) { + throw new RuntimeException(e.getMessage(), e); + } catch (StaleTableAccessException e) { + throw e; } catch (Exception e) { - throw LanceErrorMessages.failure("Failed to load Lance table metadata for " + dbName + "." + tableName, e, - tableAccess == null ? null : tableAccess.getDatasetUri(), - tableAccess == null ? namespaceStorageOptions : tableAccess.getStorageOptions(), catalogSecrets); + LanceTableAccess access = state.access; + String uri = access == null ? null : access.getDatasetUri(); + Map<String, String> options = access == null ? namespaceStorageOptions : access.getStorageOptions(); + String what = state.displayName(); + if (state.branch.isPresent() && !state.branchExists && isBranchNotFound(e, state.branch.get())) { + throw new RuntimeException("Lance branch '" + state.branch.get() + "' of " + state.tableName + + state.selector.getTag().map(tag -> " (tag '" + tag + "')").orElse("") + + " was not found" + (isNamespaceMiss(e, "table branch not found") ? " in the namespace" : ""), + sanitizedCause(e, uri, options)); + } + if (state.version.isPresent() && isVersionNotFound(e)) { + throw new RuntimeException("Lance version " + state.version.getAsLong() + " of " + what + + state.selector.getTag().map(tag -> " (tag '" + tag + "')").orElse("") + + " was not found" + (isNamespaceMiss(e, "table version not found") ? " in the namespace" : ""), + sanitizedCause(e, uri, options)); + } + String hint = access != null && access.isManagedVersioning() && isAccessDenied(e) + ? " (reading a namespace-managed Lance table may need write access to finalize a staged manifest)" + : ""; + throw LanceErrorMessages.failure("Failed to load Lance table metadata for " + what + hint, e, uri, options, + catalogSecrets); } finally { metrics.close(); } } + /** What a read has resolved so far; the catch block reports errors against it. */ + private static final class ReadState { + private final LanceRefSelector selector; + private final String tableName; + private LanceTableAccess access; + private Optional<String> branch; + /** + * Set once the branch is known to exist: the namespace recorded versions for it, or its + * latest version was checked out. Later failures are not reported as a missing branch. + */ + private boolean branchExists; + private OptionalLong version = OptionalLong.empty(); + /** The namespace's version list per chain ("" is main), fetched at most once per read. */ + private final Map<String, List<TableVersion>> namespaceVersions = new HashMap<>(); + + private ReadState(LanceRefSelector selector, String tableName) { + this.selector = selector; + this.tableName = tableName; + this.branch = selector.getBranch(); + } + + private String displayName() { + return tableName + branch.map(name -> "@" + name).orElse(""); + } + } + + private static boolean isLatestMain(LanceRefSelector selector) { + return !selector.getTag().isPresent() && !selector.getBranch().isPresent() + && !selector.getSnapshot().isPresent(); + } + + /** + * The main-chain version a selector names without looking at the latest manifest: an explicit + * version, or the latest version of a managed table, which the namespace records. + */ + private OptionalLong directMainVersion(ReadState state, LanceMetadataMetrics metrics) { + LanceRefSelector selector = state.selector; + if (selector.getTag().isPresent() || selector.getBranch().isPresent()) { + return OptionalLong.empty(); + } + if (!selector.getSnapshot().isPresent()) { + return state.access.isManagedVersioning() + ? OptionalLong.of(recordedLatestVersion(state, Optional.empty(), metrics)) + : OptionalLong.empty(); + } + TableSnapshot snapshot = selector.getSnapshot().get(); + return snapshot.getType() == TableSnapshot.VersionType.VERSION + ? OptionalLong.of(LanceSnapshotResolver.parseVersion(snapshot.getValue())) + : OptionalLong.empty(); + } + + /** Resolves the selector against the open latest main chain and reads the selected snapshot. */ + private <T> T readFromLatest(Dataset main, ReadState state, SnapshotReader<T> reader, LanceMetadataMetrics metrics) + throws Exception { + LanceRefSelector selector = state.selector; + if (selector.getTag().isPresent()) { + // Only this tag's file is read, however many tags the table has. The SDK checks the tag + // out on the branch of the version it points at; for a managed table that is an + // explicit version the namespace resolves, never a storage fallback. + String tag = selector.getTag().get(); + state.version = OptionalLong.of(metrics.measure(Stage.VERSION_RESOLVE, () -> tagVersion(main, tag, state))); + try (Dataset target = checkout(main, Ref.ofTag(tag), metrics)) { + state.branch = branchOf(target.uri(), state.access.getDatasetUri()); + return reader.read(target, accessOf(target, state), metrics); + } + } + if (state.branch.isPresent()) { + String branch = state.branch.get(); + // Check out the branch's latest version first even when a version is already known, so + // a missing branch and a missing version inside an existing branch are told apart. + Ref branchHead = Ref.ofBranch(branch); + if (state.access.isManagedVersioning()) { + branchHead = Ref.ofBranch(branch, recordedLatestVersion(state, Optional.of(branch), metrics)); + state.branchExists = true; + } + try (Dataset latest = checkout(main, branchHead, metrics)) { + state.branchExists = true; + LanceTableAccess branchAccess = accessOf(latest, state); + if (!state.version.isPresent() && selector.getSnapshot().isPresent()) { + state.version = resolveSnapshotVersion(latest, branchAccess, selector.getSnapshot().get(), state, + metrics); + } + if (!state.version.isPresent()) { + return reader.read(latest, branchAccess, metrics); + } + try (Dataset dataset = checkout(latest, Ref.ofBranch(branch, state.version.getAsLong()), metrics)) { + return reader.read(dataset, branchAccess, metrics); + } + } + } + // FOR TIME AS OF on the main chain, resolved from manifest commit times. + state.version = resolveSnapshotVersion(main, state.access, selector.getSnapshot().get(), state, metrics); + try (Dataset dataset = checkout(main, Ref.ofMain(state.version.getAsLong()), metrics)) { + return reader.read(dataset, state.access, metrics); + } + } + + private static long tagVersion(Dataset main, String tag, ReadState state) { + try { + return main.tags().getVersion(tag); + } catch (RuntimeException e) { + String rootMessage = ExceptionUtils.getRootCauseMessage(e); + if (rootMessage != null && rootMessage.contains("tag " + tag + " does not exist")) { + throw new LanceUserFacingException("Lance tag '" + tag + "' of " + state.tableName + " was not found"); + } + throw e; + } + } + + /** + * The branch a dataset checked out from the table root is on, from its root directory: the + * table root for main, {@code <root>/tree/<branch>} otherwise. Lance inserts the branch path + * before a URI's query string, so the query is compared apart. A URI that is neither is an + * error rather than main, which would hand the BE the wrong chain. + */ + static Optional<String> branchOf(String checkedOutUri, String tableUri) { + String root = StringUtils.removeEnd(StringUtils.substringBefore(tableUri, "?"), "/"); + String uri = StringUtils.removeEnd(StringUtils.substringBefore(checkedOutUri, "?"), "/"); + if (uri.equals(root)) { + return Optional.empty(); + } + String branchRoot = root + "/tree/"; + if (!uri.startsWith(branchRoot) || uri.length() == branchRoot.length()) { + // The URIs may carry credentials in their query, so they stay out of the message. + throw new IllegalStateException("Cannot tell which branch a Lance tag was checked out on"); + } + return Optional.of(uri.substring(branchRoot.length())); + } + + /** + * The access for a dataset checked out from the table: the main chain keeps the table access, + * and a branch takes the directory the SDK checked out, which is what the BE opens by URI. + */ + private static LanceTableAccess accessOf(Dataset dataset, ReadState state) { + return state.branch.isPresent() ? state.access.onBranch(state.branch.get(), dataset.uri()) : state.access; + } + + /** A selector error whose message is user-facing as is, such as a tag that does not exist. */ + private static final class LanceUserFacingException extends RuntimeException { + private LanceUserFacingException(String message) { + super(message); + } + } + + /** The SDK opened a managed table at another location than the resolved access names. */ + private static final class StaleTableAccessException extends RuntimeException { + private StaleTableAccessException() { + super("Lance table location changed since its access was resolved"); + } + } + + /** + * The newest version the namespace records for a managed chain. Doris asks for it itself: + * opening "latest" through the SDK falls back to the newest manifest in storage when the + * namespace records none, which would expose a version the namespace never published. + */ + private long recordedLatestVersion(ReadState state, Optional<String> branch, LanceMetadataMetrics metrics) { + // A read that already listed the chain's versions reuses that list. + List<TableVersion> listed = state.namespaceVersions.get(branch.orElse("")); + OptionalLong latest = listed != null + ? listed.stream().map(TableVersion::getVersion).filter(Objects::nonNull) + .mapToLong(Long::longValue).max() + : metrics.measure(Stage.VERSION_RESOLVE, + () -> namespaceClient.latestManagedVersion(state.access, branch)); + if (!latest.isPresent()) { + throw new LanceUserFacingException("Lance namespace lists no versions for " + state.tableName + + branch.map(name -> "@" + name).orElse("")); + } + return latest.getAsLong(); + } + + private RuntimeException sanitizedCause(Throwable error, String uri, Map<String, String> options) { + return new RuntimeException(LanceErrorMessages.sanitize(error, uri, options, catalogSecrets)); + } + + /** + * Checks out a ref of an already open dataset. The SDK resolves the ref itself, from the + * dataset directory or, for a namespace-managed dataset, with its own namespace client. + */ + private static Dataset checkout(Dataset dataset, Ref ref, LanceMetadataMetrics metrics) { + return metrics.measure(Stage.VERSION_RESOLVE, () -> dataset.checkout(ref)); + } + + /** + * Resolves a {@code FOR VERSION AS OF} / {@code FOR TIME AS OF} snapshot against the chain + * {@code latest} is checked out on: the main chain, or a branch when {@code access} is a + * branch access. + */ + private OptionalLong resolveSnapshotVersion(Dataset latest, LanceTableAccess access, TableSnapshot snapshot, + ReadState state, LanceMetadataMetrics metrics) { + if (snapshot.getType() == TableSnapshot.VersionType.VERSION) { + return OptionalLong.of(LanceSnapshotResolver.parseVersion(snapshot.getValue())); + } + long timestamp = parseTimeTravelTimestamp(snapshot.getValue()); + try { + return OptionalLong.of(resolveVersionAtOrBefore(latest, access, timestamp, snapshot.getValue(), state, + metrics)); + } catch (LanceSnapshotResolver.NoVersionAtOrBeforeException e) { + if (!access.getBranch().isPresent()) { + throw e; + } + // A branch's chain starts at the version it was created from and carries its own + // commit times, so an earlier timestamp has nothing to select on the branch. + throw new LanceUserFacingException("Lance branch '" + access.getBranch().get() + "' of " + + state.tableName + " has no version at or before '" + snapshot.getValue() + + "'; a branch only holds the versions from its creation on"); + } + } + + /** + * Whether a failed branch checkout means the branch does not exist. The SDK reports + * "branch <name> does not exist", a namespace "Table branch not found", or a missing manifest + * under the branch directory when nothing was ever committed there. + */ + private static boolean isBranchNotFound(Throwable throwable, String branch) { + if (ExceptionUtils.indexOfType(throwable, TableBranchNotFoundException.class) >= 0) { + return true; + } + String rootMessage = ExceptionUtils.getRootCauseMessage(throwable); + if (rootMessage == null) { + return false; + } + String lower = rootMessage.toLowerCase(Locale.ROOT); + String name = branch.toLowerCase(Locale.ROOT); + return lower.contains("table branch not found") + || lower.contains("branch " + name + " does not exist") + || (lower.contains("not found") && lower.contains("tree/" + name + "/")); + } + + /** + * Whether a not-found came from the namespace rather than storage. The SDK surfaces a + * namespace error by its display text ("Table version not found: ..."), and the Java client + * by its exception type. + */ + private static boolean isNamespaceMiss(Throwable throwable, String namespaceText) { + if (ExceptionUtils.indexOfType(throwable, TableVersionNotFoundException.class) >= 0 + || ExceptionUtils.indexOfType(throwable, TableBranchNotFoundException.class) >= 0) { + return true; + } + String rootMessage = ExceptionUtils.getRootCauseMessage(throwable); + return rootMessage != null && rootMessage.toLowerCase(Locale.ROOT).contains(namespaceText); + } + + /** An HTTP 403 as the object stores report it, or an explicit access-denied error. */ + private static final Pattern ACCESS_DENIED = Pattern.compile( + "accessdenied|access denied|permission denied|forbidden|(status|http|code)\\W{0,3}403\\b"); + + private static boolean isAccessDenied(Throwable throwable) { + String rootMessage = ExceptionUtils.getRootCauseMessage(throwable); + return rootMessage != null && ACCESS_DENIED.matcher(rootMessage.toLowerCase(Locale.ROOT)).find(); + } + + /** + * Every version the namespace records for the chain {@code access} addresses, listed once per + * read. The whole list is needed: the storage fallback filters by it, and neither the order a + * namespace returns nor monotonic commit times can be relied on to stop early. + */ + private List<TableVersion> namespaceVersions(ReadState state, LanceTableAccess access, + LanceMetadataMetrics metrics) { + return state.namespaceVersions.computeIfAbsent(access.getBranch().orElse(""), chain -> { + List<TableVersion> versions = metrics.measure(Stage.VERSION_RESOLVE, + () -> namespaceClient.listManagedVersions(access)); + if (versions.isEmpty()) { + throw new LanceUserFacingException("Lance namespace lists no versions for " + + state.tableName + (chain.isEmpty() ? "" : "@" + chain)); + } + return versions; + }); + } + + /** + * Resolves {@code FOR TIME AS OF} to a version on the chain {@code latest} is checked out on, + * from the commit times the manifests record, as Lance resolves {@code asof}. A managed table + * only selects among the versions its namespace records. + * + * <p>The version next to the selection must still exist: if the version after it, or with + * nothing listed at or before the time the one before the oldest listed, was removed by + * cleanup, the table's state at that time is unknown and the query fails instead of reading + * an older snapshot. A recorded version missing from the storage listing because its manifest + * is still staged is checked out there, which finalizes it and yields its commit time. + */ + private long resolveVersionAtOrBefore(Dataset latest, LanceTableAccess access, long timestamp, + String requestedText, ReadState state, LanceMetadataMetrics metrics) { + NavigableSet<Long> recorded = access.isManagedVersioning() + ? namespaceVersions(state, access, metrics).stream().map(TableVersion::getVersion) + .filter(Objects::nonNull).collect(Collectors.toCollection(TreeSet::new)) + : null; + long version = metrics.measure(Stage.VERSION_RESOLVE, () -> { + NavigableMap<Long, Version> listed = new TreeMap<>(); + for (Version candidate : latest.listVersions()) { + if (recorded == null || recorded.contains(candidate.getId())) { + listed.put(candidate.getId(), candidate); + } + } + long selected; + try { + selected = LanceSnapshotResolver.versionAtOrBefore(listed.values(), timestamp, requestedText); + } catch (LanceSnapshotResolver.NoVersionAtOrBeforeException e) { + if (recorded == null) { + throw e; + } + // Recorded versions older than the oldest listed one may still be staged. Walking + // back, every version passed was committed after the time, so the first one at or + // before it is the answer. + Long older = listed.isEmpty() ? recorded.last() : recorded.lower(listed.firstKey()); + while (older != null) { + Version olderVersion = recordedVersion(latest, access, older); + if (olderVersion == null) { + throw historyRemoved(older, requestedText, state); + } + if (LanceSnapshotResolver.commitMillis(olderVersion) <= timestamp) { + return older; + } + older = recorded.lower(older); + } + throw e; + } + // Lance numbers a chain's commits consecutively, so a storage chain is every number + // between the oldest and newest listed manifest; a managed chain is what is recorded. + Long next = nextVersion(selected, recorded, listed); + while (next != null) { + Version nextVersion = listed.containsKey(next) ? listed.get(next) + : recorded == null ? null : recordedVersion(latest, access, next); + if (nextVersion == null) { + throw historyRemoved(next, requestedText, state); + } + if (LanceSnapshotResolver.commitMillis(nextVersion) > timestamp) { + break; + } + selected = next; Review Comment: Follow-up after the redesign (see the PR comment): Doris no longer checks a staged version out to finalize it, so your second case now fails instead of selecting v3. With v1..v4 recorded at 10/40/20/50 ms and v3 only at a staged manifest, v3's commit time is unknown until it is finalized. It cuts the history as a removed version does, and a request at 30 ms fails with "version 3, which may hold the state at that time, cannot be read", naming both possible causes. That is how Doris's Iceberg and Paimon connectors treat a snapshot their metadata names but they cannot load. The comparator part of the fix is unchanged. One more rule changed in the same code: a version number the namespace no longer records, between versions it records, now cuts the history too; it used to be skipped, which returned an older version for a time that version covered. `versionAtOrBefore` no longer takes a checkout callback. `LanceSnapshotTest` covers both rules, and its random histories now include namespace records that end before or after the storage listing. ########## fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceCatalogClient.java: ########## @@ -235,68 +254,538 @@ public LanceTableMetadata loadBasicTableMetadata(String dbName, String tableName } public Schema loadTableSchema(String dbName, String tableName) { - return readTableSnapshot(dbName, tableName, Optional.empty(), + return readTableSnapshot(dbName, tableName, LanceRefSelector.latest(), (dataset, access, metrics) -> metrics.measure(Stage.SCHEMA, dataset::getSchema)); } public LanceTableMetadata loadTableMetadata(String dbName, String tableName, Optional<TableSnapshot> tableSnapshot) { - return loadQueryMetadata(dbName, tableName, tableSnapshot, LanceMetadataLoader.MetadataScope.WITH_INDEXES); + return loadTableMetadata(dbName, tableName, LanceRefSelector.snapshot(tableSnapshot)); + } + + public LanceTableMetadata loadTableMetadata(String dbName, String tableName, LanceRefSelector selector) { + return loadQueryMetadata(dbName, tableName, selector, LanceMetadataLoader.MetadataScope.WITH_INDEXES); } private LanceTableMetadata loadQueryMetadata(String dbName, String tableName, Optional<TableSnapshot> tableSnapshot, LanceMetadataLoader.MetadataScope mode) { - return readTableSnapshot(dbName, tableName, tableSnapshot, + return loadQueryMetadata(dbName, tableName, LanceRefSelector.snapshot(tableSnapshot), mode); + } + + private LanceTableMetadata loadQueryMetadata(String dbName, String tableName, + LanceRefSelector selector, LanceMetadataLoader.MetadataScope mode) { + return readTableSnapshot(dbName, tableName, selector, (dataset, access, metrics) -> LanceMetadataLoader.read(dataset, access, mode, metrics)); } - /** Pins one resource generation, resolved table access, and the Dataset version for the whole read. */ - private <T> T readTableSnapshot(String dbName, String tableName, Optional<TableSnapshot> tableSnapshot, + /** + * Pins one resource generation, resolved table access, and the Dataset version for the whole read. + * + * <p>The latest version of the main chain is opened once and every other selector is a + * checkout from that handle, so the SDK resolves the ref with the same commit handler + * (the namespace's, for a managed table). A tag is resolved first to the chain and version it + * points at, so a tag created on a branch selects that branch. The two shortcuts that skip the + * latest open are an explicit version on the main chain, and the latest version of a managed + * table. For a managed table, "latest" is always the newest version the namespace records, + * never the newest manifest in storage. + */ + private <T> T readTableSnapshot(String dbName, String tableName, LanceRefSelector selector, SnapshotReader<T> reader) { - LanceTableAccess tableAccess = null; + try { + return readTableSnapshotOnce(dbName, tableName, selector, reader); + } catch (StaleTableAccessException e) { + // The cached access predates a location change the SDK has already seen. Read once + // more with a fresh access, so the FE plans and the BE reads the same location. + namespaceClient.invalidateTableAccess(dbName, tableName); + try { + return readTableSnapshotOnce(dbName, tableName, selector, reader); + } catch (StaleTableAccessException again) { + throw new RuntimeException("Lance namespace reported a different location for " + dbName + "." + + tableName + " while it was being opened; retry the query"); + } + } + } + + private <T> T readTableSnapshotOnce(String dbName, String tableName, LanceRefSelector selector, + SnapshotReader<T> reader) { + ReadState state = new ReadState(selector, dbName + "." + tableName); LanceMetadataMetrics metrics = LanceMetadataMetrics.startMetadataRead(); try { T result; try (BufferAllocator allocator = namespaceAllocator.newChildAllocator( "lance-metadata-read", 0, namespaceAllocator.getLimit())) { - tableAccess = metrics.measure(Stage.TABLE_ACCESS, + state.access = metrics.measure(Stage.TABLE_ACCESS, () -> namespaceClient.resolveTableAccess(dbName, tableName)); - OptionalLong version = OptionalLong.empty(); - if (tableSnapshot.isPresent()) { - TableSnapshot snapshot = tableSnapshot.get(); - if (snapshot.getType() == TableSnapshot.VersionType.VERSION) { - version = OptionalLong.of(LanceSnapshotResolver.parseVersion(snapshot.getValue())); - } else { - long timestamp = TimeUtils.timeStringToLong(snapshot.getValue(), TimeUtils.getTimeZone()); - if (timestamp < 0) { - throw new IllegalArgumentException( - "Cannot parse Lance FOR TIME AS OF value '" + snapshot.getValue() + "'"); - } - try (Dataset latest = openDataset(allocator, tableAccess, OptionalLong.empty(), metrics)) { - version = OptionalLong.of(metrics.measure(Stage.VERSION_RESOLVE, - () -> LanceSnapshotResolver.getVersionAtOrBefore(latest, timestamp))); - } + OptionalLong direct = directMainVersion(state, metrics); + if (direct.isPresent() || isLatestMain(selector)) { + state.version = direct; + try (Dataset dataset = openDataset(allocator, state.access, direct, metrics)) { + result = reader.read(dataset, state.access, metrics); + } + } else { + OptionalLong mainVersion = state.access.isManagedVersioning() + ? OptionalLong.of(recordedLatestVersion(state, Optional.empty(), metrics)) + : OptionalLong.empty(); + try (Dataset main = openDataset(allocator, state.access, mainVersion, metrics)) { + result = readFromLatest(main, state, reader, metrics); } - } - try (Dataset dataset = openDataset(allocator, tableAccess, version, metrics)) { - result = reader.read(dataset, tableAccess, metrics); } } metrics.succeeded(); return result; + } catch (LanceUserFacingException e) { + throw new RuntimeException(e.getMessage(), e); + } catch (StaleTableAccessException e) { + throw e; } catch (Exception e) { - throw LanceErrorMessages.failure("Failed to load Lance table metadata for " + dbName + "." + tableName, e, - tableAccess == null ? null : tableAccess.getDatasetUri(), - tableAccess == null ? namespaceStorageOptions : tableAccess.getStorageOptions(), catalogSecrets); + LanceTableAccess access = state.access; + String uri = access == null ? null : access.getDatasetUri(); + Map<String, String> options = access == null ? namespaceStorageOptions : access.getStorageOptions(); + String what = state.displayName(); + if (state.branch.isPresent() && !state.branchExists && isBranchNotFound(e, state.branch.get())) { + throw new RuntimeException("Lance branch '" + state.branch.get() + "' of " + state.tableName + + state.selector.getTag().map(tag -> " (tag '" + tag + "')").orElse("") + + " was not found" + (isNamespaceMiss(e, "table branch not found") ? " in the namespace" : ""), + sanitizedCause(e, uri, options)); + } + if (state.version.isPresent() && isVersionNotFound(e)) { + throw new RuntimeException("Lance version " + state.version.getAsLong() + " of " + what + + state.selector.getTag().map(tag -> " (tag '" + tag + "')").orElse("") + + " was not found" + (isNamespaceMiss(e, "table version not found") ? " in the namespace" : ""), + sanitizedCause(e, uri, options)); + } + String hint = access != null && access.isManagedVersioning() && isAccessDenied(e) + ? " (reading a namespace-managed Lance table may need write access to finalize a staged manifest)" + : ""; + throw LanceErrorMessages.failure("Failed to load Lance table metadata for " + what + hint, e, uri, options, + catalogSecrets); } finally { metrics.close(); } } + /** What a read has resolved so far; the catch block reports errors against it. */ + private static final class ReadState { + private final LanceRefSelector selector; + private final String tableName; + private LanceTableAccess access; + private Optional<String> branch; + /** + * Set once the branch is known to exist: the namespace recorded versions for it, or its + * latest version was checked out. Later failures are not reported as a missing branch. + */ + private boolean branchExists; + private OptionalLong version = OptionalLong.empty(); + /** The namespace's version list per chain ("" is main), fetched at most once per read. */ + private final Map<String, List<TableVersion>> namespaceVersions = new HashMap<>(); + + private ReadState(LanceRefSelector selector, String tableName) { + this.selector = selector; + this.tableName = tableName; + this.branch = selector.getBranch(); + } + + private String displayName() { + return tableName + branch.map(name -> "@" + name).orElse(""); + } + } + + private static boolean isLatestMain(LanceRefSelector selector) { + return !selector.getTag().isPresent() && !selector.getBranch().isPresent() + && !selector.getSnapshot().isPresent(); + } + + /** + * The main-chain version a selector names without looking at the latest manifest: an explicit + * version, or the latest version of a managed table, which the namespace records. + */ + private OptionalLong directMainVersion(ReadState state, LanceMetadataMetrics metrics) { + LanceRefSelector selector = state.selector; + if (selector.getTag().isPresent() || selector.getBranch().isPresent()) { + return OptionalLong.empty(); + } + if (!selector.getSnapshot().isPresent()) { + return state.access.isManagedVersioning() + ? OptionalLong.of(recordedLatestVersion(state, Optional.empty(), metrics)) + : OptionalLong.empty(); + } + TableSnapshot snapshot = selector.getSnapshot().get(); + return snapshot.getType() == TableSnapshot.VersionType.VERSION + ? OptionalLong.of(LanceSnapshotResolver.parseVersion(snapshot.getValue())) + : OptionalLong.empty(); + } + + /** Resolves the selector against the open latest main chain and reads the selected snapshot. */ + private <T> T readFromLatest(Dataset main, ReadState state, SnapshotReader<T> reader, LanceMetadataMetrics metrics) + throws Exception { + LanceRefSelector selector = state.selector; + if (selector.getTag().isPresent()) { + // Only this tag's file is read, however many tags the table has. The SDK checks the tag + // out on the branch of the version it points at; for a managed table that is an + // explicit version the namespace resolves, never a storage fallback. + String tag = selector.getTag().get(); + state.version = OptionalLong.of(metrics.measure(Stage.VERSION_RESOLVE, () -> tagVersion(main, tag, state))); + try (Dataset target = checkout(main, Ref.ofTag(tag), metrics)) { + state.branch = branchOf(target.uri(), state.access.getDatasetUri()); + return reader.read(target, accessOf(target, state), metrics); + } + } + if (state.branch.isPresent()) { + String branch = state.branch.get(); + // Check out the branch's latest version first even when a version is already known, so + // a missing branch and a missing version inside an existing branch are told apart. + Ref branchHead = Ref.ofBranch(branch); + if (state.access.isManagedVersioning()) { + branchHead = Ref.ofBranch(branch, recordedLatestVersion(state, Optional.of(branch), metrics)); + state.branchExists = true; + } + try (Dataset latest = checkout(main, branchHead, metrics)) { + state.branchExists = true; + LanceTableAccess branchAccess = accessOf(latest, state); + if (!state.version.isPresent() && selector.getSnapshot().isPresent()) { + state.version = resolveSnapshotVersion(latest, branchAccess, selector.getSnapshot().get(), state, + metrics); + } + if (!state.version.isPresent()) { + return reader.read(latest, branchAccess, metrics); + } + try (Dataset dataset = checkout(latest, Ref.ofBranch(branch, state.version.getAsLong()), metrics)) { + return reader.read(dataset, branchAccess, metrics); + } + } + } + // FOR TIME AS OF on the main chain, resolved from manifest commit times. + state.version = resolveSnapshotVersion(main, state.access, selector.getSnapshot().get(), state, metrics); + try (Dataset dataset = checkout(main, Ref.ofMain(state.version.getAsLong()), metrics)) { + return reader.read(dataset, state.access, metrics); + } + } + + private static long tagVersion(Dataset main, String tag, ReadState state) { + try { + return main.tags().getVersion(tag); + } catch (RuntimeException e) { + String rootMessage = ExceptionUtils.getRootCauseMessage(e); + if (rootMessage != null && rootMessage.contains("tag " + tag + " does not exist")) { + throw new LanceUserFacingException("Lance tag '" + tag + "' of " + state.tableName + " was not found"); + } + throw e; + } + } + + /** + * The branch a dataset checked out from the table root is on, from its root directory: the + * table root for main, {@code <root>/tree/<branch>} otherwise. Lance inserts the branch path + * before a URI's query string, so the query is compared apart. A URI that is neither is an + * error rather than main, which would hand the BE the wrong chain. + */ + static Optional<String> branchOf(String checkedOutUri, String tableUri) { + String root = StringUtils.removeEnd(StringUtils.substringBefore(tableUri, "?"), "/"); + String uri = StringUtils.removeEnd(StringUtils.substringBefore(checkedOutUri, "?"), "/"); + if (uri.equals(root)) { + return Optional.empty(); + } + String branchRoot = root + "/tree/"; + if (!uri.startsWith(branchRoot) || uri.length() == branchRoot.length()) { + // The URIs may carry credentials in their query, so they stay out of the message. + throw new IllegalStateException("Cannot tell which branch a Lance tag was checked out on"); + } + return Optional.of(uri.substring(branchRoot.length())); + } + + /** + * The access for a dataset checked out from the table: the main chain keeps the table access, + * and a branch takes the directory the SDK checked out, which is what the BE opens by URI. + */ + private static LanceTableAccess accessOf(Dataset dataset, ReadState state) { + return state.branch.isPresent() ? state.access.onBranch(state.branch.get(), dataset.uri()) : state.access; + } + + /** A selector error whose message is user-facing as is, such as a tag that does not exist. */ + private static final class LanceUserFacingException extends RuntimeException { + private LanceUserFacingException(String message) { + super(message); + } + } + + /** The SDK opened a managed table at another location than the resolved access names. */ + private static final class StaleTableAccessException extends RuntimeException { + private StaleTableAccessException() { + super("Lance table location changed since its access was resolved"); + } + } + + /** + * The newest version the namespace records for a managed chain. Doris asks for it itself: + * opening "latest" through the SDK falls back to the newest manifest in storage when the + * namespace records none, which would expose a version the namespace never published. + */ + private long recordedLatestVersion(ReadState state, Optional<String> branch, LanceMetadataMetrics metrics) { + // A read that already listed the chain's versions reuses that list. + List<TableVersion> listed = state.namespaceVersions.get(branch.orElse("")); + OptionalLong latest = listed != null + ? listed.stream().map(TableVersion::getVersion).filter(Objects::nonNull) + .mapToLong(Long::longValue).max() + : metrics.measure(Stage.VERSION_RESOLVE, + () -> namespaceClient.latestManagedVersion(state.access, branch)); + if (!latest.isPresent()) { + throw new LanceUserFacingException("Lance namespace lists no versions for " + state.tableName + + branch.map(name -> "@" + name).orElse("")); + } + return latest.getAsLong(); + } + + private RuntimeException sanitizedCause(Throwable error, String uri, Map<String, String> options) { + return new RuntimeException(LanceErrorMessages.sanitize(error, uri, options, catalogSecrets)); + } + + /** + * Checks out a ref of an already open dataset. The SDK resolves the ref itself, from the + * dataset directory or, for a namespace-managed dataset, with its own namespace client. + */ + private static Dataset checkout(Dataset dataset, Ref ref, LanceMetadataMetrics metrics) { + return metrics.measure(Stage.VERSION_RESOLVE, () -> dataset.checkout(ref)); + } + + /** + * Resolves a {@code FOR VERSION AS OF} / {@code FOR TIME AS OF} snapshot against the chain + * {@code latest} is checked out on: the main chain, or a branch when {@code access} is a + * branch access. + */ + private OptionalLong resolveSnapshotVersion(Dataset latest, LanceTableAccess access, TableSnapshot snapshot, + ReadState state, LanceMetadataMetrics metrics) { + if (snapshot.getType() == TableSnapshot.VersionType.VERSION) { + return OptionalLong.of(LanceSnapshotResolver.parseVersion(snapshot.getValue())); + } + long timestamp = parseTimeTravelTimestamp(snapshot.getValue()); + try { + return OptionalLong.of(resolveVersionAtOrBefore(latest, access, timestamp, snapshot.getValue(), state, + metrics)); + } catch (LanceSnapshotResolver.NoVersionAtOrBeforeException e) { + if (!access.getBranch().isPresent()) { + throw e; + } + // A branch's chain starts at the version it was created from and carries its own + // commit times, so an earlier timestamp has nothing to select on the branch. + throw new LanceUserFacingException("Lance branch '" + access.getBranch().get() + "' of " + + state.tableName + " has no version at or before '" + snapshot.getValue() + + "'; a branch only holds the versions from its creation on"); + } + } + + /** + * Whether a failed branch checkout means the branch does not exist. The SDK reports + * "branch <name> does not exist", a namespace "Table branch not found", or a missing manifest + * under the branch directory when nothing was ever committed there. + */ + private static boolean isBranchNotFound(Throwable throwable, String branch) { + if (ExceptionUtils.indexOfType(throwable, TableBranchNotFoundException.class) >= 0) { + return true; + } + String rootMessage = ExceptionUtils.getRootCauseMessage(throwable); + if (rootMessage == null) { + return false; + } + String lower = rootMessage.toLowerCase(Locale.ROOT); + String name = branch.toLowerCase(Locale.ROOT); + return lower.contains("table branch not found") + || lower.contains("branch " + name + " does not exist") + || (lower.contains("not found") && lower.contains("tree/" + name + "/")); + } + + /** + * Whether a not-found came from the namespace rather than storage. The SDK surfaces a + * namespace error by its display text ("Table version not found: ..."), and the Java client + * by its exception type. + */ + private static boolean isNamespaceMiss(Throwable throwable, String namespaceText) { + if (ExceptionUtils.indexOfType(throwable, TableVersionNotFoundException.class) >= 0 + || ExceptionUtils.indexOfType(throwable, TableBranchNotFoundException.class) >= 0) { + return true; + } + String rootMessage = ExceptionUtils.getRootCauseMessage(throwable); + return rootMessage != null && rootMessage.toLowerCase(Locale.ROOT).contains(namespaceText); + } + + /** An HTTP 403 as the object stores report it, or an explicit access-denied error. */ + private static final Pattern ACCESS_DENIED = Pattern.compile( + "accessdenied|access denied|permission denied|forbidden|(status|http|code)\\W{0,3}403\\b"); + + private static boolean isAccessDenied(Throwable throwable) { + String rootMessage = ExceptionUtils.getRootCauseMessage(throwable); + return rootMessage != null && ACCESS_DENIED.matcher(rootMessage.toLowerCase(Locale.ROOT)).find(); + } + + /** + * Every version the namespace records for the chain {@code access} addresses, listed once per + * read. The whole list is needed: the storage fallback filters by it, and neither the order a + * namespace returns nor monotonic commit times can be relied on to stop early. + */ + private List<TableVersion> namespaceVersions(ReadState state, LanceTableAccess access, + LanceMetadataMetrics metrics) { + return state.namespaceVersions.computeIfAbsent(access.getBranch().orElse(""), chain -> { + List<TableVersion> versions = metrics.measure(Stage.VERSION_RESOLVE, + () -> namespaceClient.listManagedVersions(access)); + if (versions.isEmpty()) { + throw new LanceUserFacingException("Lance namespace lists no versions for " + + state.tableName + (chain.isEmpty() ? "" : "@" + chain)); + } + return versions; + }); + } + + /** + * Resolves {@code FOR TIME AS OF} to a version on the chain {@code latest} is checked out on, + * from the commit times the manifests record, as Lance resolves {@code asof}. A managed table + * only selects among the versions its namespace records. + * + * <p>The version next to the selection must still exist: if the version after it, or with + * nothing listed at or before the time the one before the oldest listed, was removed by + * cleanup, the table's state at that time is unknown and the query fails instead of reading + * an older snapshot. A recorded version missing from the storage listing because its manifest + * is still staged is checked out there, which finalizes it and yields its commit time. + */ + private long resolveVersionAtOrBefore(Dataset latest, LanceTableAccess access, long timestamp, + String requestedText, ReadState state, LanceMetadataMetrics metrics) { + NavigableSet<Long> recorded = access.isManagedVersioning() + ? namespaceVersions(state, access, metrics).stream().map(TableVersion::getVersion) + .filter(Objects::nonNull).collect(Collectors.toCollection(TreeSet::new)) + : null; + long version = metrics.measure(Stage.VERSION_RESOLVE, () -> { + NavigableMap<Long, Version> listed = new TreeMap<>(); + for (Version candidate : latest.listVersions()) { + if (recorded == null || recorded.contains(candidate.getId())) { + listed.put(candidate.getId(), candidate); + } + } + long selected; + try { + selected = LanceSnapshotResolver.versionAtOrBefore(listed.values(), timestamp, requestedText); + } catch (LanceSnapshotResolver.NoVersionAtOrBeforeException e) { + if (recorded == null) { + throw e; + } + // Recorded versions older than the oldest listed one may still be staged. Walking + // back, every version passed was committed after the time, so the first one at or + // before it is the answer. + Long older = listed.isEmpty() ? recorded.last() : recorded.lower(listed.firstKey()); + while (older != null) { + Version olderVersion = recordedVersion(latest, access, older); + if (olderVersion == null) { + throw historyRemoved(older, requestedText, state); + } + if (LanceSnapshotResolver.commitMillis(olderVersion) <= timestamp) { + return older; + } + older = recorded.lower(older); + } + throw e; + } + // Lance numbers a chain's commits consecutively, so a storage chain is every number + // between the oldest and newest listed manifest; a managed chain is what is recorded. + Long next = nextVersion(selected, recorded, listed); + while (next != null) { + Version nextVersion = listed.containsKey(next) ? listed.get(next) + : recorded == null ? null : recordedVersion(latest, access, next); + if (nextVersion == null) { + throw historyRemoved(next, requestedText, state); + } + if (LanceSnapshotResolver.commitMillis(nextVersion) > timestamp) { + break; + } + selected = next; + next = nextVersion(next, recorded, listed); + } + return selected; + }); + LOG.debug("Resolved Lance FOR TIME AS OF '{}' to version {} from manifest commit times", requestedText, + version); + return version; + } + + private static Long nextVersion(long version, NavigableSet<Long> recorded, NavigableMap<Long, Version> listed) { + if (recorded != null) { + return recorded.higher(version); + } + return version < listed.lastKey() ? version + 1 : null; + } + + /** + * A namespace-recorded version checked out through the namespace, or null if it is gone. A + * still-staged manifest is finalized by the checkout. + */ + private static Version recordedVersion(Dataset latest, LanceTableAccess access, long version) { + Ref ref = access.getBranch().map(name -> Ref.ofBranch(name, version)).orElseGet(() -> Ref.ofMain(version)); + try (Dataset recorded = latest.checkout(ref)) { + return recorded.getVersion(); + } catch (Exception e) { + // Also the IOException the JNI raises for a missing manifest. + if (!isVersionNotFound(e)) { + throw e; + } + return null; + } + } + + private static LanceUserFacingException historyRemoved(long version, String requestedText, ReadState state) { + return new LanceUserFacingException("Lance cannot resolve FOR TIME AS OF '" + requestedText + "' on " + + state.displayName() + ": version " + version + ", which may hold the state at that time," + + " no longer exists"); + } + + /** + * Parses a {@code FOR TIME AS OF} value in the session time zone. Second and millisecond + * precision are accepted; commit times are compared at millisecond precision, the precision a + * namespace reports them in, so a timestamp in the millisecond a commit lands in selects it. + */ + private static long parseTimeTravelTimestamp(String value) { + long timestamp = TimeUtils.timeStringToLong(value, TimeUtils.getTimeZone()); + if (timestamp < 0) { + timestamp = TimeUtils.msTimeStringToLong(value, TimeUtils.getTimeZone()); + } + if (timestamp < 0) { + throw new IllegalArgumentException("Cannot parse Lance FOR TIME AS OF value '" + value + + "', expected 'yyyy-MM-dd HH:mm:ss' or 'yyyy-MM-dd HH:mm:ss.SSS'"); + } + return timestamp; + } + + /** + * Whether a failed open of an explicitly requested version means that version does not exist. + * A namespace reports it through {@link TableVersionNotFoundException}. The storage reader + * reports it as a missing manifest under {@code _versions/} or as Lance's own version-not-found + * error; a missing dataset or an unreachable store fails differently and keeps its message. + */ + private static boolean isVersionNotFound(Throwable throwable) { + if (ExceptionUtils.indexOfType(throwable, TableVersionNotFoundException.class) >= 0) { + return true; + } + String rootMessage = ExceptionUtils.getRootCauseMessage(throwable); + if (rootMessage == null) { + return false; + } + String lower = rootMessage.toLowerCase(Locale.ROOT); + return lower.contains("version not found") + || (lower.contains("not found") && lower.contains("_versions/")); + } + private Dataset openDataset(BufferAllocator allocator, LanceTableAccess access, OptionalLong version, LanceMetadataMetrics metrics) { + ReadOptions readOptions = LanceReadOptions.forSharedSession(access.getSdkStorageOptions(), version, session); + if (access.isManagedVersioning()) { + // The SDK re-describes the table and opens the location the namespace returns, so a + // namespace that returns a relative location cannot be read in this mode. The BE + // opens the access's location, so both must still agree after that second describe. + Dataset dataset = metrics.measure(Stage.DATASET_OPEN, + () -> namespaceClient.openManagedDataset(allocator, access, readOptions, session)); + if (!StringUtils.removeEnd(dataset.uri(), "/").equals(StringUtils.removeEnd(access.getDatasetUri(), "/"))) { Review Comment: Follow-up: this fix is superseded by the redesign (see the PR comment). The FE no longer opens a managed table through the SDK's namespace client, so a read makes one `DescribeTable`, the FE opens the dataset with its location and options, and the BE receives the same access. There is no second describe to reconcile. `LanceManagedVersioningTest.testBeReadsWithTheOptionsOfTheReadsDescribe` uses a namespace that vends a new value on every describe and checks that the BE gets the value of the read's own describe. ########## fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceCatalogClient.java: ########## @@ -235,68 +252,508 @@ public LanceTableMetadata loadBasicTableMetadata(String dbName, String tableName } public Schema loadTableSchema(String dbName, String tableName) { - return readTableSnapshot(dbName, tableName, Optional.empty(), + return readTableSnapshot(dbName, tableName, LanceRefSelector.latest(), (dataset, access, metrics) -> metrics.measure(Stage.SCHEMA, dataset::getSchema)); } public LanceTableMetadata loadTableMetadata(String dbName, String tableName, Optional<TableSnapshot> tableSnapshot) { - return loadQueryMetadata(dbName, tableName, tableSnapshot, LanceMetadataLoader.MetadataScope.WITH_INDEXES); + return loadTableMetadata(dbName, tableName, LanceRefSelector.snapshot(tableSnapshot)); + } + + public LanceTableMetadata loadTableMetadata(String dbName, String tableName, LanceRefSelector selector) { + return loadQueryMetadata(dbName, tableName, selector, LanceMetadataLoader.MetadataScope.WITH_INDEXES); } private LanceTableMetadata loadQueryMetadata(String dbName, String tableName, Optional<TableSnapshot> tableSnapshot, LanceMetadataLoader.MetadataScope mode) { - return readTableSnapshot(dbName, tableName, tableSnapshot, + return loadQueryMetadata(dbName, tableName, LanceRefSelector.snapshot(tableSnapshot), mode); + } + + private LanceTableMetadata loadQueryMetadata(String dbName, String tableName, + LanceRefSelector selector, LanceMetadataLoader.MetadataScope mode) { + return readTableSnapshot(dbName, tableName, selector, (dataset, access, metrics) -> LanceMetadataLoader.read(dataset, access, mode, metrics)); } - /** Pins one resource generation, resolved table access, and the Dataset version for the whole read. */ - private <T> T readTableSnapshot(String dbName, String tableName, Optional<TableSnapshot> tableSnapshot, + /** + * Pins one resource generation, resolved table access, and the Dataset version for the whole read. + * + * <p>The latest version of the main chain is opened once and every other selector is a + * checkout from that handle, so the SDK resolves the ref with the same commit handler + * (the namespace's, for a managed table). A tag is resolved first to the chain and version it + * points at, so a tag created on a branch selects that branch. The two shortcuts that skip the + * latest open are an explicit version on the main chain, and the latest version of a managed + * table. For a managed table, "latest" is always the newest version the namespace records, + * never the newest manifest in storage. + */ + private <T> T readTableSnapshot(String dbName, String tableName, LanceRefSelector selector, SnapshotReader<T> reader) { - LanceTableAccess tableAccess = null; + ReadState state = new ReadState(selector, dbName + "." + tableName); LanceMetadataMetrics metrics = LanceMetadataMetrics.startMetadataRead(); try { T result; try (BufferAllocator allocator = namespaceAllocator.newChildAllocator( "lance-metadata-read", 0, namespaceAllocator.getLimit())) { - tableAccess = metrics.measure(Stage.TABLE_ACCESS, + state.access = metrics.measure(Stage.TABLE_ACCESS, () -> namespaceClient.resolveTableAccess(dbName, tableName)); - OptionalLong version = OptionalLong.empty(); - if (tableSnapshot.isPresent()) { - TableSnapshot snapshot = tableSnapshot.get(); - if (snapshot.getType() == TableSnapshot.VersionType.VERSION) { - version = OptionalLong.of(LanceSnapshotResolver.parseVersion(snapshot.getValue())); - } else { - long timestamp = TimeUtils.timeStringToLong(snapshot.getValue(), TimeUtils.getTimeZone()); - if (timestamp < 0) { - throw new IllegalArgumentException( - "Cannot parse Lance FOR TIME AS OF value '" + snapshot.getValue() + "'"); - } - try (Dataset latest = openDataset(allocator, tableAccess, OptionalLong.empty(), metrics)) { - version = OptionalLong.of(metrics.measure(Stage.VERSION_RESOLVE, - () -> LanceSnapshotResolver.getVersionAtOrBefore(latest, timestamp))); - } + OptionalLong direct = directMainVersion(state, metrics); + if (direct.isPresent() || isLatestMain(selector)) { + state.version = direct; + try (Dataset dataset = openDataset(allocator, state, direct, metrics)) { + result = reader.read(dataset, state.access, metrics); + } + } else { + OptionalLong mainVersion = state.access.isManagedVersioning() Review Comment: Update: with the redesign, `@branch` no longer depends on a recorded `main` head. The handle that tag and branch reads check out from is opened by URI, and the branch's head or version is asked from the namespace with the branch name. A namespace that records versions for `dev` and none for `main` now reads `@branch(dev)` and a tag pointing into it; a plain read of `main` still reports that the namespace lists no versions. A correction to my earlier reply as well: Lance 12 can open a branch by its directory URI (that is how the BE reads it); what it lacks is a Java open by branch name. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
