github-actions[bot] commented on code in PR #70830:
URL: https://github.com/apache/doris/pull/70830#discussion_r4651002518


##########
fe/fe-connector/fe-connector-lance/src/main/java/org/apache/doris/connector/lance/LancePredicateConverter.java:
##########
@@ -0,0 +1,697 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.connector.lance;
+
+import org.apache.doris.connector.spi.ConnectorType;
+import org.apache.doris.connector.spi.pushdown.ConnectorAnd;
+import org.apache.doris.connector.spi.pushdown.ConnectorColumnRef;
+import org.apache.doris.connector.spi.pushdown.ConnectorComparison;
+import org.apache.doris.connector.spi.pushdown.ConnectorExpression;
+import org.apache.doris.connector.spi.pushdown.ConnectorFunctionCall;
+import org.apache.doris.connector.spi.pushdown.ConnectorIn;
+import org.apache.doris.connector.spi.pushdown.ConnectorIsNull;
+import org.apache.doris.connector.spi.pushdown.ConnectorLike;
+import org.apache.doris.connector.spi.pushdown.ConnectorLiteral;
+import org.apache.doris.connector.spi.pushdown.ConnectorNot;
+import org.apache.doris.connector.spi.pushdown.ConnectorOr;
+
+import io.substrait.expression.Expression;
+import io.substrait.expression.ExpressionCreator;
+import io.substrait.expression.FieldReference;
+import io.substrait.extension.DefaultExtensionCatalog;
+import io.substrait.extension.SimpleExtension;
+import io.substrait.type.TypeCreator;
+import org.apache.arrow.vector.types.FloatingPointPrecision;
+import org.apache.arrow.vector.types.TimeUnit;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+import java.io.InputStream;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.math.RoundingMode;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+/**
+ * Converts the conjuncts whose Doris and Lance semantics are identical to one 
Substrait ExtendedExpression.
+ *
+ * <p>A conjunct is either converted whole or not at all: the engine removes a 
converted conjunct from the
+ * Doris residuals ({@code ScanNodePropertiesResult.withPushdownTracking}), so 
a translation that is not exact
+ * would return wrong rows. Conjuncts holding a CAST never arrive here, 
because the Lance connector does not
+ * support CAST pushdown and the engine keeps those in Doris.
+ */
+final class LancePredicateConverter {
+    private static final TypeCreator REQUIRED = TypeCreator.of(false);
+    /** Names the YAML that declares {@code array_has}; lance-c resolves 
Substrait functions by name. */
+    static final String LANCE_FUNCTIONS = 
"https://github.com/apache/doris/blob/master/";
+            + 
"fe/fe-connector/fe-connector-lance/src/main/resources/substrait/lance_functions.yaml";
+    private static final SimpleExtension.ExtensionCollection EXTENSIONS = 
loadExtensions();
+
+    private final Schema schema;
+    private final Map<String, ResolvedField> fieldsByLowerCaseName = new 
HashMap<>();
+
+    LancePredicateConverter(Schema schema) {
+        this.schema = schema;
+        List<Field> fields = schema.getFields();
+        for (int ordinal = 0; ordinal < fields.size(); ordinal++) {
+            Field field = fields.get(ordinal);
+            fieldsByLowerCaseName.putIfAbsent(
+                    field.getName().toLowerCase(Locale.ROOT), new 
ResolvedField(field, ordinal));
+        }
+    }
+
+    /**
+     * The conjuncts of the engine's filter, in the order the residual 
protocol numbers them: the children of a
+     * top-level AND, or the filter itself.
+     */
+    static List<ConnectorExpression> conjuncts(Optional<ConnectorExpression> 
filter) {
+        if (!filter.isPresent()) {
+            return Collections.emptyList();
+        }
+        return filter.get() instanceof ConnectorAnd
+                ? ((ConnectorAnd) filter.get()).getConjuncts() : 
Collections.singletonList(filter.get());
+    }
+
+    ConversionResult convert(List<ConnectorExpression> conjuncts) {
+        List<Expression> filters = new ArrayList<>();
+        List<ConnectorExpression> pushedConjuncts = new ArrayList<>();
+        Set<Integer> notPushedIndices = new TreeSet<>();
+        for (int i = 0; i < conjuncts.size(); i++) {
+            Optional<Expression> filter = convert(conjuncts.get(i));
+            if (filter.isPresent()) {
+                filters.add(filter.get());
+                pushedConjuncts.add(conjuncts.get(i));
+            } else {
+                notPushedIndices.add(i);
+            }
+        }
+        if (filters.isEmpty()) {
+            return new ConversionResult(new byte[0], "", pushedConjuncts, 
notPushedIndices);
+        }
+        Expression combined = filters.size() == 1 ? filters.get(0) : 
booleanFunction("and:bool", filters);
+        String debugPredicate = pushedConjuncts.stream()
+                .map(LancePredicateConverter::render)
+                .map(sql -> "(" + sql + ")")
+                .collect(Collectors.joining(" AND "));
+        return new ConversionResult(new 
LanceSubstraitSerializer(schema).serialize(combined), debugPredicate,
+                pushedConjuncts, notPushedIndices);
+    }
+
+    private Optional<Expression> convert(ConnectorExpression expr) {
+        if (expr instanceof ConnectorComparison) {
+            return convertComparison((ConnectorComparison) expr);
+        }
+        if (expr instanceof ConnectorAnd) {
+            return convertAll("and:bool", ((ConnectorAnd) 
expr).getConjuncts());
+        }
+        if (expr instanceof ConnectorOr) {
+            return convertAll("or:bool", ((ConnectorOr) expr).getDisjuncts());
+        }
+        if (expr instanceof ConnectorNot) {
+            return convert(((ConnectorNot) expr).getOperand())
+                    .map(operand -> booleanFunction("not:bool", 
Collections.singletonList(operand)));
+        }
+        if (expr instanceof ConnectorIn) {
+            return convertIn((ConnectorIn) expr);
+        }
+        if (expr instanceof ConnectorIsNull) {
+            return convertIsNull((ConnectorIsNull) expr);
+        }
+        if (expr instanceof ConnectorLike) {
+            return convertLike((ConnectorLike) expr);
+        }
+        if (expr instanceof ConnectorFunctionCall) {
+            return convertFunction((ConnectorFunctionCall) expr);
+        }
+        if (expr instanceof ConnectorColumnRef) {
+            return convertBooleanColumn((ConnectorColumnRef) expr);
+        }
+        return Optional.empty();
+    }
+
+    /** An AND or an OR is converted only when every operand is; the engine 
flattens nested ones. */
+    private Optional<Expression> convertAll(String function, 
List<ConnectorExpression> operands) {
+        List<Expression> converted = new ArrayList<>(operands.size());
+        for (ConnectorExpression operand : operands) {
+            Optional<Expression> expression = convert(operand);
+            if (!expression.isPresent()) {
+                return Optional.empty();
+            }
+            converted.add(expression.get());
+        }
+        return Optional.of(booleanFunction(function, converted));
+    }
+
+    private Optional<Expression> convertComparison(ConnectorComparison 
predicate) {
+        ResolvedField field = findField(predicate.getLeft());
+        if (field == null || 
!LanceSubstraitSerializer.supportsType(field.field.getType())
+                || !(predicate.getRight() instanceof ConnectorLiteral)) {

Review Comment:
   [P2] Handle literal-left comparisons in Lance pushdown. 
`ExprToConnectorExpressionConverter` preserves the SQL operand order, but 
`convertComparison` only accepts a column on the left and a literal on the 
right, so queries such as `10 < int64_value` remain residual and fail the new 
regression suite's `verifyFullyPushedDown` reversed-operand cases. Commute the 
operands and invert LT/LE/GT/GE (while preserving EQ/NE/EQ_FOR_NULL), or 
normalize this before conversion.



##########
fe/fe-connector/fe-connector-lance/src/main/java/org/apache/doris/connector/lance/LanceCatalogClient.java:
##########
@@ -132,41 +176,68 @@ LanceTableMetadata loadTableMetadata(String dbName, 
String tableName, OptionalLo
      * the commit times the manifests record; see {@link 
LanceSnapshotResolver#versionAtOrBefore}.
      */
     LanceTableMetadata loadTableMetadataAtOrBefore(String dbName, String 
tableName, long timestampMillis,
-            String requestedText) {
+            String requestedText, LanceMetadataMetrics metrics) {
         String what = dbName + "." + tableName;
-        return readTable(dbName, tableName, OptionalLong.empty(), (access, 
readAllocator) -> {
-            long version;
-            try (Dataset latest = openDataset(readAllocator, access, 
OptionalLong.empty())) {
-                try {
-                    version = 
LanceSnapshotResolver.versionAtOrBefore(latest.listVersions(), timestampMillis,
-                            requestedText);
-                } catch (LanceSnapshotResolver.NoVersionAtOrBeforeException e) 
{
-                    throw new UserFacingException("Lance table " + what + " 
has no version at or before '"
-                            + requestedText + "'");
-                } catch (LanceSnapshotResolver.HistoryRemovedException e) {
-                    throw new UserFacingException("Lance cannot resolve FOR 
TIME AS OF '" + requestedText
-                            + "' on " + what + ": version " + e.getVersion()
-                            + ", which may hold the state at that time, no 
longer exists");
-                }
-            }
-            LOG.debug("Resolved Lance FOR TIME AS OF '{}' to version {} from 
manifest commit times",
-                    requestedText, version);
-            try (Dataset dataset = openDataset(readAllocator, access, 
OptionalLong.of(version))) {
-                return readMetadata(dataset, access);
-            }
-        });
+        return readTable(dbName, tableName, OptionalLong.empty(), metrics,
+                (generation, access, readAllocator, timer) -> {
+                    long version;
+                    try (Dataset latest = openDataset(generation, 
readAllocator, access, OptionalLong.empty(),
+                            timer)) {
+                        version = timer.measure(Stage.VERSION_RESOLVE,
+                                () -> versionAtOrBefore(latest, 
timestampMillis, requestedText, what));
+                    }
+                    LOG.debug("Resolved Lance FOR TIME AS OF '{}' to version 
{} from manifest commit times",
+                            requestedText, version);
+                    try (Dataset dataset = openDataset(generation, 
readAllocator, access, OptionalLong.of(version),
+                            timer)) {
+                        return readMetadata(dataset, access, timer);
+                    }
+                });
+    }
+
+    /**
+     * REFRESH CATALOG: retires the current generation, so that the next call 
opens a new namespace, allocator and
+     * Session. Opens nothing and waits for nothing: calls in flight finish on 
the retired generation, and the
+     * last of them releases it.
+     */
+    void refresh() {
+        Generation idle = retireCurrent();
+        if (idle != null) {
+            idle.closeResources();
+        }
+    }
+
+    /**
+     * REFRESH TABLE / DATABASE: the next read describes its table again; the 
Session and its cached metadata
+     * stay. Opens nothing: before the first call nothing is cached.
+     */
+    void invalidateTableAccess() {

Review Comment:
   [P2] Invalidate the current generation atomically with refresh. 
`invalidateTableAccess()` reads `current` under the monitor, unlocks, and then 
swaps that generation's cache. A concurrent `refresh()` can retire it and open 
G2 before the swap, so the invalidation only clears retired G1; G2 can keep a 
stale URI/access until its TTL expires. Keep the generation check and cache 
swap under the same monitor, or retry against the active generation after a 
race.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to