This is an automated email from the ASF dual-hosted git repository.
cwylie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new cafc748f7ec skip expression virtual column indexes when mvd is used as
array (#15644)
cafc748f7ec is described below
commit cafc748f7eca173a92d4ba53ae6e5f67ff0f50d3
Author: Clint Wylie <[email protected]>
AuthorDate: Mon Jan 8 21:22:37 2024 -0800
skip expression virtual column indexes when mvd is used as array (#15644)
---
processing/src/main/java/org/apache/druid/math/expr/Expr.java | 10 ++++++++--
.../java/org/apache/druid/segment/filter/BaseFilterTest.java | 1 +
.../org/apache/druid/segment/filter/EqualityFilterTests.java | 10 ++++++++++
3 files changed, 19 insertions(+), 2 deletions(-)
diff --git a/processing/src/main/java/org/apache/druid/math/expr/Expr.java
b/processing/src/main/java/org/apache/druid/math/expr/Expr.java
index 9471bb24eb5..9237b0d14e9 100644
--- a/processing/src/main/java/org/apache/druid/math/expr/Expr.java
+++ b/processing/src/main/java/org/apache/druid/math/expr/Expr.java
@@ -34,6 +34,7 @@ import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.index.semantic.DictionaryEncodedValueIndex;
import org.apache.druid.segment.serde.NoIndexesColumnIndexSupplier;
+import org.apache.druid.segment.virtual.ExpressionSelectors;
import javax.annotation.Nullable;
import java.util.ArrayList;
@@ -213,6 +214,11 @@ public interface Expr extends Cacheable
);
final ColumnCapabilities capabilities =
columnIndexSelector.getColumnCapabilities(column);
+ if (!ExpressionSelectors.canMapOverDictionary(details, capabilities)) {
+ // for mvds, expression might need to evaluate entire row, but we
don't have those handy, so fall back to
+ // not using indexes
+ return NoIndexesColumnIndexSupplier.getInstance();
+ }
final ExpressionType inputType =
ExpressionType.fromColumnTypeStrict(capabilities);
final ColumnType outType;
if (outputType == null) {
@@ -509,8 +515,8 @@ public interface Expr extends Cacheable
* @see Parser#applyUnappliedBindings
* @see Parser#applyUnapplied
* @see Parser#liftApplyLambda
- * @see
org.apache.druid.segment.virtual.ExpressionSelectors#makeDimensionSelector
- * @see
org.apache.druid.segment.virtual.ExpressionSelectors#makeColumnValueSelector
+ * @see ExpressionSelectors#makeDimensionSelector
+ * @see ExpressionSelectors#makeColumnValueSelector
*/
@SuppressWarnings("JavadocReference")
class BindingAnalysis
diff --git
a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java
b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java
index 906c5cedc72..1106801729b 100644
---
a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java
@@ -147,6 +147,7 @@ public abstract class BaseFilterTest extends
InitializedNullHandlingTest
new ExpressionVirtualColumn("double-vf0-add-sub", "vf0 + (vf0 -
vf0)", ColumnType.FLOAT, TestExprMacroTable.INSTANCE),
new ExpressionVirtualColumn("double-vl0-add-sub", "vl0 + (vl0 -
vl0)", ColumnType.LONG, TestExprMacroTable.INSTANCE),
new ExpressionVirtualColumn("vdim3-concat", "dim3 + dim3",
ColumnType.LONG, TestExprMacroTable.INSTANCE),
+ new ExpressionVirtualColumn("vdim2-offset", "array_offset(dim2, 1)",
ColumnType.STRING, TestExprMacroTable.INSTANCE),
new ExpressionVirtualColumn("nestedArrayLong", "array(arrayLong)",
ColumnType.ofArray(ColumnType.LONG_ARRAY), TestExprMacroTable.INSTANCE),
new ListFilteredVirtualColumn("allow-dim0",
DefaultDimensionSpec.of("dim0"), ImmutableSet.of("3", "4"), true),
new ListFilteredVirtualColumn("deny-dim0",
DefaultDimensionSpec.of("dim0"), ImmutableSet.of("3", "4"), false),
diff --git
a/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java
b/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java
index 3a141d63758..6521d5cf957 100644
---
a/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java
+++
b/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java
@@ -483,6 +483,16 @@ public class EqualityFilterTests
: ImmutableList.of("0", "2", "3", "4")
);
}
+
+ // array_offset behaves same on both mvds and arrays
+ assertFilterMatchesSkipVectorize(
+ new EqualityFilter("vdim2-offset", ColumnType.STRING, "b", null),
+ ImmutableList.of("0")
+ );
+ assertFilterMatchesSkipVectorize(
+ NotDimFilter.of(new EqualityFilter("vdim2-offset",
ColumnType.STRING, "b", null)),
+ NullHandling.sqlCompatible() ? ImmutableList.of() :
ImmutableList.of("1", "2", "3", "4", "5")
+ );
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
