This is an automated email from the ASF dual-hosted git repository.
rongr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 6b3689c5ef [multistage] support not-in in leaf stage (#9610)
6b3689c5ef is described below
commit 6b3689c5efbedd3c179b06062dfe1f1ab13619cc
Author: Rong Rong <[email protected]>
AuthorDate: Mon Oct 17 19:24:31 2022 -0700
[multistage] support not-in in leaf stage (#9610)
Co-authored-by: Rong Rong <[email protected]>
---
.../common/function/TransformFunctionType.java | 1 +
.../transform/function/InTransformFunction.java | 12 ++---
.../transform/function/NotInTransformFunction.java | 51 ++++++++++++++++++++++
.../function/TransformFunctionFactory.java | 1 +
.../function/InTransformFunctionTest.java | 18 ++++++++
.../java/org/apache/pinot/query/QueryTestSet.java | 5 +++
6 files changed, 82 insertions(+), 6 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/function/TransformFunctionType.java
b/pinot-common/src/main/java/org/apache/pinot/common/function/TransformFunctionType.java
index f0c28f71ee..3f6ab6a144 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/function/TransformFunctionType.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/function/TransformFunctionType.java
@@ -55,6 +55,7 @@ public enum TransformFunctionType {
LESS_THAN("less_than"),
LESS_THAN_OR_EQUAL("less_than_or_equal"),
IN("in"),
+ NOT_IN("not_in"),
IS_NULL("is_null"),
IS_NOT_NULL("is_not_null"),
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/InTransformFunction.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/InTransformFunction.java
index 01720358b6..39419cd959 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/InTransformFunction.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/InTransformFunction.java
@@ -57,8 +57,8 @@ public class InTransformFunction extends
BaseTransformFunction {
@Override
public void init(List<TransformFunction> arguments, Map<String, DataSource>
dataSourceMap) {
int numArguments = arguments.size();
- Preconditions.checkArgument(numArguments >= 2,
- "At least 2 arguments are required for IN transform function:
expression, values");
+ Preconditions.checkArgument(numArguments >= 2, "At least 2 arguments are
required for [%s] "
+ + "transform function: (expression, values)", getName());
_mainFunction = arguments.get(0);
boolean allLiteralValues = true;
@@ -119,13 +119,13 @@ public class InTransformFunction extends
BaseTransformFunction {
throw new IllegalStateException();
}
} else {
-
Preconditions.checkArgument(_mainFunction.getResultMetadata().isSingleValue(),
- "The first argument for IN transform function must be single-valued
when there are non-literal values");
+
Preconditions.checkArgument(_mainFunction.getResultMetadata().isSingleValue(),
"The first argument for [%s] "
+ + "transform function must be single-valued when there are
non-literal values", getName());
_valueFunctions = new TransformFunction[numArguments - 1];
for (int i = 1; i < numArguments; i++) {
TransformFunction valueFunction = arguments.get(i);
-
Preconditions.checkArgument(valueFunction.getResultMetadata().isSingleValue(),
- "The values for IN transform function must be single-valued");
+
Preconditions.checkArgument(valueFunction.getResultMetadata().isSingleValue(),
"The values for [%s] "
+ + "transform function must be single-valued", getName());
_valueFunctions[i - 1] = valueFunction;
}
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/NotInTransformFunction.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/NotInTransformFunction.java
new file mode 100644
index 0000000000..dd09574847
--- /dev/null
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/NotInTransformFunction.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.operator.transform.function;
+
+import java.util.List;
+import java.util.Map;
+import org.apache.pinot.common.function.TransformFunctionType;
+import org.apache.pinot.core.operator.blocks.ProjectionBlock;
+import org.apache.pinot.segment.spi.datasource.DataSource;
+
+
+/**
+ * The NOT_IN transform function is the negate version of the {@link
InTransformFunction}.
+ */
+public class NotInTransformFunction extends InTransformFunction {
+
+ @Override
+ public String getName() {
+ return TransformFunctionType.NOT_IN.getName();
+ }
+
+ @Override
+ public void init(List<TransformFunction> arguments, Map<String, DataSource>
dataSourceMap) {
+ super.init(arguments, dataSourceMap);
+ }
+
+ @Override
+ public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) {
+ int[] intValuesSV = super.transformToIntValuesSV(projectionBlock);
+ for (int i = 0; i < intValuesSV.length; i++) {
+ intValuesSV[i] = 1 - intValuesSV[i];
+ }
+ return intValuesSV;
+ }
+}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunctionFactory.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunctionFactory.java
index 74a2763345..043c23cf19 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunctionFactory.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunctionFactory.java
@@ -157,6 +157,7 @@ public class TransformFunctionFactory {
typeToImplementation.put(TransformFunctionType.LESS_THAN_OR_EQUAL,
LessThanOrEqualTransformFunction.class);
typeToImplementation.put(TransformFunctionType.IN,
InTransformFunction.class);
+ typeToImplementation.put(TransformFunctionType.NOT_IN,
NotInTransformFunction.class);
// logical functions
typeToImplementation.put(TransformFunctionType.AND,
AndOperatorTransformFunction.class);
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/InTransformFunctionTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/InTransformFunctionTest.java
index 70f0864ffb..950c8da400 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/InTransformFunctionTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/InTransformFunctionTest.java
@@ -51,6 +51,24 @@ public class InTransformFunctionTest extends
BaseTransformFunctionTest {
assertEquals(intValues[i], inValues.contains(_intSVValues[i]) ? 1 : 0);
}
}
+ @Test
+ public void testIntNotInTransformFunction() {
+ String expressionStr =
+ String.format("%s NOT IN (%d, %d, %d)", INT_SV_COLUMN,
_intSVValues[2], _intSVValues[5], _intSVValues[9]);
+ ExpressionContext expression =
RequestContextUtils.getExpression(expressionStr);
+ TransformFunction transformFunction =
TransformFunctionFactory.get(expression, _dataSourceMap);
+ assertTrue(transformFunction instanceof InTransformFunction);
+ assertEquals(transformFunction.getName(),
TransformFunctionType.NOT_IN.getName());
+
+ Set<Integer> inValues = Sets.newHashSet(_intSVValues[2], _intSVValues[5],
_intSVValues[9]);
+ int[] intValues =
transformFunction.transformToIntValuesSV(_projectionBlock);
+ for (int i = 0; i < NUM_ROWS; i++) {
+ if (i != 2 && i != 5 && i != 9) {
+ assertEquals(intValues[i], 1);
+ }
+ assertEquals(intValues[i], inValues.contains(_intSVValues[i]) ? 0 : 1);
+ }
+ }
@Test
public void testIntMVInTransformFunction() {
diff --git
a/pinot-query-runtime/src/test/java/org/apache/pinot/query/QueryTestSet.java
b/pinot-query-runtime/src/test/java/org/apache/pinot/query/QueryTestSet.java
index 0456872a38..12adcd7e5c 100644
--- a/pinot-query-runtime/src/test/java/org/apache/pinot/query/QueryTestSet.java
+++ b/pinot-query-runtime/src/test/java/org/apache/pinot/query/QueryTestSet.java
@@ -95,6 +95,11 @@ public class QueryTestSet {
+ " WHERE a.col1 IN ('foo') AND b.col2 NOT IN ('')"},
// Range conditions with continuous and non-continuous range.
+ // Calcite default compilation will convert multiple `=` and `<>` into
range IN and NOT IN search predicates.
+ new Object[]{"SELECT a.col1, SUM(CASE WHEN a.col2 = 'foo' OR a.col2 =
'alice' THEN 1 ELSE 0 END) AS match_sum, "
+ + " SUM(CASE WHEN a.col2 <> 'foo' AND a.col2 <> 'alice' THEN 1
ELSE 0 END) as unmatch_sum "
+ + " FROM a WHERE a.ts >= 1600000000 GROUP BY a.col1"},
+
new Object[]{"SELECT a.col1, b.col2 FROM a JOIN b ON a.col1 = b.col1 "
+ " WHERE a.col3 IN (1, 2, 3) OR (a.col3 > 10 AND a.col3 < 50)"},
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
