This is an automated email from the ASF dual-hosted git repository.
ColinLeeo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new a4202b47f99 Fix tree-model NOT predicate pushdown (#18982)
a4202b47f99 is described below
commit a4202b47f990dd0e3160e33902dcea30b145c11d
Author: Colin Lee <[email protected]>
AuthorDate: Fri Oct 23 16:13:43 2026 +0800
Fix tree-model NOT predicate pushdown (#18982)
* Fix tree-model NOT predicate pushdown
Remove NOT from tree-model WHERE and HAVING predicates before binding and
pushdown, including the templated device path. Prevent non-invertible NOT
predicates from reaching TsFile filters.
* Address NOT predicate review feedback
Make NOT rewriting total and type-aware, fix root predicate context
propagation through unary expressions, and add regressions for NaN, nested
negation, unsupported operands, composite HAVING, and aligned templates.
* Extract time predicate from a NOT over a disjunction
NOT (A OR B) is NOT(A) AND NOT(B), so every disjunct that is a pure time
predicate still bounds the time range of the negated predicate. Hence
WHERE NOT (time > 2 OR s1 > 5) now yields time <= 2 for partition pruning
instead of scanning every time partition.
* Cover HAVING NOT over boolean aggregations
Add regressions for HAVING NOT last_value(bool) on the raw, ALIGN BY DEVICE
and aligned-template paths, which is the non-invertible HAVING NOT shape
that must stay in FilterOperator.
* Guard time-only predicates during source extraction
* Return the original predicate when NOT cannot be inverted
predicateRemoveNot() rebuilt the tree even when the operand could not be
inverted, so callers comparing instances to detect a rewrite always saw a
change and re-simplified, re-normalized and re-analyzed the predicate once
per
device. Return the original node instead so the existing short-circuit
works.
Also remove DataNodeQueryMessages.GROUP_BY_TIME_CANNOT_BE_REVERSED (en and
zh),
which is no longer referenced since ReversePredicateVisitor retains the NOT
instead of throwing.
---
.../it/query/IoTDBPredicateConversionTreeIT.java | 231 ++++++++++++++++++++-
.../iotdb/db/i18n/DataNodeQueryMessages.java | 2 -
.../iotdb/db/i18n/DataNodeQueryMessages.java | 2 -
.../queryengine/plan/analyze/AnalyzeVisitor.java | 20 +-
.../queryengine/plan/analyze/PredicateUtils.java | 98 +++++++--
.../queryengine/plan/analyze/TemplatedAnalyze.java | 9 +-
.../cartesian/BindSchemaForPredicateVisitor.java | 10 +
.../PredicateCanPushDownToSourceChecker.java | 8 +
.../visitor/predicate/ReversePredicateVisitor.java | 82 +++++++-
.../predicate/PredicateRemoveNotTest.java | 74 +++++++
.../expression/predicate/PredicateUtilsTest.java | 3 +
.../plan/analyze/PredicateUtilsTest.java | 70 +++++++
12 files changed, 579 insertions(+), 30 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBPredicateConversionTreeIT.java
b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBPredicateConversionTreeIT.java
index afe09efb481..4ca331702ae 100644
---
a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBPredicateConversionTreeIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBPredicateConversionTreeIT.java
@@ -31,9 +31,18 @@ import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import java.sql.Connection;
+import java.sql.ResultSet;
import java.sql.Statement;
+import java.util.HashSet;
+import java.util.Set;
import static org.apache.iotdb.db.it.utils.TestUtils.resultSetEqualTest;
+import static org.apache.iotdb.itbase.constant.TestConstant.DEVICE;
+import static org.apache.iotdb.itbase.constant.TestConstant.TIMESTAMP_STR;
+import static org.apache.iotdb.itbase.constant.TestConstant.count;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@RunWith(IoTDBTestRunner.class)
@@ -42,6 +51,8 @@ public class IoTDBPredicateConversionTreeIT {
private static final String DATABASE_NAME = "root.test_pred";
private static final String DEVICE_ID = DATABASE_NAME + ".d1";
+ private static final String NOT_DATABASE_NAME = "root.test_not_regression";
+ private static final String TEMPLATE_DATABASE_NAME =
"root.test_not_template";
private static final String[] SQLs =
new String[] {
@@ -57,7 +68,31 @@ public class IoTDBPredicateConversionTreeIT {
"INSERT INTO " + DEVICE_ID + "(time, int32_col, int64_col) VALUES(7,
null, null)",
"INSERT INTO " + DEVICE_ID + "(time, int32_col, int64_col) VALUES(8,
1000, 1000)",
"INSERT INTO " + DEVICE_ID + "(time, int32_col, int64_col) VALUES(9,
-29, -29)",
- "INSERT INTO " + DEVICE_ID + "(time, int32_col, int64_col) VALUES(10,
-30, -30)"
+ "INSERT INTO " + DEVICE_ID + "(time, int32_col, int64_col) VALUES(10,
-30, -30)",
+ "CREATE DATABASE " + NOT_DATABASE_NAME,
+ "CREATE TIMESERIES " + NOT_DATABASE_NAME + ".d1.s1 WITH
DATATYPE=INT32, ENCODING=PLAIN",
+ "CREATE TIMESERIES " + NOT_DATABASE_NAME + ".d1.s2 WITH
DATATYPE=INT32, ENCODING=PLAIN",
+ "CREATE TIMESERIES " + NOT_DATABASE_NAME + ".d1.b WITH
DATATYPE=BOOLEAN, ENCODING=PLAIN",
+ "INSERT INTO " + NOT_DATABASE_NAME + ".d1(time, s1, s2, b) VALUES(1,
1, 10, true)",
+ "INSERT INTO " + NOT_DATABASE_NAME + ".d1(time, s1, s2, b) VALUES(2,
6, 20, false)",
+ "INSERT INTO " + NOT_DATABASE_NAME + ".d1(time, s1, s2, b) VALUES(3,
3, 30, true)",
+ "INSERT INTO " + NOT_DATABASE_NAME + ".d1(time, s1, s2, b) VALUES(4,
8, 40, false)",
+ "CREATE TIMESERIES " + NOT_DATABASE_NAME + ".d2.s1 WITH
DATATYPE=INT32, ENCODING=PLAIN",
+ "INSERT INTO " + NOT_DATABASE_NAME + ".d2(time, s1) VALUES(1, 2)",
+ "INSERT INTO " + NOT_DATABASE_NAME + ".d2(time, s1) VALUES(2, 7)",
+ "CREATE TIMESERIES " + NOT_DATABASE_NAME + ".d3.x WITH
DATATYPE=DOUBLE, ENCODING=PLAIN",
+ "CREATE TIMESERIES " + NOT_DATABASE_NAME + ".d3.y WITH DATATYPE=INT32,
ENCODING=PLAIN",
+ "INSERT INTO " + NOT_DATABASE_NAME + ".d3(time, x, y) VALUES(1, NaN,
1)",
+ "INSERT INTO " + NOT_DATABASE_NAME + ".d3(time, x, y) VALUES(2, 3.0,
2)",
+ "INSERT INTO " + NOT_DATABASE_NAME + ".d3(time, x, y) VALUES(3, 10.0,
3)",
+ "CREATE DATABASE " + TEMPLATE_DATABASE_NAME,
+ "CREATE DEVICE TEMPLATE not_predicate_template ALIGNED (s1 INT64, b
BOOLEAN)",
+ "SET DEVICE TEMPLATE not_predicate_template TO " +
TEMPLATE_DATABASE_NAME + ".d1",
+ "SET DEVICE TEMPLATE not_predicate_template TO " +
TEMPLATE_DATABASE_NAME + ".d2",
+ "INSERT INTO " + TEMPLATE_DATABASE_NAME + ".d1(time, s1, b) ALIGNED
VALUES(1, 3, true)",
+ "INSERT INTO " + TEMPLATE_DATABASE_NAME + ".d1(time, s1, b) ALIGNED
VALUES(2, 8, false)",
+ "INSERT INTO " + TEMPLATE_DATABASE_NAME + ".d2(time, s1, b) ALIGNED
VALUES(1, 4, false)",
+ "INSERT INTO " + TEMPLATE_DATABASE_NAME + ".d2(time, s1, b) ALIGNED
VALUES(2, 9, true)"
};
@BeforeClass
@@ -290,4 +325,198 @@ public class IoTDBPredicateConversionTreeIT {
expectedHeaderInt64,
allInt64Rows);
}
+
+ @Test
+ public void testWhereAndHavingWithNot() {
+ String[] expectedWhereHeader = new String[] {TIMESTAMP_STR, DEVICE_ID +
".int32_col"};
+ resultSetEqualTest(
+ "SELECT int32_col FROM " + DEVICE_ID + " WHERE NOT (int32_col > 20)",
+ expectedWhereHeader,
+ new String[] {"1,20,", "9,-29,", "10,-30,"});
+ resultSetEqualTest(
+ "SELECT int32_col FROM " + DEVICE_ID + " WHERE NOT (int32_col > 20 AND
int32_col < 40)",
+ expectedWhereHeader,
+ new String[] {"1,20,", "8,1000,", "9,-29,", "10,-30,"});
+
+ String[] expectedHavingHeader = new String[] {TIMESTAMP_STR,
count(DEVICE_ID + ".int32_col")};
+ resultSetEqualTest(
+ "SELECT count(int32_col) FROM "
+ + DEVICE_ID
+ + " GROUP BY ([1,11),2ms) HAVING NOT (count(int32_col) > 1)",
+ expectedHavingHeader,
+ new String[] {"5,1,", "7,1,"});
+
+ String[] expectedWhereAlignByDeviceHeader = new String[] {TIMESTAMP_STR,
DEVICE, "int32_col"};
+ resultSetEqualTest(
+ "SELECT int32_col FROM root.test_pred.* WHERE NOT (int32_col > 20 AND
int32_col < 40) ALIGN BY DEVICE",
+ expectedWhereAlignByDeviceHeader,
+ new String[] {
+ "1,root.test_pred.d1,20,",
+ "8,root.test_pred.d1,1000,",
+ "9,root.test_pred.d1,-29,",
+ "10,root.test_pred.d1,-30,",
+ });
+
+ String[] expectedHavingAlignByDeviceHeader =
+ new String[] {TIMESTAMP_STR, DEVICE, count("int32_col")};
+ resultSetEqualTest(
+ "SELECT count(int32_col) FROM root.test_pred.* GROUP BY ([1,11),2ms)
HAVING NOT (count(int32_col) > 1) ALIGN BY DEVICE",
+ expectedHavingAlignByDeviceHeader,
+ new String[] {"5,root.test_pred.d1,1,", "7,root.test_pred.d1,1,"});
+ }
+
+ @Test
+ public void testTemplatedWhereWithNot() {
+ String[] expectedHeader = new String[] {TIMESTAMP_STR, DEVICE, "s1"};
+ resultSetEqualTest(
+ "SELECT s1 FROM " + TEMPLATE_DATABASE_NAME + ".* WHERE NOT (s1 > 5)
ALIGN BY DEVICE",
+ expectedHeader,
+ new String[] {
+ "1,root.test_not_template.d1,3,", "1,root.test_not_template.d2,4,",
+ });
+
+ String[] expectedBooleanHeader = new String[] {TIMESTAMP_STR, DEVICE,
"s1", "b"};
+ resultSetEqualTest(
+ "SELECT s1, b FROM " + TEMPLATE_DATABASE_NAME + ".* WHERE NOT b ALIGN
BY DEVICE",
+ expectedBooleanHeader,
+ new String[] {
+ "2,root.test_not_template.d1,8,false,",
"1,root.test_not_template.d2,4,false,",
+ });
+ }
+
+ @Test
+ public void testUnsupportedNotPredicate() {
+ String[] expectedHeader =
+ new String[] {TIMESTAMP_STR, NOT_DATABASE_NAME + ".d1.s1",
NOT_DATABASE_NAME + ".d1.b"};
+ resultSetEqualTest(
+ "SELECT s1, b FROM " + NOT_DATABASE_NAME + ".d1 WHERE NOT b OR s1 > 5",
+ expectedHeader,
+ new String[] {"2,6,false,", "4,8,false,"});
+ resultSetEqualTest(
+ "SELECT s1, b FROM " + NOT_DATABASE_NAME + ".d1 WHERE NOT b",
+ expectedHeader,
+ new String[] {"2,6,false,", "4,8,false,"});
+ resultSetEqualTest(
+ "SELECT s1, b FROM "
+ + NOT_DATABASE_NAME
+ + ".d1 WHERE NOT (CASE WHEN s1 > 5 THEN true ELSE false END) OR s2
> 35",
+ expectedHeader,
+ new String[] {"1,1,true,", "3,3,true,", "4,8,false,"});
+ }
+
+ @Test
+ public void testNotPredicateWithTimeAndMultipleSeries() {
+ resultSetEqualTest(
+ "SELECT s1, s2 FROM " + NOT_DATABASE_NAME + ".d1 WHERE NOT (time > 2
OR s1 > 5)",
+ new String[] {TIMESTAMP_STR, NOT_DATABASE_NAME + ".d1.s1",
NOT_DATABASE_NAME + ".d1.s2"},
+ new String[] {"1,1,10,"});
+
+ resultSetEqualTest(
+ "SELECT s1, s2 FROM "
+ + NOT_DATABASE_NAME
+ + ".d1 WHERE NOT (time > 2 OR s1 > 5) ALIGN BY DEVICE",
+ new String[] {TIMESTAMP_STR, DEVICE, "s1", "s2"},
+ new String[] {"1," + NOT_DATABASE_NAME + ".d1,1,10,"});
+ }
+
+ @Test
+ public void testNestedNotPredicate() {
+ String[] expectedHeader =
+ new String[] {TIMESTAMP_STR, NOT_DATABASE_NAME + ".d1.s1",
NOT_DATABASE_NAME + ".d1.s2"};
+ resultSetEqualTest(
+ "SELECT s1, s2 FROM "
+ + NOT_DATABASE_NAME
+ + ".d1 WHERE NOT (NOT (NOT (s1 > 5 AND s2 > 25)))",
+ expectedHeader,
+ new String[] {"1,1,10,", "2,6,20,", "3,3,30,"});
+ }
+
+ @Test
+ public void testNotPredicateWithNaN() throws Exception {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ try (ResultSet resultSet =
+ statement.executeQuery(
+ "SELECT x, y FROM " + NOT_DATABASE_NAME + ".d3 WHERE NOT (x > 5
OR y > 2)")) {
+ assertTrue(resultSet.next());
+ assertEquals(1, resultSet.getLong(1));
+ assertTrue(Double.isNaN(resultSet.getDouble(2)));
+ assertEquals(1, resultSet.getInt(3));
+ assertTrue(resultSet.next());
+ assertEquals(2, resultSet.getLong(1));
+ assertEquals(3.0, resultSet.getDouble(2), 0.0);
+ assertEquals(2, resultSet.getInt(3));
+ assertFalse(resultSet.next());
+ }
+
+ try (ResultSet resultSet =
+ statement.executeQuery(
+ "SELECT x, y FROM " + NOT_DATABASE_NAME + ".d3 WHERE NOT (x + y
> 5)")) {
+ assertTrue(resultSet.next());
+ assertEquals(1, resultSet.getLong(1));
+ assertTrue(Double.isNaN(resultSet.getDouble(2)));
+ assertTrue(resultSet.next());
+ assertEquals(2, resultSet.getLong(1));
+ assertFalse(resultSet.next());
+ }
+ }
+ }
+
+ @Test
+ public void testCompositeHavingNot() throws Exception {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ try (ResultSet resultSet =
+ statement.executeQuery(
+ "SELECT count(s1), sum(s2) FROM "
+ + NOT_DATABASE_NAME
+ + ".d1 HAVING NOT (count(s1) > 5 AND sum(s2) > 50)")) {
+ assertTrue(resultSet.next());
+ assertEquals(4, resultSet.getLong(1));
+ assertEquals(100.0, resultSet.getDouble(2), 0.0);
+ assertFalse(resultSet.next());
+ }
+
+ try (ResultSet resultSet =
+ statement.executeQuery(
+ "SELECT count(s1) FROM "
+ + NOT_DATABASE_NAME
+ + ".* GROUP BY ([1,5),10ms), LEVEL=1 "
+ + "HAVING NOT (count(s1) > 0 AND sum(s2) > 150)")) {
+ assertTrue(resultSet.next());
+ assertEquals(1, resultSet.getLong(1));
+ assertFalse(resultSet.next());
+ }
+ }
+ }
+
+ @Test
+ public void testHavingNotWithBooleanAggregation() {
+ resultSetEqualTest(
+ "SELECT count(s1) FROM "
+ + NOT_DATABASE_NAME
+ + ".d1 GROUP BY ([1,5),1ms) HAVING NOT last_value(b)",
+ new String[] {TIMESTAMP_STR, count(NOT_DATABASE_NAME + ".d1.s1")},
+ new String[] {"2,1,", "4,1,"});
+
+ resultSetEqualTest(
+ "SELECT count(s1) FROM "
+ + NOT_DATABASE_NAME
+ + ".d1 GROUP BY ([1,5),1ms) HAVING NOT last_value(b) ALIGN BY
DEVICE",
+ new String[] {TIMESTAMP_STR, DEVICE, count("s1")},
+ new String[] {"2," + NOT_DATABASE_NAME + ".d1,1,", "4," +
NOT_DATABASE_NAME + ".d1,1,"});
+ }
+
+ @Test
+ public void testTemplatedHavingNotWithBooleanAggregation() {
+ Set<String> expectedRows = new HashSet<>();
+ expectedRows.add("1," + TEMPLATE_DATABASE_NAME + ".d2,1,");
+ expectedRows.add("2," + TEMPLATE_DATABASE_NAME + ".d1,1,");
+ resultSetEqualTest(
+ "SELECT count(s1) FROM "
+ + TEMPLATE_DATABASE_NAME
+ + ".* GROUP BY ([1,3),1ms) HAVING NOT last_value(b) ALIGN BY
DEVICE",
+ TIMESTAMP_STR + "," + DEVICE + "," + count("s1") + ",",
+ expectedRows);
+ }
}
diff --git
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
index dabf70dd732..b8bd2a24eb2 100644
---
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
+++
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
@@ -642,8 +642,6 @@ public final class DataNodeQueryMessages {
"TIMESTAMP IN-list literal can only be LongLiteral, DoubleLiteral, or
GenericLiteral; actual type: %s";
public static final String IS_NULL_CAN_BE_PUSHED_DOWN =
"IS NULL can be pushed down";
- public static final String GROUP_BY_TIME_CANNOT_BE_REVERSED =
- "GROUP BY TIME cannot be reversed";
// --- Plan / Optimization ---
diff --git
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
index 9addf5f954a..d24c94b9b5a 100644
---
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
+++
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
@@ -622,8 +622,6 @@ public final class DataNodeQueryMessages {
"TIMESTAMP 的 IN 列表字面量只能是 LongLiteral、DoubleLiteral 或
GenericLiteral，实际类型为 %s";
public static final String IS_NULL_CAN_BE_PUSHED_DOWN =
"IS NULL 可以下推";
- public static final String GROUP_BY_TIME_CANNOT_BE_REVERSED =
- "GROUP BY TIME 不能反转";
// --- Plan / Optimization ---
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
index 018f171b132..6781cc019c7 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
@@ -1547,7 +1547,7 @@ public class AnalyzeVisitor extends
StatementVisitor<Analysis, MPPQueryContext>
while (deviceIterator.hasNext()) {
PartialPath devicePath = deviceIterator.next();
Expression whereExpression =
- analyzeWhereSplitByDevice(queryStatement, devicePath, schemaTree,
queryContext);
+ analyzeWhereSplitByDevice(analysis, queryStatement, devicePath,
schemaTree, queryContext);
if (whereExpression.equals(ConstantOperand.FALSE)) {
deviceIterator.remove();
} else if (whereExpression.equals(ConstantOperand.TRUE)) {
@@ -1580,7 +1580,7 @@ public class AnalyzeVisitor extends
StatementVisitor<Analysis, MPPQueryContext>
schemaTree,
true,
queryContext);
- Expression whereExpression =
convertConJunctionsToWhereExpression(conJunctions);
+ Expression whereExpression =
convertConJunctionsToWhereExpression(analysis, conJunctions);
if (whereExpression.equals(ConstantOperand.TRUE)) {
analysis.setWhereExpression(null);
analysis.setHasValueFilter(false);
@@ -1595,6 +1595,7 @@ public class AnalyzeVisitor extends
StatementVisitor<Analysis, MPPQueryContext>
}
private Expression analyzeWhereSplitByDevice(
+ final Analysis analysis,
final QueryStatement queryStatement,
final PartialPath devicePath,
final ISchemaTree schemaTree,
@@ -1606,16 +1607,25 @@ public class AnalyzeVisitor extends
StatementVisitor<Analysis, MPPQueryContext>
schemaTree,
true,
queryContext);
- return convertConJunctionsToWhereExpression(conJunctions);
+ return convertConJunctionsToWhereExpression(analysis, conJunctions);
}
- private Expression convertConJunctionsToWhereExpression(List<Expression>
conJunctions) {
+ private Expression convertConJunctionsToWhereExpression(
+ Analysis analysis, List<Expression> conJunctions) {
Expression predicate =
PredicateUtils.combineConjuncts(
conJunctions.stream().distinct().collect(Collectors.toList()));
predicate = PredicateUtils.simplifyPredicate(predicate);
predicate = normalizeExpression(predicate);
- return predicate;
+ analyzeExpressionType(analysis, predicate);
+ Expression predicateWithoutNot =
+ PredicateUtils.predicateRemoveNot(predicate, analysis::getType);
+ if (predicateWithoutNot != predicate) {
+ predicateWithoutNot =
PredicateUtils.simplifyPredicate(predicateWithoutNot);
+ predicateWithoutNot = normalizeExpression(predicateWithoutNot);
+ analyzeExpressionType(analysis, predicateWithoutNot);
+ }
+ return predicateWithoutNot;
}
private void analyzeDeviceViewOutput(Analysis analysis, QueryStatement
queryStatement) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/PredicateUtils.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/PredicateUtils.java
index edf52063588..c97528739bb 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/PredicateUtils.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/PredicateUtils.java
@@ -56,6 +56,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
import java.util.stream.Collectors;
public class PredicateUtils {
@@ -120,8 +121,9 @@ public class PredicateUtils {
}
return new GlobalTimePredicateExtractionResult(null, predicate, true);
} else if (predicate.getExpressionType().equals(ExpressionType.LOGIC_NOT))
{
+ Expression negatedExpression = ((UnaryExpression)
predicate).getExpression();
GlobalTimePredicateExtractionResult childResult =
- extractGlobalTimePredicate(((UnaryExpression)
predicate).getExpression());
+ extractGlobalTimePredicate(negatedExpression);
if (childResult.getGlobalTimePredicate() != null &&
!childResult.hasValueFilter()) {
return new GlobalTimePredicateExtractionResult(
ExpressionFactory.not(childResult.getGlobalTimePredicate()),
@@ -131,6 +133,14 @@ public class PredicateUtils {
// NOT(G AND R) cannot in general be represented as NOT(G) AND NOT(R). Do
// not extract a time predicate when the negated subtree contains a value
// filter.
+ //
+ // NOT(A OR B) is NOT(A) AND NOT(B) though, so every disjunct that is a
pure
+ // time predicate still bounds the time range of the whole negation, e.g.
+ // NOT (time > 2 OR s1 > 5) implies time <= 2.
+ Expression negatedTimePredicate =
extractNegatedTimePredicateFromDisjuncts(negatedExpression);
+ if (negatedTimePredicate != null) {
+ return new GlobalTimePredicateExtractionResult(negatedTimePredicate,
predicate, true);
+ }
return new GlobalTimePredicateExtractionResult(null, predicate, true);
} else if (predicate.isCompareBinaryExpression()) {
Expression leftExpression = ((BinaryExpression)
predicate).getLeftExpression();
@@ -194,6 +204,27 @@ public class PredicateUtils {
return ExpressionFactory.and(leftExpression, rightExpression);
}
+ /**
+ * Collects the negation of every disjunct that is a pure time predicate. It
is only called for
+ * the subtree negated by a {@code not()}, where De Morgan's law turns the
disjunction into a
+ * conjunction, so each collected term is a sound time bound for the whole
predicate.
+ */
+ private static Expression
extractNegatedTimePredicateFromDisjuncts(Expression predicate) {
+ if (predicate.getExpressionType().equals(ExpressionType.LOGIC_OR)) {
+ return combineConjuncts(
+ extractNegatedTimePredicateFromDisjuncts(
+ ((BinaryExpression) predicate).getLeftExpression()),
+ extractNegatedTimePredicateFromDisjuncts(
+ ((BinaryExpression) predicate).getRightExpression()),
+ null);
+ }
+ GlobalTimePredicateExtractionResult result =
extractGlobalTimePredicate(predicate);
+ if (result.getGlobalTimePredicate() != null && !result.hasValueFilter()) {
+ return ExpressionFactory.not(result.getGlobalTimePredicate());
+ }
+ return null;
+ }
+
public static final class GlobalTimePredicateExtractionResult {
private final Expression globalTimePredicate;
@@ -251,33 +282,63 @@ public class PredicateUtils {
*
* <p>eg: not(and(eq(), not(eq(y))) -> or(notEq(), eq(y))
*
- * <p>The returned predicate should have the same meaning as the original,
but without the use of
- * the not() operator.
+ * <p>The returned predicate has the same meaning as the original. A {@code
not()} is retained
+ * when the operand cannot be inverted safely, for example for
non-comparison expressions or
+ * ordered comparisons on FLOAT/DOUBLE values where NaN semantics differ.
*
* <p>See also {@link PredicateUtils#reversePredicate(Expression)}, which is
used to do the
* inversion.
*
* @param predicate the predicate to remove not() from
- * @return the predicate with all not() operators removed
+ * @return the predicate with safely invertible not() operators removed
*/
public static Expression predicateRemoveNot(Expression predicate) {
+ return predicateRemoveNot(predicate, null);
+ }
+
+ /**
+ * @param predicate the predicate to remove not() from
+ * @return the predicate with safely invertible not() operators removed. The
original instance is
+ * returned when no not() could be removed, so callers can detect that
nothing changed.
+ */
+ public static Expression predicateRemoveNot(
+ Expression predicate, Function<Expression, TSDataType> typeProvider) {
if (predicate.getExpressionType().equals(ExpressionType.LOGIC_AND)) {
- return ExpressionFactory.and(
- predicateRemoveNot(((BinaryExpression)
predicate).getLeftExpression()),
- predicateRemoveNot(((BinaryExpression)
predicate).getRightExpression()));
+ Expression leftExpression =
+ predicateRemoveNot(((BinaryExpression)
predicate).getLeftExpression(), typeProvider);
+ Expression rightExpression =
+ predicateRemoveNot(((BinaryExpression)
predicate).getRightExpression(), typeProvider);
+ if (leftExpression == ((BinaryExpression) predicate).getLeftExpression()
+ && rightExpression == ((BinaryExpression)
predicate).getRightExpression()) {
+ return predicate;
+ }
+ return ExpressionFactory.and(leftExpression, rightExpression);
} else if (predicate.getExpressionType().equals(ExpressionType.LOGIC_OR)) {
- return ExpressionFactory.or(
- predicateRemoveNot(((BinaryExpression)
predicate).getLeftExpression()),
- predicateRemoveNot(((BinaryExpression)
predicate).getRightExpression()));
+ Expression leftExpression =
+ predicateRemoveNot(((BinaryExpression)
predicate).getLeftExpression(), typeProvider);
+ Expression rightExpression =
+ predicateRemoveNot(((BinaryExpression)
predicate).getRightExpression(), typeProvider);
+ if (leftExpression == ((BinaryExpression) predicate).getLeftExpression()
+ && rightExpression == ((BinaryExpression)
predicate).getRightExpression()) {
+ return predicate;
+ }
+ return ExpressionFactory.or(leftExpression, rightExpression);
} else if (predicate.getExpressionType().equals(ExpressionType.LOGIC_NOT))
{
- return reversePredicate(((LogicNotExpression)
predicate).getExpression());
+ Expression innerExpression = ((LogicNotExpression)
predicate).getExpression();
+ Expression reversedExpression = reversePredicate(innerExpression,
typeProvider);
+ if
(reversedExpression.getExpressionType().equals(ExpressionType.LOGIC_NOT)
+ && ((LogicNotExpression) reversedExpression).getExpression() ==
innerExpression) {
+ // The operand cannot be inverted safely, so the not() is kept as is.
+ return predicate;
+ }
+ return reversedExpression;
}
return predicate;
}
/**
* Converts a predicate to its logical inverse. The returned predicate
should be equivalent to
- * not(p), but without the use of a not() operator.
+ * not(p). A {@code not()} is retained when the inverse cannot be
represented safely.
*
* <p>See also {@link PredicateUtils#predicateRemoveNot(Expression)}, which
can remove the use of
* all not() operators without inverting the overall predicate.
@@ -286,7 +347,12 @@ public class PredicateUtils {
* @return the predicate after reversing
*/
public static Expression reversePredicate(Expression predicate) {
- return new ReversePredicateVisitor().process(predicate, null);
+ return reversePredicate(predicate, null);
+ }
+
+ public static Expression reversePredicate(
+ Expression predicate, Function<Expression, TSDataType> typeProvider) {
+ return new ReversePredicateVisitor(typeProvider).process(predicate, null);
}
/**
@@ -418,7 +484,8 @@ public class PredicateUtils {
/**
* Extract the source symbol (full path for non-aligned path, device path
for aligned path) from
- * the given predicate. If the predicate contains multiple source symbols,
return null.
+ * the given predicate. If the predicate contains no source symbols or
multiple source symbols,
+ * return null.
*
* @param predicate given predicate
* @return the source symbol extracted from the given predicate
@@ -429,6 +496,9 @@ public class PredicateUtils {
sourceExpressions.stream()
.map(expression -> ((TimeSeriesOperand) expression).getPath())
.collect(Collectors.toSet());
+ if (sourcePaths.isEmpty()) {
+ return null;
+ }
Iterator<PartialPath> pathIterator = sourcePaths.iterator();
MeasurementPath firstPath = (MeasurementPath) pathIterator.next();
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAnalyze.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAnalyze.java
index 350cc5d090e..c3a3b421d93 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAnalyze.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAnalyze.java
@@ -272,12 +272,17 @@ public class TemplatedAnalyze {
analysis.getDeviceTemplate().getSchemaMap());
wherePredicate = PredicateUtils.simplifyPredicate(wherePredicate);
if (!wherePredicate.equals(ConstantOperand.TRUE)) {
- analysis.setWhereExpression(wherePredicate);
-
TSDataType outputType = analyzeExpressionForTemplatedQuery(analysis,
wherePredicate);
if (outputType != TSDataType.BOOLEAN) {
throw new SemanticException(String.format(WHERE_WRONG_TYPE_ERROR_MSG,
outputType));
}
+ Expression predicateWithoutNot =
+ PredicateUtils.predicateRemoveNot(wherePredicate, analysis::getType);
+ if (predicateWithoutNot != wherePredicate) {
+ predicateWithoutNot =
PredicateUtils.simplifyPredicate(predicateWithoutNot);
+ analyzeExpressionForTemplatedQuery(analysis, predicateWithoutNot);
+ }
+ analysis.setWhereExpression(predicateWithoutNot);
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/cartesian/BindSchemaForPredicateVisitor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/cartesian/BindSchemaForPredicateVisitor.java
index ae60ec381df..b2c1ff24c00 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/cartesian/BindSchemaForPredicateVisitor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/cartesian/BindSchemaForPredicateVisitor.java
@@ -33,6 +33,7 @@ import
org.apache.iotdb.db.queryengine.plan.expression.leaf.NullOperand;
import org.apache.iotdb.db.queryengine.plan.expression.leaf.TimeSeriesOperand;
import org.apache.iotdb.db.queryengine.plan.expression.leaf.TimestampOperand;
import
org.apache.iotdb.db.queryengine.plan.expression.multi.FunctionExpression;
+import org.apache.iotdb.db.queryengine.plan.expression.unary.UnaryExpression;
import org.apache.tsfile.external.commons.lang3.Validate;
@@ -47,6 +48,7 @@ import static
org.apache.iotdb.db.queryengine.plan.analyze.ExpressionUtils.carte
import static
org.apache.iotdb.db.queryengine.plan.analyze.ExpressionUtils.reconstructBinaryExpressionsWithMemoryCheck;
import static
org.apache.iotdb.db.queryengine.plan.analyze.ExpressionUtils.reconstructFunctionExpressionsWithMemoryCheck;
import static
org.apache.iotdb.db.queryengine.plan.analyze.ExpressionUtils.reconstructTimeSeriesOperandsWithMemoryCheck;
+import static
org.apache.iotdb.db.queryengine.plan.analyze.ExpressionUtils.reconstructUnaryExpressionsWithMemoryCheck;
import static
org.apache.iotdb.db.queryengine.plan.expression.visitor.cartesian.BindSchemaForExpressionVisitor.transformViewPath;
import static
org.apache.iotdb.db.utils.TypeInferenceUtils.bindTypeForBuiltinAggregationNonSeriesInputExpressions;
@@ -69,6 +71,14 @@ public class BindSchemaForPredicateVisitor
binaryExpression, leftExpressions, rightExpressions,
context.getQueryContext());
}
+ @Override
+ public List<Expression> visitUnaryExpression(UnaryExpression
unaryExpression, Context context) {
+ List<Expression> childExpressions =
+ process(unaryExpression.getExpression(), context.notRootClone());
+ return reconstructUnaryExpressionsWithMemoryCheck(
+ unaryExpression, childExpressions, context.getQueryContext());
+ }
+
@Override
public List<Expression> visitFunctionExpression(FunctionExpression
predicate, Context context) {
if (COUNT_TIME.equalsIgnoreCase(predicate.getFunctionName())) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/logical/PredicateCanPushDownToSourceChecker.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/logical/PredicateCanPushDownToSourceChecker.java
index 3a7c1ddbb37..dac1e59b5d9 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/logical/PredicateCanPushDownToSourceChecker.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/logical/PredicateCanPushDownToSourceChecker.java
@@ -21,6 +21,7 @@ package
org.apache.iotdb.db.queryengine.plan.expression.visitor.logical;
import org.apache.iotdb.db.queryengine.plan.expression.leaf.LeafOperand;
import org.apache.iotdb.db.queryengine.plan.expression.unary.IsNullExpression;
+import
org.apache.iotdb.db.queryengine.plan.expression.unary.LogicNotExpression;
public class PredicateCanPushDownToSourceChecker extends
LogicalAndVisitor<Void> {
@@ -37,4 +38,11 @@ public class PredicateCanPushDownToSourceChecker extends
LogicalAndVisitor<Void>
public Boolean visitLeafOperand(LeafOperand leafOperand, Void context) {
return Boolean.TRUE;
}
+
+ @Override
+ public Boolean visitLogicNotExpression(LogicNotExpression
logicNotExpression, Void context) {
+ // TsFile filters cannot represent NOT. Predicates that cannot be inverted
during analysis must
+ // be evaluated by FilterOperator instead of being pushed into a scan.
+ return Boolean.FALSE;
+ }
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/predicate/ReversePredicateVisitor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/predicate/ReversePredicateVisitor.java
index d0824863557..3659c13d86b 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/predicate/ReversePredicateVisitor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/predicate/ReversePredicateVisitor.java
@@ -19,9 +19,11 @@
package org.apache.iotdb.db.queryengine.plan.expression.visitor.predicate;
-import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
+import org.apache.iotdb.db.queryengine.plan.analyze.PredicateUtils;
import org.apache.iotdb.db.queryengine.plan.expression.Expression;
import org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory;
+import org.apache.iotdb.db.queryengine.plan.expression.ExpressionType;
+import org.apache.iotdb.db.queryengine.plan.expression.binary.BinaryExpression;
import
org.apache.iotdb.db.queryengine.plan.expression.binary.EqualToExpression;
import
org.apache.iotdb.db.queryengine.plan.expression.binary.GreaterEqualExpression;
import
org.apache.iotdb.db.queryengine.plan.expression.binary.GreaterThanExpression;
@@ -30,6 +32,7 @@ import
org.apache.iotdb.db.queryengine.plan.expression.binary.LessThanExpression
import
org.apache.iotdb.db.queryengine.plan.expression.binary.LogicAndExpression;
import
org.apache.iotdb.db.queryengine.plan.expression.binary.LogicOrExpression;
import
org.apache.iotdb.db.queryengine.plan.expression.binary.NonEqualExpression;
+import org.apache.iotdb.db.queryengine.plan.expression.leaf.ConstantOperand;
import
org.apache.iotdb.db.queryengine.plan.expression.other.GroupByTimeExpression;
import
org.apache.iotdb.db.queryengine.plan.expression.ternary.BetweenExpression;
import org.apache.iotdb.db.queryengine.plan.expression.unary.InExpression;
@@ -38,11 +41,25 @@ import
org.apache.iotdb.db.queryengine.plan.expression.unary.LikeExpression;
import
org.apache.iotdb.db.queryengine.plan.expression.unary.LogicNotExpression;
import org.apache.iotdb.db.queryengine.plan.expression.unary.RegularExpression;
+import org.apache.tsfile.enums.TSDataType;
+
+import java.util.function.Function;
+
public class ReversePredicateVisitor extends PredicateVisitor<Expression,
Void> {
+ private final Function<Expression, TSDataType> typeProvider;
+
+ public ReversePredicateVisitor() {
+ this(null);
+ }
+
+ public ReversePredicateVisitor(Function<Expression, TSDataType>
typeProvider) {
+ this.typeProvider = typeProvider;
+ }
+
@Override
public Expression visitExpression(Expression expression, Void context) {
- throw new UnsupportedOperationException();
+ return new LogicNotExpression(expression);
}
@Override
@@ -76,7 +93,7 @@ public class ReversePredicateVisitor extends
PredicateVisitor<Expression, Void>
@Override
public Expression visitLogicNotExpression(LogicNotExpression
logicNotExpression, Void context) {
- return logicNotExpression.getExpression();
+ return
PredicateUtils.predicateRemoveNot(logicNotExpression.getExpression(),
typeProvider);
}
@Override
@@ -108,6 +125,9 @@ public class ReversePredicateVisitor extends
PredicateVisitor<Expression, Void>
@Override
public Expression visitGreaterThanExpression(
GreaterThanExpression greaterThanExpression, Void context) {
+ if (!canSafelyInvertOrderedComparison(greaterThanExpression)) {
+ return new LogicNotExpression(greaterThanExpression);
+ }
return new LessEqualExpression(
greaterThanExpression.getLeftExpression(),
greaterThanExpression.getRightExpression());
}
@@ -115,12 +135,18 @@ public class ReversePredicateVisitor extends
PredicateVisitor<Expression, Void>
@Override
public Expression visitGreaterEqualExpression(
GreaterEqualExpression greaterEqualExpression, Void context) {
+ if (!canSafelyInvertOrderedComparison(greaterEqualExpression)) {
+ return new LogicNotExpression(greaterEqualExpression);
+ }
return new LessThanExpression(
greaterEqualExpression.getLeftExpression(),
greaterEqualExpression.getRightExpression());
}
@Override
public Expression visitLessThanExpression(LessThanExpression
lessThanExpression, Void context) {
+ if (!canSafelyInvertOrderedComparison(lessThanExpression)) {
+ return new LogicNotExpression(lessThanExpression);
+ }
return new GreaterEqualExpression(
lessThanExpression.getLeftExpression(),
lessThanExpression.getRightExpression());
}
@@ -128,12 +154,18 @@ public class ReversePredicateVisitor extends
PredicateVisitor<Expression, Void>
@Override
public Expression visitLessEqualExpression(
LessEqualExpression lessEqualExpression, Void context) {
+ if (!canSafelyInvertOrderedComparison(lessEqualExpression)) {
+ return new LogicNotExpression(lessEqualExpression);
+ }
return new GreaterThanExpression(
lessEqualExpression.getLeftExpression(),
lessEqualExpression.getRightExpression());
}
@Override
public Expression visitBetweenExpression(BetweenExpression
betweenExpression, Void context) {
+ if (!canSafelyInvertBetween(betweenExpression)) {
+ return new LogicNotExpression(betweenExpression);
+ }
return new BetweenExpression(
betweenExpression.getFirstExpression(),
betweenExpression.getSecondExpression(),
@@ -144,6 +176,48 @@ public class ReversePredicateVisitor extends
PredicateVisitor<Expression, Void>
@Override
public Expression visitGroupByTimeExpression(
GroupByTimeExpression groupByTimeExpression, Void context) {
- throw new
UnsupportedOperationException(DataNodeQueryMessages.GROUP_BY_TIME_CANNOT_BE_REVERSED);
+ return new LogicNotExpression(groupByTimeExpression);
+ }
+
+ private boolean canSafelyInvertOrderedComparison(Expression
comparisonExpression) {
+ BinaryExpression binaryExpression = (BinaryExpression)
comparisonExpression;
+ TSDataType leftType = getType(binaryExpression.getLeftExpression());
+ TSDataType rightType = getType(binaryExpression.getRightExpression());
+ return leftType != null
+ && rightType != null
+ && !isFloatingPoint(leftType)
+ && !isFloatingPoint(rightType);
+ }
+
+ private boolean canSafelyInvertBetween(BetweenExpression betweenExpression) {
+ TSDataType firstType = getType(betweenExpression.getFirstExpression());
+ TSDataType secondType = getType(betweenExpression.getSecondExpression());
+ TSDataType thirdType = getType(betweenExpression.getThirdExpression());
+ return firstType != null
+ && secondType != null
+ && thirdType != null
+ && !isFloatingPoint(firstType)
+ && !isFloatingPoint(secondType)
+ && !isFloatingPoint(thirdType);
+ }
+
+ private boolean isFloatingPoint(TSDataType type) {
+ return type == TSDataType.FLOAT || type == TSDataType.DOUBLE;
+ }
+
+ private TSDataType getType(Expression expression) {
+ if (typeProvider != null) {
+ TSDataType type = typeProvider.apply(expression);
+ if (type != null) {
+ return type;
+ }
+ }
+ if (expression.getExpressionType().equals(ExpressionType.TIMESTAMP)) {
+ return TSDataType.INT64;
+ }
+ if (expression.getExpressionType().equals(ExpressionType.CONSTANT)) {
+ return ((ConstantOperand) expression).getDataType();
+ }
+ return null;
}
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/expression/predicate/PredicateRemoveNotTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/expression/predicate/PredicateRemoveNotTest.java
index 6171cc48294..e20420411fa 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/expression/predicate/PredicateRemoveNotTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/expression/predicate/PredicateRemoveNotTest.java
@@ -19,17 +19,25 @@
package org.apache.iotdb.db.queryengine.expression.predicate;
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.db.queryengine.plan.expression.Expression;
+import org.apache.iotdb.db.queryengine.plan.expression.leaf.ConstantOperand;
+import org.apache.iotdb.db.queryengine.plan.expression.leaf.TimeSeriesOperand;
+
+import org.apache.tsfile.enums.TSDataType;
import org.junit.Assert;
import org.junit.Test;
import java.util.Arrays;
import java.util.LinkedHashSet;
+import java.util.function.Function;
import static
org.apache.iotdb.db.queryengine.plan.analyze.PredicateUtils.predicateRemoveNot;
import static
org.apache.iotdb.db.queryengine.plan.analyze.PredicateUtils.reversePredicate;
import static
org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.and;
import static
org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.between;
import static
org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.eq;
+import static
org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.function;
import static
org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.gt;
import static
org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.gte;
import static
org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.in;
@@ -42,9 +50,20 @@ import static
org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.
import static
org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.notIn;
import static
org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.or;
import static
org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.time;
+import static
org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.timeSeries;
public class PredicateRemoveNotTest {
+ private static Function<Expression, TSDataType> typeProvider(TSDataType
seriesType) {
+ return expression ->
+ expression instanceof TimeSeriesOperand
+ ? seriesType
+ : ((ConstantOperand) expression).getDataType();
+ }
+
+ private static final Function<Expression, TSDataType> DOUBLE_TYPE_PROVIDER =
+ typeProvider(TSDataType.DOUBLE);
+
@Test
public void testReverse() {
// test comparison
@@ -92,4 +111,59 @@ public class PredicateRemoveNotTest {
and(gt(time(), longValue(1)), lte(time(), longValue(1))),
predicateRemoveNot(and(not(lte(time(), longValue(1))), not(gt(time(),
longValue(1))))));
}
+
+ @Test
+ public void testRemoveNotKeepsUnsupportedNot() throws IllegalPathException {
+ Expression booleanSeries = timeSeries("root.sg.d.s1");
+ Expression predicate = not(booleanSeries);
+
+ // The kept not() must be the original instance so callers can detect that
nothing changed.
+ Assert.assertSame(predicate, predicateRemoveNot(predicate));
+ }
+
+ @Test
+ public void testRemoveNotKeepsFunctionPredicate() throws
IllegalPathException {
+ Expression predicate = not(function("udf", timeSeries("root.sg.d.s1")));
+
+ Assert.assertSame(predicate, predicateRemoveNot(predicate));
+ }
+
+ @Test
+ public void testRemoveNotKeepsBetweenOnDouble() throws IllegalPathException {
+ Expression series = timeSeries("root.sg.d.s1");
+ Expression predicate =
+ not(
+ between(
+ series,
+ new ConstantOperand(TSDataType.DOUBLE, "1.0"),
+ new ConstantOperand(TSDataType.DOUBLE, "5.0")));
+
+ Assert.assertSame(predicate, predicateRemoveNot(predicate,
DOUBLE_TYPE_PROVIDER));
+ }
+
+ @Test
+ public void testRemoveNotKeepsComparisonWithUnknownType() throws
IllegalPathException {
+ Expression predicate = not(gt(timeSeries("root.sg.d.s1"), longValue(1)));
+
+ // An operand of unknown type cannot be inverted safely either.
+ Assert.assertSame(predicate, predicateRemoveNot(predicate, expression ->
null));
+ }
+
+ @Test
+ public void testRemoveNotHandlesNestedNot() {
+ Expression predicate = not(not(not(and(gt(time(), longValue(1)),
lt(time(), longValue(10))))));
+ Expression expected = or(lte(time(), longValue(1)), gte(time(),
longValue(10)));
+
+ Assert.assertEquals(expected, predicateRemoveNot(predicate));
+ }
+
+ @Test
+ public void testRemoveNotUsesTypeToPreserveNaNSemantics() throws
IllegalPathException {
+ Expression series = timeSeries("root.sg.d.s1");
+ Expression predicate = not(gt(series, new
ConstantOperand(TSDataType.DOUBLE, "5.0")));
+
+ Expression result = predicateRemoveNot(predicate, DOUBLE_TYPE_PROVIDER);
+
+ Assert.assertSame(predicate, result);
+ }
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/expression/predicate/PredicateUtilsTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/expression/predicate/PredicateUtilsTest.java
index 8cd0d106047..de47e754689 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/expression/predicate/PredicateUtilsTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/expression/predicate/PredicateUtilsTest.java
@@ -28,6 +28,8 @@ import static
org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.
import static
org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.and;
import static
org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.gt;
import static
org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.intValue;
+import static
org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.longValue;
+import static
org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.time;
import static
org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.timeSeries;
import static
org.apache.iotdb.db.queryengine.plan.optimization.OptimizationTestUtil.schemaMap;
@@ -35,6 +37,7 @@ public class PredicateUtilsTest {
@Test
public void testExtractPredicateSourceSymbol() {
+ Assert.assertNull(PredicateUtils.extractPredicateSourceSymbol(gt(time(),
longValue(10))));
Assert.assertEquals(
schemaMap.get("root.sg.d1.s1"),
PredicateUtils.extractPredicateSourceSymbol(
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/PredicateUtilsTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/PredicateUtilsTest.java
index bc9d252d30e..1c7451e06c3 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/PredicateUtilsTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/PredicateUtilsTest.java
@@ -132,6 +132,69 @@ public class PredicateUtilsTest {
assertSame(timePredicate, predicate.getExpression());
}
+ @Test
+ public void testMixedNotOrExtractsNegatedTimePredicate() throws
IllegalPathException {
+ Expression timePredicate = gt(time(), longValue(1));
+ Expression valuePredicate = gt(timeSeries("root.sg.d.s1"), intValue("1"));
+ BinaryExpression child = or(timePredicate, valuePredicate);
+ LogicNotExpression predicate = not(child);
+
+ GlobalTimePredicateExtractionResult result =
+ PredicateUtils.extractGlobalTimePredicate(predicate);
+
+ // NOT(time > 1 OR s1 > 1) implies time <= 1
+ assertEquals(not(timePredicate), result.getGlobalTimePredicate());
+ assertSame(predicate, result.getResidualPredicate());
+ assertTrue(result.hasValueFilter());
+ assertSame(child, predicate.getExpression());
+ }
+
+ @Test
+ public void testNestedMixedNotOrExtractsAllNegatedTimePredicates() throws
IllegalPathException {
+ Expression firstTimePredicate = gt(time(), longValue(1));
+ Expression secondTimePredicate = lt(time(), longValue(10));
+ Expression valuePredicate = gt(timeSeries("root.sg.d.s1"), intValue("1"));
+ Expression predicate = not(or(or(firstTimePredicate, valuePredicate),
secondTimePredicate));
+
+ GlobalTimePredicateExtractionResult result =
+ PredicateUtils.extractGlobalTimePredicate(predicate);
+
+ assertEquals(
+ and(not(firstTimePredicate), not(secondTimePredicate)),
result.getGlobalTimePredicate());
+ assertSame(predicate, result.getResidualPredicate());
+ assertTrue(result.hasValueFilter());
+ }
+
+ @Test
+ public void testNegatedOrWithoutPureTimeDisjunctDoesNotExtractTimePredicate()
+ throws IllegalPathException {
+ Expression firstValuePredicate = gt(timeSeries("root.sg.d.s1"),
intValue("1"));
+ Expression secondValuePredicate = gt(timeSeries("root.sg.d.s2"),
intValue("2"));
+ LogicNotExpression predicate = not(or(firstValuePredicate,
secondValuePredicate));
+
+ GlobalTimePredicateExtractionResult result =
+ PredicateUtils.extractGlobalTimePredicate(predicate);
+
+ assertNull(result.getGlobalTimePredicate());
+ assertSame(predicate, result.getResidualPredicate());
+ assertTrue(result.hasValueFilter());
+ }
+
+ @Test
+ public void testDoubleNegatedOrDoesNotExtractTimePredicate() throws
IllegalPathException {
+ Expression timePredicate = gt(time(), longValue(1));
+ Expression valuePredicate = gt(timeSeries("root.sg.d.s1"), intValue("1"));
+ LogicNotExpression predicate = not(not(or(timePredicate, valuePredicate)));
+
+ GlobalTimePredicateExtractionResult result =
+ PredicateUtils.extractGlobalTimePredicate(predicate);
+
+ // NOT NOT (time > 1 OR s1 > 1) is a disjunction, which does not bound the
time range
+ assertNull(result.getGlobalTimePredicate());
+ assertSame(predicate, result.getResidualPredicate());
+ assertTrue(result.hasValueFilter());
+ }
+
@Test
public void testSimplifierUsesCopyOnWriteForUnaryExpression() throws
IllegalPathException {
Expression valuePredicate = gt(timeSeries("root.sg.d.s1"), intValue("1"));
@@ -169,4 +232,11 @@ public class PredicateUtilsTest {
assertSame(predicate, PredicateUtils.simplifyPredicate(predicate));
}
+
+ @Test
+ public void testLogicNotCannotPushDownToSource() throws IllegalPathException
{
+ Expression predicate = not(gt(timeSeries("root.sg.d.s1"), intValue("1")));
+
+ assertFalse(PredicateUtils.predicateCanPushDownToSource(predicate));
+ }
}
