This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new edc684e61e0 Fix the usage of idAndAttributeIndexMap in TableScanNode
edc684e61e0 is described below
commit edc684e61e0fa6879fc3df0490f7d1295d00cb10
Author: Beyyes <[email protected]>
AuthorDate: Fri Aug 16 09:29:22 2024 +0800
Fix the usage of idAndAttributeIndexMap in TableScanNode
---
...ava => IoTDBMultiIDsWithAttributesTableIT.java} | 110 ++++++++++++++++++++-
.../plan/relational/planner/RelationPlanner.java | 4 +-
.../relational/planner/node/TableScanNode.java | 4 -
.../optimizations/PushPredicateIntoTableScan.java | 44 +++++----
.../plan/relational/analyzer/AnalyzerTest.java | 16 ++-
.../plan/relational/analyzer/SortTest.java | 9 +-
6 files changed, 142 insertions(+), 45 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/orderBy/IoTDBStreamSortIT.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/orderBy/IoTDBMultiIDsWithAttributesTableIT.java
similarity index 74%
rename from
integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/orderBy/IoTDBStreamSortIT.java
rename to
integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/orderBy/IoTDBMultiIDsWithAttributesTableIT.java
index ddcbb3ecabc..c2528c32193 100644
---
a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/orderBy/IoTDBStreamSortIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/orderBy/IoTDBMultiIDsWithAttributesTableIT.java
@@ -36,13 +36,12 @@ import java.sql.Statement;
import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest;
import static org.junit.Assert.fail;
+/** In this IT, table has more than one IDs and Attributes. */
@RunWith(IoTDBTestRunner.class)
@Category({LocalStandaloneIT.class, ClusterIT.class})
-public class IoTDBStreamSortIT {
+public class IoTDBMultiIDsWithAttributesTableIT {
private static final String DATABASE_NAME = "db";
- // the data can be viewed in
- //
https://docs.google.com/spreadsheets/d/1OWA1bKraArCwWVnuTjuhJ5yLG0PFLdD78gD6FjquepI/edit#gid=0
private static final String[] sql1 =
new String[] {
"CREATE DATABASE db",
@@ -113,6 +112,111 @@ public class IoTDBStreamSortIT {
}
}
+ @Test
+ public void onlyQueryAttribute2Test() {
+ String[] expectedHeader =
+ new String[] {
+ "time", "device", "attr2", "num", "str",
+ };
+ String[] retArray =
+ new String[] {
+ "1970-01-01T00:00:00.000Z,d1,d,3,coconut,",
+ "1970-01-01T00:00:00.000Z,d2,c,3,coconut,",
+ "1970-01-01T00:00:00.020Z,d1,zz,2,pineapple,",
+ "1970-01-01T00:00:00.020Z,d2,null,2,pineapple,",
+ "1970-01-01T00:00:00.040Z,d1,a,1,apricot,",
+ "1970-01-01T00:00:00.040Z,d2,null,1,apricot,",
+ "1970-01-01T00:00:00.080Z,d1,null,9,apple,",
+ "1970-01-01T00:00:00.080Z,d2,null,9,apple,",
+ "1970-01-01T00:00:00.100Z,d1,null,8,papaya,",
+ "1970-01-01T00:00:00.100Z,d2,null,8,papaya,",
+ "1971-01-01T00:00:00.000Z,d1,d,6,banana,",
+ "1971-01-01T00:00:00.000Z,d2,c,6,banana,",
+ "1971-01-01T00:00:00.100Z,d1,zz,10,pumelo,",
+ "1971-01-01T00:00:00.100Z,d2,null,10,pumelo,",
+ "1971-01-01T00:00:00.500Z,d1,a,4,peach,",
+ "1971-01-01T00:00:00.500Z,d2,null,4,peach,",
+ "1971-01-01T00:00:01.000Z,d1,null,5,orange,",
+ "1971-01-01T00:00:01.000Z,d2,null,5,orange,",
+ "1971-01-01T00:00:10.000Z,d1,null,7,lemon,",
+ "1971-01-01T00:00:10.000Z,d2,null,7,lemon,",
+ "1971-01-01T00:01:40.000Z,d1,d,11,pitaya,",
+ "1971-01-01T00:01:40.000Z,d2,c,11,pitaya,",
+ "1971-04-26T17:46:40.000Z,d1,zz,12,strawberry,",
+ "1971-04-26T17:46:40.000Z,d2,null,12,strawberry,",
+ "1971-04-26T17:46:40.020Z,d1,a,14,cherry,",
+ "1971-04-26T17:46:40.020Z,d2,null,14,cherry,",
+ "1971-04-26T18:01:40.000Z,d1,null,13,lychee,",
+ "1971-04-26T18:01:40.000Z,d2,null,13,lychee,",
+ "1971-08-20T11:33:20.000Z,d1,null,15,watermelon,",
+ "1971-08-20T11:33:20.000Z,d2,null,15,watermelon,",
+ };
+ tableResultSetEqualTest(
+ "select time, device, attr2, num, str from table0 order by time,
device",
+ expectedHeader,
+ retArray,
+ DATABASE_NAME);
+
+ expectedHeader =
+ new String[] {
+ "time", "num", "str",
+ };
+ retArray =
+ new String[] {
+ "1970-01-01T00:00:00.040Z,1,apricot,",
+ "1971-01-01T00:00:00.500Z,4,peach,",
+ "1971-04-26T17:46:40.020Z,14,cherry,",
+ "1970-01-01T00:00:00.000Z,3,coconut,",
+ "1971-01-01T00:00:00.000Z,6,banana,",
+ "1971-01-01T00:01:40.000Z,11,pitaya,",
+ "1970-01-01T00:00:00.000Z,3,coconut,",
+ "1971-01-01T00:00:00.000Z,6,banana,",
+ "1971-01-01T00:01:40.000Z,11,pitaya,",
+ "1970-01-01T00:00:00.020Z,2,pineapple,",
+ "1971-01-01T00:00:00.100Z,10,pumelo,",
+ "1971-04-26T17:46:40.000Z,12,strawberry,",
+ "1970-01-01T00:00:00.100Z,8,papaya,",
+ "1970-01-01T00:00:00.080Z,9,apple,",
+ "1971-01-01T00:00:10.000Z,7,lemon,",
+ "1971-01-01T00:00:01.000Z,5,orange,",
+ "1971-08-20T11:33:20.000Z,15,watermelon,",
+ "1971-04-26T18:01:40.000Z,13,lychee,",
+ "1970-01-01T00:00:00.020Z,2,pineapple,",
+ "1970-01-01T00:00:00.040Z,1,apricot,",
+ "1971-01-01T00:00:00.100Z,10,pumelo,",
+ "1971-01-01T00:00:00.500Z,4,peach,",
+ "1971-04-26T17:46:40.000Z,12,strawberry,",
+ "1971-04-26T17:46:40.020Z,14,cherry,",
+ "1970-01-01T00:00:00.080Z,9,apple,",
+ "1970-01-01T00:00:00.100Z,8,papaya,",
+ "1971-01-01T00:00:01.000Z,5,orange,",
+ "1971-01-01T00:00:10.000Z,7,lemon,",
+ "1971-04-26T18:01:40.000Z,13,lychee,",
+ "1971-08-20T11:33:20.000Z,15,watermelon,",
+ };
+ tableResultSetEqualTest(
+ "select time, num, str from table0 order by attr2, device",
+ expectedHeader,
+ retArray,
+ DATABASE_NAME);
+
+ expectedHeader =
+ new String[] {
+ "time", "device", "num", "str",
+ };
+ retArray =
+ new String[] {
+ "1970-01-01T00:00:00.000Z,d1,3,coconut,",
+ "1971-01-01T00:00:00.000Z,d1,6,banana,",
+ "1971-01-01T00:01:40.000Z,d1,11,pitaya,",
+ };
+ tableResultSetEqualTest(
+ "select time, device, num, str from table0 where attr2 = 'd' order by
time,device",
+ expectedHeader,
+ retArray,
+ DATABASE_NAME);
+ }
+
@Test
public void sortAttributeTest() {
String[] expectedHeader =
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
index 9bc61233add..cdfb2023ae9 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
@@ -131,7 +131,7 @@ public class RelationPlanner extends
AstVisitor<RelationPlan, Void> {
// on the basis of that the order of fields is same with the column
category order of segments
// in DeviceEntry
Map<Symbol, Integer> idAndAttributeIndexMap = new HashMap<>();
- int idIndex = 0, attributeIndex = 0;
+ int idIndex = 0;
for (Field field : fields) {
Symbol symbol = symbolAllocator.newSymbol(field);
outputSymbolsBuilder.add(symbol);
@@ -142,8 +142,6 @@ public class RelationPlanner extends
AstVisitor<RelationPlan, Void> {
field.getName().orElse(null), field.getType(), field.isHidden(),
category));
if (category == TsTableColumnCategory.ID) {
idAndAttributeIndexMap.put(symbol, idIndex++);
- } else if (category == TsTableColumnCategory.ATTRIBUTE) {
- idAndAttributeIndexMap.put(symbol, attributeIndex++);
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java
index 770ab775c05..0e575f5dbd5 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java
@@ -371,10 +371,6 @@ public class TableScanNode extends SourceNode {
this.outputSymbols = outputSymbols;
}
- public void setAssignments(Map<Symbol, ColumnSchema> assignments) {
- this.assignments = assignments;
- }
-
public Map<Symbol, ColumnSchema> getAssignments() {
return this.assignments;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java
index 0fc783013ce..6e738a1c4aa 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java
@@ -30,6 +30,7 @@ import
org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis;
import
org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.ConvertPredicateToTimeFilterVisitor;
import
org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicateCombineIntoTableScanChecker;
import
org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoMetadataChecker;
+import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata;
import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol;
@@ -45,16 +46,17 @@ import org.apache.tsfile.utils.Pair;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
-import java.util.Locale;
+import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import static java.util.Objects.requireNonNull;
import static
org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory.ATTRIBUTE;
import static
org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory.MEASUREMENT;
+import static
org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory.TIME;
import static
org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.getTimePartitionSlotList;
-import static
org.apache.iotdb.db.queryengine.plan.expression.leaf.TimestampOperand.TIMESTAMP_EXPRESSION_STRING;
import static
org.apache.iotdb.db.queryengine.plan.relational.planner.ir.GlobalTimePredicateExtractVisitor.extractGlobalTimeFilter;
/**
@@ -199,18 +201,18 @@ public class PushPredicateIntoTableScan implements
PlanOptimizer {
}
private SplitExpression splitPredicate(TableScanNode node) {
-
- Set<String> idOrAttributeColumnNames =
- node.getIdAndAttributeIndexMap().keySet().stream()
- .map(Symbol::getName)
- .collect(Collectors.toSet());
-
- Set<String> measurementColumnNames =
- node.getAssignments().entrySet().stream()
- .filter(e ->
MEASUREMENT.equals(e.getValue().getColumnCategory()))
- .map(e -> e.getKey().getName())
- .collect(Collectors.toSet());
-
measurementColumnNames.add(TIMESTAMP_EXPRESSION_STRING.toLowerCase(Locale.ENGLISH));
+ Set<String> idOrAttributeColumnNames = new
HashSet<>(node.getAssignments().size());
+ Set<String> measurementColumnNames = new
HashSet<>(node.getAssignments().size());
+ for (Map.Entry<Symbol, ColumnSchema> entry :
node.getAssignments().entrySet()) {
+ Symbol columnSymbol = entry.getKey();
+ ColumnSchema columnSchema = entry.getValue();
+ if (MEASUREMENT.equals(columnSchema.getColumnCategory())
+ || TIME.equals(columnSchema.getColumnCategory())) {
+ measurementColumnNames.add(columnSymbol.getName());
+ } else {
+ idOrAttributeColumnNames.add(columnSymbol.getName());
+ }
+ }
List<Expression> metadataExpressions = new ArrayList<>();
List<Expression> expressionsCanPushDown = new ArrayList<>();
@@ -264,12 +266,14 @@ public class PushPredicateIntoTableScan implements
PlanOptimizer {
/** Get deviceEntries and DataPartition used in TableScan. */
private void tableMetadataIndexScan(TableScanNode node, List<Expression>
metadataExpressions) {
- List<String> attributeColumns =
- node.getOutputSymbols().stream()
- .filter(
- symbol ->
ATTRIBUTE.equals(node.getAssignments().get(symbol).getColumnCategory()))
- .map(Symbol::getName)
- .collect(Collectors.toList());
+ List<String> attributeColumns = new ArrayList<>();
+ int attributeIndex = 0;
+ for (Symbol columnName : node.getAssignments().keySet()) {
+ if
(ATTRIBUTE.equals(node.getAssignments().get(columnName).getColumnCategory())) {
+ attributeColumns.add(columnName.getName());
+ node.getIdAndAttributeIndexMap().put(columnName, attributeIndex++);
+ }
+ }
List<DeviceEntry> deviceEntries =
metadata.indexScan(
node.getQualifiedObjectName(), metadataExpressions,
attributeColumns, queryContext);
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java
index 34824d537bf..250208e6fd5 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java
@@ -267,7 +267,7 @@ public class AnalyzerTest {
@Test
public void singleTableWithFilterTest2() {
// measurement value filter, which can be pushed down to TableScanNode
- sql = "SELECT tag1, attr1, s2 FROM table1 where s1 > 1";
+ sql = "SELECT tag1, attr2, s2 FROM table1 where s1 > 1";
context = new MPPQueryContext(sql, queryId, sessionInfo, null, null);
actualAnalysis = analyzeSQL(sql, metadata, context);
assertNotNull(actualAnalysis);
@@ -282,16 +282,12 @@ public class AnalyzerTest {
assertEquals("(\"s1\" > 1)",
tableScanNode.getPushDownPredicate().toString());
assertFalse(tableScanNode.getTimePredicate().isPresent());
assertTrue(
- Stream.of(
- Symbol.of("tag1"),
- Symbol.of("tag2"),
- Symbol.of("tag3"),
- Symbol.of("attr1"),
- Symbol.of("attr2"))
+ Stream.of(Symbol.of("tag1"), Symbol.of("tag2"), Symbol.of("tag3"),
Symbol.of("attr2"))
.allMatch(tableScanNode.getIdAndAttributeIndexMap()::containsKey));
- assertEquals(Arrays.asList("tag1", "attr1", "s2"),
tableScanNode.getOutputColumnNames());
+ assertEquals(0, (int)
tableScanNode.getIdAndAttributeIndexMap().get(Symbol.of("attr2")));
+ assertEquals(Arrays.asList("tag1", "attr2", "s2"),
tableScanNode.getOutputColumnNames());
assertEquals(
- ImmutableSet.of("tag1", "attr1", "s1", "s2"),
+ ImmutableSet.of("tag1", "attr2", "s1", "s2"),
tableScanNode.getAssignments().keySet().stream()
.map(Symbol::toString)
.collect(Collectors.toSet()));
@@ -581,7 +577,7 @@ public class AnalyzerTest {
assertTrue(rootNode.getChildren().get(0) instanceof TableScanNode);
tableScanNode = (TableScanNode) rootNode.getChildren().get(0);
assertEquals(Arrays.asList("tag2", "attr2", "s2"),
tableScanNode.getOutputColumnNames());
- assertEquals(5, tableScanNode.getIdAndAttributeIndexMap().size());
+ assertEquals(4, tableScanNode.getIdAndAttributeIndexMap().size());
}
@Test
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/SortTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/SortTest.java
index 238bacc8180..e00e6bedd79 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/SortTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/SortTest.java
@@ -119,7 +119,7 @@ public class SortTest {
assertEquals("testdb.table1",
tableScanNode.getQualifiedObjectName().toString());
assertEquals(8, tableScanNode.getAssignments().size());
assertEquals(6, tableScanNode.getDeviceEntries().size());
- assertEquals(5, tableScanNode.getIdAndAttributeIndexMap().size());
+ assertEquals(4, tableScanNode.getIdAndAttributeIndexMap().size());
// TODO change scan order in logical plan
assertEquals(ASC, tableScanNode.getScanOrder());
assertEquals(0, tableScanNode.getPushDownLimit());
@@ -368,7 +368,7 @@ public class SortTest {
assertEquals("testdb.table1",
tableScanNode.getQualifiedObjectName().toString());
assertEquals(8, tableScanNode.getAssignments().size());
assertEquals(6, tableScanNode.getDeviceEntries().size());
- assertEquals(5, tableScanNode.getIdAndAttributeIndexMap().size());
+ assertEquals(4, tableScanNode.getIdAndAttributeIndexMap().size());
assertEquals(ASC, tableScanNode.getScanOrder());
assertEquals(0, tableScanNode.getPushDownLimit());
assertEquals(0, tableScanNode.getPushDownOffset());
@@ -448,7 +448,7 @@ public class SortTest {
assertEquals("testdb.table1",
tableScanNode.getQualifiedObjectName().toString());
assertEquals(8, tableScanNode.getAssignments().size());
assertEquals(6, tableScanNode.getDeviceEntries().size());
- assertEquals(5, tableScanNode.getIdAndAttributeIndexMap().size());
+ assertEquals(4, tableScanNode.getIdAndAttributeIndexMap().size());
// DistributePlan: optimize
//
`Output-Offset-Limit-Project-MergeSort-StreamSort-Project-Filter-TableScan`
@@ -614,7 +614,6 @@ public class SortTest {
assertEquals("testdb.table1",
tableScanNode.getQualifiedObjectName().toString());
assertEquals(8, tableScanNode.getAssignments().size());
assertEquals(6, tableScanNode.getDeviceEntries().size());
- assertEquals(5, tableScanNode.getIdAndAttributeIndexMap().size());
assertTrue(
tableScanNode.getPushDownLimit() == expectedPushDownLimit
&& tableScanNode.getPushDownOffset() == expectedPushDownOffset);
@@ -683,7 +682,7 @@ public class SortTest {
assertEquals("testdb.table1",
tableScanNode.getQualifiedObjectName().toString());
assertEquals(8, tableScanNode.getAssignments().size());
assertEquals(6, tableScanNode.getDeviceEntries().size());
- assertEquals(5, tableScanNode.getIdAndAttributeIndexMap().size());
+ assertEquals(4, tableScanNode.getIdAndAttributeIndexMap().size());
assertEquals(expectedPushDownLimit, tableScanNode.getPushDownLimit());
assertEquals(expectedPushDownOffset, tableScanNode.getPushDownOffset());
assertEquals(isPushLimitToEachDevice,
tableScanNode.isPushLimitToEachDevice());
