This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new a966f82d953 Fix duplicated time in result set bug
a966f82d953 is described below
commit a966f82d953f429e2146e8a19faca2e4dcded757
Author: Jackie Tien <[email protected]>
AuthorDate: Thu Feb 6 14:07:57 2025 +0800
Fix duplicated time in result set bug
---
.../relational/it/db/it/IoTDBDeletionTableIT.java | 3 -
.../it/query/recent/IoTDBNullValueIT.java | 75 ++++++++++++++++++++++
.../execution/operator/source/FileLoaderUtils.java | 4 +-
.../schemaregion/utils/ResourceByPathUtils.java | 31 ++++++---
.../impl/ReadChunkCompactionPerformer.java | 11 ++--
.../execute/utils/MultiTsFileDeviceIterator.java | 14 ++--
...BatchedFastAlignedSeriesCompactionExecutor.java | 26 ++++----
...edReadChunkAlignedSeriesCompactionExecutor.java | 40 ++++++------
.../utils/AlignedSeriesBatchCompactionUtils.java | 28 ++++----
.../fast/FastAlignedSeriesCompactionExecutor.java | 23 ++++---
.../executor/fast/SeriesCompactionExecutor.java | 10 +--
.../ReadChunkAlignedSeriesCompactionExecutor.java | 31 +++++----
.../utils/writer/FastCrossCompactionWriter.java | 10 +--
.../utils/writer/FastInnerCompactionWriter.java | 10 +--
.../memtable/AlignedReadOnlyMemChunk.java | 10 ++-
.../dataregion/memtable/TsFileProcessor.java | 7 +-
.../read/reader/chunk/DiskAlignedChunkLoader.java | 5 +-
.../read/reader/chunk/MemAlignedChunkReader.java | 4 +-
.../read/reader/chunk/MemAlignedPageReader.java | 6 +-
.../metadata/DiskAlignedChunkMetadataLoader.java | 4 +-
.../apache/iotdb/db/utils/ErrorHandlingUtils.java | 6 +-
.../apache/iotdb/db/utils/ModificationUtils.java | 9 +--
...InconsistentCompressionTypeAndEncodingTest.java | 6 +-
.../FastInnerCompactionPerformerTest.java | 6 +-
.../repair/RepairUnsortedFileCompactionTest.java | 8 +--
.../compaction/utils/BatchCompactionUtilsTest.java | 39 +++++------
pom.xml | 2 +-
27 files changed, 271 insertions(+), 157 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java
index 95eee41d007..2449a35653c 100644
---
a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java
@@ -24,8 +24,6 @@ import org.apache.iotdb.isession.ITableSession;
import org.apache.iotdb.isession.SessionDataSet;
import org.apache.iotdb.it.env.EnvFactory;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
-import org.apache.iotdb.itbase.category.ClusterIT;
-import org.apache.iotdb.itbase.category.LocalStandaloneIT;
import org.apache.iotdb.itbase.category.ManualIT;
import org.apache.iotdb.itbase.category.TableClusterIT;
import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
@@ -1647,7 +1645,6 @@ public class IoTDBDeletionTableIT {
}
@Test
- @Category({LocalStandaloneIT.class, ClusterIT.class})
public void deleteTableOfTheSameNameTest()
throws IoTDBConnectionException, StatementExecutionException {
int testNum = 24;
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullValueIT.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullValueIT.java
new file mode 100644
index 00000000000..531d88101ba
--- /dev/null
+++
b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullValueIT.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.relational.it.query.recent;
+
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.TableClusterIT;
+import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData;
+import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
+public class IoTDBNullValueIT {
+ private static final String DATABASE_NAME = "test";
+
+ private static final String[] createSqls =
+ new String[] {
+ "CREATE DATABASE " + DATABASE_NAME,
+ "USE " + DATABASE_NAME,
+ "create table table1(id1 tag, s1 string)",
+ "insert into table1 values(0, 'd1', null), (1,'d1', 1)",
+ "flush",
+ "insert into table1 values(0, 'd1', 0)",
+ "flush"
+ };
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+
EnvFactory.getEnv().getConfig().getCommonConfig().setEnableCrossSpaceCompaction(false);
+ EnvFactory.getEnv().initClusterEnvironment();
+ prepareTableData(createSqls);
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ @Test
+ public void nullTest() {
+
+ // case 1: all without time filter using previous fill without timeDuration
+ String[] expectedHeader = new String[] {"time", "id1", "s1"};
+ String[] retArray =
+ new String[] {
+ "1970-01-01T00:00:00.000Z,d1,0,", "1970-01-01T00:00:00.001Z,d1,1,",
+ };
+ tableResultSetEqualTest("select * from table1", expectedHeader, retArray,
DATABASE_NAME);
+ }
+}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java
index cd00aa5e419..213e4990a5f 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java
@@ -41,7 +41,7 @@ import
org.apache.tsfile.file.metadata.AlignedTimeSeriesMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.ITimeSeriesMetadata;
-import org.apache.tsfile.file.metadata.TableDeviceMetadata;
+import org.apache.tsfile.file.metadata.TableDeviceTimeSeriesMetadata;
import org.apache.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.tsfile.read.controller.IChunkLoader;
import org.apache.tsfile.read.filter.basic.Filter;
@@ -395,7 +395,7 @@ public class FileLoaderUtils {
AbstractAlignedTimeSeriesMetadata alignedTimeSeriesMetadata =
ignoreAllNullRows
? new AlignedTimeSeriesMetadata(timeColumnMetadata,
valueColumnMetadataList)
- : new TableDeviceMetadata(timeColumnMetadata,
valueColumnMetadataList);
+ : new TableDeviceTimeSeriesMetadata(timeColumnMetadata,
valueColumnMetadataList);
alignedTimeSeriesMetadata.setChunkMetadataLoader(
new DiskAlignedChunkMetadataLoader(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java
index dbd32b47276..12393fb2a20 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java
@@ -37,12 +37,16 @@ import org.apache.iotdb.db.utils.ModificationUtils;
import org.apache.iotdb.db.utils.datastructure.TVList;
import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
+import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata;
import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
import org.apache.tsfile.file.metadata.AlignedTimeSeriesMetadata;
import org.apache.tsfile.file.metadata.ChunkMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.ITimeSeriesMetadata;
+import org.apache.tsfile.file.metadata.TableDeviceChunkMetadata;
+import org.apache.tsfile.file.metadata.TableDeviceTimeSeriesMetadata;
import org.apache.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
@@ -108,7 +112,7 @@ class AlignedResourceByPathUtils extends
ResourceByPathUtils {
* have chunkMetadata, but query will use these, so we need to generate it
for them.
*/
@Override
- public AlignedTimeSeriesMetadata generateTimeSeriesMetadata(
+ public AbstractAlignedTimeSeriesMetadata generateTimeSeriesMetadata(
List<ReadOnlyMemChunk> readOnlyMemChunk, List<IChunkMetadata>
chunkMetadataList) {
TimeseriesMetadata timeTimeSeriesMetadata = new TimeseriesMetadata();
timeTimeSeriesMetadata.setDataSizeOfChunkMetaDataList(-1);
@@ -131,8 +135,11 @@ class AlignedResourceByPathUtils extends
ResourceByPathUtils {
boolean[] exist = new boolean[alignedFullPath.getSchemaList().size()];
boolean modified = false;
+ boolean isTable = false;
for (IChunkMetadata chunkMetadata : chunkMetadataList) {
- AlignedChunkMetadata alignedChunkMetadata = (AlignedChunkMetadata)
chunkMetadata;
+ AbstractAlignedChunkMetadata alignedChunkMetadata =
+ (AbstractAlignedChunkMetadata) chunkMetadata;
+ isTable = isTable || (alignedChunkMetadata instanceof
TableDeviceChunkMetadata);
modified = (modified || alignedChunkMetadata.isModified());
timeStatistics.mergeStatistics(alignedChunkMetadata.getTimeChunkMetadata().getStatistics());
for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) {
@@ -149,8 +156,9 @@ class AlignedResourceByPathUtils extends
ResourceByPathUtils {
for (ReadOnlyMemChunk memChunk : readOnlyMemChunk) {
if (!memChunk.isEmpty()) {
- AlignedChunkMetadata alignedChunkMetadata =
- (AlignedChunkMetadata) memChunk.getChunkMetaData();
+ AbstractAlignedChunkMetadata alignedChunkMetadata =
+ (AbstractAlignedChunkMetadata) memChunk.getChunkMetaData();
+ isTable = isTable || (alignedChunkMetadata instanceof
TableDeviceChunkMetadata);
timeStatistics.mergeStatistics(alignedChunkMetadata.getTimeChunkMetadata().getStatistics());
for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) {
if (alignedChunkMetadata.getValueChunkMetadataList().get(i) != null)
{
@@ -174,7 +182,9 @@ class AlignedResourceByPathUtils extends
ResourceByPathUtils {
}
}
- return new AlignedTimeSeriesMetadata(timeTimeSeriesMetadata,
valueTimeSeriesMetadataList);
+ return isTable
+ ? new TableDeviceTimeSeriesMetadata(timeTimeSeriesMetadata,
valueTimeSeriesMetadataList)
+ : new AlignedTimeSeriesMetadata(timeTimeSeriesMetadata,
valueTimeSeriesMetadataList);
}
@Override
@@ -268,7 +278,7 @@ class AlignedResourceByPathUtils extends
ResourceByPathUtils {
QueryContext context,
long timeLowerBound) {
- List<AlignedChunkMetadata> chunkMetadataList = new ArrayList<>();
+ List<AbstractAlignedChunkMetadata> chunkMetadataList = new ArrayList<>();
List<ChunkMetadata> timeChunkMetadataList =
writer.getVisibleMetadataList(
alignedFullPath.getDeviceId(), AlignedFullPath.VECTOR_PLACEHOLDER,
TSDataType.VECTOR);
@@ -285,7 +295,10 @@ class AlignedResourceByPathUtils extends
ResourceByPathUtils {
// only need time column
if (alignedFullPath.getMeasurementList().isEmpty()) {
chunkMetadataList.add(
- new AlignedChunkMetadata(timeChunkMetadataList.get(i),
Collections.emptyList()));
+ context.isIgnoreAllNullRows()
+ ? new AlignedChunkMetadata(timeChunkMetadataList.get(i),
Collections.emptyList())
+ : new TableDeviceChunkMetadata(
+ timeChunkMetadataList.get(i), Collections.emptyList()));
} else {
List<IChunkMetadata> valueChunkMetadata = new ArrayList<>();
// if all the sub sensors doesn't exist, it will be false
@@ -298,7 +311,9 @@ class AlignedResourceByPathUtils extends
ResourceByPathUtils {
}
if (!context.isIgnoreAllNullRows() || exits) {
chunkMetadataList.add(
- new AlignedChunkMetadata(timeChunkMetadataList.get(i),
valueChunkMetadata));
+ context.isIgnoreAllNullRows()
+ ? new AlignedChunkMetadata(timeChunkMetadataList.get(i),
valueChunkMetadata)
+ : new TableDeviceChunkMetadata(timeChunkMetadataList.get(i),
valueChunkMetadata));
}
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java
index ca592939976..c86c02e51b2 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java
@@ -36,7 +36,7 @@ import
org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.exception.write.PageException;
-import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
+import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
import org.apache.tsfile.file.metadata.ChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.read.TsFileSequenceReader;
@@ -179,8 +179,9 @@ public class ReadChunkCompactionPerformer implements
ISeqCompactionPerformer {
MultiTsFileDeviceIterator deviceIterator)
throws IOException, InterruptedException, IllegalPathException,
PageException {
checkThreadInterrupted();
- LinkedList<Pair<TsFileSequenceReader, List<AlignedChunkMetadata>>>
readerAndChunkMetadataList =
- deviceIterator.getReaderAndChunkMetadataForCurrentAlignedSeries();
+ LinkedList<Pair<TsFileSequenceReader, List<AbstractAlignedChunkMetadata>>>
+ readerAndChunkMetadataList =
+ deviceIterator.getReaderAndChunkMetadataForCurrentAlignedSeries();
if (!checkAlignedSeriesExists(readerAndChunkMetadataList)) {
return;
}
@@ -212,9 +213,9 @@ public class ReadChunkCompactionPerformer implements
ISeqCompactionPerformer {
}
private boolean checkAlignedSeriesExists(
- LinkedList<Pair<TsFileSequenceReader, List<AlignedChunkMetadata>>>
+ LinkedList<Pair<TsFileSequenceReader,
List<AbstractAlignedChunkMetadata>>>
readerAndChunkMetadataList) {
- for (Pair<TsFileSequenceReader, List<AlignedChunkMetadata>> readerListPair
:
+ for (Pair<TsFileSequenceReader, List<AbstractAlignedChunkMetadata>>
readerListPair :
readerAndChunkMetadataList) {
if (!readerListPair.right.isEmpty()) {
return true;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
index 32a1593c4a6..1c8aed4ca62 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
@@ -34,7 +34,7 @@ import
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.utils.ModificationUtils;
import org.apache.tsfile.enums.TSDataType;
-import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
+import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
import org.apache.tsfile.file.metadata.ChunkMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
@@ -384,14 +384,14 @@ public class MultiTsFileDeviceIterator implements
AutoCloseable {
* @throws IOException if io errors occurred
*/
@SuppressWarnings({"squid:S1319", "squid:S135"})
- public LinkedList<Pair<TsFileSequenceReader, List<AlignedChunkMetadata>>>
+ public LinkedList<Pair<TsFileSequenceReader,
List<AbstractAlignedChunkMetadata>>>
getReaderAndChunkMetadataForCurrentAlignedSeries() throws IOException,
IllegalPathException {
if (currentDevice == null || !currentDevice.right) {
return new LinkedList<>();
}
- LinkedList<Pair<TsFileSequenceReader, List<AlignedChunkMetadata>>>
readerAndChunkMetadataList =
- new LinkedList<>();
+ LinkedList<Pair<TsFileSequenceReader, List<AbstractAlignedChunkMetadata>>>
+ readerAndChunkMetadataList = new LinkedList<>();
for (TsFileResource tsFileResource : tsFileResourcesSortedByAsc) {
if (!deviceIteratorMap.containsKey(tsFileResource)) {
continue;
@@ -403,7 +403,7 @@ public class MultiTsFileDeviceIterator implements
AutoCloseable {
MetadataIndexNode firstMeasurementNodeOfCurrentDevice =
iterator.getFirstMeasurementNodeOfCurrentDevice();
TsFileSequenceReader reader = readerMap.get(tsFileResource);
- List<AlignedChunkMetadata> alignedChunkMetadataList =
+ List<AbstractAlignedChunkMetadata> alignedChunkMetadataList =
reader.getAlignedChunkMetadataByMetadataIndexNode(
currentDevice.left, firstMeasurementNodeOfCurrentDevice,
ignoreAllNullRows);
applyModificationForAlignedChunkMetadataList(tsFileResource,
alignedChunkMetadataList);
@@ -420,7 +420,7 @@ public class MultiTsFileDeviceIterator implements
AutoCloseable {
* @param alignedChunkMetadataList list of aligned chunk metadata
*/
private void applyModificationForAlignedChunkMetadataList(
- TsFileResource tsFileResource, List<AlignedChunkMetadata>
alignedChunkMetadataList)
+ TsFileResource tsFileResource, List<AbstractAlignedChunkMetadata>
alignedChunkMetadataList)
throws IllegalPathException {
if (alignedChunkMetadataList.isEmpty()) {
// all the value chunks is empty chunk
@@ -437,7 +437,7 @@ public class MultiTsFileDeviceIterator implements
AutoCloseable {
tsFileResource, r -> new
ArrayList<>(tsFileResource.getAllModEntries()));
// construct the input params List<List<Modification>> for
QueryUtils.modifyAlignedChunkMetaData
- AlignedChunkMetadata alignedChunkMetadata =
alignedChunkMetadataList.get(0);
+ AbstractAlignedChunkMetadata alignedChunkMetadata =
alignedChunkMetadataList.get(0);
List<IChunkMetadata> valueChunkMetadataList =
alignedChunkMetadata.getValueChunkMetadataList();
// match time column modifications
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedFastAlignedSeriesCompactionExecutor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedFastAlignedSeriesCompactionExecutor.java
index c2044da7f63..636fba8c2ca 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedFastAlignedSeriesCompactionExecutor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedFastAlignedSeriesCompactionExecutor.java
@@ -43,7 +43,7 @@ import
org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory;
import org.apache.tsfile.common.constant.TsFileConstant;
import org.apache.tsfile.exception.write.PageException;
-import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
+import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
import org.apache.tsfile.file.metadata.ChunkMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
@@ -72,7 +72,7 @@ public class BatchedFastAlignedSeriesCompactionExecutor
private final List<IMeasurementSchema> valueMeasurementSchemas;
private final List<TsFileResource> sortedSourceFiles;
- private final Map<TsFileResource, List<AlignedChunkMetadata>>
alignedChunkMetadataCache;
+ private final Map<TsFileResource, List<AbstractAlignedChunkMetadata>>
alignedChunkMetadataCache;
private final BatchCompactionPlan batchCompactionPlan;
private final int batchSize =
IoTDBDescriptor.getInstance().getConfig().getCompactionMaxAlignedSeriesNumInOneBatch();
@@ -110,11 +110,11 @@ public class BatchedFastAlignedSeriesCompactionExecutor
this.batchCompactionPlan = new BatchCompactionPlan();
}
- private List<AlignedChunkMetadata>
getAlignedChunkMetadataListBySelectedValueColumn(
+ private List<AbstractAlignedChunkMetadata>
getAlignedChunkMetadataListBySelectedValueColumn(
TsFileResource tsFileResource, List<IMeasurementSchema>
selectedValueMeasurementSchemas)
throws IOException, IllegalPathException {
// 1. get Full AlignedChunkMetadata from cache
- List<AlignedChunkMetadata> alignedChunkMetadataList = null;
+ List<AbstractAlignedChunkMetadata> alignedChunkMetadataList = null;
if (alignedChunkMetadataCache.containsKey(tsFileResource)) {
alignedChunkMetadataList = alignedChunkMetadataCache.get(tsFileResource);
} else {
@@ -124,8 +124,8 @@ public class BatchedFastAlignedSeriesCompactionExecutor
}
// 2. generate AlignedChunkMetadata list by selected value columns
- List<AlignedChunkMetadata> filteredAlignedChunkMetadataList = new
ArrayList<>();
- for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList)
{
+ List<AbstractAlignedChunkMetadata> filteredAlignedChunkMetadataList = new
ArrayList<>();
+ for (AbstractAlignedChunkMetadata alignedChunkMetadata :
alignedChunkMetadataList) {
filteredAlignedChunkMetadataList.add(
AlignedSeriesBatchCompactionUtils.filterAlignedChunkMetadataByIndex(
alignedChunkMetadata,
batchColumnSelection.getSelectedColumnIndexList()));
@@ -264,8 +264,8 @@ public class BatchedFastAlignedSeriesCompactionExecutor
}
@Override
- protected List<AlignedChunkMetadata>
getAlignedChunkMetadataList(TsFileResource resource)
- throws IOException, IllegalPathException {
+ protected List<AbstractAlignedChunkMetadata> getAlignedChunkMetadataList(
+ TsFileResource resource) throws IOException, IllegalPathException {
return getAlignedChunkMetadataListBySelectedValueColumn(resource,
measurementSchemas);
}
@@ -301,10 +301,10 @@ public class BatchedFastAlignedSeriesCompactionExecutor
IChunkMetadata batchedAlignedChunkMetadata =
alignedPageElement.getChunkMetadataElement().chunkMetadata;
TsFileResource resource =
alignedPageElement.getChunkMetadataElement().fileElement.resource;
- List<AlignedChunkMetadata> alignedChunkMetadataListOfFile =
+ List<AbstractAlignedChunkMetadata> alignedChunkMetadataListOfFile =
alignedChunkMetadataCache.get(resource);
- AlignedChunkMetadata originAlignedChunkMetadata = null;
- for (AlignedChunkMetadata alignedChunkMetadata :
alignedChunkMetadataListOfFile) {
+ AbstractAlignedChunkMetadata originAlignedChunkMetadata = null;
+ for (AbstractAlignedChunkMetadata alignedChunkMetadata :
alignedChunkMetadataListOfFile) {
if (alignedChunkMetadata.getOffsetOfChunkHeader()
== batchedAlignedChunkMetadata.getOffsetOfChunkHeader()) {
originAlignedChunkMetadata = alignedChunkMetadata;
@@ -383,8 +383,8 @@ public class BatchedFastAlignedSeriesCompactionExecutor
}
@Override
- protected List<AlignedChunkMetadata>
getAlignedChunkMetadataList(TsFileResource resource)
- throws IOException, IllegalPathException {
+ protected List<AbstractAlignedChunkMetadata> getAlignedChunkMetadataList(
+ TsFileResource resource) throws IOException, IllegalPathException {
return getAlignedChunkMetadataListBySelectedValueColumn(resource,
measurementSchemas);
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedReadChunkAlignedSeriesCompactionExecutor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedReadChunkAlignedSeriesCompactionExecutor.java
index c0d16841ab1..e343ea59b92 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedReadChunkAlignedSeriesCompactionExecutor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedReadChunkAlignedSeriesCompactionExecutor.java
@@ -38,7 +38,7 @@ import
org.apache.iotdb.db.storageengine.dataregion.compaction.io.CompactionTsFi
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.tsfile.exception.write.PageException;
-import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
+import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
import org.apache.tsfile.file.metadata.ChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.statistics.Statistics;
@@ -67,13 +67,14 @@ public class BatchedReadChunkAlignedSeriesCompactionExecutor
private final int batchSize =
IoTDBDescriptor.getInstance().getConfig().getCompactionMaxAlignedSeriesNumInOneBatch();
private final AlignedSeriesBatchCompactionUtils.BatchColumnSelection
batchColumnSelection;
- private final LinkedList<Pair<TsFileSequenceReader,
List<AlignedChunkMetadata>>>
+ private final LinkedList<Pair<TsFileSequenceReader,
List<AbstractAlignedChunkMetadata>>>
originReaderAndChunkMetadataList;
public BatchedReadChunkAlignedSeriesCompactionExecutor(
IDeviceID device,
TsFileResource targetResource,
- LinkedList<Pair<TsFileSequenceReader, List<AlignedChunkMetadata>>>
readerAndChunkMetadataList,
+ LinkedList<Pair<TsFileSequenceReader,
List<AbstractAlignedChunkMetadata>>>
+ readerAndChunkMetadataList,
CompactionTsFileWriter writer,
CompactionTaskSummary summary,
boolean ignoreAllNullRows)
@@ -113,7 +114,7 @@ public class BatchedReadChunkAlignedSeriesCompactionExecutor
selectedColumnSchemaList =
batchColumnSelection.getCurrentSelectedColumnSchemaList();
}
- LinkedList<Pair<TsFileSequenceReader, List<AlignedChunkMetadata>>>
+ LinkedList<Pair<TsFileSequenceReader, List<AbstractAlignedChunkMetadata>>>
batchedReaderAndChunkMetadataList =
filterAlignedChunkMetadataList(readerAndChunkMetadataList,
selectedColumnIndexList);
@@ -137,7 +138,7 @@ public class BatchedReadChunkAlignedSeriesCompactionExecutor
private void compactLeftBatches() throws PageException, IOException {
while (batchColumnSelection.hasNext()) {
batchColumnSelection.next();
- LinkedList<Pair<TsFileSequenceReader, List<AlignedChunkMetadata>>>
+ LinkedList<Pair<TsFileSequenceReader,
List<AbstractAlignedChunkMetadata>>>
groupReaderAndChunkMetadataList =
filterAlignedChunkMetadataList(
readerAndChunkMetadataList,
batchColumnSelection.getSelectedColumnIndexList());
@@ -155,16 +156,19 @@ public class
BatchedReadChunkAlignedSeriesCompactionExecutor
}
}
- private LinkedList<Pair<TsFileSequenceReader, List<AlignedChunkMetadata>>>
+ private LinkedList<Pair<TsFileSequenceReader,
List<AbstractAlignedChunkMetadata>>>
filterAlignedChunkMetadataList(
- List<Pair<TsFileSequenceReader, List<AlignedChunkMetadata>>>
readerAndChunkMetadataList,
+ List<Pair<TsFileSequenceReader, List<AbstractAlignedChunkMetadata>>>
+ readerAndChunkMetadataList,
List<Integer> selectedMeasurementIndexs) {
- LinkedList<Pair<TsFileSequenceReader, List<AlignedChunkMetadata>>>
+ LinkedList<Pair<TsFileSequenceReader, List<AbstractAlignedChunkMetadata>>>
groupReaderAndChunkMetadataList = new LinkedList<>();
- for (Pair<TsFileSequenceReader, List<AlignedChunkMetadata>> pair :
readerAndChunkMetadataList) {
- List<AlignedChunkMetadata> alignedChunkMetadataList = pair.getRight();
- List<AlignedChunkMetadata> selectedColumnAlignedChunkMetadataList = new
LinkedList<>();
- for (AlignedChunkMetadata alignedChunkMetadata :
alignedChunkMetadataList) {
+ for (Pair<TsFileSequenceReader, List<AbstractAlignedChunkMetadata>> pair :
+ readerAndChunkMetadataList) {
+ List<AbstractAlignedChunkMetadata> alignedChunkMetadataList =
pair.getRight();
+ List<AbstractAlignedChunkMetadata>
selectedColumnAlignedChunkMetadataList =
+ new LinkedList<>();
+ for (AbstractAlignedChunkMetadata alignedChunkMetadata :
alignedChunkMetadataList) {
selectedColumnAlignedChunkMetadataList.add(
AlignedSeriesBatchCompactionUtils.filterAlignedChunkMetadataByIndex(
alignedChunkMetadata, selectedMeasurementIndexs));
@@ -181,7 +185,7 @@ public class BatchedReadChunkAlignedSeriesCompactionExecutor
public FirstBatchedReadChunkAlignedSeriesCompactionExecutor(
IDeviceID device,
TsFileResource targetResource,
- LinkedList<Pair<TsFileSequenceReader, List<AlignedChunkMetadata>>>
+ LinkedList<Pair<TsFileSequenceReader,
List<AbstractAlignedChunkMetadata>>>
readerAndChunkMetadataList,
CompactionTsFileWriter writer,
CompactionTaskSummary summary,
@@ -224,8 +228,8 @@ public class BatchedReadChunkAlignedSeriesCompactionExecutor
String file = timePage.getFile();
ChunkMetadata timeChunkMetadata = timePage.getChunkMetadata();
- List<AlignedChunkMetadata> alignedChunkMetadataList =
Collections.emptyList();
- for (Pair<TsFileSequenceReader, List<AlignedChunkMetadata>> pair :
+ List<AbstractAlignedChunkMetadata> alignedChunkMetadataList =
Collections.emptyList();
+ for (Pair<TsFileSequenceReader, List<AbstractAlignedChunkMetadata>> pair
:
originReaderAndChunkMetadataList) {
TsFileSequenceReader reader = pair.getLeft();
if (reader.getFileName().equals(file)) {
@@ -234,8 +238,8 @@ public class BatchedReadChunkAlignedSeriesCompactionExecutor
}
}
- AlignedChunkMetadata originAlignedChunkMetadata = null;
- for (AlignedChunkMetadata alignedChunkMetadata :
alignedChunkMetadataList) {
+ AbstractAlignedChunkMetadata originAlignedChunkMetadata = null;
+ for (AbstractAlignedChunkMetadata alignedChunkMetadata :
alignedChunkMetadataList) {
if (alignedChunkMetadata.getOffsetOfChunkHeader()
== timeChunkMetadata.getOffsetOfChunkHeader()) {
originAlignedChunkMetadata = alignedChunkMetadata;
@@ -303,7 +307,7 @@ public class BatchedReadChunkAlignedSeriesCompactionExecutor
public FollowingBatchedReadChunkAlignedSeriesGroupCompactionExecutor(
IDeviceID device,
TsFileResource targetResource,
- LinkedList<Pair<TsFileSequenceReader, List<AlignedChunkMetadata>>>
+ LinkedList<Pair<TsFileSequenceReader,
List<AbstractAlignedChunkMetadata>>>
readerAndChunkMetadataList,
CompactionTsFileWriter writer,
CompactionTaskSummary summary,
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/AlignedSeriesBatchCompactionUtils.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/AlignedSeriesBatchCompactionUtils.java
index 25613fc2a1a..c3c58e5bb6a 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/AlignedSeriesBatchCompactionUtils.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/AlignedSeriesBatchCompactionUtils.java
@@ -21,7 +21,7 @@ package
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.ex
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.ModifiedStatus;
-import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
+import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
import org.apache.tsfile.file.metadata.ChunkMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.read.TsFileSequenceReader;
@@ -43,17 +43,18 @@ public class AlignedSeriesBatchCompactionUtils {
private AlignedSeriesBatchCompactionUtils() {}
public static void markAlignedChunkHasDeletion(
- LinkedList<Pair<TsFileSequenceReader, List<AlignedChunkMetadata>>>
+ LinkedList<Pair<TsFileSequenceReader,
List<AbstractAlignedChunkMetadata>>>
readerAndChunkMetadataList) {
- for (Pair<TsFileSequenceReader, List<AlignedChunkMetadata>> pair :
readerAndChunkMetadataList) {
- List<AlignedChunkMetadata> alignedChunkMetadataList = pair.getRight();
+ for (Pair<TsFileSequenceReader, List<AbstractAlignedChunkMetadata>> pair :
+ readerAndChunkMetadataList) {
+ List<AbstractAlignedChunkMetadata> alignedChunkMetadataList =
pair.getRight();
markAlignedChunkHasDeletion(alignedChunkMetadataList);
}
}
public static void markAlignedChunkHasDeletion(
- List<AlignedChunkMetadata> alignedChunkMetadataList) {
- for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList)
{
+ List<AbstractAlignedChunkMetadata> alignedChunkMetadataList) {
+ for (AbstractAlignedChunkMetadata alignedChunkMetadata :
alignedChunkMetadataList) {
IChunkMetadata timeChunkMetadata =
alignedChunkMetadata.getTimeChunkMetadata();
for (IChunkMetadata iChunkMetadata :
alignedChunkMetadata.getValueChunkMetadataList()) {
if (iChunkMetadata != null && iChunkMetadata.isModified()) {
@@ -68,8 +69,8 @@ public class AlignedSeriesBatchCompactionUtils {
return chunkMetadata.getMeasurementUid().isEmpty();
}
- public static AlignedChunkMetadata filterAlignedChunkMetadataByIndex(
- AlignedChunkMetadata alignedChunkMetadata, List<Integer>
selectedMeasurements) {
+ public static AbstractAlignedChunkMetadata filterAlignedChunkMetadataByIndex(
+ AbstractAlignedChunkMetadata alignedChunkMetadata, List<Integer>
selectedMeasurements) {
IChunkMetadata[] valueChunkMetadataArr = new
IChunkMetadata[selectedMeasurements.size()];
List<IChunkMetadata> originValueChunkMetadataList =
alignedChunkMetadata.getValueChunkMetadataList();
@@ -77,12 +78,13 @@ public class AlignedSeriesBatchCompactionUtils {
int columnIndex = selectedMeasurements.get(i);
valueChunkMetadataArr[i] = originValueChunkMetadataList.get(columnIndex);
}
- return new AlignedChunkMetadata(
+ return alignedChunkMetadata.createNewChunkMetadata(
alignedChunkMetadata.getTimeChunkMetadata(),
Arrays.asList(valueChunkMetadataArr));
}
- public static AlignedChunkMetadata fillAlignedChunkMetadataBySchemaList(
- AlignedChunkMetadata originAlignedChunkMetadata,
List<IMeasurementSchema> schemaList) {
+ public static AbstractAlignedChunkMetadata
fillAlignedChunkMetadataBySchemaList(
+ AbstractAlignedChunkMetadata originAlignedChunkMetadata,
+ List<IMeasurementSchema> schemaList) {
List<IChunkMetadata> originValueChunkMetadataList =
originAlignedChunkMetadata.getValueChunkMetadataList();
IChunkMetadata[] newValueChunkMetadataArr = new
IChunkMetadata[schemaList.size()];
@@ -109,14 +111,14 @@ public class AlignedSeriesBatchCompactionUtils {
currentValueChunkMetadataIndex++;
}
}
- return new AlignedChunkMetadata(
+ return originAlignedChunkMetadata.createNewChunkMetadata(
originAlignedChunkMetadata.getTimeChunkMetadata(),
Arrays.asList(newValueChunkMetadataArr));
}
public static ModifiedStatus calculateAlignedPageModifiedStatus(
long startTime,
long endTime,
- AlignedChunkMetadata originAlignedChunkMetadata,
+ AbstractAlignedChunkMetadata originAlignedChunkMetadata,
boolean ignoreAllNullRows) {
ModifiedStatus timePageModifiedStatus =
checkIsModified(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java
index 64fb585542d..b489630894b 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java
@@ -44,10 +44,12 @@ import org.apache.tsfile.common.constant.TsFileConstant;
import org.apache.tsfile.exception.write.PageException;
import org.apache.tsfile.file.header.ChunkHeader;
import org.apache.tsfile.file.header.PageHeader;
+import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
import org.apache.tsfile.file.metadata.ChunkMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.TableDeviceChunkMetadata;
import org.apache.tsfile.read.TsFileSequenceReader;
import org.apache.tsfile.read.common.Chunk;
import org.apache.tsfile.utils.Pair;
@@ -155,7 +157,8 @@ public class FastAlignedSeriesCompactionExecutor extends
SeriesCompactionExecuto
throws IOException, IllegalPathException {
for (FileElement fileElement : fileElements) {
TsFileResource resource = fileElement.resource;
- List<AlignedChunkMetadata> alignedChunkMetadataList =
getAlignedChunkMetadataList(resource);
+ List<AbstractAlignedChunkMetadata> alignedChunkMetadataList =
+ getAlignedChunkMetadataList(resource);
if (alignedChunkMetadataList.isEmpty()) {
// all chunks have been deleted in this file or current file does not
contain this aligned
@@ -174,7 +177,7 @@ public class FastAlignedSeriesCompactionExecutor extends
SeriesCompactionExecuto
}
}
- protected List<AlignedChunkMetadata>
getAlignedChunkMetadataList(TsFileResource resource)
+ protected List<AbstractAlignedChunkMetadata>
getAlignedChunkMetadataList(TsFileResource resource)
throws IOException, IllegalPathException {
// read time chunk metadatas and value chunk metadatas in the current file
List<IChunkMetadata> timeChunkMetadatas = null;
@@ -216,7 +219,7 @@ public class FastAlignedSeriesCompactionExecutor extends
SeriesCompactionExecuto
}
}
- List<AlignedChunkMetadata> alignedChunkMetadataList = new ArrayList<>();
+ List<AbstractAlignedChunkMetadata> alignedChunkMetadataList = new
ArrayList<>();
// if current file contains this aligned device,then construct aligned
chunk metadatas
if (timeChunkMetadatas != null) {
for (int i = 0; i < timeChunkMetadatas.size(); i++) {
@@ -228,8 +231,10 @@ public class FastAlignedSeriesCompactionExecutor extends
SeriesCompactionExecuto
valueChunkMetadataList.add(chunkMetadata.get(i));
}
}
- AlignedChunkMetadata alignedChunkMetadata =
- new AlignedChunkMetadata(timeChunkMetadatas.get(i),
valueChunkMetadataList);
+ AbstractAlignedChunkMetadata alignedChunkMetadata =
+ ignoreAllNullRows
+ ? new AlignedChunkMetadata(timeChunkMetadatas.get(i),
valueChunkMetadataList)
+ : new TableDeviceChunkMetadata(timeChunkMetadatas.get(i),
valueChunkMetadataList);
alignedChunkMetadataList.add(alignedChunkMetadata);
}
@@ -345,8 +350,8 @@ public class FastAlignedSeriesCompactionExecutor extends
SeriesCompactionExecuto
@Override
void readChunk(ChunkMetadataElement chunkMetadataElement) throws IOException
{
updateSummary(chunkMetadataElement, ChunkStatus.READ_IN);
- AlignedChunkMetadata alignedChunkMetadata =
- (AlignedChunkMetadata) chunkMetadataElement.chunkMetadata;
+ AbstractAlignedChunkMetadata alignedChunkMetadata =
+ (AbstractAlignedChunkMetadata) chunkMetadataElement.chunkMetadata;
TsFileSequenceReader reader =
readerCacheMap.get(chunkMetadataElement.fileElement.resource);
chunkMetadataElement.chunk =
readChunk(reader, (ChunkMetadata)
alignedChunkMetadata.getTimeChunkMetadata());
@@ -418,8 +423,8 @@ public class FastAlignedSeriesCompactionExecutor extends
SeriesCompactionExecuto
protected ModifiedStatus isPageModified(PageElement pageElement) {
long startTime = pageElement.getStartTime();
long endTime = pageElement.getEndTime();
- AlignedChunkMetadata alignedChunkMetadata =
- (AlignedChunkMetadata)
pageElement.getChunkMetadataElement().chunkMetadata;
+ AbstractAlignedChunkMetadata alignedChunkMetadata =
+ (AbstractAlignedChunkMetadata)
pageElement.getChunkMetadataElement().chunkMetadata;
return
AlignedSeriesBatchCompactionUtils.calculateAlignedPageModifiedStatus(
startTime, endTime, alignedChunkMetadata, ignoreAllNullRows);
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java
index 83dab2e7690..c2a5c6c7558 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java
@@ -36,7 +36,7 @@ import org.apache.iotdb.db.utils.ModificationUtils;
import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory;
import org.apache.tsfile.exception.write.PageException;
-import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
+import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.read.TimeValuePair;
@@ -495,14 +495,14 @@ public abstract class SeriesCompactionExecutor {
case READ_IN:
summary.increaseProcessChunkNum(
isAligned
- ? ((AlignedChunkMetadata) chunkMetadataElement.chunkMetadata)
+ ? ((AbstractAlignedChunkMetadata)
chunkMetadataElement.chunkMetadata)
.getValueChunkMetadataList()
.size()
+ 1
: 1);
if (isAligned) {
for (IChunkMetadata valueChunkMetadata :
- ((AlignedChunkMetadata) chunkMetadataElement.chunkMetadata)
+ ((AbstractAlignedChunkMetadata)
chunkMetadataElement.chunkMetadata)
.getValueChunkMetadataList()) {
if (valueChunkMetadata == null) {
continue;
@@ -517,7 +517,7 @@ public abstract class SeriesCompactionExecutor {
case DIRECTORY_FLUSH:
if (isAligned) {
summary.increaseDirectlyFlushChunkNum(
- ((AlignedChunkMetadata) (chunkMetadataElement.chunkMetadata))
+ ((AbstractAlignedChunkMetadata)
(chunkMetadataElement.chunkMetadata))
.getValueChunkMetadataList()
.size()
+ 1);
@@ -528,7 +528,7 @@ public abstract class SeriesCompactionExecutor {
case DESERIALIZE_CHUNK:
if (isAligned) {
summary.increaseDeserializedChunkNum(
- ((AlignedChunkMetadata) (chunkMetadataElement.chunkMetadata))
+ ((AbstractAlignedChunkMetadata)
(chunkMetadataElement.chunkMetadata))
.getValueChunkMetadataList()
.size()
+ 1);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java
index 12c205c472e..17281c87051 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java
@@ -39,7 +39,7 @@ import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.exception.write.PageException;
import org.apache.tsfile.file.header.ChunkHeader;
import org.apache.tsfile.file.header.PageHeader;
-import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
+import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
import org.apache.tsfile.file.metadata.ChunkMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
@@ -68,7 +68,7 @@ import java.util.stream.Collectors;
public class ReadChunkAlignedSeriesCompactionExecutor {
protected final IDeviceID device;
- protected final LinkedList<Pair<TsFileSequenceReader,
List<AlignedChunkMetadata>>>
+ protected final LinkedList<Pair<TsFileSequenceReader,
List<AbstractAlignedChunkMetadata>>>
readerAndChunkMetadataList;
protected final TsFileResource targetResource;
protected final CompactionTsFileWriter writer;
@@ -85,7 +85,8 @@ public class ReadChunkAlignedSeriesCompactionExecutor {
public ReadChunkAlignedSeriesCompactionExecutor(
IDeviceID device,
TsFileResource targetResource,
- LinkedList<Pair<TsFileSequenceReader, List<AlignedChunkMetadata>>>
readerAndChunkMetadataList,
+ LinkedList<Pair<TsFileSequenceReader,
List<AbstractAlignedChunkMetadata>>>
+ readerAndChunkMetadataList,
CompactionTsFileWriter writer,
CompactionTaskSummary summary,
boolean ignoreAllNullRows)
@@ -108,7 +109,8 @@ public class ReadChunkAlignedSeriesCompactionExecutor {
public ReadChunkAlignedSeriesCompactionExecutor(
IDeviceID device,
TsFileResource targetResource,
- LinkedList<Pair<TsFileSequenceReader, List<AlignedChunkMetadata>>>
readerAndChunkMetadataList,
+ LinkedList<Pair<TsFileSequenceReader,
List<AbstractAlignedChunkMetadata>>>
+ readerAndChunkMetadataList,
CompactionTsFileWriter writer,
CompactionTaskSummary summary,
IMeasurementSchema timeSchema,
@@ -131,11 +133,11 @@ public class ReadChunkAlignedSeriesCompactionExecutor {
private void collectValueColumnSchemaList() throws IOException {
Map<String, IMeasurementSchema> measurementSchemaMap = new HashMap<>();
for (int i = this.readerAndChunkMetadataList.size() - 1; i >= 0; i--) {
- Pair<TsFileSequenceReader, List<AlignedChunkMetadata>> pair =
+ Pair<TsFileSequenceReader, List<AbstractAlignedChunkMetadata>> pair =
this.readerAndChunkMetadataList.get(i);
CompactionTsFileReader reader = (CompactionTsFileReader) pair.getLeft();
- List<AlignedChunkMetadata> alignedChunkMetadataList = pair.getRight();
- for (AlignedChunkMetadata alignedChunkMetadata :
alignedChunkMetadataList) {
+ List<AbstractAlignedChunkMetadata> alignedChunkMetadataList =
pair.getRight();
+ for (AbstractAlignedChunkMetadata alignedChunkMetadata :
alignedChunkMetadataList) {
if (alignedChunkMetadata == null) {
continue;
}
@@ -176,10 +178,11 @@ public class ReadChunkAlignedSeriesCompactionExecutor {
}
private void fillAlignedChunkMetadataToMatchSchemaList() {
- for (Pair<TsFileSequenceReader, List<AlignedChunkMetadata>> pair :
readerAndChunkMetadataList) {
- List<AlignedChunkMetadata> alignedChunkMetadataList = pair.getRight();
+ for (Pair<TsFileSequenceReader, List<AbstractAlignedChunkMetadata>> pair :
+ readerAndChunkMetadataList) {
+ List<AbstractAlignedChunkMetadata> alignedChunkMetadataList =
pair.getRight();
for (int i = 0; i < alignedChunkMetadataList.size(); i++) {
- AlignedChunkMetadata alignedChunkMetadata =
alignedChunkMetadataList.get(i);
+ AbstractAlignedChunkMetadata alignedChunkMetadata =
alignedChunkMetadataList.get(i);
alignedChunkMetadataList.set(
i,
AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList(
@@ -193,15 +196,15 @@ public class ReadChunkAlignedSeriesCompactionExecutor {
}
public void execute() throws IOException, PageException {
- for (Pair<TsFileSequenceReader, List<AlignedChunkMetadata>> readerListPair
:
+ for (Pair<TsFileSequenceReader, List<AbstractAlignedChunkMetadata>>
readerListPair :
readerAndChunkMetadataList) {
TsFileSequenceReader reader = readerListPair.left;
- List<AlignedChunkMetadata> alignedChunkMetadataList =
readerListPair.right;
+ List<AbstractAlignedChunkMetadata> alignedChunkMetadataList =
readerListPair.right;
if (reader instanceof CompactionTsFileReader) {
((CompactionTsFileReader) reader).markStartOfAlignedSeries();
}
- for (AlignedChunkMetadata alignedChunkMetadata :
alignedChunkMetadataList) {
+ for (AbstractAlignedChunkMetadata alignedChunkMetadata :
alignedChunkMetadataList) {
compactWithAlignedChunk(reader, alignedChunkMetadata);
}
if (reader instanceof CompactionTsFileReader) {
@@ -215,7 +218,7 @@ public class ReadChunkAlignedSeriesCompactionExecutor {
}
private void compactWithAlignedChunk(
- TsFileSequenceReader reader, AlignedChunkMetadata alignedChunkMetadata)
+ TsFileSequenceReader reader, AbstractAlignedChunkMetadata
alignedChunkMetadata)
throws IOException, PageException {
ChunkLoader timeChunk =
getChunkLoader(reader, (ChunkMetadata)
alignedChunkMetadata.getTimeChunkMetadata());
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java
index ae5bc2e7591..d4b20039109 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java
@@ -26,7 +26,7 @@ import
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.tsfile.exception.write.PageException;
import org.apache.tsfile.file.header.PageHeader;
-import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
+import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
import org.apache.tsfile.file.metadata.ChunkMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.read.TsFileSequenceReader;
@@ -97,8 +97,8 @@ public class FastCrossCompactionWriter extends
AbstractCrossCompactionWriter {
@Override
public boolean flushAlignedChunk(ChunkMetadataElement chunkMetadataElement,
int subTaskId)
throws IOException {
- AlignedChunkMetadata alignedChunkMetadata =
- (AlignedChunkMetadata) chunkMetadataElement.chunkMetadata;
+ AbstractAlignedChunkMetadata alignedChunkMetadata =
+ (AbstractAlignedChunkMetadata) chunkMetadataElement.chunkMetadata;
IChunkMetadata timeChunkMetadata =
alignedChunkMetadata.getTimeChunkMetadata();
List<IChunkMetadata> valueChunkMetadatas =
alignedChunkMetadata.getValueChunkMetadataList();
Chunk timeChunk = chunkMetadataElement.chunk;
@@ -132,8 +132,8 @@ public class FastCrossCompactionWriter extends
AbstractCrossCompactionWriter {
int subTaskId,
AbstractCompactionFlushController flushController)
throws IOException {
- AlignedChunkMetadata alignedChunkMetadata =
- (AlignedChunkMetadata) chunkMetadataElement.chunkMetadata;
+ AbstractAlignedChunkMetadata alignedChunkMetadata =
+ (AbstractAlignedChunkMetadata) chunkMetadataElement.chunkMetadata;
IChunkMetadata timeChunkMetadata =
alignedChunkMetadata.getTimeChunkMetadata();
List<IChunkMetadata> valueChunkMetadatas =
alignedChunkMetadata.getValueChunkMetadataList();
List<Chunk> valueChunks = chunkMetadataElement.valueChunks;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastInnerCompactionWriter.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastInnerCompactionWriter.java
index e9709af3dbf..39e621cfaf4 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastInnerCompactionWriter.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastInnerCompactionWriter.java
@@ -26,7 +26,7 @@ import
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.tsfile.exception.write.PageException;
import org.apache.tsfile.file.header.PageHeader;
-import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
+import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
import org.apache.tsfile.file.metadata.ChunkMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.read.common.Chunk;
@@ -89,8 +89,8 @@ public class FastInnerCompactionWriter extends
AbstractInnerCompactionWriter {
@Override
public boolean flushAlignedChunk(ChunkMetadataElement chunkMetadataElement,
int subTaskId)
throws IOException {
- AlignedChunkMetadata alignedChunkMetadata =
- (AlignedChunkMetadata) chunkMetadataElement.chunkMetadata;
+ AbstractAlignedChunkMetadata alignedChunkMetadata =
+ (AbstractAlignedChunkMetadata) chunkMetadataElement.chunkMetadata;
IChunkMetadata timeChunkMetadata =
alignedChunkMetadata.getTimeChunkMetadata();
List<IChunkMetadata> valueChunkMetadatas =
alignedChunkMetadata.getValueChunkMetadataList();
Chunk timeChunk = chunkMetadataElement.chunk;
@@ -121,8 +121,8 @@ public class FastInnerCompactionWriter extends
AbstractInnerCompactionWriter {
int subTaskId,
AbstractCompactionFlushController flushController)
throws IOException {
- AlignedChunkMetadata alignedChunkMetadata =
- (AlignedChunkMetadata) chunkMetadataElement.chunkMetadata;
+ AbstractAlignedChunkMetadata alignedChunkMetadata =
+ (AbstractAlignedChunkMetadata) chunkMetadataElement.chunkMetadata;
IChunkMetadata timeChunkMetadata =
alignedChunkMetadata.getTimeChunkMetadata();
List<IChunkMetadata> valueChunkMetadatas =
alignedChunkMetadata.getValueChunkMetadataList();
List<Chunk> valueChunks = chunkMetadataElement.valueChunks;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java
index 569db7347ab..a3eb66b018d 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java
@@ -30,6 +30,7 @@ import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
import org.apache.tsfile.file.metadata.ChunkMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
+import org.apache.tsfile.file.metadata.TableDeviceChunkMetadata;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.file.metadata.statistics.Statistics;
import org.apache.tsfile.read.common.TimeRange;
@@ -75,10 +76,11 @@ public class AlignedReadOnlyMemChunk extends
ReadOnlyMemChunk {
timeColumnDeletion,
valueColumnsDeletionList,
context.isIgnoreAllNullRows());
- initAlignedChunkMetaFromTsBlock();
+ initAlignedChunkMetaFromTsBlock(context.isIgnoreAllNullRows());
}
- private void initAlignedChunkMetaFromTsBlock() throws QueryProcessException {
+ private void initAlignedChunkMetaFromTsBlock(boolean ignoreAllNullRows)
+ throws QueryProcessException {
// Time chunk
Statistics timeStatistics = Statistics.getStatsByType(TSDataType.VECTOR);
IChunkMetadata timeChunkMetadata =
@@ -160,7 +162,9 @@ public class AlignedReadOnlyMemChunk extends
ReadOnlyMemChunk {
}
}
IChunkMetadata alignedChunkMetadata =
- new AlignedChunkMetadata(timeChunkMetadata, valueChunkMetadataList);
+ ignoreAllNullRows
+ ? new AlignedChunkMetadata(timeChunkMetadata,
valueChunkMetadataList)
+ : new TableDeviceChunkMetadata(timeChunkMetadata,
valueChunkMetadataList);
alignedChunkMetadata.setChunkLoader(new MemAlignedChunkLoader(context,
this));
alignedChunkMetadata.setVersion(Long.MAX_VALUE);
cachedMetaData = alignedChunkMetadata;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
index 3b2c5fe2944..b6f5629528f 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
@@ -80,6 +80,7 @@ import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
import org.apache.tsfile.file.metadata.ChunkMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
@@ -1788,7 +1789,7 @@ public class TsFileProcessor {
private void processAlignedChunkMetaDataFromFlushedMemTable(
IDeviceID deviceID,
- AlignedChunkMetadata alignedChunkMetadata,
+ AbstractAlignedChunkMetadata alignedChunkMetadata,
Map<String, List<IChunkMetadata>> measurementToChunkMetaMap,
Map<String, List<IChunkHandle>> measurementToChunkHandleMap,
String filePath) {
@@ -1841,10 +1842,10 @@ public class TsFileProcessor {
Map<String, List<IChunkMetadata>> measurementToChunkMetaList,
Map<String, List<IChunkHandle>> measurementToChunkHandleList) {
for (IChunkMetadata chunkMetadata : chunkMetadataList) {
- if (chunkMetadata instanceof AlignedChunkMetadata) {
+ if (chunkMetadata instanceof AbstractAlignedChunkMetadata) {
processAlignedChunkMetaDataFromFlushedMemTable(
deviceID,
- (AlignedChunkMetadata) chunkMetadata,
+ (AbstractAlignedChunkMetadata) chunkMetadata,
measurementToChunkMetaList,
measurementToChunkHandleList,
this.tsFileResource.getTsFilePath());
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java
index adef246ec8d..a7c6eb96d42 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java
@@ -25,7 +25,7 @@ import org.apache.iotdb.db.storageengine.buffer.ChunkCache;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
-import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
+import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
import org.apache.tsfile.file.metadata.ChunkMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.read.common.Chunk;
@@ -76,7 +76,8 @@ public class DiskAlignedChunkLoader implements IChunkLoader {
throws IOException {
long t1 = System.nanoTime();
try {
- AlignedChunkMetadata alignedChunkMetadata = (AlignedChunkMetadata)
chunkMetaData;
+ AbstractAlignedChunkMetadata alignedChunkMetadata =
+ (AbstractAlignedChunkMetadata) chunkMetaData;
ChunkMetadata timeChunkMetadata = (ChunkMetadata)
alignedChunkMetadata.getTimeChunkMetadata();
Chunk timeChunk =
ChunkCache.getInstance()
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java
index 33c85408f0d..12ac281dffc 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java
@@ -21,7 +21,7 @@ package
org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk;
import
org.apache.iotdb.db.storageengine.dataregion.memtable.AlignedReadOnlyMemChunk;
-import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
+import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
import org.apache.tsfile.read.common.BatchData;
import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.read.reader.IChunkReader;
@@ -42,7 +42,7 @@ public class MemAlignedChunkReader implements IChunkReader {
Collections.singletonList(
new MemAlignedPageReader(
readableChunk.getTsBlock(),
- (AlignedChunkMetadata) readableChunk.getChunkMetaData(),
+ (AbstractAlignedChunkMetadata)
readableChunk.getChunkMetaData(),
globalTimeFilter));
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java
index 473cd7366c3..6adfe09c054 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java
@@ -22,7 +22,7 @@ package
org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk;
import org.apache.tsfile.block.column.Column;
import org.apache.tsfile.block.column.ColumnBuilder;
import org.apache.tsfile.enums.TSDataType;
-import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
+import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
import org.apache.tsfile.file.metadata.statistics.Statistics;
import org.apache.tsfile.read.common.BatchData;
import org.apache.tsfile.read.common.BatchDataFactory;
@@ -45,7 +45,7 @@ import static
org.apache.tsfile.read.reader.series.PaginationController.UNLIMITE
public class MemAlignedPageReader implements IPageReader {
private final TsBlock tsBlock;
- private final AlignedChunkMetadata chunkMetadata;
+ private final AbstractAlignedChunkMetadata chunkMetadata;
private Filter recordFilter;
private PaginationController paginationController =
UNLIMITED_PAGINATION_CONTROLLER;
@@ -53,7 +53,7 @@ public class MemAlignedPageReader implements IPageReader {
private TsBlockBuilder builder;
public MemAlignedPageReader(
- TsBlock tsBlock, AlignedChunkMetadata chunkMetadata, Filter
recordFilter) {
+ TsBlock tsBlock, AbstractAlignedChunkMetadata chunkMetadata, Filter
recordFilter) {
this.tsBlock = tsBlock;
this.chunkMetadata = chunkMetadata;
this.recordFilter = recordFilter;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java
index afc57079c88..f243b51ee1a 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java
@@ -26,8 +26,8 @@ import
org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.DiskAligne
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.utils.ModificationUtils;
+import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata;
-import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.file.metadata.ITimeSeriesMetadata;
import org.apache.tsfile.read.controller.IChunkMetadataLoader;
@@ -83,7 +83,7 @@ public class DiskAlignedChunkMetadataLoader implements
IChunkMetadataLoader {
public List<IChunkMetadata> loadChunkMetadataList(ITimeSeriesMetadata
timeSeriesMetadata) {
final long t1 = System.nanoTime();
try {
- List<AlignedChunkMetadata> alignedChunkMetadataList =
+ List<AbstractAlignedChunkMetadata> alignedChunkMetadataList =
((AbstractAlignedTimeSeriesMetadata)
timeSeriesMetadata).getCopiedChunkMetadataList();
// when alignedChunkMetadataList.size() == 1, it means that the chunk
statistics is same as
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ErrorHandlingUtils.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ErrorHandlingUtils.java
index 247a0a3fe82..57527c85af8 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ErrorHandlingUtils.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ErrorHandlingUtils.java
@@ -106,7 +106,11 @@ public class ErrorHandlingUtils {
|| status.getCode() == TSStatusCode.ILLEGAL_PATH.getStatusCode()
|| status.getCode() ==
TSStatusCode.NUMERIC_VALUE_OUT_OF_RANGE.getStatusCode()
|| status.getCode() ==
TSStatusCode.DIVISION_BY_ZERO.getStatusCode()
- || status.getCode() ==
TSStatusCode.DATE_OUT_OF_RANGE.getStatusCode()) {
+ || status.getCode() ==
TSStatusCode.DATE_OUT_OF_RANGE.getStatusCode()
+ || status.getCode() ==
TSStatusCode.TABLE_NOT_EXISTS.getStatusCode()
+ || status.getCode() ==
TSStatusCode.TABLE_ALREADY_EXISTS.getStatusCode()
+ || status.getCode() ==
TSStatusCode.COLUMN_NOT_EXISTS.getStatusCode()
+ || status.getCode() ==
TSStatusCode.COLUMN_ALREADY_EXISTS.getStatusCode()) {
LOGGER.info(message);
} else {
LOGGER.warn(message, e);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java
index 59966207648..f9a6d28e2b9 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java
@@ -25,6 +25,7 @@ import
org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry;
import
org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry;
import
org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry;
+import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
@@ -101,7 +102,7 @@ public class ModificationUtils {
}
private static void modifyValueColumns(
- AlignedChunkMetadata metaData, List<List<ModEntry>>
valueColumnsModifications) {
+ AbstractAlignedChunkMetadata metaData, List<List<ModEntry>>
valueColumnsModifications) {
List<IChunkMetadata> valueChunkMetadataList =
metaData.getValueChunkMetadataList();
// deal with each sub sensor
for (int j = 0; j < valueChunkMetadataList.size(); j++) {
@@ -116,7 +117,7 @@ public class ModificationUtils {
}
private static boolean areAllValueColumnsDeleted(
- AlignedChunkMetadata alignedChunkMetadata, boolean modified) {
+ AbstractAlignedChunkMetadata alignedChunkMetadata, boolean modified) {
// the whole aligned path need to be removed, only set to be true if all
the sub sensors
// are deleted and ignoreAllNullRows is true
@@ -156,11 +157,11 @@ public class ModificationUtils {
}
public static void modifyAlignedChunkMetaData(
- List<AlignedChunkMetadata> chunkMetaData,
+ List<? extends AbstractAlignedChunkMetadata> chunkMetaData,
List<ModEntry> timeColumnModifications,
List<List<ModEntry>> valueColumnsModifications,
boolean ignoreAllNullRows) {
- for (AlignedChunkMetadata metaData : chunkMetaData) {
+ for (AbstractAlignedChunkMetadata metaData : chunkMetaData) {
IChunkMetadata timeColumnChunkMetadata = metaData.getTimeChunkMetadata();
for (ModEntry modification : timeColumnModifications) {
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithInconsistentCompressionTypeAndEncodingTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithInconsistentCompressionTypeAndEncodingTest.java
index 55cb8625da3..f62480c5529 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithInconsistentCompressionTypeAndEncodingTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithInconsistentCompressionTypeAndEncodingTest.java
@@ -32,7 +32,7 @@ import
org.apache.tsfile.exception.write.WriteProcessException;
import org.apache.tsfile.file.MetaMarker;
import org.apache.tsfile.file.header.ChunkHeader;
import org.apache.tsfile.file.header.PageHeader;
-import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
+import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
import org.apache.tsfile.file.metadata.ChunkMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
@@ -622,9 +622,9 @@ public class
FastCompactionPerformerWithInconsistentCompressionTypeAndEncodingTe
throws IOException {
Map<String, CompressionType> compressionTypeMap = new HashMap<>();
for (IDeviceID device : reader.getAllDevices()) {
- List<AlignedChunkMetadata> alignedChunkMetadataList =
+ List<AbstractAlignedChunkMetadata> alignedChunkMetadataList =
reader.getAlignedChunkMetadata(device, true);
- for (AlignedChunkMetadata alignedChunkMetadata :
alignedChunkMetadataList) {
+ for (AbstractAlignedChunkMetadata alignedChunkMetadata :
alignedChunkMetadataList) {
IChunkMetadata timeChunkMetadata =
alignedChunkMetadata.getTimeChunkMetadata();
List<IChunkMetadata> valueChunkMetadataList =
alignedChunkMetadata.getValueChunkMetadataList();
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastInnerCompactionPerformerTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastInnerCompactionPerformerTest.java
index 8c9088f5ceb..f2537e9bfdd 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastInnerCompactionPerformerTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastInnerCompactionPerformerTest.java
@@ -39,7 +39,7 @@ import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.exception.write.WriteProcessException;
-import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
+import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
import org.apache.tsfile.file.metadata.ChunkMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
@@ -2047,10 +2047,10 @@ public class FastInnerCompactionPerformerTest extends
AbstractCompactionTest {
TsFileResource targetResource = tsFileManager.getTsFileList(false).get(0);
try (TsFileSequenceReader reader = new
TsFileSequenceReader(targetResource.getTsFilePath())) {
- List<AlignedChunkMetadata> chunkMetadataList =
+ List<AbstractAlignedChunkMetadata> chunkMetadataList =
reader.getAlignedChunkMetadata(
IDeviceID.Factory.DEFAULT_FACTORY.create("root.testsg.d1"),
true);
- for (AlignedChunkMetadata alignedChunkMetadata : chunkMetadataList) {
+ for (AbstractAlignedChunkMetadata alignedChunkMetadata :
chunkMetadataList) {
ChunkMetadata timeChunkMetadata =
(ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata();
Chunk timeChunk = reader.readMemChunk(timeChunkMetadata);
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java
index a9c559a00a5..f2e39d8f866 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java
@@ -46,7 +46,7 @@ import
org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex;
import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils;
import org.apache.tsfile.exception.write.WriteProcessException;
-import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
+import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
import org.apache.tsfile.file.metadata.ChunkMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
@@ -682,10 +682,10 @@ public class RepairUnsortedFileCompactionTest extends
AbstractRepairDataTest {
Assert.assertTrue(task.start());
TsFileResource target = tsFileManager.getTsFileList(false).get(0);
try (TsFileSequenceReader reader = new
TsFileSequenceReader(target.getTsFilePath())) {
- List<AlignedChunkMetadata> chunkMetadataList =
+ List<AbstractAlignedChunkMetadata> chunkMetadataList =
reader.getAlignedChunkMetadata(
IDeviceID.Factory.DEFAULT_FACTORY.create("root.testsg.d1"),
true);
- for (AlignedChunkMetadata alignedChunkMetadata : chunkMetadataList) {
+ for (AbstractAlignedChunkMetadata alignedChunkMetadata :
chunkMetadataList) {
ChunkMetadata timeChunkMetadata =
(ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata();
Chunk timeChunk = reader.readMemChunk(timeChunkMetadata);
@@ -796,7 +796,7 @@ public class RepairUnsortedFileCompactionTest extends
AbstractRepairDataTest {
Assert.assertTrue(task.start());
TsFileResource target = tsFileManager.getTsFileList(false).get(0);
try (TsFileSequenceReader reader = new
TsFileSequenceReader(target.getTsFilePath())) {
- List<AlignedChunkMetadata> chunkMetadataList =
+ List<AbstractAlignedChunkMetadata> chunkMetadataList =
reader.getAlignedChunkMetadata(
IDeviceID.Factory.DEFAULT_FACTORY.create("root.testsg.d1"),
true);
Assert.assertEquals(3, chunkMetadataList.size());
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/BatchCompactionUtilsTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/BatchCompactionUtilsTest.java
index ef804437361..c257ee4ec20 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/BatchCompactionUtilsTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/BatchCompactionUtilsTest.java
@@ -34,6 +34,7 @@ import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.exception.write.PageException;
import org.apache.tsfile.exception.write.WriteProcessException;
import org.apache.tsfile.file.header.PageHeader;
+import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
import org.apache.tsfile.file.metadata.ChunkMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
@@ -107,7 +108,7 @@ public class BatchCompactionUtilsTest extends
AbstractCompactionTest {
true);
try (TsFileSequenceReader reader =
new TsFileSequenceReader(seqResource1.getTsFile().getAbsolutePath())) {
- AlignedChunkMetadata alignedChunkMetadata =
+ AbstractAlignedChunkMetadata alignedChunkMetadata =
reader
.getAlignedChunkMetadata(
IDeviceID.Factory.DEFAULT_FACTORY.create("root.testsg.d0"),
true)
@@ -311,14 +312,14 @@ public class BatchCompactionUtilsTest extends
AbstractCompactionTest {
new MeasurementSchema("s1", TSDataType.INT32),
new MeasurementSchema("s2", TSDataType.INT32),
new MeasurementSchema("s4", TSDataType.INT32));
- AlignedChunkMetadata newAlignedChunkMetadata =
+ AbstractAlignedChunkMetadata newAlignedChunkMetadata =
AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList(
alignedChunkMetadata, measurementSchemas);
Assert.assertEquals(
+ Arrays.asList("s0", "s1", "s2", "s4"),
newAlignedChunkMetadata.getValueChunkMetadataList().stream()
.map(IChunkMetadata::getMeasurementUid)
- .collect(Collectors.toList()),
- Arrays.asList("s0", "s1", "s2", "s4"));
+ .collect(Collectors.toList()));
}
@Test
@@ -333,14 +334,14 @@ public class BatchCompactionUtilsTest extends
AbstractCompactionTest {
Arrays.asList(
new MeasurementSchema("s0", TSDataType.INT32),
new MeasurementSchema("s4", TSDataType.INT32));
- AlignedChunkMetadata newAlignedChunkMetadata =
+ AbstractAlignedChunkMetadata newAlignedChunkMetadata =
AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList(
alignedChunkMetadata, measurementSchemas);
Assert.assertEquals(
+ Arrays.asList(null, "s4"),
newAlignedChunkMetadata.getValueChunkMetadataList().stream()
.map(chunkMetadata -> chunkMetadata == null ? null :
chunkMetadata.getMeasurementUid())
- .collect(Collectors.toList()),
- Arrays.asList(null, "s4"));
+ .collect(Collectors.toList()));
}
@Test
@@ -368,23 +369,23 @@ public class BatchCompactionUtilsTest extends
AbstractCompactionTest {
new MeasurementSchema("s2", TSDataType.INT32),
new MeasurementSchema("s3", TSDataType.INT32),
new MeasurementSchema("s4", TSDataType.INT32));
- AlignedChunkMetadata newAlignedChunkMetadata1 =
+ AbstractAlignedChunkMetadata newAlignedChunkMetadata1 =
AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList(
alignedChunkMetadata1, measurementSchemas);
Assert.assertEquals(
+ Arrays.asList("s0", "s1", "s2", null, null),
newAlignedChunkMetadata1.getValueChunkMetadataList().stream()
.map(chunkMetadata -> chunkMetadata == null ? null :
chunkMetadata.getMeasurementUid())
- .collect(Collectors.toList()),
- Arrays.asList("s0", "s1", "s2", null, null));
+ .collect(Collectors.toList()));
- AlignedChunkMetadata newAlignedChunkMetadata2 =
+ AbstractAlignedChunkMetadata newAlignedChunkMetadata2 =
AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList(
alignedChunkMetadata2, measurementSchemas);
Assert.assertEquals(
+ Arrays.asList(null, null, null, "s3", "s4"),
newAlignedChunkMetadata2.getValueChunkMetadataList().stream()
.map(chunkMetadata -> chunkMetadata == null ? null :
chunkMetadata.getMeasurementUid())
- .collect(Collectors.toList()),
- Arrays.asList(null, null, null, "s3", "s4"));
+ .collect(Collectors.toList()));
}
@Test
@@ -411,22 +412,22 @@ public class BatchCompactionUtilsTest extends
AbstractCompactionTest {
Arrays.asList(
new MeasurementSchema("s2", TSDataType.INT32),
new MeasurementSchema("s4", TSDataType.INT32));
- AlignedChunkMetadata newAlignedChunkMetadata1 =
+ AbstractAlignedChunkMetadata newAlignedChunkMetadata1 =
AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList(
alignedChunkMetadata1, measurementSchemas);
Assert.assertEquals(
+ Arrays.asList("s2", null),
newAlignedChunkMetadata1.getValueChunkMetadataList().stream()
.map(chunkMetadata -> chunkMetadata == null ? null :
chunkMetadata.getMeasurementUid())
- .collect(Collectors.toList()),
- Arrays.asList("s2", null));
+ .collect(Collectors.toList()));
- AlignedChunkMetadata newAlignedChunkMetadata2 =
+ AbstractAlignedChunkMetadata newAlignedChunkMetadata2 =
AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList(
alignedChunkMetadata2, measurementSchemas);
Assert.assertEquals(
+ Arrays.asList(null, "s4"),
newAlignedChunkMetadata2.getValueChunkMetadataList().stream()
.map(chunkMetadata -> chunkMetadata == null ? null :
chunkMetadata.getMeasurementUid())
- .collect(Collectors.toList()),
- Arrays.asList(null, "s4"));
+ .collect(Collectors.toList()));
}
}
diff --git a/pom.xml b/pom.xml
index 40164fc32ed..1df4651c4d7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -167,7 +167,7 @@
<thrift.version>0.14.1</thrift.version>
<xz.version>1.9</xz.version>
<zstd-jni.version>1.5.6-3</zstd-jni.version>
- <tsfile.version>2.0.0-250118-SNAPSHOT</tsfile.version>
+ <tsfile.version>2.1.0-250206-SNAPSHOT</tsfile.version>
</properties>
<!--
if we claim dependencies in dependencyManagement, then we do not claim
