This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch ty/iotdb_tsfile_v4 in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 842ca96d6e4ac6bd5fd40f63baa51966c3a8997f Merge: 0dfeac67 0d0e58a3 Author: JackieTien97 <[email protected]> AuthorDate: Sat Apr 20 10:22:22 2024 +0800 merge tsfile_v4 common/pom.xml | 2 +- .../org/apache/tsfile/block/column/Column.java | 20 + examples/pom.xml | 4 +- .../tsfile/TsFileWriteAlignedWithTSRecord.java | 4 +- .../tsfile/TsFileWriteAlignedWithTablet.java | 6 +- .../org/apache/tsfile/TsFileWriteWithTSRecord.java | 4 +- .../org/apache/tsfile/TsFileWriteWithTablet.java | 4 +- pom.xml | 2 +- tsfile/pom.xml | 4 +- .../org/apache/tsfile/common/cache/LRUCache.java | 4 + .../apache/tsfile/common/conf/TSFileConfig.java | 5 +- .../BufferDeserializer.java} | 18 +- .../tsfile/compatibility/CompatibilityUtils.java | 74 +++ .../tsfile/compatibility/DeserializeConfig.java | 102 ++++ .../StreamDeserializer.java} | 19 +- .../tsfile/exception/TsFileRuntimeException.java | 4 + .../FileVersionTooOldException.java} | 14 +- .../read/NoColumnException.java} | 17 +- .../ReadProcessException.java} | 17 +- .../read/UnsupportedOrderingException.java} | 17 +- ...ception.java => ConflictDataTypeException.java} | 10 +- ...rementException.java => NoDeviceException.java} | 6 +- .../exception/write/NoMeasurementException.java | 4 +- ...urementException.java => NoTableException.java} | 9 +- .../tsfile/file/header/ChunkGroupHeader.java | 28 +- .../apache/tsfile/file/metadata/ChunkMetadata.java | 18 + .../file/metadata/DeviceMetadataIndexEntry.java | 13 +- .../org/apache/tsfile/file/metadata/IDeviceID.java | 11 - .../tsfile/file/metadata/LogicalTableSchema.java | 82 +++ .../file/metadata/MetadataIndexConstructor.java | 11 + .../tsfile/file/metadata/MetadataIndexNode.java | 20 +- .../apache/tsfile/file/metadata/PlainDeviceID.java | 89 +++- .../tsfile/file/metadata/StringArrayDeviceID.java | 2 +- .../apache/tsfile/file/metadata/TableSchema.java | 148 ++++++ .../tsfile/file/metadata/TsFileMetadata.java | 75 ++- .../apache/tsfile/read/TsFileSequenceReader.java | 558 +++++++++++++++------ ...leSequenceReaderTimeseriesMetadataIterator.java | 23 +- .../java/org/apache/tsfile/read/common/Path.java | 48 +- .../apache/tsfile/read/common/block/TsBlock.java | 44 +- .../read/common/block/column/BinaryColumn.java | 21 +- .../read/common/block/column/BooleanColumn.java | 21 +- .../common/block/column/ColumnFactory.java} | 32 +- .../read/common/block/column/DoubleColumn.java | 21 +- .../read/common/block/column/FloatColumn.java | 21 +- .../tsfile/read/common/block/column/IntColumn.java | 21 +- .../read/common/block/column/LongColumn.java | 21 +- .../read/common/block/column/NullColumn.java | 10 +- .../block/column/RunLengthEncodedColumn.java | 12 +- .../read/common/block/column/TimeColumn.java | 14 +- .../tsfile/read/controller/DeviceMetaIterator.java | 133 +++++ .../tsfile/read/controller/IMetadataQuerier.java | 20 + .../read/controller/MetadataQuerierByFileImpl.java | 65 ++- .../expression/ExpressionTree.java} | 20 +- .../read/query/executor/TableQueryExecutor.java | 154 ++++++ .../read/query/executor/task/DeviceQueryTask.java | 73 +++ .../query/executor/task/DeviceTaskIterator.java | 62 +++ .../reader/block/DeviceOrderedTsBlockReader.java | 96 ++++ .../reader/block/SingleDeviceTsBlockReader.java | 407 +++++++++++++++ .../reader/block/TsBlockReader.java} | 32 +- .../reader/series/AbstractFileSeriesReader.java | 6 + .../read/reader/series/FileSeriesReader.java | 3 + .../org/apache/tsfile/utils/MeasurementGroup.java | 14 +- .../org/apache/tsfile/utils/ReadWriteIOUtils.java | 4 +- .../apache/tsfile/utils/TsFileGeneratorUtils.java | 16 +- .../java/org/apache/tsfile/utils/WriteUtils.java | 28 ++ .../java/org/apache/tsfile/write/TsFileWriter.java | 341 +++++++------ .../write/chunk/AlignedChunkGroupWriterImpl.java | 52 +- .../tsfile/write/chunk/IChunkGroupWriter.java | 16 +- .../chunk/NonAlignedChunkGroupWriterImpl.java | 28 +- .../org/apache/tsfile/write/record/TSRecord.java | 3 +- .../org/apache/tsfile/write/record/Tablet.java | 249 +++++++-- .../org/apache/tsfile/write/schema/Schema.java | 70 ++- .../write/writer/RestorableTsFileIOWriter.java | 11 +- .../apache/tsfile/write/writer/TsFileIOWriter.java | 59 ++- .../write/writer/TsFileIOWriterEndFileTest.java | 4 +- .../write/writer/tsmiterator/TSMIterator.java | 11 +- .../tsfile/compatibility/CompatibilityTest.java | 74 +++ .../tsfile/exception/TsFileExceptionTest.java | 2 +- .../tsfile/file/metadata/TsFileMetadataTest.java | 4 +- .../tsfile/file/metadata/utils/TestHelper.java | 14 +- .../apache/tsfile/file/metadata/utils/Utils.java | 10 +- .../org/apache/tsfile/read/GetAllDevicesTest.java | 14 +- ...easurementChunkMetadataListMapIteratorTest.java | 7 +- .../org/apache/tsfile/read/TsFileReaderTest.java | 9 +- .../tsfile/read/TsFileSequenceReaderTest.java | 14 +- .../org/apache/tsfile/read/common/PathTest.java | 47 +- .../read/filter/IExpressionOptimizerTest.java | 32 +- .../apache/tsfile/tableview/PerformanceTest.java | 372 ++++++++++++++ .../org/apache/tsfile/tableview/TableViewTest.java | 344 +++++++++++++ .../org/apache/tsfile/utils/FileGenerator.java | 6 +- .../java/org/apache/tsfile/utils/RecordUtils.java | 5 +- .../tsfile/utils/TsFileGeneratorForTest.java | 5 +- .../tsfile/write/DefaultSchemaTemplateTest.java | 5 +- .../tsfile/write/MetadataIndexConstructorTest.java | 31 +- .../apache/tsfile/write/TsFileIOWriterTest.java | 24 +- .../apache/tsfile/write/TsFileWriteApiTest.java | 36 +- .../org/apache/tsfile/write/TsFileWriterTest.java | 20 +- .../java/org/apache/tsfile/write/WriteTest.java | 11 +- .../org/apache/tsfile/write/record/TabletTest.java | 5 +- .../write/schema/converter/SchemaBuilderTest.java | 4 +- .../write/writer/RestorableTsFileIOWriterTest.java | 8 +- .../writer/TsFileIOWriterMemoryControlTest.java | 13 +- tsfile/src/test/resources/v3TsFile | Bin 0 -> 378 bytes 103 files changed, 3984 insertions(+), 843 deletions(-) diff --cc common/pom.xml index ad785213,8e61f807..4723ab8c --- a/common/pom.xml +++ b/common/pom.xml @@@ -24,7 -24,7 +24,7 @@@ <parent> <groupId>org.apache.tsfile</groupId> <artifactId>tsfile-parent</artifactId> - <version>1.0.1-table-SNAPSHOT</version> - <version>1.0.1-SNAPSHOT</version> ++ <version>1.0.1-tsfilev4-SNAPSHOT</version> </parent> <artifactId>common</artifactId> <name>TsFile: Common API</name> diff --cc examples/pom.xml index a5e32728,aa9f958d..e83b16c8 --- a/examples/pom.xml +++ b/examples/pom.xml @@@ -24,7 -24,7 +24,7 @@@ <parent> <groupId>org.apache.tsfile</groupId> <artifactId>tsfile-parent</artifactId> - <version>1.0.1-table-SNAPSHOT</version> - <version>1.0.1-SNAPSHOT</version> ++ <version>1.0.1-tsfilev4-SNAPSHOT</version> </parent> <artifactId>examples</artifactId> <packaging>pom</packaging> @@@ -33,7 -33,7 +33,7 @@@ <dependency> <groupId>org.apache.tsfile</groupId> <artifactId>tsfile</artifactId> - <version>1.0.1-table-SNAPSHOT</version> - <version>1.0.1-SNAPSHOT</version> ++ <version>1.0.1-tsfilev4-SNAPSHOT</version> </dependency> </dependencies> <build> diff --cc pom.xml index f3b0df28,ce766e3c..d9773ce7 --- a/pom.xml +++ b/pom.xml @@@ -28,7 -28,7 +28,7 @@@ </parent> <groupId>org.apache.tsfile</groupId> <artifactId>tsfile-parent</artifactId> - <version>1.0.1-table-SNAPSHOT</version> - <version>1.0.1-SNAPSHOT</version> ++ <version>1.0.1-tsfilev4-SNAPSHOT</version> <packaging>pom</packaging> <name>Apache TsFile Project Parent POM</name> <modules> diff --cc tsfile/pom.xml index 86dada8c,a8fb83af..24896e69 --- a/tsfile/pom.xml +++ b/tsfile/pom.xml @@@ -24,7 -24,7 +24,7 @@@ <parent> <groupId>org.apache.tsfile</groupId> <artifactId>tsfile-parent</artifactId> - <version>1.0.1-table-SNAPSHOT</version> - <version>1.0.1-SNAPSHOT</version> ++ <version>1.0.1-tsfilev4-SNAPSHOT</version> </parent> <artifactId>tsfile</artifactId> <name>TsFile: TsFile</name> @@@ -38,7 -38,7 +38,7 @@@ <dependency> <groupId>org.apache.tsfile</groupId> <artifactId>common</artifactId> - <version>1.0.1-table-SNAPSHOT</version> - <version>1.0.1-SNAPSHOT</version> ++ <version>1.0.1-tsfilev4-SNAPSHOT</version> </dependency> <dependency> <groupId>com.github.luben</groupId> diff --cc tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java index 0f2100cf,23c68b0a..4473a4ec --- a/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java +++ b/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java @@@ -73,8 -73,8 +73,9 @@@ public class TSFileConfig implements Se /** The primitive array capacity threshold. */ public static final int ARRAY_CAPACITY_THRESHOLD = 1000; + - public static final int DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME = 3; + // TODO: configurable but unchangeable + public static int DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME = 3; /** Memory size threshold for flushing to disk, default value is 128MB. */ private int groupSizeInByte = 128 * 1024 * 1024; /** The memory size for each series writer to pack page, default value is 64KB. */ diff --cc tsfile/src/main/java/org/apache/tsfile/utils/WriteUtils.java index aa91e228,2fb13466..542d252f --- a/tsfile/src/main/java/org/apache/tsfile/utils/WriteUtils.java +++ b/tsfile/src/main/java/org/apache/tsfile/utils/WriteUtils.java @@@ -16,11 -16,38 +16,39 @@@ * specific language governing permissions and limitations * under the License. */ + package org.apache.tsfile.utils; + import org.apache.tsfile.file.metadata.IDeviceID; + import org.apache.tsfile.write.record.Tablet; + + import java.util.ArrayList; + import java.util.List; + public class WriteUtils { + /** + * A Tablet for the table-view insert interface may contain more than one device. This method + * splits a Tablet by different deviceIds so that the caller can insert them device-by-device. + * + * @return (deviceId, endRowNum) pairs + */ + public static List<Pair<IDeviceID, Integer>> splitTabletByDevice(Tablet tablet) { + List<Pair<IDeviceID, Integer>> result = new ArrayList<>(); + IDeviceID lastDeviceID = null; + for (int i = 0; i < tablet.rowSize; i++) { + final IDeviceID currDeviceID = tablet.getDeviceID(i); + if (!currDeviceID.equals(lastDeviceID)) { + if (lastDeviceID != null) { + result.add(new Pair<>(lastDeviceID, i)); + } + lastDeviceID = currDeviceID; + } + } + result.add(new Pair<>(lastDeviceID, tablet.rowSize)); + return result; + } + public static int compareStrings(String a, String b) { if (a == null && b == null) { return 0;
