This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch rel/0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.12 by this push:
new d725ab3 [To rel/0.12][IOTDB-1903] Fix IndexOutOfRangeException when
starting IoTDB (#4312)
d725ab3 is described below
commit d725ab3158537567a792151b3223ccafc6d62296
Author: liuxuxin <[email protected]>
AuthorDate: Fri Nov 5 21:33:48 2021 +0800
[To rel/0.12][IOTDB-1903] Fix IndexOutOfRangeException when starting IoTDB
(#4312)
---
pom.xml | 56 +-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 17 +-
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 4 +-
.../read/TsFileSequenceReaderSelfCheckTest.java | 799 +++++++++++++++++++++
4 files changed, 852 insertions(+), 24 deletions(-)
diff --git a/pom.xml b/pom.xml
index f50d5eb..c8dd8a9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -19,7 +19,8 @@
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache</groupId>
@@ -31,7 +32,9 @@
<version>0.12.4-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache IoTDB Project Parent POM</name>
- <description>This is the top level project that builds, packages the
tsfile, iotdb engine, jdbc, and integration libs.</description>
+ <description>This is the top level project that builds, packages the
tsfile, iotdb engine, jdbc, and integration
+ libs.
+ </description>
<licenses>
<license>
<name>The Apache License, Version 2.0</name>
@@ -147,7 +150,8 @@
<!-- Exclude all generated code -->
<sonar.exclusions>**/generated-sources</sonar.exclusions>
<sonar.java.checkstyle.reportPaths>target/checkstyle-report.xml</sonar.java.checkstyle.reportPaths>
-
<sonar.coverage.jacoco.xmlReportPaths>target/jacoco-merged-reports/jacoco.xml</sonar.coverage.jacoco.xmlReportPaths>
+
<sonar.coverage.jacoco.xmlReportPaths>target/jacoco-merged-reports/jacoco.xml
+ </sonar.coverage.jacoco.xmlReportPaths>
<sonar.junit.reportPaths>target/surefire-reports,target/failsafe-reports</sonar.junit.reportPaths>
<!-- By default, the argLine is empty-->
<gson.version>2.8.6</gson.version>
@@ -806,7 +810,8 @@
</requireJavaVersion>
<!-- Disabled for now as it breaks the ability
to build single modules -->
<!--reactorModuleConvergence/-->
- <banVulnerable
implementation="org.sonatype.ossindex.maven.enforcer.BanVulnerableDependencies"/>
+ <banVulnerable
+
implementation="org.sonatype.ossindex.maven.enforcer.BanVulnerableDependencies"/>
</rules>
</configuration>
</execution>
@@ -1014,7 +1019,9 @@
</activation>
<properties>
<os.classifier>windows-x86_64</os.classifier>
-
<thrift.download-url>http://artfiles.org/apache.org/thrift/${thrift.version}/thrift-${thrift.version}.exe</thrift.download-url>
+ <thrift.download-url>
+
http://artfiles.org/apache.org/thrift/${thrift.version}/thrift-${thrift.version}.exe
+ </thrift.download-url>
<thrift.executable>thrift-${thrift.version}-win-x86_64.exe</thrift.executable>
<thrift.skip-making-executable>true</thrift.skip-making-executable>
<thrift.exec-cmd.executable>echo</thrift.exec-cmd.executable>
@@ -1031,7 +1038,9 @@
</activation>
<properties>
<os.classifier>linux-x86_64</os.classifier>
-
<thrift.download-url>https://github.com/apache/iotdb-bin-resources/raw/main/compile-tools/thrift-0.14-ubuntu</thrift.download-url>
+ <thrift.download-url>
+
https://github.com/apache/iotdb-bin-resources/raw/main/compile-tools/thrift-0.14-ubuntu
+ </thrift.download-url>
<thrift.executable>thrift_0.14.1_linux.exe</thrift.executable>
<thrift.skip-making-executable>false</thrift.skip-making-executable>
<thrift.exec-cmd.executable>chmod</thrift.exec-cmd.executable>
@@ -1047,7 +1056,9 @@
</activation>
<properties>
<os.classifier>mac-x86_64</os.classifier>
-
<thrift.download-url>https://github.com/apache/iotdb-bin-resources/raw/main/compile-tools/thrift-0.14-MacOS</thrift.download-url>
+ <thrift.download-url>
+
https://github.com/apache/iotdb-bin-resources/raw/main/compile-tools/thrift-0.14-MacOS
+ </thrift.download-url>
<thrift.executable>thrift_0.14.1_mac.exe</thrift.executable>
<thrift.skip-making-executable>false</thrift.skip-making-executable>
<thrift.exec-cmd.executable>chmod</thrift.exec-cmd.executable>
@@ -1097,7 +1108,8 @@
</file>
</activation>
<properties>
-
<thrift.exec.absolute.path>${project.build.directory}/tools/${thrift.executable}</thrift.exec.absolute.path>
+
<thrift.exec.absolute.path>${project.build.directory}/tools/${thrift.executable}
+ </thrift.exec.absolute.path>
</properties>
<build>
<plugins>
@@ -1166,7 +1178,8 @@
<generator>py</generator>
<thriftExecutable>${thrift.exec.absolute.path}</thriftExecutable>
<thriftSourceRoot>${basedir}/src/main/thrift</thriftSourceRoot>
-
<outputDirectory>${project.build.directory}/generated-sources-python/</outputDirectory>
+
<outputDirectory>${project.build.directory}/generated-sources-python/
+ </outputDirectory>
</configuration>
</execution>
<execution>
@@ -1318,7 +1331,9 @@
<goal>prepare-agent</goal>
</goals>
<configuration>
-
<destFile>${project.build.directory}/${project.build.finalName}-jacoco-unit-tests.exec</destFile>
+ <destFile>
+
${project.build.directory}/${project.build.finalName}-jacoco-unit-tests.exec
+ </destFile>
<propertyName>surefire.jacoco.args</propertyName>
</configuration>
</execution>
@@ -1331,7 +1346,9 @@
<goal>check</goal>
</goals>
<configuration>
-
<dataFile>${project.build.directory}/${project.build.finalName}-jacoco-unit-tests.exec</dataFile>
+ <dataFile>
+
${project.build.directory}/${project.build.finalName}-jacoco-unit-tests.exec
+ </dataFile>
<outputDirectory>${project.build.directory}/jacoco-unit-reports</outputDirectory>
</configuration>
</execution>
@@ -1343,7 +1360,9 @@
<goal>prepare-agent</goal>
</goals>
<configuration>
-
<destFile>${project.build.directory}/${project.build.finalName}-jacoco-integration-tests.exec</destFile>
+ <destFile>
+
${project.build.directory}/${project.build.finalName}-jacoco-integration-tests.exec
+ </destFile>
<propertyName>failsafe.jacoco.args</propertyName>
</configuration>
</execution>
@@ -1355,8 +1374,11 @@
<goal>check</goal>
</goals>
<configuration>
-
<dataFile>${project.build.directory}/${project.build.finalName}-jacoco-integration-tests.exec</dataFile>
-
<outputDirectory>${project.build.directory}/jacoco-integration-reports</outputDirectory>
+ <dataFile>
+
${project.build.directory}/${project.build.finalName}-jacoco-integration-tests.exec
+ </dataFile>
+
<outputDirectory>${project.build.directory}/jacoco-integration-reports
+ </outputDirectory>
</configuration>
</execution>
<execution>
@@ -1374,7 +1396,8 @@
</includes>
</fileSet>
</fileSets>
-
<destFile>${project.build.directory}/${project.build.finalName}-merged.exec</destFile>
+
<destFile>${project.build.directory}/${project.build.finalName}-merged.exec
+ </destFile>
</configuration>
</execution>
<execution>
@@ -1385,7 +1408,8 @@
<goal>check</goal>
</goals>
<configuration>
-
<dataFile>${project.build.directory}/${project.build.finalName}-merged.exec</dataFile>
+
<dataFile>${project.build.directory}/${project.build.finalName}-merged.exec
+ </dataFile>
<outputDirectory>${project.build.directory}/jacoco-merged-reports</outputDirectory>
</configuration>
</execution>
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index aee2d92..d174ca0 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -1136,6 +1136,7 @@ public class TsFileSequenceReader implements
AutoCloseable {
}
// last chunk group Metadata
chunkGroupMetadataList.add(new ChunkGroupMetadata(lastDeviceId,
chunkMetadataList));
+ lastDeviceId = null;
}
truncatedSize = this.position() - 1;
} catch (Exception e) {
@@ -1145,12 +1146,16 @@ public class TsFileSequenceReader implements
AutoCloseable {
this.position(),
e.getMessage());
}
- if (loadLastChunkMetadata
- && !chunkGroupMetadataList
- .get(chunkGroupMetadataList.size() - 1)
- .getDevice()
- .equals(lastDeviceId)) {
- chunkGroupMetadataList.add(new ChunkGroupMetadata(lastDeviceId,
chunkMetadataList));
+ if (loadLastChunkMetadata && lastDeviceId != null) {
+ // add last chunk group metadata list
+ if (chunkMetadataList.size() > 0) {
+ chunkGroupMetadataList.add(new ChunkGroupMetadata(lastDeviceId,
chunkMetadataList));
+ if (newSchema != null) {
+ for (MeasurementSchema tsSchema : measurementSchemaList) {
+ newSchema.putIfAbsent(new Path(lastDeviceId,
tsSchema.getMeasurementId()), tsSchema);
+ }
+ }
+ }
}
// Despite the completeness of the data section, we will discard current
FileMetadata
// so that we can continue to write data into this tsfile.
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
index d0818f3..b009d09 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
@@ -377,11 +377,11 @@ public class TsFileIOWriter {
out.close();
}
- void writeSeparatorMaskForTest() throws IOException {
+ public void writeSeparatorMaskForTest() throws IOException {
out.write(new byte[] {MetaMarker.SEPARATOR});
}
- void writeChunkGroupMarkerForTest() throws IOException {
+ public void writeChunkGroupMarkerForTest() throws IOException {
out.write(new byte[] {MetaMarker.CHUNK_GROUP_HEADER});
}
diff --git
a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderSelfCheckTest.java
b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderSelfCheckTest.java
new file mode 100644
index 0000000..ac8274f
--- /dev/null
+++
b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderSelfCheckTest.java
@@ -0,0 +1,799 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.tsfile.read;
+
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.constant.TestConstant;
+import org.apache.iotdb.tsfile.file.MetaMarker;
+import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetadata;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.file.metadata.statistics.FloatStatistics;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.utils.TsFileGeneratorForTest;
+import org.apache.iotdb.tsfile.write.TsFileWriter;
+import org.apache.iotdb.tsfile.write.record.TSRecord;
+import org.apache.iotdb.tsfile.write.record.datapoint.FloatDataPoint;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class TsFileSequenceReaderSelfCheckTest {
+ private static final String FILE_NAME =
+ TestConstant.BASE_OUTPUT_PATH.concat(System.currentTimeMillis() +
"-1-0-0.tsfile");
+ File file = new File(FILE_NAME);
+
+ @Before
+ public void setUp() throws IOException {
+ if (!file.getParentFile().exists()) {
+ Assert.assertTrue(file.getParentFile().mkdirs());
+ }
+ if (file.exists()) {
+ Assert.assertTrue(file.delete());
+ }
+ }
+
+ @After
+ public void tearDown() {
+ if (file.exists()) {
+ Assert.assertTrue(file.delete());
+ }
+ }
+
+ // File with wrong head magic string
+ @Test
+ public void testCheckWithBadHeadMagic() throws Exception {
+ try (FileWriter writer = new FileWriter(file)) {
+ writer.write("TEST_STRING");
+ }
+ try (TsFileSequenceReader reader = new
TsFileSequenceReader(file.getPath())) {
+ Map<Path, MeasurementSchema> newSchema = new HashMap<>();
+ List<ChunkGroupMetadata> chunkGroupMetadataList = new ArrayList<>();
+ long truncateSize = reader.selfCheck(newSchema, chunkGroupMetadataList,
false, true);
+ Assert.assertEquals(truncateSize, TsFileCheckStatus.INCOMPATIBLE_FILE);
+ Assert.assertEquals(newSchema.size(), 0);
+ Assert.assertEquals(chunkGroupMetadataList.size(), 0);
+ }
+ }
+
+ // test file with magic string and version number
+ @Test
+ public void testCheckWithOnlyHeadMagic() throws Exception {
+ TsFileWriter writer = new TsFileWriter(file);
+ // we have to flush using inner API.
+ writer.close();
+ try (TsFileSequenceReader reader = new
TsFileSequenceReader(file.getPath())) {
+ Map<Path, MeasurementSchema> newSchema = new HashMap<>();
+ List<ChunkGroupMetadata> chunkGroupMetadataList = new ArrayList<>();
+ long checkStatus = reader.selfCheck(newSchema, chunkGroupMetadataList,
false, true);
+ Assert.assertEquals(checkStatus,
TSFileConfig.MAGIC_STRING.getBytes().length + Byte.BYTES);
+ Assert.assertEquals(newSchema.size(), 0);
+ Assert.assertEquals(chunkGroupMetadataList.size(), 0);
+ }
+ }
+
+ // test file with magic string, version number and a marker of ChunkHeader
+ @Test
+ public void testCheckWithOnlyFirstMask() throws Exception {
+ TsFileWriter writer = new TsFileWriter(file);
+ // we have to flush using inner API.
+ writer.getIOWriter().getIOWriterOut().write(new byte[]
{MetaMarker.CHUNK_HEADER});
+ writer.getIOWriter().close();
+ try (TsFileSequenceReader reader = new
TsFileSequenceReader(file.getPath())) {
+ Map<Path, MeasurementSchema> newSchema = new HashMap<>();
+ List<ChunkGroupMetadata> chunkGroupMetadataList = new ArrayList<>();
+ long checkStatus = reader.selfCheck(newSchema, chunkGroupMetadataList,
false, true);
+ Assert.assertEquals(checkStatus,
TSFileConfig.MAGIC_STRING.getBytes().length + Byte.BYTES);
+ Assert.assertEquals(newSchema.size(), 0);
+ Assert.assertEquals(chunkGroupMetadataList.size(), 0);
+ }
+ }
+
+ // Test file with magic string, version number, marker of ChunkHeader,
incomplete chunk header
+ @Test
+ public void testCheckWithIncompleteChunkHeader() throws Exception {
+ TsFileGeneratorForTest.writeFileWithOneIncompleteChunkHeader(file);
+ try (TsFileSequenceReader reader = new
TsFileSequenceReader(file.getPath())) {
+ Map<Path, MeasurementSchema> newSchema = new HashMap<>();
+ List<ChunkGroupMetadata> chunkGroupMetadataList = new ArrayList<>();
+ long checkStatus = reader.selfCheck(newSchema, chunkGroupMetadataList,
false, true);
+ Assert.assertEquals(checkStatus,
TSFileConfig.MAGIC_STRING.getBytes().length + Byte.BYTES);
+ Assert.assertEquals(newSchema.size(), 0);
+ Assert.assertEquals(chunkGroupMetadataList.size(), 0);
+ }
+ }
+
+ // Test file with magic string, version number, marker of ChunkHeader, one
chunkHeader
+ // And do not load the last chunk group metadata
+ @Test
+ public void testCheckWithOnlyOneChunkHeader() throws Exception {
+ TsFileWriter writer = new TsFileWriter(file);
+ writer.getIOWriter().startChunkGroup("root.sg1.d1");
+ writer
+ .getIOWriter()
+ .startFlushChunk(
+ new MeasurementSchema("s1", TSDataType.FLOAT, TSEncoding.PLAIN),
+ CompressionType.SNAPPY,
+ TSDataType.FLOAT,
+ TSEncoding.PLAIN,
+ new FloatStatistics(),
+ 100,
+ 10);
+ writer.getIOWriter().close();
+
+ try (TsFileSequenceReader reader = new
TsFileSequenceReader(file.getPath())) {
+ Map<Path, MeasurementSchema> newSchema = new HashMap<>();
+ List<ChunkGroupMetadata> chunkGroupMetadataList = new ArrayList<>();
+ long checkStatus = reader.selfCheck(newSchema, chunkGroupMetadataList,
false, false);
+ Assert.assertEquals(checkStatus,
TSFileConfig.MAGIC_STRING.getBytes().length + Byte.BYTES);
+ Assert.assertEquals(newSchema.size(), 0);
+ Assert.assertEquals(chunkGroupMetadataList.size(), 0);
+ }
+ }
+
+ // Test file with magic string, version number, marker of ChunkHeader, one
chunkHeader
+ // And load the last chunk group metadata
+ @Test
+ public void testCheckWithOnlyOneChunkHeaderAndLoadLastChunkGroupMetadata()
throws Exception {
+ TsFileWriter writer = new TsFileWriter(file);
+ writer.getIOWriter().startChunkGroup("root.sg1.d1");
+ writer
+ .getIOWriter()
+ .startFlushChunk(
+ new MeasurementSchema("s1", TSDataType.FLOAT, TSEncoding.PLAIN),
+ CompressionType.SNAPPY,
+ TSDataType.FLOAT,
+ TSEncoding.PLAIN,
+ new FloatStatistics(),
+ 100,
+ 10);
+ writer.getIOWriter().close();
+
+ try (TsFileSequenceReader reader = new
TsFileSequenceReader(file.getPath())) {
+ Map<Path, MeasurementSchema> newSchema = new HashMap<>();
+ List<ChunkGroupMetadata> chunkGroupMetadataList = new ArrayList<>();
+ long checkStatus = reader.selfCheck(newSchema, chunkGroupMetadataList,
false, true);
+ Assert.assertEquals(checkStatus,
TSFileConfig.MAGIC_STRING.getBytes().length + Byte.BYTES);
+ Assert.assertEquals(newSchema.size(), 0);
+ Assert.assertEquals(chunkGroupMetadataList.size(), 0);
+ }
+ }
+
+ /*
+ * test check with tsfile with one ChunkGroup, and not marker behind this
chunk group
+ */
+ @Test
+ public void testCheckWithOnlyOneChunkGroupAndNoMarker() throws Exception {
+ TsFileWriter writer = new TsFileWriter(file);
+ writer.registerTimeseries(
+ new Path("d1", "s1"), new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d1", "s2"), new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.write(
+ new TSRecord(1, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.write(
+ new TSRecord(2, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.flushAllChunkGroups();
+ writer.getIOWriter().close();
+ try (TsFileSequenceReader reader = new
TsFileSequenceReader(file.getPath())) {
+ Map<Path, MeasurementSchema> newSchema = new HashMap<>();
+ List<ChunkGroupMetadata> chunkGroupMetadataList = new ArrayList<>();
+ long checkStatus = reader.selfCheck(newSchema, chunkGroupMetadataList,
false, false);
+ Assert.assertEquals(TSFileConfig.MAGIC_STRING.getBytes().length +
Byte.BYTES, checkStatus);
+ Assert.assertEquals(newSchema.size(), 0);
+ Assert.assertEquals(chunkGroupMetadataList.size(), 0);
+ }
+ }
+
+ /*
+ * test check with tsfile with one ChunkGroup, and not marker behind this
chunk group. And load the last chunk group metadata.
+ */
+ @Test
+ public void
testCheckWithOnlyOneChunkGroupAndNoMarkerAndLoadLastChunkGroupMetadata()
+ throws Exception {
+ TsFileWriter writer = new TsFileWriter(file);
+ writer.registerTimeseries(
+ new Path("d1", "s1"), new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d1", "s2"), new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.write(
+ new TSRecord(1, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.write(
+ new TSRecord(2, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.flushAllChunkGroups();
+ writer.getIOWriter().close();
+ try (TsFileSequenceReader reader = new
TsFileSequenceReader(file.getPath())) {
+ Map<Path, MeasurementSchema> newSchema = new HashMap<>();
+ List<ChunkGroupMetadata> chunkGroupMetadataList = new ArrayList<>();
+ long checkStatus = reader.selfCheck(newSchema, chunkGroupMetadataList,
false, true);
+ Assert.assertEquals(TSFileConfig.MAGIC_STRING.getBytes().length +
Byte.BYTES, checkStatus);
+ Assert.assertEquals(newSchema.size(), 2);
+ Assert.assertEquals(chunkGroupMetadataList.size(), 1);
+ ChunkGroupMetadata chunkGroupMetadata = chunkGroupMetadataList.get(0);
+ Assert.assertEquals(chunkGroupMetadata.getChunkMetadataList().size(), 2);
+ }
+ }
+
+ /** Test check with one complete chunk group and marker */
+ @Test
+ public void testCheckWithOneCompleteChunkGroupAndMarker() throws Exception {
+ TsFileWriter writer = new TsFileWriter(file);
+ writer.registerTimeseries(
+ new Path("d1", "s1"), new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d1", "s2"), new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.write(
+ new TSRecord(1, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.write(
+ new TSRecord(2, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.flushAllChunkGroups();
+ long pos = writer.getIOWriter().getPos();
+ writer.getIOWriter().writeChunkGroupMarkerForTest();
+ writer.getIOWriter().close();
+ try (TsFileSequenceReader reader = new
TsFileSequenceReader(file.getPath())) {
+ Map<Path, MeasurementSchema> newSchema = new HashMap<>();
+ List<ChunkGroupMetadata> chunkGroupMetadataList = new ArrayList<>();
+ long checkStatus = reader.selfCheck(newSchema, chunkGroupMetadataList,
false, false);
+ Assert.assertEquals(pos, checkStatus);
+ Assert.assertEquals(newSchema.size(), 2);
+ Assert.assertEquals(chunkGroupMetadataList.size(), 1);
+ ChunkGroupMetadata chunkGroupMetadata = chunkGroupMetadataList.get(0);
+ Assert.assertEquals(chunkGroupMetadata.getChunkMetadataList().size(), 2);
+ }
+ }
+
+ /** Test check with one complete chunk group and marker, and load the last
chunk group */
+ @Test
+ public void
testCheckWithOneCompleteChunkGroupAndMarkerAndLoadLastChunkGroup() throws
Exception {
+ TsFileWriter writer = new TsFileWriter(file);
+ writer.registerTimeseries(
+ new Path("d1", "s1"), new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d1", "s2"), new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.write(
+ new TSRecord(1, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.write(
+ new TSRecord(2, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.flushAllChunkGroups();
+ long pos = writer.getIOWriter().getPos();
+ writer.getIOWriter().writeChunkGroupMarkerForTest();
+ writer.getIOWriter().close();
+ try (TsFileSequenceReader reader = new
TsFileSequenceReader(file.getPath())) {
+ Map<Path, MeasurementSchema> newSchema = new HashMap<>();
+ List<ChunkGroupMetadata> chunkGroupMetadataList = new ArrayList<>();
+ long checkStatus = reader.selfCheck(newSchema, chunkGroupMetadataList,
false, true);
+ Assert.assertEquals(pos, checkStatus);
+ Assert.assertEquals(newSchema.size(), 2);
+ Assert.assertEquals(chunkGroupMetadataList.size(), 1);
+ ChunkGroupMetadata chunkGroupMetadata = chunkGroupMetadataList.get(0);
+ Assert.assertEquals(chunkGroupMetadata.getChunkMetadataList().size(), 2);
+ }
+ }
+
+ /*
+ * Test file with magic string, version number, marker of ChunkHeader, one
complete chunk group, one chunk group with the last chunk uncompleted. And do
not load the last chunk metadata.
+ */
+ @Test
+ public void testCheckWithOneCompleteChunkGroupAndOneUncompletedChunkGroup()
throws Exception {
+ TsFileWriter writer = new TsFileWriter(file);
+ writer.registerTimeseries(
+ new Path("d1", "s1"), new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d1", "s2"), new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.write(
+ new TSRecord(1, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.write(
+ new TSRecord(2, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.flushAllChunkGroups();
+ long pos1 = writer.getIOWriter().getPos();
+ writer.registerTimeseries(
+ new Path("d2", "s1"), new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d2", "s2"), new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.write(
+ new TSRecord(1, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.flushAllChunkGroups();
+ long pos2 = writer.getIOWriter().getPos();
+ // let's delete one byte. the version is broken
+ writer.getIOWriter().getIOWriterOut().truncate(pos2 - 1);
+ writer.getIOWriter().close();
+
+ try (TsFileSequenceReader reader = new
TsFileSequenceReader(file.getPath())) {
+ Map<Path, MeasurementSchema> newSchema = new HashMap<>();
+ List<ChunkGroupMetadata> chunkGroupMetadataList = new ArrayList<>();
+ long checkStatus = reader.selfCheck(newSchema, chunkGroupMetadataList,
false, false);
+ Assert.assertEquals(checkStatus, pos1);
+ Assert.assertEquals(newSchema.size(), 2);
+ Assert.assertEquals(chunkGroupMetadataList.size(), 1);
+ ChunkGroupMetadata chunkGroupMetadata = chunkGroupMetadataList.get(0);
+ Assert.assertEquals(chunkGroupMetadata.getChunkMetadataList().size(), 2);
+ }
+ }
+
+ /*
+ * Test file with magic string, version number, marker of ChunkHeader, one
complete chunk group, one chunk group with the last chunk uncompleted. And load
the last chunk metadata.
+ */
+ @Test
+ public void
+
testCheckWithOneCompleteChunkGroupAndOneUncompletedChunkGroupAndLoadLastChunkGroupMetadata()
+ throws Exception {
+ TsFileWriter writer = new TsFileWriter(file);
+ writer.registerTimeseries(
+ new Path("d1", "s1"), new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d1", "s2"), new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.write(
+ new TSRecord(1, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.write(
+ new TSRecord(2, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.flushAllChunkGroups();
+ long pos1 = writer.getIOWriter().getPos();
+ writer.registerTimeseries(
+ new Path("d2", "s1"), new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d2", "s2"), new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.write(
+ new TSRecord(1, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.flushAllChunkGroups();
+ long pos2 = writer.getIOWriter().getPos();
+ // let's delete one byte. the version is broken
+ writer.getIOWriter().getIOWriterOut().truncate(pos2 - 1);
+ writer.getIOWriter().close();
+
+ try (TsFileSequenceReader reader = new
TsFileSequenceReader(file.getPath())) {
+ Map<Path, MeasurementSchema> newSchema = new HashMap<>();
+ List<ChunkGroupMetadata> chunkGroupMetadataList = new ArrayList<>();
+ long checkStatus = reader.selfCheck(newSchema, chunkGroupMetadataList,
false, true);
+ Assert.assertEquals(checkStatus, pos1);
+ Assert.assertEquals(newSchema.size(), 2);
+ Assert.assertEquals(chunkGroupMetadataList.size(), 2);
+ ChunkGroupMetadata firstChunkGroupMetadata =
chunkGroupMetadataList.get(0);
+ ChunkGroupMetadata secondChunkGroupMetadata =
chunkGroupMetadataList.get(1);
+ // the first chunk group contains the chunk metadata for two chunks
+ Assert.assertEquals(2,
firstChunkGroupMetadata.getChunkMetadataList().size());
+ // the second chunk group contains only chunk metadata for the first
chunk
+ Assert.assertEquals(1,
secondChunkGroupMetadata.getChunkMetadataList().size());
+ }
+ }
+ /*
+ * Test check with two chunk group and marker
+ */
+ @Test
+ public void testCheckWithTwoChunkGroupAndMarker() throws Exception {
+ TsFileWriter writer = new TsFileWriter(file);
+ writer.registerTimeseries(
+ new Path("d1", "s1"), new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d1", "s2"), new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d2", "s1"), new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d2", "s2"), new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.write(
+ new TSRecord(1, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.write(
+ new TSRecord(2, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+
+ writer.write(
+ new TSRecord(1, "d2")
+ .addTuple(new FloatDataPoint("s1", 6))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.write(
+ new TSRecord(2, "d2")
+ .addTuple(new FloatDataPoint("s1", 6))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.flushAllChunkGroups();
+ long pos1 = writer.getIOWriter().getPos();
+ writer.getIOWriter().writeChunkGroupMarkerForTest();
+ writer.getIOWriter().close();
+ try (TsFileSequenceReader reader = new
TsFileSequenceReader(file.getPath())) {
+ Map<Path, MeasurementSchema> newSchema = new HashMap<>();
+ List<ChunkGroupMetadata> chunkGroupMetadataList = new ArrayList<>();
+ long checkStatus = reader.selfCheck(newSchema, chunkGroupMetadataList,
false, false);
+ Assert.assertEquals(checkStatus, pos1);
+ Assert.assertEquals(newSchema.size(), 4);
+ Assert.assertEquals(chunkGroupMetadataList.size(), 2);
+ ChunkGroupMetadata firstChunkGroupMetadata =
chunkGroupMetadataList.get(0);
+ ChunkGroupMetadata secondChunkGroupMetadata =
chunkGroupMetadataList.get(1);
+ // the first chunk group contains the chunk metadata for two chunks
+ Assert.assertEquals(2,
firstChunkGroupMetadata.getChunkMetadataList().size());
+ // the second chunk group contains only chunk metadata for the first
chunk
+ Assert.assertEquals(2,
secondChunkGroupMetadata.getChunkMetadataList().size());
+ }
+ }
+
+ /*
+ * Test check with two chunk group and marker, and load last chunk group
+ */
+ @Test
+ public void testCheckWithTwoChunkGroupAndMarkerAndLoadLastChunkGroup()
throws Exception {
+ TsFileWriter writer = new TsFileWriter(file);
+ writer.registerTimeseries(
+ new Path("d1", "s1"), new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d1", "s2"), new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d2", "s1"), new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d2", "s2"), new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.write(
+ new TSRecord(1, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.write(
+ new TSRecord(2, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+
+ writer.write(
+ new TSRecord(1, "d2")
+ .addTuple(new FloatDataPoint("s1", 6))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.write(
+ new TSRecord(2, "d2")
+ .addTuple(new FloatDataPoint("s1", 6))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.flushAllChunkGroups();
+ long pos1 = writer.getIOWriter().getPos();
+ writer.getIOWriter().writeChunkGroupMarkerForTest();
+ writer.getIOWriter().close();
+ try (TsFileSequenceReader reader = new
TsFileSequenceReader(file.getPath())) {
+ Map<Path, MeasurementSchema> newSchema = new HashMap<>();
+ List<ChunkGroupMetadata> chunkGroupMetadataList = new ArrayList<>();
+ long checkStatus = reader.selfCheck(newSchema, chunkGroupMetadataList,
false, true);
+ Assert.assertEquals(checkStatus, pos1);
+ Assert.assertEquals(newSchema.size(), 4);
+ Assert.assertEquals(chunkGroupMetadataList.size(), 2);
+ ChunkGroupMetadata firstChunkGroupMetadata =
chunkGroupMetadataList.get(0);
+ ChunkGroupMetadata secondChunkGroupMetadata =
chunkGroupMetadataList.get(1);
+ // the first chunk group contains the chunk metadata for two chunks
+ Assert.assertEquals(2,
firstChunkGroupMetadata.getChunkMetadataList().size());
+ // the second chunk group contains only chunk metadata for the first
chunk
+ Assert.assertEquals(2,
secondChunkGroupMetadata.getChunkMetadataList().size());
+ }
+ }
+
+ /*
+ * Test check with two chunk group, and no marker behind the last chunk group
+ */
+ @Test
+ public void testCheckWithTwoChunkGroupAndNotMarkerBehind() throws Exception {
+ TsFileWriter writer = new TsFileWriter(file);
+ writer.registerTimeseries(
+ new Path("d1", "s1"), new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d1", "s2"), new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d2", "s1"), new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d2", "s2"), new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.write(
+ new TSRecord(1, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.write(
+ new TSRecord(2, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+
+ writer.write(
+ new TSRecord(1, "d2")
+ .addTuple(new FloatDataPoint("s1", 6))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.write(
+ new TSRecord(2, "d2")
+ .addTuple(new FloatDataPoint("s1", 6))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.flushAllChunkGroups();
+ long pos1 = writer.getIOWriter().getPos();
+ writer.getIOWriter().writeSeparatorMaskForTest();
+ writer.getIOWriter().close();
+ try (TsFileSequenceReader reader = new
TsFileSequenceReader(file.getPath())) {
+ Map<Path, MeasurementSchema> newSchema = new HashMap<>();
+ List<ChunkGroupMetadata> chunkGroupMetadataList = new ArrayList<>();
+ long checkStatus = reader.selfCheck(newSchema, chunkGroupMetadataList,
false, false);
+ Assert.assertEquals(checkStatus, pos1);
+ Assert.assertEquals(newSchema.size(), 4);
+ Assert.assertEquals(chunkGroupMetadataList.size(), 2);
+ ChunkGroupMetadata firstChunkGroupMetadata =
chunkGroupMetadataList.get(0);
+ ChunkGroupMetadata secondChunkGroupMetadata =
chunkGroupMetadataList.get(1);
+ // the first chunk group contains the chunk metadata for two chunks
+ Assert.assertEquals(2,
firstChunkGroupMetadata.getChunkMetadataList().size());
+ // the second chunk group contains only chunk metadata for the first
chunk
+ Assert.assertEquals(2,
secondChunkGroupMetadata.getChunkMetadataList().size());
+ }
+ }
+
+ /**
+ * Test check with two chunk group, and no marker behind the last chunk
group, and load the last
+ * chunk group metadata
+ */
+ @Test
+ public void
testCheckWithTwoChunkGroupAndNotMarkerBehindAndLoadLastChunkGroupMetadata()
+ throws Exception {
+ TsFileWriter writer = new TsFileWriter(file);
+ writer.registerTimeseries(
+ new Path("d1", "s1"), new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d1", "s2"), new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d2", "s1"), new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d2", "s2"), new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.write(
+ new TSRecord(1, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.write(
+ new TSRecord(2, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+
+ writer.write(
+ new TSRecord(1, "d2")
+ .addTuple(new FloatDataPoint("s1", 6))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.write(
+ new TSRecord(2, "d2")
+ .addTuple(new FloatDataPoint("s1", 6))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.flushAllChunkGroups();
+ long pos1 = writer.getIOWriter().getPos();
+ writer.getIOWriter().writeSeparatorMaskForTest();
+ writer.getIOWriter().close();
+ try (TsFileSequenceReader reader = new
TsFileSequenceReader(file.getPath())) {
+ Map<Path, MeasurementSchema> newSchema = new HashMap<>();
+ List<ChunkGroupMetadata> chunkGroupMetadataList = new ArrayList<>();
+ long checkStatus = reader.selfCheck(newSchema, chunkGroupMetadataList,
false, true);
+ Assert.assertEquals(checkStatus, pos1);
+ Assert.assertEquals(newSchema.size(), 4);
+ Assert.assertEquals(chunkGroupMetadataList.size(), 2);
+ ChunkGroupMetadata firstChunkGroupMetadata =
chunkGroupMetadataList.get(0);
+ ChunkGroupMetadata secondChunkGroupMetadata =
chunkGroupMetadataList.get(1);
+ // the first chunk group contains the chunk metadata for two chunks
+ Assert.assertEquals(2,
firstChunkGroupMetadata.getChunkMetadataList().size());
+ // the second chunk group contains only chunk metadata for the first
chunk
+ Assert.assertEquals(2,
secondChunkGroupMetadata.getChunkMetadataList().size());
+ }
+ }
+
+ /**
+ * Test check with two chunk group, and a marker of FileMetadata
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testCheckWithTwoChunkGroupAndFileMetadataMarker() throws
Exception {
+ TsFileWriter writer = new TsFileWriter(file);
+ writer.registerTimeseries(
+ new Path("d1", "s1"), new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d1", "s2"), new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d2", "s1"), new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d2", "s2"), new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.write(
+ new TSRecord(1, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.write(
+ new TSRecord(2, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+
+ writer.write(
+ new TSRecord(1, "d2")
+ .addTuple(new FloatDataPoint("s1", 6))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.write(
+ new TSRecord(2, "d2")
+ .addTuple(new FloatDataPoint("s1", 6))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.flushAllChunkGroups();
+ long pos1 = writer.getIOWriter().getPos();
+ writer.getIOWriter().writeSeparatorMaskForTest();
+ writer.getIOWriter().writeSeparatorMaskForTest();
+ writer.getIOWriter().close();
+ try (TsFileSequenceReader reader = new
TsFileSequenceReader(file.getPath())) {
+ Map<Path, MeasurementSchema> newSchema = new HashMap<>();
+ List<ChunkGroupMetadata> chunkGroupMetadataList = new ArrayList<>();
+ long checkStatus = reader.selfCheck(newSchema, chunkGroupMetadataList,
false, false);
+ Assert.assertEquals(checkStatus, pos1);
+ Assert.assertEquals(newSchema.size(), 4);
+ Assert.assertEquals(chunkGroupMetadataList.size(), 2);
+ ChunkGroupMetadata firstChunkGroupMetadata =
chunkGroupMetadataList.get(0);
+ ChunkGroupMetadata secondChunkGroupMetadata =
chunkGroupMetadataList.get(1);
+ // the first chunk group contains the chunk metadata for two chunks
+ Assert.assertEquals(2,
firstChunkGroupMetadata.getChunkMetadataList().size());
+ // the second chunk group contains only chunk metadata for the first
chunk
+ Assert.assertEquals(2,
secondChunkGroupMetadata.getChunkMetadataList().size());
+ }
+ }
+
+ /**
+ * Test check with two chunk group, and a marker of FileMetadata, and load
the last chunk group
+ * metadata
+ *
+ * @throws Exception
+ */
+ @Test
+ public void
testCheckWithTwoChunkGroupAndFileMetadataMarkerAndLoadLastChunkGroup()
+ throws Exception {
+ TsFileWriter writer = new TsFileWriter(file);
+ writer.registerTimeseries(
+ new Path("d1", "s1"), new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d1", "s2"), new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d2", "s1"), new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d2", "s2"), new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.write(
+ new TSRecord(1, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.write(
+ new TSRecord(2, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+
+ writer.write(
+ new TSRecord(1, "d2")
+ .addTuple(new FloatDataPoint("s1", 6))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.write(
+ new TSRecord(2, "d2")
+ .addTuple(new FloatDataPoint("s1", 6))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.flushAllChunkGroups();
+ long pos1 = writer.getIOWriter().getPos();
+ writer.getIOWriter().writeSeparatorMaskForTest();
+ writer.getIOWriter().writeSeparatorMaskForTest();
+ writer.getIOWriter().close();
+ try (TsFileSequenceReader reader = new
TsFileSequenceReader(file.getPath())) {
+ Map<Path, MeasurementSchema> newSchema = new HashMap<>();
+ List<ChunkGroupMetadata> chunkGroupMetadataList = new ArrayList<>();
+ long checkStatus = reader.selfCheck(newSchema, chunkGroupMetadataList,
false, true);
+ Assert.assertEquals(checkStatus, pos1);
+ Assert.assertEquals(newSchema.size(), 4);
+ Assert.assertEquals(chunkGroupMetadataList.size(), 2);
+ ChunkGroupMetadata firstChunkGroupMetadata =
chunkGroupMetadataList.get(0);
+ ChunkGroupMetadata secondChunkGroupMetadata =
chunkGroupMetadataList.get(1);
+ // the first chunk group contains the chunk metadata for two chunks
+ Assert.assertEquals(2,
firstChunkGroupMetadata.getChunkMetadataList().size());
+ // the second chunk group contains only chunk metadata for the first
chunk
+ Assert.assertEquals(2,
secondChunkGroupMetadata.getChunkMetadataList().size());
+ }
+ }
+
+ /**
+ * Test check with a complete file
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testCheckWithCompleteFile() throws Exception {
+ TsFileWriter writer = new TsFileWriter(file);
+ writer.registerTimeseries(
+ new Path("d1", "s1"), new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d1", "s2"), new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.write(
+ new TSRecord(1, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.write(
+ new TSRecord(2, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.flushAllChunkGroups();
+ long pos = writer.getIOWriter().getPos();
+ writer.close();
+ try (TsFileSequenceReader reader = new
TsFileSequenceReader(file.getPath())) {
+ Map<Path, MeasurementSchema> newSchema = new HashMap<>();
+ List<ChunkGroupMetadata> chunkGroupMetadataList = new ArrayList<>();
+ long checkStatus = reader.selfCheck(newSchema, chunkGroupMetadataList,
false, false);
+ Assert.assertEquals(checkStatus, pos);
+ Assert.assertEquals(newSchema.size(), 2);
+ Assert.assertEquals(chunkGroupMetadataList.size(), 1);
+ ChunkGroupMetadata firstChunkGroupMetadata =
chunkGroupMetadataList.get(0);
+ Assert.assertEquals(2,
firstChunkGroupMetadata.getChunkMetadataList().size());
+ }
+ }
+
+ /**
+ * Test check with a complete file, and load last chunk group metadata
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testCheckWithCompleteFileAndLoadLastChunkGroupMetadata() throws
Exception {
+ TsFileWriter writer = new TsFileWriter(file);
+ writer.registerTimeseries(
+ new Path("d1", "s1"), new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.registerTimeseries(
+ new Path("d1", "s2"), new MeasurementSchema("s2", TSDataType.FLOAT,
TSEncoding.RLE));
+ writer.write(
+ new TSRecord(1, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.write(
+ new TSRecord(2, "d1")
+ .addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.flushAllChunkGroups();
+ long pos = writer.getIOWriter().getPos();
+ writer.close();
+ try (TsFileSequenceReader reader = new
TsFileSequenceReader(file.getPath())) {
+ Map<Path, MeasurementSchema> newSchema = new HashMap<>();
+ List<ChunkGroupMetadata> chunkGroupMetadataList = new ArrayList<>();
+ long checkStatus = reader.selfCheck(newSchema, chunkGroupMetadataList,
false, true);
+ Assert.assertEquals(checkStatus, pos);
+ Assert.assertEquals(newSchema.size(), 2);
+ Assert.assertEquals(chunkGroupMetadataList.size(), 1);
+ ChunkGroupMetadata firstChunkGroupMetadata =
chunkGroupMetadataList.get(0);
+ Assert.assertEquals(2,
firstChunkGroupMetadata.getChunkMetadataList().size());
+ }
+ }
+}
