This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 4c09acd Fix ForceAppendTsFileWriter (#1275)
4c09acd is described below
commit 4c09acd5b62557ce01f518c09059e2580b0803cb
Author: Haonan <[email protected]>
AuthorDate: Thu May 28 20:35:37 2020 +0800
Fix ForceAppendTsFileWriter (#1275)
* fix ForceAppendTsFileWriter
---
docs/UserGuide/Client/Programming - TsFile API.md | 19 ++++
.../UserGuide/Client/Programming - TsFile API.md | 19 +++-
.../iotdb/tsfile/TsFileForceAppendWrite.java | 108 +++++++++++++++++++++
.../write/writer/ForceAppendTsFileWriter.java | 19 ++++
.../write/writer/ForceAppendTsFileWriterTest.java | 42 ++++++++
5 files changed, 206 insertions(+), 1 deletion(-)
diff --git a/docs/UserGuide/Client/Programming - TsFile API.md
b/docs/UserGuide/Client/Programming - TsFile API.md
index fbf0eb7..8010d50 100644
--- a/docs/UserGuide/Client/Programming - TsFile API.md
+++ b/docs/UserGuide/Client/Programming - TsFile API.md
@@ -263,6 +263,21 @@ A TsFile can be generated by following three steps and the
complete code will be
```
public void close() throws IOException
```
+We are also able to write data into a closed TsFile.
+
+* Use `ForceAppendTsFileWriter` to open a closed file.
+
+```
+public ForceAppendTsFileWriter(File file) throws IOException
+```
+* call `doTruncate` truncate the part of Metadata
+
+* Then use `ForceAppendTsFileWriter` to construct a new `TsFileWriter`
+
+```
+public TsFileWriter(TsFileIOWriter fileWriter) throws IOException
+```
+Please note, we should redo the step of adding measurements before writing new
data to the TsFile.
#### Example for writing a TsFile
@@ -280,6 +295,10 @@ You could write a TsFile by constructing **Tablet** if you
have the **aligned**
A more thorough example can be found at
`/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileWriteWithTablet.java`
+You could write data into a closed TsFile by using **ForceAppendTsFileWriter**.
+
+A more thorough example can be found at
`/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileForceAppendWrite.java`
+
### Interface for Reading TsFile
#### Before the Start
diff --git a/docs/zh/UserGuide/Client/Programming - TsFile API.md
b/docs/zh/UserGuide/Client/Programming - TsFile API.md
index edc1337..64e5679 100644
--- a/docs/zh/UserGuide/Client/Programming - TsFile API.md
+++ b/docs/zh/UserGuide/Client/Programming - TsFile API.md
@@ -185,7 +185,7 @@ TsFile可以通过以下三个步骤生成，完整的代码参见"写入 TsFile
* config : TsFile 的一些配置项
-* 第二部，添加测量值(measurement)
+* 第二步，添加测量值(measurement)
你也可以先创建一个`Schema`类的实例然后把它传递给`TsFileWriter`类的构造函数
@@ -262,6 +262,22 @@ TsFile可以通过以下三个步骤生成，完整的代码参见"写入 TsFile
public void close() throws IOException
```
+我们也支持将数据写入已关闭的 TsFile 文件中。
+
+* 使用`ForceAppendTsFileWriter`打开已经关闭的文件。
+
+```
+public ForceAppendTsFileWriter(File file) throws IOException
+```
+* 调用 `doTruncate` 去掉文件的Metadata部分
+
+* 然后使用 `ForceAppendTsFileWriter` 构造另一个`TsFileWriter`
+
+```
+public TsFileWriter(TsFileIOWriter fileWriter) throws IOException
+```
+请注意 此时需要重新添加测量值(measurement) 再进行上述写入操作。
+
#### 写入 TsFile 示例
您需要安装 TsFile 到本地的 Maven 仓库中。
@@ -279,6 +295,7 @@ mvn clean install -pl tsfile -am -DskipTests
更详细的例子可以在`/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileWriteWithTablet.java`中查看
+在已关闭的TsFile
文件中写入新数据的详细例子可以在`/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileForceAppendWrite.java`中查看
### 读取 TsFile 接口
diff --git
a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileForceAppendWrite.java
b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileForceAppendWrite.java
new file mode 100644
index 0000000..be25806
--- /dev/null
+++
b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileForceAppendWrite.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.tsfile;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.write.TsFileWriter;
+import org.apache.iotdb.tsfile.write.record.TSRecord;
+import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
+import org.apache.iotdb.tsfile.write.record.datapoint.LongDataPoint;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+import org.apache.iotdb.tsfile.write.writer.ForceAppendTsFileWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TsFileForceAppendWrite {
+
+ private static final Logger logger =
LoggerFactory.getLogger(TsFileForceAppendWrite.class);
+
+ public static void main(String[] args) throws IOException {
+ String path = "test.tsfile";
+ File f = FSFactoryProducer.getFSFactory().getFile(path);
+ if (f.exists()) {
+ Files.delete(f.toPath());
+ }
+ try (TsFileWriter tsFileWriter = new TsFileWriter(f)) {
+
+ // add measurements into file schema
+ for (int i = 0; i < 4; i++) {
+ tsFileWriter.registerTimeseries(new Path(Constant.DEVICE_PREFIX + i,
Constant.SENSOR_1),
+ new MeasurementSchema(Constant.SENSOR_1, TSDataType.INT64,
TSEncoding.RLE));
+ tsFileWriter.registerTimeseries(new Path(Constant.DEVICE_PREFIX + i,
Constant.SENSOR_2),
+ new MeasurementSchema(Constant.SENSOR_2, TSDataType.INT64,
TSEncoding.RLE));
+ tsFileWriter.registerTimeseries(new Path(Constant.DEVICE_PREFIX + i,
Constant.SENSOR_3),
+ new MeasurementSchema(Constant.SENSOR_3, TSDataType.INT64,
TSEncoding.RLE));
+ }
+
+ // construct TSRecord
+ for (int i = 0; i < 100; i++) {
+ TSRecord tsRecord = new TSRecord(i, Constant.DEVICE_PREFIX + (i % 4));
+ DataPoint dPoint1 = new LongDataPoint(Constant.SENSOR_1, i);
+ DataPoint dPoint2 = new LongDataPoint(Constant.SENSOR_2, i);
+ DataPoint dPoint3 = new LongDataPoint(Constant.SENSOR_3, i);
+ tsRecord.addTuple(dPoint1);
+ tsRecord.addTuple(dPoint2);
+ tsRecord.addTuple(dPoint3);
+
+ // write TSRecord
+ tsFileWriter.write(tsRecord);
+ }
+ } catch (Exception e) {
+ logger.error("meet error in TsFileWrite ", e);
+ }
+
+ // open the closed file with ForceAppendTsFileWriter
+ ForceAppendTsFileWriter fwriter = new ForceAppendTsFileWriter(f);
+ fwriter.doTruncate();
+ try (TsFileWriter tsFileWriter1 = new TsFileWriter(fwriter)) {
+ // add measurements into file schema
+ for (int i = 0; i < 4; i++) {
+ tsFileWriter1.registerTimeseries(new Path(Constant.DEVICE_PREFIX + i,
Constant.SENSOR_1),
+ new MeasurementSchema(Constant.SENSOR_1, TSDataType.INT64,
TSEncoding.RLE));
+ tsFileWriter1.registerTimeseries(new Path(Constant.DEVICE_PREFIX + i,
Constant.SENSOR_2),
+ new MeasurementSchema(Constant.SENSOR_2, TSDataType.INT64,
TSEncoding.RLE));
+ tsFileWriter1.registerTimeseries(new Path(Constant.DEVICE_PREFIX + i,
Constant.SENSOR_3),
+ new MeasurementSchema(Constant.SENSOR_3, TSDataType.INT64,
TSEncoding.RLE));
+ }
+ // construct TSRecord
+ for (int i = 100; i < 120; i++) {
+ TSRecord tsRecord = new TSRecord(i, Constant.DEVICE_PREFIX + (i % 4));
+ DataPoint dPoint1 = new LongDataPoint(Constant.SENSOR_1, i);
+ DataPoint dPoint2 = new LongDataPoint(Constant.SENSOR_2, i);
+ DataPoint dPoint3 = new LongDataPoint(Constant.SENSOR_3, i);
+ tsRecord.addTuple(dPoint1);
+ tsRecord.addTuple(dPoint2);
+ tsRecord.addTuple(dPoint3);
+
+ // write TSRecord
+ tsFileWriter1.write(tsRecord);
+ }
+ } catch (Exception e) {
+ logger.error("meet error in TsFileWrite ", e);
+ }
+ fwriter.close();
+ }
+}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java
index f23e99b..d677e38 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java
@@ -20,7 +20,12 @@ package org.apache.iotdb.tsfile.write.writer;
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.iotdb.tsfile.exception.write.TsFileNotCompleteException;
+import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetadata;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.slf4j.Logger;
@@ -57,6 +62,20 @@ public class ForceAppendTsFileWriter extends TsFileIOWriter {
TsFileMetadata tsFileMetadata = reader.readFileMetadata();
// truncate metadata and marker
truncatePosition = tsFileMetadata.getMetaOffset();
+
+ canWrite = true;
+ versionInfo = tsFileMetadata.getVersionInfo();
+ totalChunkNum = tsFileMetadata.getTotalChunkNum();
+ invalidChunkNum = tsFileMetadata.getInvalidChunkNum();
+
+ List<String> devices = reader.getAllDevices();
+ for (String device : devices) {
+ List<ChunkMetadata> chunkMetadataList = new ArrayList<>();
+ reader.readChunkMetadataInDevice(device).values()
+ .forEach(chunkMetadataList::addAll);
+ ChunkGroupMetadata chunkGroupMetadata = new ChunkGroupMetadata(device,
chunkMetadataList);
+ chunkGroupMetadataList.add(chunkGroupMetadata);
+ }
}
}
diff --git
a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriterTest.java
b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriterTest.java
index 694d7c1..aff1a20 100644
---
a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriterTest.java
+++
b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriterTest.java
@@ -19,15 +19,24 @@
package org.apache.iotdb.tsfile.write.writer;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.iotdb.tsfile.constant.TestConstant;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
import org.apache.iotdb.tsfile.fileSystem.fsFactory.FSFactory;
+import org.apache.iotdb.tsfile.read.ReadOnlyTsFile;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.read.common.RowRecord;
+import org.apache.iotdb.tsfile.read.expression.QueryExpression;
+import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
import org.apache.iotdb.tsfile.write.TsFileWriter;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.record.datapoint.FloatDataPoint;
@@ -56,6 +65,39 @@ public class ForceAppendTsFileWriterTest {
writer.close();
ForceAppendTsFileWriter fwriter = new ForceAppendTsFileWriter(file);
assertEquals(firstMetadataPosition, fwriter.getTruncatePosition());
+ fwriter.doTruncate();
+
+ // write more data into this TsFile
+ writer = new TsFileWriter(fwriter);
+ writer.registerTimeseries(new Path("d1.s1"),
+ new MeasurementSchema("s1", TSDataType.FLOAT, TSEncoding.RLE));
+ writer.registerTimeseries(new Path("d1.s2"),
+ new MeasurementSchema("s2", TSDataType.FLOAT, TSEncoding.RLE));
+ writer.write(new TSRecord(3, "d1").addTuple(new FloatDataPoint("s1", 5))
+ .addTuple(new FloatDataPoint("s2", 4)));
+ writer.close();
+ ReadOnlyTsFile readOnlyTsFile = new ReadOnlyTsFile(new
TsFileSequenceReader(file.getPath()));
+ List<Path> pathList = new ArrayList<>();
+ pathList.add(new Path("d1", "s1"));
+ pathList.add(new Path("d1", "s2"));
+ QueryExpression queryExpression = QueryExpression.create(pathList, null);
+ QueryDataSet dataSet = readOnlyTsFile.query(queryExpression);
+ RowRecord record = dataSet.next();
+ assertEquals(1, record.getTimestamp());
+ assertEquals(5.0f, record.getFields().get(0).getFloatV(), 0.001);
+ assertEquals(4.0f, record.getFields().get(1).getFloatV(), 0.001);
+ record = dataSet.next();
+ assertEquals(2, record.getTimestamp());
+ assertEquals(5.0f, record.getFields().get(0).getFloatV(), 0.001);
+ assertEquals(4.0f, record.getFields().get(1).getFloatV(), 0.001);
+ record = dataSet.next();
+ assertEquals(3, record.getTimestamp());
+ assertEquals(5.0f, record.getFields().get(0).getFloatV(), 0.001);
+ assertEquals(4.0f, record.getFields().get(1).getFloatV(), 0.001);
+ readOnlyTsFile.close();
+ assertFalse(dataSet.hasNext());
+
+ assertTrue(file.delete());
}
}
