This is an automated email from the ASF dual-hosted git repository. qiaojialin pushed a commit to branch opt_tsfile_insert_tablet in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 507ad7e8e0708c05e3a5e3ff2680800b928f2472 Author: qiaojialin <[email protected]> AuthorDate: Tue Jan 12 14:45:33 2021 +0800 enable TsFile insertTablet with only a template --- .../apache/iotdb/tsfile/write/TsFileWriter.java | 13 ++- .../tsfile/write/DefaultDeviceTemplateTest.java | 110 +++++++++++++++++++++ 2 files changed, 120 insertions(+), 3 deletions(-) diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java index aa31866..daa9f65 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java @@ -225,9 +225,16 @@ public class TsFileWriter implements AutoCloseable { // add all SeriesWriter of measurements in this Tablet to this ChunkGroupWriter for (MeasurementSchema timeseries : tablet.getSchemas()) { String measurementId = timeseries.getMeasurementId(); - if (schema.containsTimeseries(new Path(deviceId, measurementId))) { - groupWriter.tryToAddSeriesWriter(schema.getSeriesSchema(new Path(deviceId, measurementId)), - pageSize); + Path path = new Path(deviceId, measurementId); + if (schema.containsTimeseries(path)) { + groupWriter.tryToAddSeriesWriter(schema.getSeriesSchema(path), pageSize); + } else if (schema.getDeviceTemplates() != null && schema.getDeviceTemplates().size() == 1) { + // use the default template without needing to register device + Map<String, MeasurementSchema> template = schema.getDeviceTemplates() + .entrySet().iterator().next().getValue(); + if (template.containsKey(path.getMeasurement())) { + groupWriter.tryToAddSeriesWriter(template.get(path.getMeasurement()), pageSize); + } } else { throw new NoMeasurementException("input measurement is invalid: " + measurementId); } diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/DefaultDeviceTemplateTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/DefaultDeviceTemplateTest.java new file mode 100644 index 0000000..dcf325a --- /dev/null +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/DefaultDeviceTemplateTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.tsfile.write; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.iotdb.tsfile.exception.write.WriteProcessException; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.read.ReadOnlyTsFile; +import org.apache.iotdb.tsfile.read.TsFileSequenceReader; +import org.apache.iotdb.tsfile.read.common.Path; +import org.apache.iotdb.tsfile.read.expression.QueryExpression; +import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet; +import org.apache.iotdb.tsfile.write.record.Tablet; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import org.junit.Assert; +import org.junit.Test; + +public class DefaultDeviceTemplateTest { + + @Test + public void testUsingDefaultDeviceTemplate() throws IOException, WriteProcessException { + File file = new File("target/defaultDeviceTemplate.tsfile"); + try (TsFileWriter writer = new TsFileWriter(file)) { + MeasurementSchema s1 = new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.PLAIN); + MeasurementSchema s2 = new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.PLAIN); + + List<MeasurementSchema> schemaList = new ArrayList<>(); + schemaList.add(s1); + schemaList.add(s2); + + Map<String, MeasurementSchema> schema = new HashMap<>(); + schema.put("s1", s1); + schema.put("s2", s2); + + writer.registerDeviceTemplate("defaultTemplate", schema); + + Tablet tablet = new Tablet("d1", schemaList); + long[] timestamps = tablet.timestamps; + Object[] values = tablet.values; + + long timestamp = 1; + long value = 1L; + + for (int r = 0; r < 10; r++, value++) { + int row = tablet.rowSize++; + timestamps[row] = timestamp++; + for (int i = 0; i < 2; i++) { + long[] sensor = (long[]) values[i]; + sensor[row] = value; + } + // write Tablet to TsFile + if (tablet.rowSize == tablet.getMaxRowNumber()) { + writer.write(tablet); + tablet.reset(); + } + } + // write Tablet to TsFile + if (tablet.rowSize != 0) { + writer.write(tablet); + tablet.reset(); + } + } + + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getPath()); + ReadOnlyTsFile readTsFile = new ReadOnlyTsFile(reader)) { + + // use these paths(all measurements) for all the queries + ArrayList<Path> paths = new ArrayList<>(); + paths.add(new Path("d1", "s1")); + + QueryExpression queryExpression = QueryExpression.create(paths, null); + QueryDataSet queryDataSet = readTsFile.query(queryExpression); + int count = 0; + while (queryDataSet.hasNext()) { + queryDataSet.next(); + count ++; + } + + Assert.assertEquals(10, count); + } + + + Files.deleteIfExists(file.toPath()); + + } + +}
