This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch bw-rename in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit de07555b66e2f05ca1ce2c991964f59f611277c7 Author: JackieTien97 <[email protected]> AuthorDate: Sat Sep 23 10:32:08 2023 +0800 rename --- example/tsfile/pom.xml | 52 +++ .../java/org/apache/iotdb/tsfile/SeriesRename.java | 441 +++++++++++++++++++++ .../analyze/cache/partition/PartitionCache.java | 8 +- .../apache/iotdb/commons/conf/CommonConfig.java | 2 +- .../iotdb/tsfile/read/TsFileSequenceReader.java | 4 +- .../iotdb/tsfile/read/reader/page/PageReader.java | 3 + .../apache/iotdb/tsfile/write/TsFileWriter.java | 4 +- .../iotdb/tsfile/write/chunk/ChunkWriterImpl.java | 16 +- .../chunk/NonAlignedChunkGroupWriterImpl.java | 2 +- .../iotdb/tsfile/write/writer/TsFileIOWriter.java | 10 +- 10 files changed, 521 insertions(+), 21 deletions(-) diff --git a/example/tsfile/pom.xml b/example/tsfile/pom.xml index c8ba35c0a28..ced2d825699 100644 --- a/example/tsfile/pom.xml +++ b/example/tsfile/pom.xml @@ -35,5 +35,57 @@ <artifactId>tsfile</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>node-commons</artifactId> + <version>${project.version}</version> + </dependency> </dependencies> + <profiles> + <profile> + <id>get-jar-with-dependencies</id> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <version>3.1.1</version> + <configuration> + <descriptorRefs> + <descriptorRef>jar-with-dependencies</descriptorRef> + </descriptorRefs> + <finalName>${project.build.finalName}-jar-with-dependencies</finalName> + <appendAssemblyId>false</appendAssemblyId> + <archive> + <manifest> + <mainClass>org.apache.iotdb.tsfile.SeriesRename</mainClass> + <addClasspath>true</addClasspath> + <classpathPrefix>lib/</classpathPrefix> + </manifest> + </archive> + </configuration> + <executions> + <execution> + <id>make-assembly</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>3.3</version> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-resources-plugin</artifactId> + <version>2.4.3</version> + </plugin> + </plugins> + </build> + </profile> + </profiles> </project> diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/SeriesRename.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/SeriesRename.java new file mode 100644 index 00000000000..9b2d847b2fd --- /dev/null +++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/SeriesRename.java @@ -0,0 +1,441 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tsfile; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.tsfile.common.conf.TSFileConfig; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; +import org.apache.iotdb.tsfile.encoding.decoder.Decoder; +import org.apache.iotdb.tsfile.exception.write.WriteProcessException; +import org.apache.iotdb.tsfile.file.MetaMarker; +import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader; +import org.apache.iotdb.tsfile.file.header.ChunkHeader; +import org.apache.iotdb.tsfile.file.header.PageHeader; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.read.TsFileSequenceReader; +import org.apache.iotdb.tsfile.read.common.block.TsBlock; +import org.apache.iotdb.tsfile.read.common.block.column.Column; +import org.apache.iotdb.tsfile.read.common.block.column.DoubleColumn; +import org.apache.iotdb.tsfile.read.common.block.column.FloatColumn; +import org.apache.iotdb.tsfile.read.common.block.column.IntColumn; +import org.apache.iotdb.tsfile.read.common.block.column.LongColumn; +import org.apache.iotdb.tsfile.read.reader.page.PageReader; +import org.apache.iotdb.tsfile.utils.Pair; +import org.apache.iotdb.tsfile.write.TsFileWriter; +import org.apache.iotdb.tsfile.write.record.Tablet; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Stream; + +import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX; + +public class SeriesRename { + + public static void main(String[] args) { + long startTime = System.nanoTime(); + File[] tsfileDirs = new File[args.length]; + for (int i = 0, n = args.length; i < n; i++) { + tsfileDirs[i] = new File(args[i]); + } + List<String> tsfileList = new ArrayList<>(); + for (File tsfileDir : tsfileDirs) { + if (tsfileDir.exists()) { + if (tsfileDir.isDirectory()) { + try (Stream<Path> paths = Files.walk(tsfileDir.toPath())) { + paths + .map(Path::toString) + .filter(f -> f.endsWith(TSFILE_SUFFIX)) + .forEach(tsfileList::add); + } catch (IOException e) { + System.out.println("Collect tsfile failed!"); + e.printStackTrace(); + } + } else if (tsfileDir.isFile() && tsfileDir.getName().endsWith(TSFILE_SUFFIX)) { + tsfileList.add(tsfileDir.getName()); + } else { + System.out.println("skip invalid tsfileDir: " + tsfileDir); + } + } + } + + System.out.println("Total tsfile number: " + tsfileList.size() + System.lineSeparator()); + + List<String> failedTsFileList = new ArrayList<>(); + + int completed = 0; + int failed = 0; + for (String tsfileName : tsfileList) { + boolean succeed = false; + try (TsFileSequenceReader reader = new TsFileSequenceReader(tsfileName); + TsFileWriter tsFileWriter = new TsFileWriter(new File(tsfileName + ".rename"))) { + // Sequential reading of one ChunkGroup now follows this order: + // first the CHUNK_GROUP_HEADER, then SeriesChunks (headers and data) in one ChunkGroup + // Because we do not know how many chunks a ChunkGroup may have, we should read one byte + // (the + // marker) ahead and judge accordingly. + reader.position((long) TSFileConfig.MAGIC_STRING.getBytes().length + 1); + String currentDevice = ""; + byte marker; + + Set<String> seenPath = new HashSet<>(); + while ((marker = reader.readMarker()) != MetaMarker.SEPARATOR) { + switch (marker) { + case MetaMarker.CHUNK_HEADER: + case MetaMarker.TIME_CHUNK_HEADER: + case MetaMarker.VALUE_CHUNK_HEADER: + case MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER: + case MetaMarker.ONLY_ONE_PAGE_TIME_CHUNK_HEADER: + case MetaMarker.ONLY_ONE_PAGE_VALUE_CHUNK_HEADER: + ChunkHeader header = reader.readChunkHeader(marker); + String measurementName = header.getMeasurementID(); + if (header.getDataSize() == 0) { + // empty value chunk + break; + } + Decoder defaultTimeDecoder = + Decoder.getDecoderByType( + TSEncoding.valueOf( + TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), + TSDataType.INT64); + Decoder valueDecoder = + Decoder.getDecoderByType(header.getEncodingType(), header.getDataType()); + int dataSize = header.getDataSize(); + if (header.getDataType() == TSDataType.VECTOR) { + throw new IllegalStateException( + "there exist aligned timeseries: " + currentDevice + "." + measurementName); + } + Optional<Pair<String, MeasurementSchema>> pair = + rename( + currentDevice, + measurementName, + header.getDataType(), + header.getEncodingType(), + header.getCompressionType()); + if (!pair.isPresent()) { + System.out.println( + "ignore invalid time series: " + + currentDevice + + "." + + measurementName + + ", datatype: " + + header.getDataType()); + } else { + if (seenPath.add(pair.get().left + "." + pair.get().right.getMeasurementId())) { + tsFileWriter.registerTimeseries( + new org.apache.iotdb.tsfile.read.common.Path(pair.get().left), + pair.get().right); + } + } + while (dataSize > 0) { + valueDecoder.reset(); + PageHeader pageHeader = + reader.readPageHeader( + header.getDataType(), + (header.getChunkType() & 0x3F) == MetaMarker.CHUNK_HEADER); + ByteBuffer pageData = reader.readPage(pageHeader, header.getCompressionType()); + PageReader pageReader = + new PageReader( + pageHeader, + pageData, + header.getDataType(), + valueDecoder, + defaultTimeDecoder, + null); + TsBlock tsBlock = pageReader.getAllSatisfiedData(); + dataSize -= pageHeader.getSerializedPageSize(); + if (pair.isPresent()) { + writeToNewTsFile(tsBlock, tsFileWriter, pair.get().left, pair.get().right); + } + } + break; + case MetaMarker.CHUNK_GROUP_HEADER: + ChunkGroupHeader chunkGroupHeader = reader.readChunkGroupHeader(); + currentDevice = chunkGroupHeader.getDeviceID(); + break; + case MetaMarker.OPERATION_INDEX_RANGE: + reader.readPlanIndex(); + break; + default: + MetaMarker.handleUnexpectedMarker(marker); + } + } + succeed = true; + completed++; + } catch (Exception e) { + System.out.println("error happened while renaming " + tsfileName); + failedTsFileList.add(tsfileName); + e.printStackTrace(); + failed++; + } + + if (succeed) { + System.out.println("successfully rename tsfile: " + tsfileName); + try { + Files.deleteIfExists(Paths.get(tsfileName)); + } catch (IOException e) { + System.out.println("failed to delete: " + tsfileName); + } + } else { + try { + Files.deleteIfExists(Paths.get(tsfileName + ".rename")); + } catch (IOException e) { + System.out.println("failed to delete tmp file: " + tsfileName + ".rename"); + } + } + System.out.println( + "rename completed: " + + completed + + ", failed: " + + failed + + ", total: " + + tsfileList.size() + + ", progress: " + + (((double) completed + failed) / tsfileList.size()) + + ", elapse time: " + + (System.nanoTime() - startTime) / 1_000_000_000 + + "s"); + } + } + + private static Optional<Pair<String, MeasurementSchema>> rename( + String device, + String measurement, + TSDataType dataType, + TSEncoding encoding, + CompressionType compressionType) { + if (device.length() <= 17) { + return Optional.empty(); + } + String newDevice; + MeasurementSchema measurementSchema; + PartialPath devicePath; + try { + devicePath = new PartialPath(device); + } catch (IllegalPathException e) { + return Optional.empty(); + } + + String[] deviceNodes = devicePath.getNodes(); + if (deviceNodes == null + || deviceNodes.length < 4 + || !"root".equals(deviceNodes[0]) + || !"bw".equals(deviceNodes[1]) + || !"baoshan".equals(deviceNodes[2])) { + return Optional.empty(); + } + + if ("S".equals(deviceNodes[3])) { + if (dataType != TSDataType.FLOAT + || "value".equalsIgnoreCase(measurement) + || deviceNodes.length != 6) { + return Optional.empty(); + } + newDevice = + "root.bw.baoshan." + deviceNodes[4] + "." + deviceNodes[5] + "." + measurement + ".S"; + measurementSchema = new MeasurementSchema("value", dataType, encoding, compressionType); + return Optional.of(new Pair<>(newDevice, measurementSchema)); + } else if ("I".equals(deviceNodes[3])) { + if (deviceNodes.length != 7) { + return Optional.empty(); + } + + if ("cl".equals(measurement)) { + if (dataType != TSDataType.INT32) { + return Optional.empty(); + } + } else if ("hz".equals(measurement)) { + if (dataType != TSDataType.INT32) { + return Optional.empty(); + } + } else if ("end_time".equals(measurement)) { + if (dataType != TSDataType.INT64) { + return Optional.empty(); + } + } else { + return Optional.empty(); + } + newDevice = + "root.bw.baoshan." + deviceNodes[4] + "." + deviceNodes[5] + "." + deviceNodes[6] + ".I"; + measurementSchema = new MeasurementSchema(measurement, dataType, encoding, compressionType); + return Optional.of(new Pair<>(newDevice, measurementSchema)); + } else if ("F".equals(deviceNodes[3])) { + if (dataType != TSDataType.DOUBLE + || "value".equalsIgnoreCase(measurement) + || deviceNodes.length != 5) { + return Optional.empty(); + } + newDevice = "root.bw.baoshan." + deviceNodes[4] + ".`00`" + "." + measurement; + measurementSchema = + new MeasurementSchema("value", TSDataType.FLOAT, encoding, compressionType); + return Optional.of(new Pair<>(newDevice, measurementSchema)); + } else { + if (dataType != TSDataType.FLOAT + || "value".equalsIgnoreCase(measurement) + || deviceNodes.length != 5) { + return Optional.empty(); + } + newDevice = "root.bw.baoshan." + deviceNodes[3] + "." + deviceNodes[4] + "." + measurement; + measurementSchema = + new MeasurementSchema("value", TSDataType.FLOAT, encoding, compressionType); + return Optional.of(new Pair<>(newDevice, measurementSchema)); + } + + // switch (device.charAt(16)) { + // case 'S': + // if (device.charAt(17) != '.') { + // return Optional.empty(); + // } + // if (dataType != TSDataType.FLOAT || "value".equalsIgnoreCase(measurement)) { + // return Optional.empty(); + // } + // builder = new StringBuilder(); + // builder.append(device); + // newDevice = processDeviceWithS(measurement, builder); + // measurementSchema = new MeasurementSchema("value", dataType, encoding, + // compressionType); + // return Optional.of(new Pair<>(newDevice, measurementSchema)); + // case 'I': + // if (device.charAt(17) != '.') { + // return Optional.empty(); + // } + // builder = new StringBuilder(); + // builder.append(device); + // + // newDevice = processDeviceWithI(builder); + // measurementSchema = new MeasurementSchema(measurement, dataType, encoding, + // compressionType); + // + // return Optional.of(new Pair<>(newDevice, measurementSchema)); + // case 'F': + // if (device.charAt(17) != '.') { + // return Optional.empty(); + // } + // if (dataType != TSDataType.DOUBLE || "value".equalsIgnoreCase(measurement)) { + // return Optional.empty(); + // } + // String[] nodes = device.split("\\."); + // if (nodes.length < 4) { + // return Optional.empty(); + // } + // builder = new StringBuilder(); + // builder.append(device); + // newDevice = processDeviceWithF(measurement, nodes[4].length(), builder); + // measurementSchema = + // new MeasurementSchema("value", TSDataType.FLOAT, encoding, compressionType); + // return Optional.of(new Pair<>(newDevice, measurementSchema)); + // + // default: + // if ("value".equalsIgnoreCase(measurement) + // || device.endsWith(".I") + // || dataType != TSDataType.FLOAT) { + // return Optional.empty(); + // } + // builder = new StringBuilder(); + // builder.append(device); + // newDevice = processDevice(measurement, builder); + // measurementSchema = new MeasurementSchema("value", dataType, encoding, + // compressionType); + // return Optional.of(new Pair<>(newDevice, measurementSchema)); + // } + } + + // private static String processDeviceWithF( + // String measurement, int deviceCodeLength, StringBuilder builder) { + // return builder + // .delete(16, 18) + // .insert(16 + deviceCodeLength, ".`00`") + // .append('.') + // .append(measurement) + // .toString(); + // } + // + // private static String processDevice(String measurement, StringBuilder builder) { + // return builder.append('.').append(measurement).toString(); + // } + // + // private static String processDeviceWithS(String measurement, StringBuilder builder) { + // return builder.delete(16, 18).append('.').append(measurement).append(".S").toString(); + // } + // + // private static String processDeviceWithI(StringBuilder builder) { + // return builder.delete(16, 18).append(".I").toString(); + // } + + private static void writeToNewTsFile( + TsBlock tsBlock, + TsFileWriter tsFileWriter, + String newDeviceId, + MeasurementSchema measurementSchema) + throws IOException, WriteProcessException { + Tablet tablet = + new Tablet( + newDeviceId, Collections.singletonList(measurementSchema), tsBlock.getPositionCount()); + tablet.rowSize = tsBlock.getPositionCount(); + Column valueColumn = tsBlock.getColumn(0); + if (valueColumn instanceof FloatColumn) { + long[] timestamps = tablet.timestamps; + float[] values = (float[]) tablet.values[0]; + for (int i = 0, n = tsBlock.getPositionCount(); i < n; i++) { + timestamps[i] = tsBlock.getTimeByIndex(i); + values[i] = valueColumn.getFloat(i); + } + } else if (valueColumn instanceof DoubleColumn) { + long[] timestamps = tablet.timestamps; + float[] values = (float[]) tablet.values[0]; + for (int i = 0, n = tsBlock.getPositionCount(); i < n; i++) { + timestamps[i] = tsBlock.getTimeByIndex(i); + values[i] = (float) valueColumn.getDouble(i); + } + } else if (valueColumn instanceof IntColumn) { + long[] timestamps = tablet.timestamps; + int[] values = (int[]) tablet.values[0]; + for (int i = 0, n = tsBlock.getPositionCount(); i < n; i++) { + timestamps[i] = tsBlock.getTimeByIndex(i); + values[i] = valueColumn.getInt(i); + } + } else if (valueColumn instanceof LongColumn) { + long[] timestamps = tablet.timestamps; + long[] values = (long[]) tablet.values[0]; + for (int i = 0, n = tsBlock.getPositionCount(); i < n; i++) { + timestamps[i] = tsBlock.getTimeByIndex(i); + values[i] = valueColumn.getLong(i); + } + } else { + throw new IllegalArgumentException( + "data type should not be: " + valueColumn.getClass().getName()); + } + tsFileWriter.write(tablet); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java index 478033b335e..1ad7409d033 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java @@ -315,9 +315,11 @@ public class PartitionCache { boolean tryToFetch, boolean isAutoCreate) { // miss when devicePath contains * - for (String devicePath : devicePaths) { - if (devicePath.contains("*")) { - return; + if (!isAutoCreate) { + for (String devicePath : devicePaths) { + if (devicePath.contains("*")) { + return; + } } } // first try to hit database in fast-fail way diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index 00d2e86d421..928294cefb6 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -135,7 +135,7 @@ public class CommonConfig { private long timePartitionInterval = 604_800_000; /** This variable set timestamp precision as millisecond, microsecond or nanosecond. */ - private String timestampPrecision = "ms"; + private String timestampPrecision = "ns"; /** * The name of the directory that stores the tsfiles temporarily hold or generated by the pipe diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java index e8925543076..5cfbd798aad 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java @@ -136,7 +136,7 @@ public class TsFileSequenceReader implements AutoCloseable { */ public TsFileSequenceReader(String file, boolean loadMetadataSize) throws IOException { if (resourceLogger.isDebugEnabled()) { - resourceLogger.debug("{} reader is opened. {}", file, getClass().getName()); + // resourceLogger.debug("{} reader is opened. {}", file, getClass().getName()); } this.file = file; tsFileInput = FSFactoryProducer.getFileInputFactory().getTsFileInput(file); @@ -1424,7 +1424,7 @@ public class TsFileSequenceReader implements AutoCloseable { @Override public void close() throws IOException { if (resourceLogger.isDebugEnabled()) { - resourceLogger.debug("{} reader is closed.", file); + // resourceLogger.debug("{} reader is closed.", file); } this.tsFileInput.close(); } diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java index d01705adc12..5aaaec36bf8 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java @@ -166,6 +166,9 @@ public class PageReader implements IPageReader { private boolean pageSatisfy() { Statistics statistics = getStatistics(); + if (statistics == null) { + return true; + } if (filter == null || filter.allSatisfy(statistics)) { long rowCount = statistics.getCount(); if (paginationController.hasCurOffset(rowCount)) { diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java index 2ab6780f184..54e65d37161 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java @@ -546,7 +546,7 @@ public class TsFileWriter implements AutoCloseable { long memSize = calculateMemSizeForAllGroup(); assert memSize > 0; if (memSize > chunkGroupSizeThreshold) { - LOG.debug("start to flush chunk groups, memory space occupy:{}", memSize); + // LOG.debug("start to flush chunk groups, memory space occupy:{}", memSize); recordCountForNextMemCheck = recordCount * chunkGroupSizeThreshold / memSize; return flushAllChunkGroups(); } else { @@ -623,7 +623,7 @@ public class TsFileWriter implements AutoCloseable { */ @Override public void close() throws IOException { - LOG.info("start close file"); + // LOG.info("start close file"); flushAllChunkGroups(); fileWriter.endFile(); } diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java index 585620eefd7..8219531520d 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java @@ -254,7 +254,8 @@ public class ChunkWriterImpl implements IChunkWriter { */ private void checkPageSizeAndMayOpenANewPage() { if (pageWriter.getPointNumber() == maxNumberOfPointsInPage) { - logger.debug("current line count reaches the upper bound, write page {}", measurementSchema); + // logger.debug("current line count reaches the upper bound, write page {}", + // measurementSchema); writePageToPageBuffer(); } else if (pageWriter.getPointNumber() >= valueCountInOnePageForNextCheck) { // need to check memory size @@ -262,12 +263,13 @@ public class ChunkWriterImpl implements IChunkWriter { long currentPageSize = pageWriter.estimateMaxMemSize(); if (currentPageSize > pageSizeThreshold) { // memory size exceeds threshold // we will write the current page - logger.debug( - "enough size, write page {}, pageSizeThreshold:{}, currentPateSize:{}, valueCountInOnePage:{}", - measurementSchema.getMeasurementId(), - pageSizeThreshold, - currentPageSize, - pageWriter.getPointNumber()); + // logger.debug( + // "enough size, write page {}, pageSizeThreshold:{}, currentPateSize:{}, + // valueCountInOnePage:{}", + // measurementSchema.getMeasurementId(), + // pageSizeThreshold, + // currentPageSize, + // pageWriter.getPointNumber()); writePageToPageBuffer(); valueCountInOnePageForNextCheck = MINIMUM_RECORD_COUNT_FOR_CHECK; } else { diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java index 4d207c98f3f..429e06bab2c 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java @@ -137,7 +137,7 @@ public class NonAlignedChunkGroupWriterImpl implements IChunkGroupWriter { @Override public long flushToFileWriter(TsFileIOWriter fileWriter) throws IOException { - LOG.debug("start flush device id:{}", deviceId); + // LOG.debug("start flush device id:{}", deviceId); // make sure all the pages have been compressed into buffers, so that we can get correct // groupWriter.getCurrentChunkGroupSize(). sealAllChunks(); diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java index 6a6a29221c2..8e2296992cf 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java @@ -130,7 +130,7 @@ public class TsFileIOWriter implements AutoCloseable { this.out = FSFactoryProducer.getFileOutputFactory().getTsFileOutput(file.getPath(), false); this.file = file; if (resourceLogger.isDebugEnabled()) { - resourceLogger.debug("{} writer is opened.", file.getName()); + // resourceLogger.debug("{} writer is opened.", file.getName()); } startFile(); } @@ -178,7 +178,7 @@ public class TsFileIOWriter implements AutoCloseable { public int startChunkGroup(String deviceId) throws IOException { this.currentChunkGroupDeviceId = deviceId; if (logger.isDebugEnabled()) { - logger.debug("start chunk group:{}, file position {}", deviceId, out.getPosition()); + // logger.debug("start chunk group:{}, file position {}", deviceId, out.getPosition()); } chunkMetadataList = new ArrayList<>(); ChunkGroupHeader chunkGroupHeader = new ChunkGroupHeader(currentChunkGroupDeviceId); @@ -327,7 +327,7 @@ public class TsFileIOWriter implements AutoCloseable { long footerIndex = out.getPosition(); if (logger.isDebugEnabled()) { - logger.debug("start to flush the footer,file pos:{}", footerIndex); + // logger.debug("start to flush the footer,file pos:{}", footerIndex); } // write magic string @@ -336,7 +336,7 @@ public class TsFileIOWriter implements AutoCloseable { // close file out.close(); if (resourceLogger.isDebugEnabled() && file != null) { - resourceLogger.debug("{} writer is closed.", file.getName()); + // resourceLogger.debug("{} writer is closed.", file.getName()); } if (file != null) { File chunkMetadataFile = new File(file.getAbsolutePath() + CHUNK_METADATA_TEMP_FILE_SUFFIX); @@ -346,7 +346,7 @@ public class TsFileIOWriter implements AutoCloseable { } canWrite = false; long cost = System.currentTimeMillis() - startTime; - logger.info("Time for flushing metadata is {} ms", cost); + // logger.info("Time for flushing metadata is {} ms", cost); } private void checkInMemoryPathCount() {
