This is an automated email from the ASF dual-hosted git repository. rong pushed a commit to branch 0.12-debug-compaction in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 13b85caa56b76a8e248d1e0a24f13e66bb366379 Author: Steve Yurong Su <[email protected]> AuthorDate: Thu Jun 3 22:35:13 2021 +0800 add logs --- .../main/java/org/apache/iotdb/SessionExample.java | 50 +++++++++++----------- .../engine/compaction/utils/CompactionUtils.java | 9 ++-- .../org/apache/iotdb/db/metadata/MManager.java | 2 +- .../iotdb/tsfile/read/TsFileSequenceReader.java | 4 ++ .../iotdb/tsfile/write/writer/TsFileIOWriter.java | 49 ++++++++++++++++++++- 5 files changed, 84 insertions(+), 30 deletions(-) diff --git a/example/session/src/main/java/org/apache/iotdb/SessionExample.java b/example/session/src/main/java/org/apache/iotdb/SessionExample.java index 386b4e4..b91d49f 100644 --- a/example/session/src/main/java/org/apache/iotdb/SessionExample.java +++ b/example/session/src/main/java/org/apache/iotdb/SessionExample.java @@ -66,30 +66,30 @@ public class SessionExample { } createTimeseries(); - createMultiTimeseries(); - insertRecord(); + // createMultiTimeseries(); + // insertRecord(); insertTablet(); - insertTablets(); - insertRecords(); - nonQuery(); - query(); - queryWithTimeout(); - rawDataQuery(); - queryByIterator(); - deleteData(); - deleteTimeseries(); - setTimeout(); - - sessionEnableRedirect = new Session(LOCAL_HOST, 6667, "root", "root"); - sessionEnableRedirect.setEnableQueryRedirection(true); - sessionEnableRedirect.open(false); - - // set session fetchSize - sessionEnableRedirect.setFetchSize(10000); - - insertRecord4Redirect(); - query4Redirect(); - sessionEnableRedirect.close(); + // insertTablets(); + // insertRecords(); + // nonQuery(); + // query(); + // queryWithTimeout(); + // rawDataQuery(); + // queryByIterator(); + // deleteData(); + // deleteTimeseries(); + // setTimeout(); + // + // sessionEnableRedirect = new Session(LOCAL_HOST, 6667, "root", "root"); + // sessionEnableRedirect.setEnableQueryRedirection(true); + // sessionEnableRedirect.open(false); + // + // // set session fetchSize + // sessionEnableRedirect.setFetchSize(10000); + // + // insertRecord4Redirect(); + // query4Redirect(); + // sessionEnableRedirect.close(); session.close(); } @@ -331,7 +331,7 @@ public class SessionExample { // Method 1 to add tablet data long timestamp = System.currentTimeMillis(); - for (long row = 0; row < 100; row++) { + for (long row = 0; row < 10000000; row++) { int rowIndex = tablet.rowSize++; tablet.addTimestamp(rowIndex, timestamp); for (int s = 0; s < 3; s++) { @@ -354,7 +354,7 @@ public class SessionExample { long[] timestamps = tablet.timestamps; Object[] values = tablet.values; - for (long time = 0; time < 100; time++) { + for (long time = 0; time < 1000; time++) { int row = tablet.rowSize++; timestamps[row] = time; for (int i = 0; i < 3; i++) { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java index e056fe0..9b5c7e9 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java @@ -196,8 +196,10 @@ public class CompactionUtils { if (isChunkMetadataEmpty) { return; } - IChunkWriter chunkWriter; - chunkWriter = + logger.warn( + "++++++ new ChunkWriterImpl in writeByDeserializePageMerge: " + + new PartialPath(device, entry.getKey()).getFullPath()); + IChunkWriter chunkWriter = new ChunkWriterImpl( IoTDB.metaManager.getSeriesSchema(new PartialPath(device), entry.getKey()), true); @@ -299,7 +301,8 @@ public class CompactionUtils { if (chunkMetadataListIteratorCache.get(reader).hasNext()) { sensorChunkMetadataListMap = chunkMetadataListIteratorCache.get(reader).next(); for (String sensor : sensorChunkMetadataListMap.keySet()) { - logger.error("chunkMetadataListIterator get measurement:{}", device + "." + sensor); + logger.warn( + "+++++ chunkMetadataListIterator get measurement:{}.{}", device, sensor); } chunkMetadataListCacheForMerge.put(reader, sensorChunkMetadataListMap); } else { diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java index b74ea34..ce44d49 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java @@ -998,7 +998,7 @@ public class MManager { if (leaf != null) { return ((MeasurementMNode) leaf).getSchema(); } - logger.error("cannot get schema {}", device.getDevice() + "." + measurement); + logger.warn("+++++ cannot get schema {}", device.getDevice() + "." + measurement); return null; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java index 211a688..17ffdc9 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java @@ -1323,6 +1323,10 @@ public class TsFileSequenceReader implements AutoCloseable { measurementChunkMetadataList .computeIfAbsent(timeseriesMetadata.getMeasurementId(), m -> new ArrayList<>()) .addAll(timeseriesMetadata.getChunkMetadataList()); + logger.warn( + "----- getMeasurementChunkMetadataListMapIterator#next(): {}.{}", + device, + timeseriesMetadata.getMeasurementId()); } return measurementChunkMetadataList; } catch (IOException e) { diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java index 3df2eb9..a7373ab 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java @@ -34,6 +34,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer; +import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.Chunk; import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.utils.BytesUtils; @@ -47,11 +48,15 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; import java.util.TreeMap; /** @@ -232,14 +237,19 @@ public class TsFileIOWriter { // serialize the SEPARATOR of MetaData ReadWriteIOUtils.write(MetaMarker.SEPARATOR, out.wrapAsStream()); + logger.warn("-------------START: group ChunkMetadata by series-----------------"); + Set<Path> expectedPaths = new HashSet<>(); // group ChunkMetadata by series Map<Path, List<ChunkMetadata>> chunkMetadataListMap = new TreeMap<>(); for (ChunkGroupMetadata chunkGroupMetadata : chunkGroupMetadataList) { for (ChunkMetadata chunkMetadata : chunkGroupMetadata.getChunkMetadataList()) { Path series = new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid()); + expectedPaths.add(series); + logger.warn("++++++ endFile: {}", series.getFullPath()); chunkMetadataListMap.computeIfAbsent(series, k -> new ArrayList<>()).add(chunkMetadata); } } + logger.warn("-------------END: group ChunkMetadata by series-------------------"); MetadataIndexNode metadataIndex = flushMetadataIndex(chunkMetadataListMap); TsFileMetadata tsFileMetaData = new TsFileMetadata(); @@ -274,7 +284,44 @@ public class TsFileIOWriter { if (resourceLogger.isDebugEnabled() && file != null) { resourceLogger.debug("{} writer is closed.", file.getName()); } - canWrite = false; + + logger.warn("-------------START: write check-----------------"); + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath(), true)) { + Set<Path> actualPaths1 = new HashSet<>(); + Set<Path> actualPaths2 = new HashSet<>(); + + for (Entry<String, List<TimeseriesMetadata>> entry : + reader.getAllTimeseriesMetadata().entrySet()) { + for (TimeseriesMetadata timeseriesMetadata : entry.getValue()) { + actualPaths1.add(new Path(entry.getKey(), timeseriesMetadata.getMeasurementId())); + } + + Iterator<Map<String, List<ChunkMetadata>>> iterator = + reader.getMeasurementChunkMetadataListMapIterator(entry.getKey()); + while (iterator.hasNext()) { + for (String m : iterator.next().keySet()) { + actualPaths2.add(new Path(entry.getKey(), m)); + } + } + } + + if (!actualPaths1.equals(expectedPaths)) { + logger.error("!!!!!!!!!!!! - 1 reader.getAllTimeseriesMetadata()"); + logger.error(Arrays.toString(expectedPaths.toArray())); + logger.error(Arrays.toString(actualPaths1.toArray())); + logger.error("!!!!!!!!!!!!"); + } + + if (!actualPaths2.equals(expectedPaths)) { + logger.error("!!!!!!!!!!!! - 2 reader.getMeasurementChunkMetadataListMapIterator"); + logger.error(Arrays.toString(expectedPaths.toArray())); + logger.error(Arrays.toString(actualPaths2.toArray())); + logger.error("!!!!!!!!!!!!"); + } + } finally { + logger.warn("-------------END: write check-----------------"); + canWrite = false; + } } /**
