This is an automated email from the ASF dual-hosted git repository. rong pushed a commit to branch 0.12-debug-compaction in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1901c0a6237c3595a03de7c24dc88d2f0ed4c3fc Author: zhanglingzhe0820 <[email protected]> AuthorDate: Thu Jun 3 21:24:24 2021 +0800 [DEBUG] Compaction error schema add log (#3324) Co-authored-by: zhanglingzhe <[email protected]> --- .../level/LevelCompactionTsFileManagement.java | 4 ++-- .../db/engine/compaction/utils/CompactionUtils.java | 21 +++++++++------------ .../java/org/apache/iotdb/db/metadata/MManager.java | 1 + .../db/engine/compaction/CompactionChunkTest.java | 2 +- .../compaction/LevelCompactionRecoverTest.java | 12 +++++------- 5 files changed, 18 insertions(+), 22 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java index dc37e8e..cbc8b24 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java @@ -29,7 +29,7 @@ import org.apache.iotdb.db.engine.compaction.utils.CompactionUtils; import org.apache.iotdb.db.engine.modification.Modification; import org.apache.iotdb.db.engine.modification.ModificationFile; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; -import org.apache.iotdb.db.exception.metadata.IllegalPathException; +import org.apache.iotdb.db.exception.metadata.MetadataException; import org.apache.iotdb.db.query.control.FileReaderManager; import org.apache.iotdb.db.utils.TestOnly; import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer; @@ -526,7 +526,7 @@ public class LevelCompactionTsFileManagement extends TsFileManagement { } } } - } catch (IOException | IllegalPathException | InterruptedException e) { + } catch (IOException | InterruptedException | MetadataException e) { logger.error("recover level tsfile management error ", e); } finally { if (logFile.exists()) { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java index a6aa68c..e056fe0 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java @@ -25,7 +25,6 @@ import org.apache.iotdb.db.engine.merge.manage.MergeManager; import org.apache.iotdb.db.engine.modification.Modification; import org.apache.iotdb.db.engine.modification.ModificationFile; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; -import org.apache.iotdb.db.exception.metadata.IllegalPathException; import org.apache.iotdb.db.exception.metadata.MetadataException; import org.apache.iotdb.db.metadata.PartialPath; import org.apache.iotdb.db.service.IoTDB; @@ -178,7 +177,7 @@ public class CompactionUtils { RestorableTsFileIOWriter writer, Map<String, List<Modification>> modificationCache, List<Modification> modifications) - throws IOException, IllegalPathException { + throws IOException, MetadataException { Map<Long, TimeValuePair> timeValuePairMap = new TreeMap<>(); Map<TsFileSequenceReader, List<ChunkMetadata>> readerChunkMetadataMap = entry.getValue(); readByDeserializePageMerge( @@ -198,15 +197,10 @@ public class CompactionUtils { return; } IChunkWriter chunkWriter; - try { - chunkWriter = - new ChunkWriterImpl( - IoTDB.metaManager.getSeriesSchema(new PartialPath(device), entry.getKey()), true); - } catch (MetadataException e) { - // this may caused in IT by restart - logger.error("{} get schema {} error, skip this sensor", device, entry.getKey(), e); - return; - } + chunkWriter = + new ChunkWriterImpl( + IoTDB.metaManager.getSeriesSchema(new PartialPath(device), entry.getKey()), true); + for (TimeValuePair timeValuePair : timeValuePairMap.values()) { writeTVPair(timeValuePair, chunkWriter); targetResource.updateStartTime(device, timeValuePair.getTimestamp()); @@ -259,7 +253,7 @@ public class CompactionUtils { Set<String> devices, boolean sequence, List<Modification> modifications) - throws IOException, IllegalPathException { + throws IOException, MetadataException { RestorableTsFileIOWriter writer = new RestorableTsFileIOWriter(targetResource.getTsFile()); Map<String, TsFileSequenceReader> tsFileSequenceReaderMap = new HashMap<>(); Map<String, List<Modification>> modificationCache = new HashMap<>(); @@ -304,6 +298,9 @@ public class CompactionUtils { if (sensorChunkMetadataListMap.size() <= 0) { if (chunkMetadataListIteratorCache.get(reader).hasNext()) { sensorChunkMetadataListMap = chunkMetadataListIteratorCache.get(reader).next(); + for (String sensor : sensorChunkMetadataListMap.keySet()) { + logger.error("chunkMetadataListIterator get measurement:{}", device + "." + sensor); + } chunkMetadataListCacheForMerge.put(reader, sensorChunkMetadataListMap); } else { continue; diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java index 8224995..b74ea34 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java @@ -998,6 +998,7 @@ public class MManager { if (leaf != null) { return ((MeasurementMNode) leaf).getSchema(); } + logger.error("cannot get schema {}", device.getDevice() + "." + measurement); return null; } diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionChunkTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionChunkTest.java index 5e7276a..bf958c4 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionChunkTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionChunkTest.java @@ -152,7 +152,7 @@ public class CompactionChunkTest extends LevelCompactionTest { } @Test - public void testDeserializeMerge() throws IOException, IllegalPathException { + public void testDeserializeMerge() throws IOException, MetadataException { Map<String, Map<TsFileSequenceReader, List<ChunkMetadata>>> measurementChunkMetadataMap = new HashMap<>(); List<TsFileResource> sourceTsfileResources = seqResources.subList(0, 2); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionRecoverTest.java index 7adffc2..473777e 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionRecoverTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionRecoverTest.java @@ -80,7 +80,7 @@ public class LevelCompactionRecoverTest extends LevelCompactionTest { /** compaction recover merge finished */ @Test - public void testCompactionMergeRecoverMergeFinished() throws IOException, IllegalPathException { + public void testCompactionMergeRecoverMergeFinished() throws IOException, MetadataException { LevelCompactionTsFileManagement levelCompactionTsFileManagement = new LevelCompactionTsFileManagement(COMPACTION_TEST_SG, tempSGDir.getPath()); levelCompactionTsFileManagement.addAll(seqResources, true); @@ -171,7 +171,7 @@ public class LevelCompactionRecoverTest extends LevelCompactionTest { /** compaction recover merge finished, delete one offset */ @Test public void testCompactionMergeRecoverMergeFinishedAndDeleteOneOffset() - throws IOException, IllegalPathException { + throws IOException, MetadataException { LevelCompactionTsFileManagement levelCompactionTsFileManagement = new LevelCompactionTsFileManagement(COMPACTION_TEST_SG, tempSGDir.getPath()); levelCompactionTsFileManagement.addAll(seqResources, true); @@ -286,7 +286,7 @@ public class LevelCompactionRecoverTest extends LevelCompactionTest { /** compaction recover merge finished, delete one device - offset */ @Test public void testCompactionMergeRecoverMergeFinishedAndDeleteOneDeviceWithOffset() - throws IOException, IllegalPathException { + throws IOException, MetadataException { LevelCompactionTsFileManagement levelCompactionTsFileManagement = new LevelCompactionTsFileManagement(COMPACTION_TEST_SG, tempSGDir.getPath()); levelCompactionTsFileManagement.addAll(seqResources, true); @@ -406,8 +406,7 @@ public class LevelCompactionRecoverTest extends LevelCompactionTest { /** compaction recover merge finished,unseq */ @Test - public void testCompactionMergeRecoverMergeFinishedUnseq() - throws IOException, IllegalPathException { + public void testCompactionMergeRecoverMergeFinishedUnseq() throws IOException, MetadataException { LevelCompactionTsFileManagement levelCompactionTsFileManagement = new LevelCompactionTsFileManagement(COMPACTION_TEST_SG, tempSGDir.getPath()); levelCompactionTsFileManagement.addAll(seqResources, true); @@ -638,8 +637,7 @@ public class LevelCompactionRecoverTest extends LevelCompactionTest { /** compaction recover merge finished but no finish log */ @Test - public void testCompactionMergeRecoverMergeFinishedNoLog() - throws IOException, IllegalPathException { + public void testCompactionMergeRecoverMergeFinishedNoLog() throws IOException, MetadataException { LevelCompactionTsFileManagement levelCompactionTsFileManagement = new LevelCompactionTsFileManagement(COMPACTION_TEST_SG, tempSGDir.getPath()); levelCompactionTsFileManagement.addAll(seqResources, true);
