This is an automated email from the ASF dual-hosted git repository. spricoder pushed a commit to branch optimize/log-v2 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 939736ac14470b44b2bead2f563943ed7b704291 Author: spricoder <[email protected]> AuthorDate: Wed Jan 3 17:20:16 2024 +0800 Optimize log according to real scenarios after checking --- .../db/storageengine/dataregion/DataRegion.java | 10 ------- .../dataregion/flush/CompressionRatio.java | 3 -- .../dataregion/flush/MemTableFlushTask.java | 24 ---------------- .../dataregion/memtable/TsFileProcessor.java | 32 ++++++---------------- .../iotdb/tsfile/write/writer/TsFileIOWriter.java | 3 -- 5 files changed, 9 insertions(+), 63 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 2819d0a4a00..84b61aca712 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -2359,16 +2359,6 @@ public class DataRegion implements IDataRegionForQuery { } } if (summary.hasSubmitTask()) { - logger.info( - "[CompactionScheduler][{}] selected sequence InnerSpaceCompactionTask num is {}," - + " selected unsequence InnerSpaceCompactionTask num is {}," - + " selected CrossSpaceCompactionTask num is {}," - + " selected InsertionCrossSpaceCompactionTask num is {}", - dataRegionId, - summary.getSubmitSeqInnerSpaceCompactionTaskNum(), - summary.getSubmitUnseqInnerSpaceCompactionTaskNum(), - summary.getSubmitCrossSpaceCompactionTaskNum(), - summary.getSubmitInsertionCrossSpaceCompactionTaskNum()); CompactionMetrics.getInstance().updateCompactionTaskSelectionNum(summary); } return trySubmitCount; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java index dfc37ba428e..5d6c40e1487 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java @@ -95,9 +95,6 @@ public class CompressionRatio { String.format( Locale.ENGLISH, RATIO_FILE_PATH_FORMAT, totalMemorySize.get(), totalDiskSize)); persist(oldFile, newFile); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Compression ratio is {}", (double) totalMemorySize.get() / totalDiskSize); - } } /** Get the average compression ratio for all closed files */ diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java index 454702e72f3..46c919835a8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java @@ -110,14 +110,6 @@ public class MemTableFlushTask { memTable.getSeriesNumber() == 0 ? 0 : memTable.getTotalPointsNum() / memTable.getSeriesNumber(); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug( - "The memTable size of SG {} is {}, the avg series points num in chunk is {}, total timeseries number is {}", - storageGroup, - memTable.memSize(), - avgSeriesPointsNum, - memTable.getSeriesNumber()); - } WRITING_METRICS.recordFlushingMemTableStatus( storageGroup, memTable.memSize(), @@ -215,14 +207,6 @@ public class MemTableFlushTask { MetricLevel.CORE, Tag.NAME.toString(), "flush"); - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug( - "Database {} memtable {} flushing a memtable has finished! Time consumption: {}ms", - storageGroup, - memTable, - System.currentTimeMillis() - start); - } } /** encoding task (second task of pipeline) */ @@ -288,14 +272,6 @@ public class MemTableFlushTask { } recordFlushPointsMetric(); - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug( - "Database {}, flushing memtable {} into disk: Encoding data cost " + "{} ms.", - storageGroup, - writer.getFile().getName(), - memSerializeTime); - } WRITING_METRICS.recordFlushCost(WritingMetrics.FLUSH_STAGE_ENCODING, memSerializeTime); } }; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index 1f11f696abb..244bc276342 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -792,12 +792,6 @@ public class TsFileProcessor { return false; } if (workMemTable.shouldFlush()) { - if (logger.isDebugEnabled()) { - logger.debug( - "The memtable size {} of tsfile {} reaches the mem control threshold", - workMemTable.memSize(), - tsFileResource.getTsFile().getAbsolutePath()); - } return true; } if (!enableMemControl && workMemTable.memSize() >= getMemtableSizeThresholdBasedOnSeriesNum()) { @@ -873,9 +867,9 @@ public class TsFileProcessor { FLUSH_QUERY_WRITE_LOCKED, storageGroupName, tsFileResource.getTsFile().getName()); } try { - if (logger.isInfoEnabled()) { + if (logger.isDebugEnabled()) { if (workMemTable != null) { - logger.info( + logger.debug( "{}: flush a working memtable in async close tsfile {}, memtable size: {}, tsfile " + "size: {}, plan index: [{}, {}], progress index: {}", storageGroupName, @@ -886,7 +880,7 @@ public class TsFileProcessor { workMemTable.getMaxPlanIndex(), tsFileResource.getMaxProgressIndex()); } else { - logger.info( + logger.debug( "{}: flush a NotifyFlushMemTable in async close tsfile {}, tsfile size: {}", storageGroupName, tsFileResource.getTsFile().getAbsolutePath(), @@ -1346,14 +1340,12 @@ public class TsFileProcessor { private void updateCompressionRatio() { try { double compressionRatio = ((double) totalMemTableSize) / writer.getPos(); - if (logger.isDebugEnabled()) { - logger.debug( - "The compression ratio of tsfile {} is {}, totalMemTableSize: {}, the file size: {}", - writer.getFile().getAbsolutePath(), - compressionRatio, - totalMemTableSize, - writer.getPos()); - } + logger.info( + "The compression ratio of tsfile {} is {}, totalMemTableSize: {}, the file size: {}", + writer.getFile().getAbsolutePath(), + compressionRatio, + totalMemTableSize, + writer.getPos()); String dataRegionId = dataRegionInfo.getDataRegion().getDataRegionId(); WritingMetrics.getInstance() .recordTsFileCompressionRatioOfFlushingMemTable(dataRegionId, compressionRatio); @@ -1369,14 +1361,8 @@ public class TsFileProcessor { /** end file and write some meta */ private void endFile() throws IOException, TsFileProcessorException { - if (logger.isDebugEnabled()) { - logger.debug("Start to end file {}", tsFileResource); - } writer.endFile(); tsFileResource.serialize(); - if (logger.isDebugEnabled()) { - logger.debug("Ended file {}", tsFileResource); - } // remove this processor from Closing list in StorageGroupProcessor, // mark the TsFileResource closed, no need writer anymore for (CloseFileListener closeFileListener : closeFileListeners) { diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java index 12b79a9fc80..658a16deda6 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java @@ -321,7 +321,6 @@ public class TsFileIOWriter implements AutoCloseable { */ @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning public void endFile() throws IOException { - long startTime = System.currentTimeMillis(); checkInMemoryPathCount(); readChunkMetadataAndConstructIndexTree(); @@ -348,8 +347,6 @@ public class TsFileIOWriter implements AutoCloseable { } } canWrite = false; - long cost = System.currentTimeMillis() - startTime; - logger.debug("Time for flushing metadata is {} ms", cost); } private void checkInMemoryPathCount() {
