This is an automated email from the ASF dual-hosted git repository. zyk pushed a commit to branch rc/1.1.0 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 63796d5fafc2ebb718c0fa30f3dc43ab559e0b93 Author: 周沛辰 <[email protected]> AuthorDate: Fri Mar 17 16:15:24 2023 +0800 [To rel/1.1][IOTDB-5651] Fix compaction UT --- .../execute/task/CrossSpaceCompactionTask.java | 7 +- .../execute/task/InnerSpaceCompactionTask.java | 264 ++++++++++----------- 2 files changed, 135 insertions(+), 136 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java index 887b930406..e29c083c70 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java @@ -213,10 +213,6 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask { long unsequenceFileSize = deleteOldFiles(selectedUnsequenceFiles); CompactionUtils.deleteCompactionModsFile(selectedSequenceFiles, selectedUnsequenceFiles); - if (logFile.exists()) { - FileUtils.delete(logFile); - } - // update the metrics finally in case of any exception occurs for (TsFileResource targetResource : targetTsfileResourceList) { if (!targetResource.isDeleted()) { @@ -246,6 +242,9 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask { (selectedSeqFileSize + selectedUnseqFileSize) / 1024 / 1024 / costTime, summary); } + if (logFile.exists()) { + FileUtils.delete(logFile); + } } catch (Throwable throwable) { // catch throwable to handle OOM errors if (!(throwable instanceof InterruptedException)) { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java index a9f7aca321..4c871843fa 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java @@ -59,6 +59,7 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask { protected int sumOfCompactionCount; protected long maxFileVersion; protected int maxCompactionCount; + private File logFile; protected TsFileResourceList tsFileResourceList; protected List<TsFileResource> targetTsFileList; @@ -116,153 +117,152 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask { sequence ? "Sequence" : "Unsequence", selectedTsFileResourceList.size(), selectedFileSize / 1024 / 1024); + try { targetTsFileResource = TsFileNameGenerator.getInnerCompactionTargetFileResource( selectedTsFileResourceList, sequence); - } catch (IOException e) { - LOGGER.error("Failed to get target file for {}", selectedTsFileResourceList, e); - return; - } - File logFile = - new File( - dataDirectory - + File.separator - + targetTsFileResource.getTsFile().getName() - + CompactionLogger.INNER_COMPACTION_LOG_NAME_SUFFIX); - try (CompactionLogger compactionLogger = new CompactionLogger(logFile)) { - // Here is tmpTargetFile, which is xxx.target - targetTsFileList = new ArrayList<>(Collections.singletonList(targetTsFileResource)); - compactionLogger.logFiles(selectedTsFileResourceList, CompactionLogger.STR_SOURCE_FILES); - compactionLogger.logFiles(targetTsFileList, CompactionLogger.STR_TARGET_FILES); - - LOGGER.info( - "{}-{} [Compaction] compaction with {}", - storageGroupName, - dataRegionId, - selectedTsFileResourceList); - - // carry out the compaction - performer.setSourceFiles(selectedTsFileResourceList); - // As elements in targetFiles may be removed in ReadPointCompactionPerformer, we should use a - // mutable list instead of Collections.singletonList() - performer.setTargetFiles(targetTsFileList); - performer.setSummary(summary); - performer.perform(); - - CompactionUtils.moveTargetFile(targetTsFileList, true, storageGroupName + "-" + dataRegionId); - - LOGGER.info( - "{}-{} [InnerSpaceCompactionTask] start to rename mods file", - storageGroupName, - dataRegionId); - CompactionUtils.combineModsInInnerCompaction( - selectedTsFileResourceList, targetTsFileResource); - - if (Thread.currentThread().isInterrupted() || summary.isCancel()) { - throw new InterruptedException( - String.format("%s-%s [Compaction] abort", storageGroupName, dataRegionId)); - } + logFile = + new File( + dataDirectory + + File.separator + + targetTsFileResource.getTsFile().getName() + + CompactionLogger.INNER_COMPACTION_LOG_NAME_SUFFIX); + try (CompactionLogger compactionLogger = new CompactionLogger(logFile)) { + // Here is tmpTargetFile, which is xxx.target + targetTsFileList = new ArrayList<>(Collections.singletonList(targetTsFileResource)); + compactionLogger.logFiles(selectedTsFileResourceList, CompactionLogger.STR_SOURCE_FILES); + compactionLogger.logFiles(targetTsFileList, CompactionLogger.STR_TARGET_FILES); + + LOGGER.info( + "{}-{} [Compaction] compaction with {}", + storageGroupName, + dataRegionId, + selectedTsFileResourceList); + + // carry out the compaction + performer.setSourceFiles(selectedTsFileResourceList); + // As elements in targetFiles may be removed in ReadPointCompactionPerformer, we should use + // a + // mutable list instead of Collections.singletonList() + performer.setTargetFiles(targetTsFileList); + performer.setSummary(summary); + performer.perform(); + + CompactionUtils.moveTargetFile( + targetTsFileList, true, storageGroupName + "-" + dataRegionId); + + LOGGER.info( + "{}-{} [InnerSpaceCompactionTask] start to rename mods file", + storageGroupName, + dataRegionId); + CompactionUtils.combineModsInInnerCompaction( + selectedTsFileResourceList, targetTsFileResource); - // replace the old files with new file, the new is in same position as the old - if (sequence) { - tsFileManager.replace( - selectedTsFileResourceList, - Collections.emptyList(), - targetTsFileList, - timePartition, - true); - } else { - tsFileManager.replace( - Collections.emptyList(), - selectedTsFileResourceList, - targetTsFileList, - timePartition, - false); - } + if (Thread.currentThread().isInterrupted() || summary.isCancel()) { + throw new InterruptedException( + String.format("%s-%s [Compaction] abort", storageGroupName, dataRegionId)); + } - if (targetTsFileResource.isDeleted()) { - compactionLogger.logFile(targetTsFileResource, CompactionLogger.STR_DELETED_TARGET_FILES); - } + // replace the old files with new file, the new is in same position as the old + if (sequence) { + tsFileManager.replace( + selectedTsFileResourceList, + Collections.emptyList(), + targetTsFileList, + timePartition, + true); + } else { + tsFileManager.replace( + Collections.emptyList(), + selectedTsFileResourceList, + targetTsFileList, + timePartition, + false); + } - if (IoTDBDescriptor.getInstance().getConfig().isEnableCompactionValidation() - && !CompactionUtils.validateTsFileResources( - tsFileManager, storageGroupName, timePartition)) { - LOGGER.error( - "Failed to pass compaction validation, source files is: {}, target files is {}", - selectedTsFileResourceList, - targetTsFileList); - throw new RuntimeException("Failed to pass compaction validation"); - } + if (targetTsFileResource.isDeleted()) { + compactionLogger.logFile(targetTsFileResource, CompactionLogger.STR_DELETED_TARGET_FILES); + } - LOGGER.info( - "{}-{} [Compaction] Compacted target files, try to get the write lock of source files", - storageGroupName, - dataRegionId); + if (IoTDBDescriptor.getInstance().getConfig().isEnableCompactionValidation() + && !CompactionUtils.validateTsFileResources( + tsFileManager, storageGroupName, timePartition)) { + LOGGER.error( + "Failed to pass compaction validation, source files is: {}, target files is {}", + selectedTsFileResourceList, + targetTsFileList); + throw new RuntimeException("Failed to pass compaction validation"); + } - // release the read lock of all source files, and get the write lock of them to delete them - for (int i = 0; i < selectedTsFileResourceList.size(); ++i) { - selectedTsFileResourceList.get(i).readUnlock(); - isHoldingReadLock[i] = false; - selectedTsFileResourceList.get(i).writeLock(); - isHoldingWriteLock[i] = true; - } + LOGGER.info( + "{}-{} [Compaction] Compacted target files, try to get the write lock of source files", + storageGroupName, + dataRegionId); + + // release the read lock of all source files, and get the write lock of them to delete them + for (int i = 0; i < selectedTsFileResourceList.size(); ++i) { + selectedTsFileResourceList.get(i).readUnlock(); + isHoldingReadLock[i] = false; + selectedTsFileResourceList.get(i).writeLock(); + isHoldingWriteLock[i] = true; + } - if (targetTsFileResource.getTsFile().exists() - && targetTsFileResource.getTsFile().length() - < TSFileConfig.MAGIC_STRING.getBytes().length * 2L + Byte.BYTES) { - // the file size is smaller than magic string and version number - throw new TsFileNotCompleteException( - String.format( - "target file %s is smaller than magic string and version number size", - targetTsFileResource)); - } + if (targetTsFileResource.getTsFile().exists() + && targetTsFileResource.getTsFile().length() + < TSFileConfig.MAGIC_STRING.getBytes().length * 2L + Byte.BYTES) { + // the file size is smaller than magic string and version number + throw new TsFileNotCompleteException( + String.format( + "target file %s is smaller than magic string and version number size", + targetTsFileResource)); + } - LOGGER.info( - "{}-{} [Compaction] compaction finish, start to delete old files", - storageGroupName, - dataRegionId); - // delete the old files - long totalSizeOfDeletedFile = 0L; - for (TsFileResource resource : selectedTsFileResourceList) { - totalSizeOfDeletedFile += resource.getTsFileSize(); - } - CompactionUtils.deleteTsFilesInDisk( - selectedTsFileResourceList, storageGroupName + "-" + dataRegionId); - CompactionUtils.deleteModificationForSourceFile( - selectedTsFileResourceList, storageGroupName + "-" + dataRegionId); + LOGGER.info( + "{}-{} [Compaction] compaction finish, start to delete old files", + storageGroupName, + dataRegionId); + // delete the old files + long totalSizeOfDeletedFile = 0L; + for (TsFileResource resource : selectedTsFileResourceList) { + totalSizeOfDeletedFile += resource.getTsFileSize(); + } + CompactionUtils.deleteTsFilesInDisk( + selectedTsFileResourceList, storageGroupName + "-" + dataRegionId); + CompactionUtils.deleteModificationForSourceFile( + selectedTsFileResourceList, storageGroupName + "-" + dataRegionId); + + // inner space compaction task has only one target file + if (!targetTsFileResource.isDeleted()) { + TsFileMetricManager.getInstance() + .addFile(targetTsFileResource.getTsFile().length(), sequence); + + // set target resource to CLOSED, so that it can be selected to compact + targetTsFileResource.setStatus(TsFileResourceStatus.CLOSED); + } else { + // target resource is empty after compaction, then delete it + targetTsFileResource.remove(); + } + TsFileMetricManager.getInstance() + .deleteFile(totalSizeOfDeletedFile, sequence, selectedTsFileResourceList.size()); + + CompactionMetricsManager.getInstance().updateSummary(summary); + double costTime = (System.currentTimeMillis() - startTime) / 1000.0d; + LOGGER.info( + "{}-{} [Compaction] {} InnerSpaceCompaction task finishes successfully, target file is {}," + + "time cost is {} s, compaction speed is {} MB/s, {}", + storageGroupName, + dataRegionId, + sequence ? "Sequence" : "Unsequence", + targetTsFileResource.getTsFile().getName(), + costTime, + selectedFileSize / 1024.0d / 1024.0d / costTime, + summary); + } if (logFile.exists()) { FileUtils.delete(logFile); } - - // inner space compaction task has only one target file - if (!targetTsFileResource.isDeleted()) { - TsFileMetricManager.getInstance() - .addFile(targetTsFileResource.getTsFile().length(), sequence); - - // set target resource to CLOSED, so that it can be selected to compact - targetTsFileResource.setStatus(TsFileResourceStatus.CLOSED); - } else { - // target resource is empty after compaction, then delete it - targetTsFileResource.remove(); - } - TsFileMetricManager.getInstance() - .deleteFile(totalSizeOfDeletedFile, sequence, selectedTsFileResourceList.size()); - - CompactionMetricsManager.getInstance().updateSummary(summary); - - double costTime = (System.currentTimeMillis() - startTime) / 1000.0d; - LOGGER.info( - "{}-{} [Compaction] {} InnerSpaceCompaction task finishes successfully, target file is {}," - + "time cost is {} s, compaction speed is {} MB/s, {}", - storageGroupName, - dataRegionId, - sequence ? "Sequence" : "Unsequence", - targetTsFileResource.getTsFile().getName(), - costTime, - selectedFileSize / 1024.0d / 1024.0d / costTime, - summary); } catch (Throwable throwable) { // catch throwable to handle OOM errors if (!(throwable instanceof InterruptedException)) {
