This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new d72335967c4 [IOTDB-5933] Pipe: update progress index after compaction
(#9970)
d72335967c4 is described below
commit d72335967c497b45df052b9d5eea679ee345dadb
Author: 周沛辰 <[email protected]>
AuthorDate: Mon May 29 20:54:51 2023 +0800
[IOTDB-5933] Pipe: update progress index after compaction (#9970)
---
.../execute/performer/impl/FastCompactionPerformer.java | 7 +------
.../execute/task/CrossSpaceCompactionTask.java | 2 ++
.../execute/task/InnerSpaceCompactionTask.java | 2 ++
.../compaction/execute/utils/CompactionUtils.java | 17 +++++++++++++++--
...sSpaceCompactionWithFastPerformerValidationTest.java | 4 +---
.../compaction/utils/MultiTsFileDeviceIteratorTest.java | 8 ++------
6 files changed, 23 insertions(+), 17 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/performer/impl/FastCompactionPerformer.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/performer/impl/FastCompactionPerformer.java
index 3319594effa..37f7b2f0c1e 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/performer/impl/FastCompactionPerformer.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/performer/impl/FastCompactionPerformer.java
@@ -20,7 +20,6 @@ package
org.apache.iotdb.db.engine.compaction.execute.performer.impl;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.exception.IllegalPathException;
-import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import
org.apache.iotdb.db.engine.compaction.execute.performer.ICrossCompactionPerformer;
import
org.apache.iotdb.db.engine.compaction.execute.performer.ISeqCompactionPerformer;
@@ -36,7 +35,6 @@ import
org.apache.iotdb.db.engine.compaction.execute.utils.writer.FastInnerCompa
import org.apache.iotdb.db.engine.compaction.schedule.CompactionTaskManager;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
-import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.exception.WriteProcessException;
import org.apache.iotdb.tsfile.exception.write.PageException;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
@@ -100,8 +98,7 @@ public class FastCompactionPerformer
}
@Override
- public void perform()
- throws IOException, MetadataException, StorageEngineException,
InterruptedException {
+ public void perform() throws Exception {
this.subTaskSummary.setTemporalFileNum(targetFiles.size());
try (MultiTsFileDeviceIterator deviceIterator =
new MultiTsFileDeviceIterator(seqFiles, unseqFiles,
readerCacheMap);
@@ -140,8 +137,6 @@ public class FastCompactionPerformer
}
compactionWriter.endFile();
CompactionUtils.updatePlanIndexes(targetFiles, seqFiles, unseqFiles);
- } catch (Exception e) {
- throw new RuntimeException(e);
} finally {
// readers of source files have been closed in MultiTsFileDeviceIterator
// clean cache
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java
index 506ddbc4997..b6c18d3a487 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java
@@ -155,6 +155,8 @@ public class CrossSpaceCompactionTask extends
AbstractCompactionTask {
performer.setSummary(summary);
performer.perform();
+ CompactionUtils.updateProgressIndex(
+ targetTsfileResourceList, selectedSequenceFiles,
selectedUnsequenceFiles);
CompactionUtils.moveTargetFile(
targetTsfileResourceList, false, storageGroupName + "-" +
dataRegionId);
CompactionUtils.combineModsInCrossCompaction(
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java
index b212be69f8c..8de1947e210 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java
@@ -150,6 +150,8 @@ public class InnerSpaceCompactionTask extends
AbstractCompactionTask {
performer.setSummary(summary);
performer.perform();
+ CompactionUtils.updateProgressIndex(
+ targetTsFileList, selectedTsFileResourceList,
Collections.emptyList());
CompactionUtils.moveTargetFile(
targetTsFileList, true, storageGroupName + "-" + dataRegionId);
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/CompactionUtils.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/CompactionUtils.java
index 5e873bea69f..a891e332e6b 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/CompactionUtils.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/CompactionUtils.java
@@ -234,6 +234,20 @@ public class CompactionUtils {
}
}
+ public static void updateProgressIndex(
+ List<TsFileResource> targetResources,
+ List<TsFileResource> seqResources,
+ List<TsFileResource> unseqResources) {
+ for (TsFileResource targetResource : targetResources) {
+ for (TsFileResource unseqResource : unseqResources) {
+
targetResource.updateProgressIndex(unseqResource.getMaxProgressIndexAfterClose());
+ }
+ for (TsFileResource seqResource : seqResources) {
+
targetResource.updateProgressIndex(seqResource.getMaxProgressIndexAfterClose());
+ }
+ }
+ }
+
public static void updatePlanIndexes(
List<TsFileResource> targetResources,
List<TsFileResource> seqResources,
@@ -244,8 +258,7 @@ public class CompactionUtils {
// however, since the data of unseq files are mixed together, we won't be
able to know
// which files are exactly contained in the new file, so we have to record
all unseq files
// in the new file
- for (int i = 0; i < targetResources.size(); i++) {
- TsFileResource targetResource = targetResources.get(i);
+ for (TsFileResource targetResource : targetResources) {
for (TsFileResource unseqResource : unseqResources) {
targetResource.updatePlanIndexes(unseqResource);
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
index b2b1279cee3..e8fab937603 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
@@ -2124,9 +2124,7 @@ public class
CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
* task.
*/
@Test
- public void testSelectingFilesWhenSomeFilesBeingDeleted()
- throws MetadataException, IOException, WriteProcessException,
StorageEngineException,
- InterruptedException, MergeException {
+ public void testSelectingFilesWhenSomeFilesBeingDeleted() throws Exception {
registerTimeseriesInMManger(5, 10, true);
createFiles(5, 10, 5, 1000, 0, 0, 100, 100, false, true);
createFiles(1, 5, 10, 4500, 500, 500, 0, 100, false, false);
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/MultiTsFileDeviceIteratorTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/MultiTsFileDeviceIteratorTest.java
index 440ec5a3296..7a38dea7999 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/MultiTsFileDeviceIteratorTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/MultiTsFileDeviceIteratorTest.java
@@ -917,9 +917,7 @@ public class MultiTsFileDeviceIteratorTest extends
AbstractCompactionTest {
* nonAligned property. Check whether the deviceID and its property can be
obtained correctly.
*/
@Test
- public void getDeletedDevicesWithSameNameFromSeqFilesByFastPerformer()
- throws MetadataException, IOException, WriteProcessException,
StorageEngineException,
- InterruptedException {
+ public void getDeletedDevicesWithSameNameFromSeqFilesByFastPerformer()
throws Exception {
TSFileDescriptor.getInstance().getConfig().setMaxDegreeOfIndexNode(3);
int oldAlignedDeviceOffset = TsFileGeneratorUtils.alignDeviceOffset;
TsFileGeneratorUtils.alignDeviceOffset = 0;
@@ -1064,9 +1062,7 @@ public class MultiTsFileDeviceIteratorTest extends
AbstractCompactionTest {
* aligned property. Check whether the deviceID and its property can be
obtained correctly.
*/
@Test
- public void getDeletedDevicesWithSameNameFromSeqFilesByFastPerformer2()
- throws MetadataException, IOException, WriteProcessException,
StorageEngineException,
- InterruptedException {
+ public void getDeletedDevicesWithSameNameFromSeqFilesByFastPerformer2()
throws Exception {
TSFileDescriptor.getInstance().getConfig().setMaxDegreeOfIndexNode(3);
int oldAlignedDeviceOffset = TsFileGeneratorUtils.alignDeviceOffset;
TsFileGeneratorUtils.alignDeviceOffset = 0;
