This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch compaction_review
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/compaction_review by this push:
new acf66b67751 merge selection logic
acf66b67751 is described below
commit acf66b67751681b1d9763ba1051d1214daee6164
Author: jt2594838 <[email protected]>
AuthorDate: Fri Jun 14 19:26:06 2024 +0800
merge selection logic
---
.../impl/RewriteCrossSpaceCompactionSelector.java | 32 +------------------
.../utils/CrossSpaceCompactionCandidate.java | 36 ++++++++++++++++++++++
2 files changed, 37 insertions(+), 31 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
index 1904723ce73..aecb7dbd024 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
@@ -193,19 +193,6 @@ public class RewriteCrossSpaceCompactionSelector
implements ICrossSpaceSelector
List<TsFileResource> targetSeqFiles =
split.seqFiles.stream().map(c ->
c.resource).collect(Collectors.toList());
- if (!split.atLeastOneSeqFileSelected) {
- LOGGER.debug("Unseq file {} does not overlap with any seq files.",
unseqFile);
- TsFileResourceCandidate latestSealedSeqFile =
- getLatestSealedSeqFile(candidate.getSeqFileCandidates());
- if (latestSealedSeqFile == null) {
- break;
- }
- if (!latestSealedSeqFile.selected) {
- targetSeqFiles.add(latestSealedSeqFile.resource);
- latestSealedSeqFile.markAsSelected();
- }
- }
-
List<TsFileResource> newSelectedSeqResources = new
ArrayList<>(taskResource.getSeqFiles());
newSelectedSeqResources.addAll(targetSeqFiles);
List<TsFileResource> newSelectedUnseqResources =
@@ -230,24 +217,7 @@ public class RewriteCrossSpaceCompactionSelector
implements ICrossSpaceSelector
return taskResource;
}
- private TsFileResourceCandidate getLatestSealedSeqFile(
- List<TsFileResourceCandidate> seqResourceCandidateList) {
- for (int i = seqResourceCandidateList.size() - 1; i >= 0; i--) {
- TsFileResourceCandidate seqResourceCandidate =
seqResourceCandidateList.get(i);
- if (seqResourceCandidate.resource.isClosed()) {
- // We must select the latest sealed and valid seq file to compact
with, in order to avoid
- // overlapping of the new compacted files with the subsequent seq
files.
- if (seqResourceCandidate.isValidCandidate) {
- LOGGER.debug(
- "Select one valid seq file {} for nonOverlap unseq file to
compact with.",
- seqResourceCandidate.resource);
- return seqResourceCandidate;
- }
- break;
- }
- }
- return null;
- }
+
// TODO: (xingtanzjr) need to confirm whether we should strictly guarantee
the conditions
// If we guarantee the condition strictly, the smallest collection of cross
task resource may not
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/utils/CrossSpaceCompactionCandidate.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/utils/CrossSpaceCompactionCandidate.java
index 8cf29534bba..7935ea4ddf8 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/utils/CrossSpaceCompactionCandidate.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/utils/CrossSpaceCompactionCandidate.java
@@ -28,12 +28,17 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* CrossSpaceCompactionResource manages files and caches of readers to avoid
unnecessary object
* creations and file openings.
*/
public class CrossSpaceCompactionCandidate {
+
+ private static final Logger LOGGER =
LoggerFactory.getLogger(CrossSpaceCompactionCandidate.class);
+
private List<TsFileResourceCandidate> seqFiles;
private List<TsFileResourceCandidate> unseqFiles;
@@ -163,9 +168,40 @@ public class CrossSpaceCompactionCandidate {
unseqFile.markAsSelected();
nextSplit = tmpSplit;
nextUnseqFileIndex++;
+
+ if (!tmpSplit.atLeastOneSeqFileSelected) {
+ // the file only contains exclusive devices (devices that do not exist
in other files)
+ // TODO: move this part inside candidate.hasNextSplit()
+ LOGGER.debug("Unseq file {} does not overlap with any seq files.",
unseqFile);
+ TsFileResourceCandidate latestSealedSeqFile =
+ getLatestSealedSeqFile(getSeqFileCandidates());
+ if (latestSealedSeqFile == null) {
+ return false;
+ }
+ tmpSplit.addSeqFileIfNotSelected(latestSealedSeqFile);
+ }
return true;
}
+ private TsFileResourceCandidate getLatestSealedSeqFile(
+ List<TsFileResourceCandidate> seqResourceCandidateList) {
+ for (int i = seqResourceCandidateList.size() - 1; i >= 0; i--) {
+ TsFileResourceCandidate seqResourceCandidate =
seqResourceCandidateList.get(i);
+ if (seqResourceCandidate.resource.isClosed()) {
+ // We must select the latest sealed and valid seq file to compact
with, in order to avoid
+ // overlapping of the new compacted files with the subsequent seq
files.
+ if (seqResourceCandidate.isValidCandidate) {
+ LOGGER.debug(
+ "Select one valid seq file {} for nonOverlap unseq file to
compact with.",
+ seqResourceCandidate.resource);
+ return seqResourceCandidate;
+ }
+ break;
+ }
+ }
+ return null;
+ }
+
private List<TsFileResourceCandidate> copySeqResource(List<TsFileResource>
seqFiles) {
List<TsFileResourceCandidate> ret = new ArrayList<>();
for (TsFileResource resource : seqFiles) {
