wg1026688210 commented on code in PR #1268:
URL: https://github.com/apache/incubator-paimon/pull/1268#discussion_r1211274808
##########
paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMeta.java:
##########
@@ -137,53 +139,68 @@ public String toString() {
* <p>NOTE: This method is atomic.
*/
public static List<ManifestFileMeta> merge(
- List<ManifestFileMeta> metas,
+ List<ManifestFileMeta> input,
ManifestFile manifestFile,
long suggestedMetaSize,
int suggestedMinMetaCount,
long manifestFullCompactionSize,
RowType partitionType) {
- List<ManifestFileMeta> result = new ArrayList<>();
// these are the newly created manifest files, clean them up if
exception occurs
List<ManifestFileMeta> newMetas = new ArrayList<>();
- List<ManifestFileMeta> candidates = new ArrayList<>();
- long totalSize = 0;
try {
- List<ManifestFileMeta> manifestFileMetas =
+ Optional<List<ManifestFileMeta>> fullCompacted =
tryFullCompaction(
- metas,
+ input,
newMetas,
manifestFile,
- partitionType,
suggestedMetaSize,
- manifestFullCompactionSize);
- // merge existing manifests first
- for (ManifestFileMeta manifest : manifestFileMetas) {
- totalSize += manifest.fileSize;
- candidates.add(manifest);
- if (totalSize >= suggestedMetaSize) {
- // reach suggested file size, perform merging and produce
new file
- mergeCandidates(candidates, manifestFile, result,
newMetas);
- candidates.clear();
- totalSize = 0;
- }
- }
-
- // merge the last bit of manifests if there are too many
- if (candidates.size() >= suggestedMinMetaCount) {
- mergeCandidates(candidates, manifestFile, result, newMetas);
- } else {
- result.addAll(candidates);
- }
+ manifestFullCompactionSize,
+ partitionType);
+ return fullCompacted.orElseGet(
+ () ->
+ tryMinorCompaction(
+ input,
+ newMetas,
+ manifestFile,
+ suggestedMetaSize,
+ suggestedMinMetaCount));
} catch (Throwable e) {
// exception occurs, clean up and rethrow
for (ManifestFileMeta manifest : newMetas) {
manifestFile.delete(manifest.fileName);
}
throw e;
}
+ }
+ private static List<ManifestFileMeta> tryMinorCompaction(
+ List<ManifestFileMeta> input,
+ List<ManifestFileMeta> newMetas,
+ ManifestFile manifestFile,
+ long suggestedMetaSize,
+ int suggestedMinMetaCount) {
+ List<ManifestFileMeta> result = new ArrayList<>();
+ List<ManifestFileMeta> candidates = new ArrayList<>();
+ long totalSize = 0;
+ // merge existing small manifest files
+ for (ManifestFileMeta manifest : input) {
+ totalSize += manifest.fileSize;
+ candidates.add(manifest);
+ if (totalSize >= suggestedMetaSize) {
+ // reach suggested file size, perform merging and produce new
file
+ mergeCandidates(candidates, manifestFile, result, newMetas);
+ candidates.clear();
+ totalSize = 0;
+ }
+ }
+
+ // merge the last bit of manifests if there are too many
+ if (candidates.size() >= suggestedMinMetaCount) {
+ mergeCandidates(candidates, manifestFile, result, newMetas);
Review Comment:
Do we need to keep this logic after full compaction
##########
paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMeta.java:
##########
@@ -280,31 +299,65 @@ public static List<ManifestFileMeta> tryFullCompaction(
}
}
} else {
- // There is no DELETE Entry in Delta,Base don't need compaction
+ // There is no DELETE Entry in Delta, Base don't need
compaction
j = base.size();
result.addAll(base);
}
}
- Map<ManifestEntry.Identifier, ManifestEntry> fullMerged = new
LinkedHashMap<>();
+ // 2.2. try to skip base files by reading entries
+
Review Comment:
Do we need to add a condition if (partitionType.getFieldCount() ==0)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
