JingsongLi commented on code in PR #1268:
URL: https://github.com/apache/incubator-paimon/pull/1268#discussion_r1211311517
##########
paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMeta.java:
##########
@@ -137,53 +139,68 @@ public String toString() {
* <p>NOTE: This method is atomic.
*/
public static List<ManifestFileMeta> merge(
- List<ManifestFileMeta> metas,
+ List<ManifestFileMeta> input,
ManifestFile manifestFile,
long suggestedMetaSize,
int suggestedMinMetaCount,
long manifestFullCompactionSize,
RowType partitionType) {
- List<ManifestFileMeta> result = new ArrayList<>();
// these are the newly created manifest files, clean them up if
exception occurs
List<ManifestFileMeta> newMetas = new ArrayList<>();
- List<ManifestFileMeta> candidates = new ArrayList<>();
- long totalSize = 0;
try {
- List<ManifestFileMeta> manifestFileMetas =
+ Optional<List<ManifestFileMeta>> fullCompacted =
tryFullCompaction(
- metas,
+ input,
newMetas,
manifestFile,
- partitionType,
suggestedMetaSize,
- manifestFullCompactionSize);
- // merge existing manifests first
- for (ManifestFileMeta manifest : manifestFileMetas) {
- totalSize += manifest.fileSize;
- candidates.add(manifest);
- if (totalSize >= suggestedMetaSize) {
- // reach suggested file size, perform merging and produce
new file
- mergeCandidates(candidates, manifestFile, result,
newMetas);
- candidates.clear();
- totalSize = 0;
- }
- }
-
- // merge the last bit of manifests if there are too many
- if (candidates.size() >= suggestedMinMetaCount) {
- mergeCandidates(candidates, manifestFile, result, newMetas);
- } else {
- result.addAll(candidates);
- }
+ manifestFullCompactionSize,
+ partitionType);
+ return fullCompacted.orElseGet(
+ () ->
+ tryMinorCompaction(
+ input,
+ newMetas,
+ manifestFile,
+ suggestedMetaSize,
+ suggestedMinMetaCount));
} catch (Throwable e) {
// exception occurs, clean up and rethrow
for (ManifestFileMeta manifest : newMetas) {
manifestFile.delete(manifest.fileName);
}
throw e;
}
+ }
+ private static List<ManifestFileMeta> tryMinorCompaction(
+ List<ManifestFileMeta> input,
+ List<ManifestFileMeta> newMetas,
+ ManifestFile manifestFile,
+ long suggestedMetaSize,
+ int suggestedMinMetaCount) {
+ List<ManifestFileMeta> result = new ArrayList<>();
+ List<ManifestFileMeta> candidates = new ArrayList<>();
+ long totalSize = 0;
+ // merge existing small manifest files
+ for (ManifestFileMeta manifest : input) {
+ totalSize += manifest.fileSize;
+ candidates.add(manifest);
+ if (totalSize >= suggestedMetaSize) {
+ // reach suggested file size, perform merging and produce new
file
+ mergeCandidates(candidates, manifestFile, result, newMetas);
+ candidates.clear();
+ totalSize = 0;
+ }
+ }
+
+ // merge the last bit of manifests if there are too many
+ if (candidates.size() >= suggestedMinMetaCount) {
+ mergeCandidates(candidates, manifestFile, result, newMetas);
Review Comment:
If we've done a full compaction, there shouldn't be any small files that
need to be merged
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
