JingsongLi commented on code in PR #2076:
URL: https://github.com/apache/incubator-paimon/pull/2076#discussion_r1349469790
##########
paimon-core/src/main/java/org/apache/paimon/index/PartitionIndex.java:
##########
@@ -66,41 +72,45 @@ public int assign(int hash, IntPredicate bucketFilter) {
}
// 2. find bucket from existing buckets
- for (Integer bucket : bucketInformation.keySet()) {
- if (bucketFilter.test(bucket)) {
- // it is my bucket
- Long number = bucketInformation.get(bucket);
- if (number < targetBucketRowNumber) {
- bucketInformation.put(bucket, number + 1);
- hash2Bucket.put(hash, bucket.shortValue());
- return bucket;
- }
+ Iterator<Map.Entry<Integer, Long>> iterator =
+ nonFullBucketInformation.entrySet().iterator();
+ while (iterator.hasNext()) {
+ Map.Entry<Integer, Long> entry = iterator.next();
+ Integer bucket = entry.getKey();
+ Long number = entry.getValue();
+ if (number < targetBucketRowNumber) {
+ entry.setValue(number + 1);
+ hash2Bucket.put(hash, bucket.shortValue());
+ return bucket;
+ } else {
+ iterator.remove();
}
}
// 3. create a new bucket
for (int i = 0; i < Short.MAX_VALUE; i++) {
- if (bucketFilter.test(i) && !bucketInformation.containsKey(i)) {
+ if (bucketFilter.test(i) && !totalBucket.contains(i)) {
hash2Bucket.put(hash, (short) i);
- bucketInformation.put(i, 1L);
+ nonFullBucketInformation.put(i, 1L);
+ totalBucket.add(i);
return i;
}
}
@SuppressWarnings("OptionalGetWithoutIsPresent")
- int maxBucket =
-
bucketInformation.keySet().stream().mapToInt(Integer::intValue).max().getAsInt();
+ int maxBucket =
totalBucket.stream().mapToInt(Integer::intValue).max().getAsInt();
throw new RuntimeException(
String.format(
- "To more bucket %s, you should increase target bucket
row number %s.",
+ "Too more bucket %s, you should increase target bucket
row number %s.",
maxBucket, targetBucketRowNumber));
}
public static PartitionIndex loadIndex(
IndexFileHandler indexFileHandler,
BinaryRow partition,
long targetBucketRowNumber,
- IntPredicate loadFilter) {
+ IntPredicate loadFilter,
+ IntPredicate bucketFilter) {
Review Comment:
Have you consider rescale case?
##########
paimon-core/src/main/java/org/apache/paimon/index/PartitionIndex.java:
##########
@@ -66,41 +72,45 @@ public int assign(int hash, IntPredicate bucketFilter) {
}
// 2. find bucket from existing buckets
- for (Integer bucket : bucketInformation.keySet()) {
- if (bucketFilter.test(bucket)) {
- // it is my bucket
- Long number = bucketInformation.get(bucket);
- if (number < targetBucketRowNumber) {
- bucketInformation.put(bucket, number + 1);
- hash2Bucket.put(hash, bucket.shortValue());
- return bucket;
- }
+ Iterator<Map.Entry<Integer, Long>> iterator =
+ nonFullBucketInformation.entrySet().iterator();
+ while (iterator.hasNext()) {
+ Map.Entry<Integer, Long> entry = iterator.next();
+ Integer bucket = entry.getKey();
+ Long number = entry.getValue();
+ if (number < targetBucketRowNumber) {
+ entry.setValue(number + 1);
+ hash2Bucket.put(hash, bucket.shortValue());
+ return bucket;
+ } else {
+ iterator.remove();
}
}
// 3. create a new bucket
for (int i = 0; i < Short.MAX_VALUE; i++) {
- if (bucketFilter.test(i) && !bucketInformation.containsKey(i)) {
+ if (bucketFilter.test(i) && !totalBucket.contains(i)) {
hash2Bucket.put(hash, (short) i);
- bucketInformation.put(i, 1L);
+ nonFullBucketInformation.put(i, 1L);
+ totalBucket.add(i);
return i;
}
}
@SuppressWarnings("OptionalGetWithoutIsPresent")
- int maxBucket =
-
bucketInformation.keySet().stream().mapToInt(Integer::intValue).max().getAsInt();
+ int maxBucket =
totalBucket.stream().mapToInt(Integer::intValue).max().getAsInt();
throw new RuntimeException(
String.format(
- "To more bucket %s, you should increase target bucket
row number %s.",
+ "Too more bucket %s, you should increase target bucket
row number %s.",
maxBucket, targetBucketRowNumber));
}
public static PartitionIndex loadIndex(
IndexFileHandler indexFileHandler,
BinaryRow partition,
long targetBucketRowNumber,
- IntPredicate loadFilter) {
+ IntPredicate loadFilter,
+ IntPredicate bucketFilter) {
Review Comment:
Have you considerred rescale case?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
