gavinchou commented on code in PR #53250:
URL: https://github.com/apache/doris/pull/53250#discussion_r2206229941
##########
fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java:
##########
@@ -191,65 +192,98 @@ private static long getNextPartitionSize(ArrayList<Long>
historyPartitionsSize)
}
}
- private static int getBucketsNum(DynamicPartitionProperty property,
OlapTable table,
+ private static Pair<Integer, Integer>
getBucketsNum(DynamicPartitionProperty property, OlapTable table,
String partitionName, String nowPartitionName, boolean
executeFirstTime) {
// if execute first time, all partitions no contain data
if (!table.isAutoBucket() || executeFirstTime) {
- return property.getBuckets();
+ return Pair.of(property.getBuckets(), 0);
}
- // auto bucket
- // get all history partitions
- RangePartitionInfo info = (RangePartitionInfo)
(table.getPartitionInfo());
- List<Map.Entry<Long, PartitionItem>> idToItems = new
ArrayList<>(info.getIdToItem(false).entrySet());
- idToItems.sort(Comparator.comparing(o -> ((RangePartitionItem)
o.getValue()).getItems().upperEndpoint()));
- List<Partition> partitions = idToItems.stream()
- .map(entry -> table.getPartition(entry.getKey()))
- .filter(partition -> partition != null &&
!partition.getName().equals(nowPartitionName))
- .collect(Collectors.toList());
- List<Long> visibleVersions = null;
+ List<Partition> partitions = getHistoricalPartitions(table,
nowPartitionName);
+ List<Long> visibleVersions;
try {
visibleVersions = Partition.getVisibleVersions(partitions);
} catch (RpcException e) {
- LOG.warn("autobucket use property's buckets get visible version
fail, table: [{}-{}], "
+ LOG.warn("auto bucket use property's buckets get visible version
fail, table: [{}-{}], "
+ "partition: {}, buckets num: {}, exception: ",
table.getName(), table.getId(), partitionName,
property.getBuckets(), e);
- return property.getBuckets();
+ return Pair.of(property.getBuckets(), 0);
+ }
+
+ List<Partition> hasDataPartitions = filterDataPartitions(partitions,
visibleVersions);
+ if (hasDataPartitions.isEmpty()) {
+ return handleNoDataPartitions(table, partitionName,
property.getBuckets());
}
- List<Partition> hasDataPartitions = Lists.newArrayList();
+ return calculateBuckets(hasDataPartitions);
+ }
+
+ private static List<Partition> getHistoricalPartitions(OlapTable table,
String nowPartitionName) {
+ RangePartitionInfo info = (RangePartitionInfo)
(table.getPartitionInfo());
+ List<Map.Entry<Long, PartitionItem>> idToItems = new
ArrayList<>(info.getIdToItem(false).entrySet());
+ idToItems.sort(Comparator.comparing(o -> ((RangePartitionItem)
o.getValue()).getItems().upperEndpoint()));
+ return idToItems.stream()
+ .map(entry -> table.getPartition(entry.getKey()))
+ .filter(partition -> partition != null &&
!partition.getName().equals(nowPartitionName))
+ .collect(Collectors.toList());
+ }
+
+ private static List<Partition> filterDataPartitions(List<Partition>
partitions, List<Long> visibleVersions) {
+ Preconditions.checkState(partitions.size() == visibleVersions.size(),
+ String.format("partitions size %d not eq visibleVersions size
%d, impossible",
+ partitions.size(), visibleVersions.size()));
+ List<Partition> hasDataPartitions = new ArrayList<>();
for (int i = 0; i < partitions.size(); i++) {
if (visibleVersions.get(i) >= 2) {
hasDataPartitions.add(partitions.get(i));
}
}
+ return hasDataPartitions;
+ }
- // no exist history partition data
- if (hasDataPartitions.isEmpty()) {
- LOG.info("autobucket use property's buckets due to all partitions
no data, table: [{}-{}], "
- + "partition: {}, buckets num: {}",
- table.getName(), table.getId(), partitionName,
property.getBuckets());
- return property.getBuckets();
+ private static Pair<Integer, Integer> handleNoDataPartitions(OlapTable
table,
+ String
partitionName, int defaultBuckets) {
+ LOG.info("auto bucket use property's buckets due to all partitions no
data, table: [{}-{}], "
+ + "partition: {}, buckets num: {}", table.getName(),
table.getId(), partitionName, defaultBuckets);
+ return Pair.of(defaultBuckets, 0);
+ }
+
+ private static Pair<Integer, Integer> calculateBuckets(List<Partition>
hasDataPartitions) {
+ List<Long> partitionSizeArray = new ArrayList<>();
+ List<Long> sizeUnknownArray = new ArrayList<>();
+
+ for (Partition hasDataPartition : hasDataPartitions) {
+ long partitionSize =
hasDataPartition.getDataSizeExcludeEmptyReplica(true);
+ if (partitionSize == 0) {
Review Comment:
<= 0
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
