github-actions[bot] commented on code in PR #63894:
URL: https://github.com/apache/doris/pull/63894#discussion_r3322812383
##########
fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java:
##########
@@ -5451,82 +5452,44 @@ public TGetOlapTableMetaResult
getOlapTableMeta(TGetOlapTableMetaRequest request
MetaContext metaContext = new MetaContext();
metaContext.setMetaVersion(FeConstants.meta_version);
metaContext.setThreadLocalInfo();
- table.readLock();
try (ByteArrayOutputStream bOutputStream = new
ByteArrayOutputStream(8192)) {
- OlapTable copyTable = table.copyTableMeta();
- try (DataOutputStream out = new
DataOutputStream(bOutputStream)) {
- copyTable.write(out);
- out.flush();
- result.setTableMeta(bOutputStream.toByteArray());
- }
- Set<Long> updatedPartitionIds =
Sets.newHashSet(table.getPartitionIds());
- List<TPartitionMeta> partitionMetas =
request.getPartitionsSize() == 0 ? Lists.newArrayList()
- : request.getPartitions();
- for (TPartitionMeta partitionMeta : partitionMetas) {
- if (request.getTableId() != table.getId()) {
- result.addToRemovedPartitions(partitionMeta.getId());
- continue;
- }
- Partition partition =
table.getPartition(partitionMeta.getId());
- if (partition == null) {
- result.addToRemovedPartitions(partitionMeta.getId());
- continue;
- }
- if (partition.getVisibleVersion() ==
partitionMeta.getVisibleVersion()
- && partition.getVisibleVersionTime() ==
partitionMeta.getVisibleVersionTime()) {
- updatedPartitionIds.remove(partitionMeta.getId());
- }
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("receive getOlapTableMeta db: {} table:{}
update partitions: {} removed partition:{}",
- request.getDb(), request.getTable(),
updatedPartitionIds.size(),
- result.getRemovedPartitionsSize());
- }
- for (Long partitionId : updatedPartitionIds) {
- bOutputStream.reset();
- Partition partition = table.getPartition(partitionId);
+ Set<Long> updatedPartitionIds;
+ Set<Long> updatedTempPartitionIds;
+ List<TPartitionMeta> partitionChecksumCandidates =
Lists.newArrayList();
+ List<TPartitionMeta> tempPartitionChecksumCandidates =
Lists.newArrayList();
+ table.readLock();
+ try {
+ OlapTable copyTable = table.copyTableMeta();
try (DataOutputStream out = new
DataOutputStream(bOutputStream)) {
- Text.writeString(out,
GsonUtils.GSON.toJson(partition));
+ copyTable.write(out);
out.flush();
-
result.addToUpdatedPartitions(ByteBuffer.wrap(bOutputStream.toByteArray()));
- }
- }
- // temp partitions
- updatedPartitionIds =
Sets.newHashSet(table.getTempPartitions().getPartitionIds());
- partitionMetas = request.getTempPartitionsSize() == 0 ?
Lists.newArrayList()
- : request.getTempPartitions();
- for (TPartitionMeta partitionMeta : partitionMetas) {
- if (request.getTableId() != table.getId()) {
-
result.addToRemovedTempPartitions(partitionMeta.getId());
- continue;
- }
- Partition tempPartition =
table.getTempPartitions().getPartition(partitionMeta.getId());
- if (tempPartition == null) {
-
result.addToRemovedTempPartitions(partitionMeta.getId());
- continue;
- }
- if (tempPartition.getVisibleVersion() ==
partitionMeta.getVisibleVersion()
- && tempPartition.getVisibleVersionTime() ==
partitionMeta.getVisibleVersionTime()) {
- updatedPartitionIds.remove(partitionMeta.getId());
+ result.setTableMeta(bOutputStream.toByteArray());
}
+ updatedPartitionIds =
Sets.newHashSet(table.getPartitionIds());
+ collectPartitionMetaChanges(table, request.getTableId(),
request.getPartitions(), false,
+ updatedPartitionIds, partitionChecksumCandidates,
result);
+ updatedTempPartitionIds =
Sets.newHashSet(table.getTempPartitions().getPartitionIds());
+ collectPartitionMetaChanges(table, request.getTableId(),
request.getTempPartitions(), true,
+ updatedTempPartitionIds,
tempPartitionChecksumCandidates, result);
+ } finally {
+ table.readUnlock();
}
+
+ filterMatchedPartitionsByChecksum(table,
partitionChecksumCandidates, false, updatedPartitionIds,
+ result);
Review Comment:
This now releases `table.readLock()` after serializing `table_meta` and
before deciding/sending the partition payloads. That can return a mixed
snapshot if DDL runs between these phases. For example, the first locked block
serializes `partitionInfo` with partition `p`, then a concurrent `DROP
PARTITION p` commits, and this later checksum/filter phase observes `p == null`
and adds it to `removed_partitions`. The client builds `RemoteOlapTable` from
the old `table_meta`, then `rebuildPartitions()` only removes `p` from
`idToPartition`/`nameToPartition`; it does not remove the stale entry from
`partitionInfo`, so pruning/planning can see table-level partition metadata
that no longer has a corresponding `Partition`. The old implementation avoided
this by holding one read lock for table meta and all partition lists. Please
keep the whole response based on a single locked snapshot, or take an immutable
snapshot of both table-level metadata and partition payload/removal decisions
before
releasing the lock.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
