github-actions[bot] commented on code in PR #67725:
URL: https://github.com/apache/doris/pull/67725#discussion_r4226435242
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/PluginDrivenMvccExternalTable.java:
##########
@@ -714,11 +692,142 @@ static boolean schemaCacheDisabled(Connector connector) {
@Override
public Map<String, PartitionItem>
getNameToPartitionItems(Optional<MvccSnapshot> snapshot) {
+ if (supportsConnectorPartitionPruning()) {
+ PluginDrivenMvccSnapshot pin = getOrMaterialize(snapshot);
+ if (pin.isPartitionViewMaterialized()) {
+ return pin.getNameToPartitionItem();
+ }
+ if (pin.isPartitionViewUnavailable()) {
+ return Collections.emptyMap();
+ }
+ Optional<ExternalTablePreloadInfo> sharedView =
sharedLatestPreloadInfo(snapshot);
+ if (sharedView.isPresent() &&
sharedView.get().hasScanPartitionView()
+ && sharedView.get().getScanPartitionView().isPresent()) {
+ return sharedView.get().getScanPartitionView().get();
+ }
+ // The latest Hive query pin intentionally carries no partition
map so selective scans can send a
+ // predicate to HMS first. Consumers that explicitly ask for a
partition map (MTMV alignment,
+ // no-filter scan finalization, and a connector-declined pruning
fallback) require the real full
+ // view instead of treating that query-only pin as an empty table.
An unavailable shared scan view
+ // is not an authoritative empty map either: checked MTMV
consumers reject it in
+ // getAndCopyPartitionItems, while other explicit map consumers
retain this fail-loud full-view path.
+ return super.getNameToPartitionItems(snapshot);
+ }
return getOrMaterialize(snapshot).getNameToPartitionItem();
}
@Override
- public Map<String, PartitionItem>
getAndCopyPartitionItems(Optional<MvccSnapshot> snapshot) {
+ public Optional<Map<String, PartitionItem>>
getNameToPartitionItemsForScan(Optional<MvccSnapshot> snapshot) {
+ if (snapshot.isPresent() && snapshot.get() instanceof
PluginDrivenMvccSnapshot
+ && ((PluginDrivenMvccSnapshot)
snapshot.get()).isPartitionViewUnavailable()) {
+ return Optional.empty();
+ }
+ if (supportsConnectorPartitionPruning()) {
+ PluginDrivenMvccSnapshot pin = getOrMaterialize(snapshot);
+ if (pin.isPartitionViewMaterialized()) {
+ // This statement's pin already carries a materialized view:
reuse it instead of paying another
+ // connector round-trip that could observe a different remote
generation.
+ return Optional.of(pin.getNameToPartitionItem());
+ }
+ }
+ return super.getNameToPartitionItemsForScan(snapshot);
+ }
+
+ @Override
+ public Optional<ConnectorFilteredPartitionView>
applyPartitionFilterForScan(Optional<MvccSnapshot> snapshot,
+ ConnectorExpression partitionFilter) {
+ if (snapshot.isPresent() && snapshot.get() instanceof
PluginDrivenMvccSnapshot
+ && ((PluginDrivenMvccSnapshot)
snapshot.get()).isPartitionViewUnavailable()) {
+ return Optional.empty();
+ }
+ return super.applyPartitionFilterForScan(snapshot, partitionFilter);
+ }
+
+ /**
+ * Threads this statement's MVCC pin onto the handle the partition view is
enumerated from, so a
+ * time-travel / {@code @options} query never prunes against the latest
generation: the data scan reads the
+ * pinned snapshot, and a latest view can be missing (or contain)
partitions it will never read.
+ */
+ @Override
+ protected ConnectorTableHandle pinPartitionViewHandle(ConnectorTableHandle
handle,
+ ConnectorMetadata metadata, ConnectorSession session,
Optional<MvccSnapshot> snapshot) {
+ if (snapshot.isPresent() && snapshot.get() instanceof
PluginDrivenMvccSnapshot) {
+ return metadata.applySnapshot(session, handle,
+ ((PluginDrivenMvccSnapshot)
snapshot.get()).getConnectorSnapshot());
+ }
+ return handle;
+ }
+
+ /**
+ * Materializes the complete partition view for an MTMV refresh before it
acquires base-table locks.
+ * The lightweight Hive query pin keeps the connector snapshot/freshness
kind but deliberately omits the
+ * partition map; MTMV alignment needs that map and must not load it while
holding internal table locks.
+ */
+ public MvccSnapshot materializePartitionViewForMtmv(MvccSnapshot snapshot)
{
+ if (!supportsConnectorPartitionPruning() || !(snapshot instanceof
PluginDrivenMvccSnapshot)) {
+ return snapshot;
+ }
+ PluginDrivenMvccSnapshot pin = (PluginDrivenMvccSnapshot) snapshot;
+ if (!pin.isPartitionViewDeferred()) {
+ return pin;
+ }
+ Map<String, PartitionItem> partitionItems =
super.getNameToPartitionItems(Optional.of(pin));
+ Map<String, Long> partitionLastModified = new HashMap<>();
+ for (String partitionName : partitionItems.keySet()) {
+ partitionLastModified.put(partitionName,
ConnectorPartitionInfo.UNKNOWN);
+ }
+ return new PluginDrivenMvccSnapshot(pin.getConnectorSnapshot(),
partitionItems, partitionLastModified,
+ null);
+ }
+
+ /**
+ * Returns the latest-view preload record materialized by the statement
before internal table locks were
+ * acquired, or empty when this request does not refer to that latest view.
+ *
+ * <p>The record, rather than only its map, preserves the intentionally
unwarmed filtered-latest state, an
+ * unavailable scan view, and a materialized map. Query-time async-MV
validation and union compensation ask
+ * for partition metadata while the planner holds internal read locks.
Reusing the materialized pre-lock
+ * generation avoids a full HMS listing under those locks; preserving the
other two states lets the checked
+ * MTMV path reject the candidate instead of listing partitions under the
locks or converting "scan every
+ * partition" into an authoritative empty universe. A
+ * supplied pin is accepted only when it is the statement's own latest
pin; historical/time-travel pins and
+ * calls without a statement context keep their existing snapshot
semantics.</p>
+ */
+ private Optional<ExternalTablePreloadInfo>
sharedLatestPreloadInfo(Optional<MvccSnapshot> snapshot) {
+ ConnectContext connectContext = ConnectContext.get();
+ StatementContext statementContext = connectContext == null ? null :
connectContext.getStatementContext();
+ if (statementContext == null) {
+ return Optional.empty();
+ }
+ Optional<ExternalTablePreloadInfo> preloadInfo =
statementContext.getExternalTablePreloadInfo(getId());
+ if (!preloadInfo.isPresent() ||
!preloadInfo.get().hasLatestOnlyRelation()) {
+ return Optional.empty();
+ }
+ Optional<MvccSnapshot> latestSnapshot =
statementContext.getSnapshot(this);
+ boolean isLatestRequest = !snapshot.isPresent()
+ || (latestSnapshot.isPresent() && latestSnapshot.get() ==
snapshot.get());
+ return isLatestRequest ? preloadInfo : Optional.empty();
+ }
+
+ @Override
Review Comment:
[P2] Keep a full partition view available for filtered async-MV mapping. For
the reduced plan Filter(h.p = 2) -> Hive h, CollectRelation marks h filtered,
so the pre-lock full-view warmup leaves scanPartitionView unset. During
partitioned async-MV union compensation, MTMV.calculatePartitionMappings asks
this method for the related-table items and reaches this new exception;
AbstractMaterializedViewRule catches it and skips the otherwise eligible MV
candidate. The query then scans the base table even though the rewrite was
previously available. Supply a consistent view before MV mapping and cover a
filtered query requiring union compensation.
##########
fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java:
##########
@@ -1289,6 +1281,106 @@ private List<ConnectorPartitionInfo>
listPartitionsUncached(HiveTableHandle hive
return result;
}
+ private PartitionPruningResult prunePartitions(ConnectorSession session,
HiveTableHandle hiveHandle,
+ ConnectorExpression expression) {
+ List<String> partKeyNames = hiveHandle.getPartitionKeyNames();
+ Map<String, List<String>> partitionPredicates =
extractPartitionPredicates(expression, partKeyNames);
+ if (partitionPredicates.isEmpty()) {
+ return null;
+ }
+
+ String hmsFilter = buildHmsPartitionFilter(partKeyNames,
hiveHandle.getPartitionKeyTypes(),
+ hiveHandle.getPartitionKeyHiveTypes(), partitionPredicates);
+ if (hmsFilter != null) {
+ int predicateValueCount =
partitionPredicates.values().stream().mapToInt(List::size).sum();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("HMS partition filter request for {}.{}
predicateValues={} filter={}",
+ hiveHandle.getDbName(), hiveHandle.getTableName(),
predicateValueCount,
+ summarizeHmsFilterForDebug(hmsFilter));
Review Comment:
[P1] Do not trust successful integral JDO filtering as complete. With Hive
metastore.integral.jdo.pushdown=true and direct SQL disabled or failing, a
valid INT partition stored as p=01 is omitted from a successful HMS response to
p = 1: [Hive documents this leading-zero
limitation](https://github.com/apache/hive/blob/rel/release-3.1.3/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java#L3168-L3179),
and its [JDO equality
filter](https://github.com/apache/hive/blob/rel/release-3.1.3/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java#L2236-L2293)
compares the rendered partition-name fragment as text. This return accepts the
incomplete set as the logical and physical scan view, so the query misses
matching rows; the local fallback cannot run because the RPC succeeded. Use
full-name listing plus typed pruning for integral keys unless safe server
semantics are known, and test p=01 alongside p=1 with JDO
pushdown enabled.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
