This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch branch-catalog-spi
in repository https://gitbox.apache.org/repos/asf/doris.git
from f1d808e3cf6 [feat](catalog) port upstream #65329 iceberg nested column
schema change to the connector SPI (Tier 2)
add 23b06e04767 [fix](catalog) catalog-SPI abstraction-mismatch review:
paimon time-travel crash, partition/schema fidelity, SPI convergence &
dead-code cleanup (#65970)
No new revisions were added by this update.
Summary of changes:
fe/be-java-extensions/paimon-scanner/pom.xml | 65 ++++++
fe/be-java-extensions/preload-extensions/pom.xml | 22 ++
.../connector/api/ConnectorStatisticsOps.java | 21 ++
.../doris/connector/api/ConnectorTableOps.java | 37 +++
.../connector/api/ddl/ConnectorBucketSpec.java | 19 +-
.../connector/api/handle/ConnectorTransaction.java | 53 -----
.../connector/api/handle/ConnectorWriteHandle.java | 15 +-
.../api/handle/RewriteCapableTransaction.java | 52 +++++
.../WriteBlockAllocatingConnectorTransaction.java | 40 ++++
.../api/mvcc/ConnectorMvccPartitionView.java | 47 +++-
.../connector/api/mvcc/ConnectorMvccSnapshot.java | 32 +++
.../connector/api/pushdown/ConnectorDomain.java | 116 ----------
.../api/pushdown/ConnectorFilterConstraint.java | 34 +--
.../connector/api/pushdown/ConnectorRange.java | 144 ------------
.../connector/api/scan/ConnectorDeleteFile.java | 79 -------
.../connector/api/scan/ConnectorScanRange.java | 8 -
.../api/mvcc/ConnectorMvccSnapshotTest.java | 46 ++++
.../apache/doris/connector/hive/HiveConnector.java | 35 ++-
.../connector/hive/HiveConnectorMetadata.java | 14 +-
.../doris/connector/hive/HiveFileListingCache.java | 5 +-
.../connector/hive/HiveReadTransactionManager.java | 10 +-
.../connector/hive/HiveWritePlanProvider.java | 6 +-
.../hive/HiveConnectorCapabilitiesTest.java | 4 +-
.../hive/HiveConnectorClientCacheTest.java | 4 +-
.../hive/HiveConnectorInvalidateTest.java | 2 +-
.../hive/HiveConnectorMetadataColumnStatsTest.java | 2 +-
.../HiveConnectorMetadataPartitionListTest.java | 2 +-
...HiveConnectorMetadataSiblingDelegationTest.java | 9 +-
...HiveConnectorMetadataTableHandleDivertTest.java | 4 +-
.../hive/HiveConnectorProcedureOpsDivertTest.java | 4 +-
.../hive/HiveConnectorScanProviderDivertTest.java | 4 +-
.../connector/hive/HiveConnectorSiblingTest.java | 8 +-
.../hive/HiveConnectorThreeWayRoutingTest.java | 2 +-
.../hive/HiveConnectorWriteProviderDivertTest.java | 4 +-
.../connector/hive/HiveFileListingCacheTest.java | 3 +-
.../hive/HiveTableFormatDetectionTest.java | 2 +-
.../doris/connector/hms/CachingHmsClient.java | 12 +-
.../connector/hudi/HudiConnectorMetadata.java | 10 +
.../connector/hudi/HudiPartitionPruningTest.java | 27 +++
.../doris/connector/iceberg/IcebergCatalogOps.java | 28 ++-
.../iceberg/IcebergConnectorMetadata.java | 43 +++-
.../iceberg/IcebergConnectorTransaction.java | 3 +-
.../connector/iceberg/IcebergPartitionUtils.java | 9 +-
.../connector/iceberg/IcebergSchemaUtils.java | 18 ++
...BackedIcebergCatalogOpsColumnEvolutionTest.java | 51 ++++-
...cebergConnectorMetadataColumnEvolutionTest.java | 15 ++
.../iceberg/IcebergPartitionUtilsTest.java | 18 +-
.../connector/iceberg/IcebergSchemaUtilsTest.java | 70 ++++++
.../iceberg/RecordingIcebergCatalogOps.java | 4 +-
.../maxcompute/MaxComputeConnectorTransaction.java | 9 +-
.../connector/paimon/PaimonConnectorMetadata.java | 89 +++++++-
.../doris/connector/paimon/PaimonTypeMapping.java | 20 +-
.../PaimonConnectorMetadataPartitionTest.java | 58 ++++-
.../paimon/PaimonTypeMappingReadTest.java | 27 +++
.../paimon/PaimonTypeMappingToPaimonTest.java | 22 +-
.../converter/ConnectorColumnConverter.java | 8 +-
.../mvcc/PluginDrivenMvccExternalTable.java | 24 +-
.../datasource/mvcc/PluginDrivenMvccSnapshot.java | 48 +++-
.../plugin/PluginDrivenExternalTable.java | 50 ++++
.../datasource/scan/PluginDrivenScanNode.java | 105 ++++++---
.../org/apache/doris/mtmv/MTMVRelatedTableIf.java | 14 ++
.../org/apache/doris/nereids/StatementContext.java | 27 +++
.../commands/execute/ConnectorExecuteAction.java | 2 +-
.../commands/execute/ConnectorRewriteDriver.java | 14 +-
.../org/apache/doris/qe/cache/CacheAnalyzer.java | 37 ++-
.../apache/doris/service/FrontendServiceImpl.java | 5 +-
.../PluginDrivenTransactionManager.java | 45 ++--
.../org/apache/doris/transaction/Transaction.java | 23 --
.../WriteBlockAllocatingTransaction.java | 43 ++++
.../fake/ConnectorTransactionDefaultsTest.java | 24 +-
.../converter/ConnectorColumnConverterTest.java | 20 ++
.../mvcc/PluginDrivenMvccExternalTableTest.java | 23 +-
.../qe/cache/PluginTableCacheAnalyzerTest.java | 31 +++
.../doris/service/FrontendServiceImplTest.java | 11 +-
.../PluginDrivenTransactionManagerTest.java | 74 +++---
plan-doc/catalog-spi-mismatch/HANDOFF.md | 100 ++++++++
plan-doc/catalog-spi-mismatch/TASKLIST.md | 221 ++++++++++++++++++
.../catalog-spi-mismatch/analysis-00-rollup.md | 93 ++++++++
.../analysis-A-literal-predicate.md | 84 +++++++
.../analysis-B-schema-column-identity.md | 253 +++++++++++++++++++++
.../catalog-spi-mismatch/analysis-C-partition.md | 164 +++++++++++++
.../analysis-D-scan-split-file.md | 129 +++++++++++
.../analysis-E-stats-mvcc-timetravel.md | 221 ++++++++++++++++++
.../analysis-F-write-txn-ddl.md | 126 ++++++++++
.../analysis-G-reader-path-dispatch.md | 110 +++++++++
.../catalog-spi-abstraction-mismatch-survey.md | 252 ++++++++++++++++++++
.../design-transaction-capability-convergence.md | 75 ++++++
.../fix-getcolumnhandles-snapshot-design.md | 65 ++++++
.../fix-hudi-decimal-partition-predicate-design.md | 51 +++++
...fix-hudi-decimal-partition-predicate-summary.md | 26 +++
.../fix-iceberg-sqlcache-gate-design.md | 57 +++++
.../fix-iceberg-sqlcache-gate-summary.md | 34 +++
.../fix-iceberg-write-default-design.md | 63 +++++
.../fix-iceberg-write-default-summary.md | 28 +++
.../fix-paimon-nested-struct-comment-design.md | 86 +++++++
.../fix-paimon-nested-struct-comment-summary.md | 36 +++
.../fix-paimon-partition-values-tvf-design.md | 78 +++++++
.../fix-paimon-partition-values-tvf-summary.md | 40 ++++
.../iceberg/test_iceberg_sqlcache.groovy | 114 ++++++++++
.../iceberg/test_iceberg_write_default.groovy | 87 +++++++
.../test_paimon_nested_struct_comment.groovy | 97 ++++++++
.../paimon/test_paimon_time_travel_rename.groovy | 113 +++++++++
.../paimon/test_paimon_time_travel_rowcount.groovy | 78 +++++++
103 files changed, 4284 insertions(+), 712 deletions(-)
create mode 100644
fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/handle/RewriteCapableTransaction.java
create mode 100644
fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/handle/WriteBlockAllocatingConnectorTransaction.java
delete mode 100644
fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/pushdown/ConnectorDomain.java
delete mode 100644
fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/pushdown/ConnectorRange.java
delete mode 100644
fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/scan/ConnectorDeleteFile.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/transaction/WriteBlockAllocatingTransaction.java
create mode 100644 plan-doc/catalog-spi-mismatch/HANDOFF.md
create mode 100644 plan-doc/catalog-spi-mismatch/TASKLIST.md
create mode 100644 plan-doc/catalog-spi-mismatch/analysis-00-rollup.md
create mode 100644
plan-doc/catalog-spi-mismatch/analysis-A-literal-predicate.md
create mode 100644
plan-doc/catalog-spi-mismatch/analysis-B-schema-column-identity.md
create mode 100644 plan-doc/catalog-spi-mismatch/analysis-C-partition.md
create mode 100644 plan-doc/catalog-spi-mismatch/analysis-D-scan-split-file.md
create mode 100644
plan-doc/catalog-spi-mismatch/analysis-E-stats-mvcc-timetravel.md
create mode 100644 plan-doc/catalog-spi-mismatch/analysis-F-write-txn-ddl.md
create mode 100644
plan-doc/catalog-spi-mismatch/analysis-G-reader-path-dispatch.md
create mode 100644
plan-doc/catalog-spi-mismatch/catalog-spi-abstraction-mismatch-survey.md
create mode 100644
plan-doc/catalog-spi-mismatch/design-transaction-capability-convergence.md
create mode 100644
plan-doc/catalog-spi-mismatch/fix-getcolumnhandles-snapshot-design.md
create mode 100644
plan-doc/catalog-spi-mismatch/fix-hudi-decimal-partition-predicate-design.md
create mode 100644
plan-doc/catalog-spi-mismatch/fix-hudi-decimal-partition-predicate-summary.md
create mode 100644
plan-doc/catalog-spi-mismatch/fix-iceberg-sqlcache-gate-design.md
create mode 100644
plan-doc/catalog-spi-mismatch/fix-iceberg-sqlcache-gate-summary.md
create mode 100644
plan-doc/catalog-spi-mismatch/fix-iceberg-write-default-design.md
create mode 100644
plan-doc/catalog-spi-mismatch/fix-iceberg-write-default-summary.md
create mode 100644
plan-doc/catalog-spi-mismatch/fix-paimon-nested-struct-comment-design.md
create mode 100644
plan-doc/catalog-spi-mismatch/fix-paimon-nested-struct-comment-summary.md
create mode 100644
plan-doc/catalog-spi-mismatch/fix-paimon-partition-values-tvf-design.md
create mode 100644
plan-doc/catalog-spi-mismatch/fix-paimon-partition-values-tvf-summary.md
create mode 100644
regression-test/suites/external_table_p0/iceberg/test_iceberg_sqlcache.groovy
create mode 100644
regression-test/suites/external_table_p0/iceberg/test_iceberg_write_default.groovy
create mode 100644
regression-test/suites/external_table_p0/paimon/test_paimon_nested_struct_comment.groovy
create mode 100644
regression-test/suites/external_table_p0/paimon/test_paimon_time_travel_rename.groovy
create mode 100644
regression-test/suites/external_table_p0/paimon/test_paimon_time_travel_rowcount.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
