This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch branch-catalog-spi
in repository https://gitbox.apache.org/repos/asf/doris.git
from 420483b0c84 [test](nereids) rebase-adapt PullUpJoinFromUnionAllTest to
the connector-migrated Hudi scan path
add 0c082d67e52 [opt](catalog) fe-connector: unify the connector cache
foundation, memoize per-statement hot paths, fix iceberg case-insensitive
column DDL (#66029)
No new revisions were added by this update.
Summary of changes:
.../connector/api/ConnectorStatementScopes.java | 76 ++
.../api/ConnectorStatementScopesTest.java | 225 ++++
.../apache/doris/connector/cache/CacheSpec.java | 17 +
...nViewCache.java => ConnectorMetadataCache.java} | 47 +-
...ionViewCacheKey.java => ConnectorTableKey.java} | 16 +-
.../doris/connector/cache/CacheSpecTest.java | 30 +
...heTest.java => ConnectorMetadataCacheTest.java} | 34 +-
.../doris/connector/es/EsConnectorMetadata.java | 40 +-
.../doris/connector/es/EsMetadataFetcher.java | 14 +-
.../doris/connector/es/EsScanPlanProvider.java | 32 +-
.../doris/connector/es/EsStatementScope.java | 56 +
.../doris/connector/es/EsScanPlanProviderTest.java | 276 +++-
.../doris/connector/es/EsStatementScopeTest.java | 54 +
.../apache/doris/connector/hive/HiveConnector.java | 10 +-
.../connector/hive/HiveConnectorMetadata.java | 12 +-
...iveConnectorMetadataPartitionViewCacheTest.java | 18 +-
.../hive/HiveConnectorPartitionViewCacheTest.java | 18 +-
.../apache/doris/connector/hudi/HudiConnector.java | 71 +-
.../connector/hudi/HudiConnectorMetadata.java | 81 +-
.../doris/connector/hudi/HudiScanPlanProvider.java | 2 +-
.../doris/connector/hudi/HudiStatementScope.java | 78 ++
.../connector/hudi/HudiConnectorHmsCacheTest.java | 265 ++++
.../hudi/HudiConnectorOwnsHandleTest.java | 4 +-
.../hudi/HudiReadOnlyWriteRejectTest.java | 4 +-
.../connector/hudi/HudiStatementMemoTest.java | 227 ++++
.../doris/connector/iceberg/IcebergCatalogOps.java | 13 +-
.../connector/iceberg/IcebergCommentCache.java | 11 +-
.../doris/connector/iceberg/IcebergConnector.java | 41 +-
.../iceberg/IcebergConnectorMetadata.java | 27 +-
.../iceberg/IcebergConnectorTransaction.java | 12 +-
.../connector/iceberg/IcebergFormatCache.java | 7 +-
.../iceberg/IcebergLatestSnapshotCache.java | 9 +-
.../iceberg/IcebergNestedColumnEvolution.java | 45 +-
.../connector/iceberg/IcebergPartitionCache.java | 8 +-
.../connector/iceberg/IcebergScanPlanProvider.java | 2 +-
.../connector/iceberg/IcebergStatementScope.java | 28 +-
.../doris/connector/iceberg/IcebergTableCache.java | 8 +-
.../iceberg/IcebergWritePlanProvider.java | 4 +-
.../action/IcebergExecuteActionFactory.java | 4 +-
.../action/IcebergRewriteDataFilesAction.java | 2 +-
...BackedIcebergCatalogOpsColumnEvolutionTest.java | 112 ++
.../iceberg/IcebergConnectorCacheTest.java | 12 +-
...ergConnectorMetadataPartitionViewCacheTest.java | 26 +-
.../iceberg/IcebergStatementScopeTest.java | 72 +
.../connector/jdbc/JdbcConnectorMetadata.java | 17 +-
.../connector/jdbc/JdbcWritePlanProvider.java | 6 +
.../connector/jdbc/JdbcConnectorMetadataTest.java | 186 +++
.../connector/jdbc/JdbcWritePlanProviderTest.java | 79 ++
.../maxcompute/MaxComputeConnectorMetadata.java | 44 +-
.../maxcompute/MaxComputeConnectorTransaction.java | 10 +-
.../maxcompute/MaxComputeWritePlanProvider.java | 7 +-
.../MaxComputeConnectorMetadataHandleMemoTest.java | 230 ++++
.../doris/connector/paimon/PaimonConnector.java | 15 +-
.../connector/paimon/PaimonConnectorMetadata.java | 72 +-
.../paimon/PaimonLatestSnapshotCache.java | 9 +-
.../connector/paimon/PaimonConnectorCacheTest.java | 12 +-
...monConnectorMetadataPartitionViewCacheTest.java | 124 +-
fe/fe-connector/pom.xml | 19 -
.../connector/ConnectorStatementScopeTest.java | 55 +-
plan-doc/HANDOFF.md | 66 +-
.../00-research-report.md | 168 +++
plan-doc/connector-cache-unification/HANDOFF.md | 129 ++
plan-doc/connector-cache-unification/README.md | 51 +
.../appendix-A-framework-taxonomy.md | 128 ++
.../appendix-B-trino-reference.md | 140 ++
.../appendix-C-funnel-universality.md | 77 ++
.../appendix-D-critic-punchlist.md | 20 +
.../connector-cache-unification/connectors/es.md | 103 ++
.../connector-cache-unification/connectors/hive.md | 97 ++
.../connector-cache-unification/connectors/hudi.md | 94 ++
.../connector-cache-unification/connectors/jdbc.md | 90 ++
.../connectors/maxcompute.md | 95 ++
.../connectors/paimon.md | 86 ++
.../connectors/trino.md | 100 ++
.../data/connector-audits.json | 1419 ++++++++++++++++++++
.../designs/foundation-design-FINAL.md | 195 +++
.../designs/foundation-design-draft.md | 259 ++++
.../designs/review-1.md | 103 ++
.../designs/review-2.md | 44 +
.../designs/review-3.md | 59 +
plan-doc/connector-cache-unification/progress.md | 186 +++
plan-doc/connector-cache-unification/tasklist.md | 114 ++
plan-doc/perf-hotpath-es/HANDOFF.md | 47 +
plan-doc/perf-hotpath-es/README.md | 19 +
plan-doc/perf-hotpath-es/designs/round-1-design.md | 51 +
plan-doc/perf-hotpath-es/progress.md | 18 +
plan-doc/perf-hotpath-es/tasklist.md | 22 +
plan-doc/perf-hotpath-hudi/HANDOFF.md | 37 +
plan-doc/perf-hotpath-hudi/README.md | 19 +
.../designs/round-1-memo-hms-cache-design.md | 104 ++
.../designs/round-1-pieceB-flagship-impl-notes.md | 141 ++
plan-doc/perf-hotpath-hudi/progress.md | 49 +
plan-doc/perf-hotpath-hudi/tasklist.md | 25 +
plan-doc/perf-hotpath-jdbc/README.md | 29 +
plan-doc/perf-hotpath-maxcompute/HANDOFF.md | 46 +
plan-doc/perf-hotpath-maxcompute/README.md | 19 +
.../designs/round-1-handle-memo-design.md | 70 +
plan-doc/perf-hotpath-maxcompute/progress.md | 15 +
plan-doc/perf-hotpath-maxcompute/tasklist.md | 20 +
plan-doc/perf-hotpath-paimon/README.md | 30 +
tools/check-authz-cache-sharding.sh | 110 --
tools/check-authz-cache-sharding.test.sh | 108 --
102 files changed, 7223 insertions(+), 614 deletions(-)
create mode 100644
fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/ConnectorStatementScopes.java
create mode 100644
fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/ConnectorStatementScopesTest.java
rename
fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/{ConnectorPartitionViewCache.java
=> ConnectorMetadataCache.java} (65%)
rename
fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/{PartitionViewCacheKey.java
=> ConnectorTableKey.java} (84%)
rename
fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/{ConnectorPartitionViewCacheTest.java
=> ConnectorMetadataCacheTest.java} (89%)
create mode 100644
fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsStatementScope.java
create mode 100644
fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsStatementScopeTest.java
create mode 100644
fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiStatementScope.java
create mode 100644
fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorHmsCacheTest.java
create mode 100644
fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiStatementMemoTest.java
create mode 100644
fe/fe-connector/fe-connector-maxcompute/src/test/java/org/apache/doris/connector/maxcompute/MaxComputeConnectorMetadataHandleMemoTest.java
create mode 100644 plan-doc/connector-cache-unification/00-research-report.md
create mode 100644 plan-doc/connector-cache-unification/HANDOFF.md
create mode 100644 plan-doc/connector-cache-unification/README.md
create mode 100644
plan-doc/connector-cache-unification/appendix-A-framework-taxonomy.md
create mode 100644
plan-doc/connector-cache-unification/appendix-B-trino-reference.md
create mode 100644
plan-doc/connector-cache-unification/appendix-C-funnel-universality.md
create mode 100644
plan-doc/connector-cache-unification/appendix-D-critic-punchlist.md
create mode 100644 plan-doc/connector-cache-unification/connectors/es.md
create mode 100644 plan-doc/connector-cache-unification/connectors/hive.md
create mode 100644 plan-doc/connector-cache-unification/connectors/hudi.md
create mode 100644 plan-doc/connector-cache-unification/connectors/jdbc.md
create mode 100644
plan-doc/connector-cache-unification/connectors/maxcompute.md
create mode 100644 plan-doc/connector-cache-unification/connectors/paimon.md
create mode 100644 plan-doc/connector-cache-unification/connectors/trino.md
create mode 100644
plan-doc/connector-cache-unification/data/connector-audits.json
create mode 100644
plan-doc/connector-cache-unification/designs/foundation-design-FINAL.md
create mode 100644
plan-doc/connector-cache-unification/designs/foundation-design-draft.md
create mode 100644 plan-doc/connector-cache-unification/designs/review-1.md
create mode 100644 plan-doc/connector-cache-unification/designs/review-2.md
create mode 100644 plan-doc/connector-cache-unification/designs/review-3.md
create mode 100644 plan-doc/connector-cache-unification/progress.md
create mode 100644 plan-doc/connector-cache-unification/tasklist.md
create mode 100644 plan-doc/perf-hotpath-es/HANDOFF.md
create mode 100644 plan-doc/perf-hotpath-es/README.md
create mode 100644 plan-doc/perf-hotpath-es/designs/round-1-design.md
create mode 100644 plan-doc/perf-hotpath-es/progress.md
create mode 100644 plan-doc/perf-hotpath-es/tasklist.md
create mode 100644 plan-doc/perf-hotpath-hudi/HANDOFF.md
create mode 100644 plan-doc/perf-hotpath-hudi/README.md
create mode 100644
plan-doc/perf-hotpath-hudi/designs/round-1-memo-hms-cache-design.md
create mode 100644
plan-doc/perf-hotpath-hudi/designs/round-1-pieceB-flagship-impl-notes.md
create mode 100644 plan-doc/perf-hotpath-hudi/progress.md
create mode 100644 plan-doc/perf-hotpath-hudi/tasklist.md
create mode 100644 plan-doc/perf-hotpath-jdbc/README.md
create mode 100644 plan-doc/perf-hotpath-maxcompute/HANDOFF.md
create mode 100644 plan-doc/perf-hotpath-maxcompute/README.md
create mode 100644
plan-doc/perf-hotpath-maxcompute/designs/round-1-handle-memo-design.md
create mode 100644 plan-doc/perf-hotpath-maxcompute/progress.md
create mode 100644 plan-doc/perf-hotpath-maxcompute/tasklist.md
create mode 100644 plan-doc/perf-hotpath-paimon/README.md
delete mode 100755 tools/check-authz-cache-sharding.sh
delete mode 100755 tools/check-authz-cache-sharding.test.sh
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
