github-actions[bot] commented on code in PR #67996: URL: https://github.com/apache/doris/pull/67996#discussion_r4058957886
########## fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCacheSizeEstimator.java: ########## @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.cache.JvmSizeUtils; +import org.apache.doris.connector.cache.MetaCacheSizeEstimate; +import org.apache.doris.connector.cache.ReflectiveObjectSizeEstimator; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.fs.Path; +import org.apache.paimon.table.CatalogEnvironment; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.FormatTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.iceberg.IcebergTable; +import org.apache.paimon.table.lance.LanceTable; +import org.apache.paimon.table.object.ObjectTable; + +import java.net.URI; + +/** + * Retained-size formulas for Paimon table-cache entries. + * + * <p>The table's shallow size includes references to FileIO, catalog loaders, and lock factories, + * but their graphs are catalog-scoped executable services rather than entry-owned metadata. Walking + * those graphs both double-counts shared state and reaches strongly encapsulated JDK objects. The + * estimator therefore expands only immutable metadata owned by the entry. + */ +final class PaimonCacheSizeEstimator { + private PaimonCacheSizeEstimator() { + } + + static MetaCacheSizeEstimate estimateTable(Identifier key, Table table, long entryOverheadBytes) { + long bytes = add(entryOverheadBytes, ReflectiveObjectSizeEstimator.estimateComplete(key)); + bytes = add(bytes, JvmSizeUtils.instanceSize(table.getClass())); + if (table instanceof FileStoreTable) { + FileStoreTable fileStoreTable = (FileStoreTable) table; + bytes = add(bytes, ReflectiveObjectSizeEstimator.estimateComplete(fileStoreTable.schema())); Review Comment: [P1] Include the retained decorator and fallback graphs in the estimate A live entry can be `PrivilegedFileStoreTable(FallbackReadFileStoreTable(main, fallback))` (the shape documented in `PaimonTableDecorators`). This branch counts only the outer runtime object's shallow size plus delegated `schema()`, `location()`, and `catalogEnvironment()`; those expose the main table, while the cached object still strongly retains both table branches, the decorator objects, and the privilege checker's per-entry map snapshot. That state is omitted even though the estimate is marked complete, so enough decorated entries can exceed `external_meta_cache_max_weight` or catalog `meta.cache.max-weight`. This is distinct from the earlier hidden-lambda thread: ordinary tables are now admitted, but these supported shapes are admitted with an incomplete weight. Please account the retained decorator/both-branch graph with identity deduplication (or reject unsupported shapes) and add weight-governed decorated-table tests. ########## fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonMetaCacheCatalog.java: ########## @@ -0,0 +1,342 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.JvmSizeUtils; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimators; +import org.apache.doris.connector.cache.ScopePath; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogLoader; +import org.apache.paimon.catalog.Database; +import org.apache.paimon.catalog.DelegateCatalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.catalog.PropertyChange; +import org.apache.paimon.fs.Path; +import org.apache.paimon.options.CatalogOptions; +import org.apache.paimon.options.MemorySize; +import org.apache.paimon.options.Options; +import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Caffeine; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.system.SystemTableLoader; +import org.apache.paimon.utils.SegmentsCache; + +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongSupplier; + +/** + * Doris-owned replacement for Paimon's {@code CachingCatalog}. Table and database entries live in + * {@link CatalogMetaCache}, so a Doris catalog/database/table invalidation fences every matching + * in-flight load and cached value. + * + * <p>The user's {@code paimon.cache-enabled} and access/write expiry settings remain authoritative. + * The Paimon SDK wrapper itself is disabled because a second hidden table cache cannot participate + * in Doris invalidation. Under a Doris weight budget, mutable SDK snapshot/stats/manifest caches are + * not attached: their post-publication growth cannot be reweighed by the enclosing budget. + */ +final class PaimonMetaCacheCatalog extends DelegateCatalog { + + private static final int DATABASE_CACHE_CAPACITY = 100; + private static final long TABLE_ENTRY_OVERHEAD_BYTES = JvmSizeUtils.saturatedAdd( + JvmSizeUtils.instanceSize(ExpiringValue.class), JvmSizeUtils.instanceSize(AtomicLong.class)); + + private final CatalogMetaCache metaCache; + private final MetaCache<Identifier, ExpiringValue<Table>> tableCache; + private final MetaCache<String, ExpiringValue<Database>> databaseCache; + private final SegmentsCache<Path> manifestCache; + private final long tableExpireAfterAccessNanos; + private final long databaseExpireAfterAccessNanos; + private final long expireAfterWriteNanos; + private final int snapshotMaxNumPerTable; + private final boolean attachSdkCaches; + private final LongSupplier nanoTime; + + static Catalog tryToCreate(Catalog wrapped, CatalogMetaCache metaCache, int tableCacheMaxSize, + long tableCacheTtlSecond, Options catalogOptions, boolean cacheEnabled, + boolean hasEnclosingWeightLimit) { + return new PaimonMetaCacheCatalog(wrapped, metaCache, tableCacheMaxSize, + tableCacheTtlSecond, catalogOptions, cacheEnabled, hasEnclosingWeightLimit, System::nanoTime); + } + + PaimonMetaCacheCatalog(Catalog wrapped, CatalogMetaCache metaCache, int tableCacheMaxSize, + long tableCacheTtlSecond, Options catalogOptions, boolean hasEnclosingWeightLimit, + LongSupplier nanoTime) { + this(wrapped, metaCache, tableCacheMaxSize, tableCacheTtlSecond, catalogOptions, + true, hasEnclosingWeightLimit, nanoTime); + } + + private PaimonMetaCacheCatalog(Catalog wrapped, CatalogMetaCache metaCache, int tableCacheMaxSize, + long tableCacheTtlSecond, Options catalogOptions, boolean cacheEnabled, + boolean hasEnclosingWeightLimit, LongSupplier nanoTime) { + super(wrapped); + this.metaCache = metaCache; + this.nanoTime = nanoTime; + + Duration expireAfterAccess = catalogOptions.get(CatalogOptions.CACHE_EXPIRE_AFTER_ACCESS); + Duration expireAfterWrite = catalogOptions.get(CatalogOptions.CACHE_EXPIRE_AFTER_WRITE); + if (cacheEnabled) { + requirePositive(expireAfterAccess, CatalogOptions.CACHE_EXPIRE_AFTER_ACCESS.key()); + requirePositive(expireAfterWrite, CatalogOptions.CACHE_EXPIRE_AFTER_WRITE.key()); + } + long paimonAccessNanos = cacheEnabled ? expireAfterAccess.toNanos() : Long.MAX_VALUE; + this.tableExpireAfterAccessNanos = cacheEnabled && tableCacheTtlSecond > 0 + ? Math.min(paimonAccessNanos, Duration.ofSeconds(tableCacheTtlSecond).toNanos()) + : paimonAccessNanos; + this.databaseExpireAfterAccessNanos = paimonAccessNanos; + this.expireAfterWriteNanos = cacheEnabled ? expireAfterWrite.toNanos() : Long.MAX_VALUE; + + CacheSpec tableSpec = CacheSpec.of(cacheEnabled, + cacheEnabled && tableCacheTtlSecond > 0 + ? CacheSpec.CACHE_NO_TTL : CacheSpec.CACHE_TTL_DISABLE_CACHE, + tableCacheMaxSize); + this.tableCache = metaCache.create(MetaCacheDefinition + .<Identifier, ExpiringValue<Table>>builder("paimon-table", tableSpec, + id -> ScopePath.table(id.getDatabaseName(), id.getTableName())) + .sizeEstimator((id, value) -> PaimonCacheSizeEstimator.estimateTable( + id, value.value, TABLE_ENTRY_OVERHEAD_BYTES)) + .build()); + CacheSpec dbSpec = CacheSpec.of(cacheEnabled, cacheEnabled + ? CacheSpec.CACHE_NO_TTL : CacheSpec.CACHE_TTL_DISABLE_CACHE, DATABASE_CACHE_CAPACITY); + this.databaseCache = metaCache.create(MetaCacheDefinition + .<String, ExpiringValue<Database>>builder("paimon-database", dbSpec, ScopePath::database) + .sizeEstimator(MetaCacheSizeEstimators.reflective()) + .build()); + + this.attachSdkCaches = cacheEnabled && !hasEnclosingWeightLimit; + this.manifestCache = attachSdkCaches ? buildManifestCache(catalogOptions) : null; + this.snapshotMaxNumPerTable = catalogOptions.get( + CatalogOptions.CACHE_SNAPSHOT_MAX_NUM_PER_TABLE); + } + + @Override + public Table getTable(Identifier identifier) throws TableNotExistException { + if (identifier.isSystemTable()) { + Identifier origin = new Identifier(identifier.getDatabaseName(), identifier.getTableName(), + identifier.getBranchName(), null); + Table originTable = getTable(origin); + if (!(originTable instanceof FileStoreTable)) { + return super.getTable(identifier); + } + Table systemTable = SystemTableLoader.load(identifier.getSystemTableName(), + (FileStoreTable) originTable); + if (systemTable == null) { + throw new TableNotExistException(identifier); + } + return systemTable; + } + + while (true) { + long now = nanoTime.getAsLong(); + ExpiringValue<Table> cached = tableCache.getIfPresent(identifier); + if (cached != null) { + if (cached.tryAccess(now, tableExpireAfterAccessNanos, expireAfterWriteNanos)) { + return cached.value; + } + tableCache.compareAndSet(identifier, cached, null); + continue; + } + try { + return tableCache.get(identifier, ignored -> { + try { + Table loaded = attachPerTableCaches(super.getTable(identifier)); Review Comment: [P1] Do not cache the snapshot privilege checker Before this change the HMS stack was `PrivilegedCatalog(CachingCatalog(raw))`, so every `getTable` wrapped the cached raw table with a fresh `getPrivilegeChecker()`. The new outer cache stores the result of `PrivilegedCatalog#getTable` here instead. In Paimon 1.3.1 that `PrivilegedFileStoreTable` retains a `PrivilegeCheckerImpl` whose privilege map was loaded once; after an admin revokes SELECT through another catalog, a hot entry therefore keeps authorizing scans with the pre-revocation map until expiry or explicit refresh. Please keep the privilege wrapper outside the table cache (or cache only the undecorated table and apply a fresh checker per lookup), and add a regression that warms a table, revokes SELECT, and verifies the next scan is denied. ########## fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonMetaCacheCatalog.java: ########## @@ -0,0 +1,342 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.JvmSizeUtils; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimators; +import org.apache.doris.connector.cache.ScopePath; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogLoader; +import org.apache.paimon.catalog.Database; +import org.apache.paimon.catalog.DelegateCatalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.catalog.PropertyChange; +import org.apache.paimon.fs.Path; +import org.apache.paimon.options.CatalogOptions; +import org.apache.paimon.options.MemorySize; +import org.apache.paimon.options.Options; +import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Caffeine; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.system.SystemTableLoader; +import org.apache.paimon.utils.SegmentsCache; + +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongSupplier; + +/** + * Doris-owned replacement for Paimon's {@code CachingCatalog}. Table and database entries live in + * {@link CatalogMetaCache}, so a Doris catalog/database/table invalidation fences every matching + * in-flight load and cached value. + * + * <p>The user's {@code paimon.cache-enabled} and access/write expiry settings remain authoritative. + * The Paimon SDK wrapper itself is disabled because a second hidden table cache cannot participate + * in Doris invalidation. Under a Doris weight budget, mutable SDK snapshot/stats/manifest caches are + * not attached: their post-publication growth cannot be reweighed by the enclosing budget. + */ +final class PaimonMetaCacheCatalog extends DelegateCatalog { + + private static final int DATABASE_CACHE_CAPACITY = 100; + private static final long TABLE_ENTRY_OVERHEAD_BYTES = JvmSizeUtils.saturatedAdd( + JvmSizeUtils.instanceSize(ExpiringValue.class), JvmSizeUtils.instanceSize(AtomicLong.class)); + + private final CatalogMetaCache metaCache; + private final MetaCache<Identifier, ExpiringValue<Table>> tableCache; + private final MetaCache<String, ExpiringValue<Database>> databaseCache; + private final SegmentsCache<Path> manifestCache; + private final long tableExpireAfterAccessNanos; + private final long databaseExpireAfterAccessNanos; + private final long expireAfterWriteNanos; + private final int snapshotMaxNumPerTable; + private final boolean attachSdkCaches; + private final LongSupplier nanoTime; + + static Catalog tryToCreate(Catalog wrapped, CatalogMetaCache metaCache, int tableCacheMaxSize, + long tableCacheTtlSecond, Options catalogOptions, boolean cacheEnabled, + boolean hasEnclosingWeightLimit) { + return new PaimonMetaCacheCatalog(wrapped, metaCache, tableCacheMaxSize, + tableCacheTtlSecond, catalogOptions, cacheEnabled, hasEnclosingWeightLimit, System::nanoTime); + } + + PaimonMetaCacheCatalog(Catalog wrapped, CatalogMetaCache metaCache, int tableCacheMaxSize, + long tableCacheTtlSecond, Options catalogOptions, boolean hasEnclosingWeightLimit, + LongSupplier nanoTime) { + this(wrapped, metaCache, tableCacheMaxSize, tableCacheTtlSecond, catalogOptions, + true, hasEnclosingWeightLimit, nanoTime); + } + + private PaimonMetaCacheCatalog(Catalog wrapped, CatalogMetaCache metaCache, int tableCacheMaxSize, + long tableCacheTtlSecond, Options catalogOptions, boolean cacheEnabled, + boolean hasEnclosingWeightLimit, LongSupplier nanoTime) { + super(wrapped); + this.metaCache = metaCache; + this.nanoTime = nanoTime; + + Duration expireAfterAccess = catalogOptions.get(CatalogOptions.CACHE_EXPIRE_AFTER_ACCESS); + Duration expireAfterWrite = catalogOptions.get(CatalogOptions.CACHE_EXPIRE_AFTER_WRITE); + if (cacheEnabled) { + requirePositive(expireAfterAccess, CatalogOptions.CACHE_EXPIRE_AFTER_ACCESS.key()); + requirePositive(expireAfterWrite, CatalogOptions.CACHE_EXPIRE_AFTER_WRITE.key()); + } + long paimonAccessNanos = cacheEnabled ? expireAfterAccess.toNanos() : Long.MAX_VALUE; + this.tableExpireAfterAccessNanos = cacheEnabled && tableCacheTtlSecond > 0 + ? Math.min(paimonAccessNanos, Duration.ofSeconds(tableCacheTtlSecond).toNanos()) Review Comment: [P2] Avoid overflow for accepted cache TTLs `meta.cache.paimon.table.ttl-second` accepts any `long >= -1`. For example, `9223372037` passes statement-time validation and `resolveTableCacheTtlSecond`, but `Duration.ofSeconds(9223372037L).toNanos()` throws `ArithmeticException` here while the catalog is created lazily. The direct conversions of Paimon's access/write durations have the same representational limit. Please either reject values above the nanosecond horizon during statement-time validation or use a saturated conversion so every accepted persisted setting remains loadable, and cover the boundary just above `Long.MAX_VALUE` nanoseconds. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
