github-actions[bot] commented on code in PR #66717: URL: https://github.com/apache/doris/pull/66717#discussion_r3826250213
########## fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java: ########## @@ -0,0 +1,363 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.DelegatedFileStoreTable; +import org.apache.paimon.table.FallbackReadFileStoreTable; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.RowType; + +import java.util.List; +import java.util.Map; + +/** + * Publication-time approximate weight formulas for Paimon table handles and snapshot projections. + * + * <p>The formulas follow a coarse cardinality model: a rounded-up constant per stable logical + * dimension (schema field, logical type node, option, key, partition, ...) plus the + * skew-sensitive string payload the loader already materialized. The per-node constants absorb + * the lazy state Paimon materializes after admission (the four RowType lookup maps, the store + * graph and its derived RowType copies) instead of modeling those objects individually, so + * weights track metadata size without depending on SDK-private layouts. {@code max-weight} is + * an estimated admission budget, not an exact heap limit. Estimation never opens the table + * store and performs no IO. + */ +final class PaimonCacheSizeEstimator { + // Bounds accounting work per publication; far above real schemas and option maps. + private static final long MAX_TABLE_ACCOUNTING_ELEMENTS = 50_000L; + private static final int MAX_TYPE_ACCOUNTING_DEPTH = 256; + + private static final long KEY_BASE_WEIGHT = 256L; + private static final long SNAPSHOT_BASE_WEIGHT = 4L * 1024L; + private static final long TABLE_BASE_WEIGHT = 16L * 1024L; + // PaimonTableCacheValue and its size-estimate holder. + private static final long TABLE_VALUE_BASE_WEIGHT = 128L; + // One schema field (DataField) at any depth, including its share of the enclosing RowType's + // four lazily built lookup maps, boxed ids and the field copies the lazily created store + // graph derives from it (append-only row copy, trimmed key/value types, merge row type). + private static final long FIELD_NODE_WEIGHT = 576L; + // One bare nested type node (array/map/multiset/vector element types and unknown future + // DataType implementations), including its store-graph copies; charged generically instead + // of disabling weighted caching for unknown types. + private static final long TYPE_NODE_WEIGHT = 128L; + // One nested RowType container: the RowType, its field list, its four lazily built lookup + // maps and the container copies the store graph derives. + private static final long ROW_CONTAINER_WEIGHT = 2048L; + // One schema/table-level entry: option map node, key list slot and boxes. + private static final long OPTION_WEIGHT = 192L; + private static final long KEY_WEIGHT = 192L; + // Wrapper tables (privileged, fallback-read) around the concrete FileStoreTable. + private static final long WRAPPER_WEIGHT = 512L; + // Each concrete FileStoreTable strongly owns its FileIO graph (configuration, and for REST + // catalogs the lazily fetched, periodically rotated vended token). Paimon's FileIO exposes + // no IO-free way to read that state, so a generous fixed allowance is charged per owner: + // rotation replaces the retained token rather than growing it, and typical token/config + // payloads stay far below this bound. + private static final long FILE_IO_WEIGHT = 16L * 1024L; + private static final long PARTITION_WEIGHT = 320L; + private static final long PARTITION_ITEM_WEIGHT = 768L; + + private PaimonCacheSizeEstimator() { + } + + /** + * Retained weight of the base table entry. The table handle is owned independently of the + * snapshot projections that reference it (they may pin an older generation), so the same + * table graph is charged to both owners rather than shared. + */ + static MetaCacheSizeEstimate estimateTableEntry(NameMapping key, PaimonTableCacheValue value) { + String unsupported = unsupportedReason(value.getPaimonTable()); + if (unsupported != null) { + return MetaCacheSizeEstimate.incomplete(unsupported); + } + long bytes = MetaCacheWeightUtils.saturatedAdd( + KEY_BASE_WEIGHT, MetaCacheWeightUtils.estimatedNameMappingBytes(key)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, TABLE_VALUE_BASE_WEIGHT); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, value.getRetainedTablePayloadBytes()); + return MetaCacheSizeEstimate.complete( + MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(value.getPaimonTable()))); + } + + static MetaCacheSizeEstimate estimateSnapshotEntry( + PaimonSnapshotEntryKey key, PaimonSnapshotCacheValue value) { + Table table = value.getSnapshot().getTable(); + String unsupported = unsupportedReason(table); + if (unsupported != null) { + return MetaCacheSizeEstimate.incomplete(unsupported); + } + long bytes = MetaCacheWeightUtils.saturatedAdd( + KEY_BASE_WEIGHT, MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, SNAPSHOT_BASE_WEIGHT); + bytes = addCount(bytes, value.getPartitionInfo().getNameToPartition().size(), PARTITION_WEIGHT); + bytes = addCount(bytes, + value.getPartitionInfo().getNameToPartitionItem().size(), PARTITION_ITEM_WEIGHT); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, value.getPartitionInfo().getRetainedPayloadBytes()); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, value.getRetainedTablePayloadBytes()); + return MetaCacheSizeEstimate.complete( + MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(table))); + } + + private static String unsupportedReason(Table table) { + if (table == null) { + return "unsupported_paimon_table:null"; + } + if (unwrap(table) == null) { + return "unsupported_paimon_table:" + table.getClass().getName(); + } + return null; + } + + /** The concrete FileStoreTable behind any known wrapper chain, or null. */ + private static FileStoreTable unwrap(Table table) { + if (table instanceof DelegatedFileStoreTable) { + return unwrap(((DelegatedFileStoreTable) table).wrapped()); + } + return table instanceof FileStoreTable ? (FileStoreTable) table : null; + } + + /** Uses TableSchema cardinalities only and deliberately never calls FileStoreTable.store(). */ + private static long estimateTable(Table table) { + long bytes = 0L; + Table current = table; + while (true) { + if (current instanceof FallbackReadFileStoreTable) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, WRAPPER_WEIGHT); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, estimateTable(((FallbackReadFileStoreTable) current).other())); + current = ((FallbackReadFileStoreTable) current).wrapped(); + continue; + } + if (current instanceof DelegatedFileStoreTable) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, WRAPPER_WEIGHT); + current = ((DelegatedFileStoreTable) current).wrapped(); + continue; + } + break; + } + if (!(current instanceof FileStoreTable)) { + return bytes; + } + FileStoreTable fileStoreTable = (FileStoreTable) current; + TableSchema schema = fileStoreTable.schema(); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, TABLE_BASE_WEIGHT); Review Comment: [P2] Govern the Paimon metadata caches retained by this handle Paimon 1.4.2 enables `CachingCatalog` by default. Its returned table carries per-table snapshot/statistics caches plus the catalog's shared manifest cache (defaults: 20 snapshots, 5 statistics entries, and 128 MiB of manifests); the direct base handle also receives the shared DV cache, while table copies retain the first three caches. This estimator charges schema/store/FileIO and then reports a complete weight, but `latestSnapshot()`, statistics reads, and scans can populate those caches after admission, outside the Doris entry/catalog/global `max-weight` reservations. Since the manifest owner is shared, please either disable Paimon's overlapping catalog cache while Doris weight governance is enabled, account it once at the catalog owner, or fail closed for handles carrying ungoverned cache state; add a cache-enabled post-admission growth regression. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
