github-actions[bot] commented on code in PR #66717: URL: https://github.com/apache/doris/pull/66717#discussion_r3825158024
########## fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java: ########## @@ -0,0 +1,356 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.DelegatedFileStoreTable; +import org.apache.paimon.table.FallbackReadFileStoreTable; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.RowType; + +import java.util.List; +import java.util.Map; + +/** + * Publication-time approximate weight formulas for Paimon table handles and snapshot projections. + * + * <p>The formulas follow a coarse cardinality model: a rounded-up constant per stable logical + * dimension (schema field, logical type node, option, key, partition, ...) plus the + * skew-sensitive string payload the loader already materialized. The per-node constants absorb + * the lazy state Paimon materializes after admission (the four RowType lookup maps, the store + * graph and its derived RowType copies) instead of modeling those objects individually, so + * weights track metadata size without depending on SDK-private layouts. {@code max-weight} is + * an estimated admission budget, not an exact heap limit. Estimation never opens the table + * store and performs no IO. + */ +final class PaimonCacheSizeEstimator { + // Bounds accounting work per publication; far above real schemas and option maps. + private static final long MAX_TABLE_ACCOUNTING_ELEMENTS = 50_000L; + private static final int MAX_TYPE_ACCOUNTING_DEPTH = 256; + + private static final long KEY_BASE_WEIGHT = 256L; + private static final long SNAPSHOT_BASE_WEIGHT = 4L * 1024L; + private static final long TABLE_BASE_WEIGHT = 16L * 1024L; + // PaimonTableCacheValue and its size-estimate holder. + private static final long TABLE_VALUE_BASE_WEIGHT = 128L; + // One schema field (DataField) at any depth, including its share of the enclosing RowType's + // four lazily built lookup maps, boxed ids and the field copies the lazily created store + // graph derives from it (append-only row copy, trimmed key/value types, merge row type). + private static final long FIELD_NODE_WEIGHT = 576L; + // One bare nested type node (array/map/multiset/vector element types and unknown future + // DataType implementations), including its store-graph copies; charged generically instead + // of disabling weighted caching for unknown types. + private static final long TYPE_NODE_WEIGHT = 128L; + // One nested RowType container: the RowType, its field list, its four lazily built lookup + // maps and the container copies the store graph derives. + private static final long ROW_CONTAINER_WEIGHT = 2048L; + // One schema/table-level entry: option map node, key list slot and boxes. + private static final long OPTION_WEIGHT = 192L; + private static final long KEY_WEIGHT = 192L; + // Wrapper tables (privileged, fallback-read) around the concrete FileStoreTable. + private static final long WRAPPER_WEIGHT = 512L; + private static final long PARTITION_WEIGHT = 320L; + private static final long PARTITION_ITEM_WEIGHT = 768L; + + private PaimonCacheSizeEstimator() { + } + + /** + * Retained weight of the base table entry. The table handle is owned independently of the + * snapshot projections that reference it (they may pin an older generation), so the same + * table graph is charged to both owners rather than shared. + */ + static MetaCacheSizeEstimate estimateTableEntry(NameMapping key, PaimonTableCacheValue value) { + String unsupported = unsupportedReason(value.getPaimonTable()); + if (unsupported != null) { + return MetaCacheSizeEstimate.incomplete(unsupported); + } + long bytes = MetaCacheWeightUtils.saturatedAdd( + KEY_BASE_WEIGHT, MetaCacheWeightUtils.estimatedNameMappingBytes(key)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, TABLE_VALUE_BASE_WEIGHT); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, value.getRetainedTablePayloadBytes()); + return MetaCacheSizeEstimate.complete( + MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(value.getPaimonTable()))); + } + + static MetaCacheSizeEstimate estimateSnapshotEntry( + PaimonSnapshotEntryKey key, PaimonSnapshotCacheValue value) { + Table table = value.getSnapshot().getTable(); + String unsupported = unsupportedReason(table); + if (unsupported != null) { + return MetaCacheSizeEstimate.incomplete(unsupported); + } + long bytes = MetaCacheWeightUtils.saturatedAdd( + KEY_BASE_WEIGHT, MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, SNAPSHOT_BASE_WEIGHT); + bytes = addCount(bytes, value.getPartitionInfo().getNameToPartition().size(), PARTITION_WEIGHT); + bytes = addCount(bytes, + value.getPartitionInfo().getNameToPartitionItem().size(), PARTITION_ITEM_WEIGHT); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, value.getPartitionInfo().getRetainedPayloadBytes()); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, value.getRetainedTablePayloadBytes()); + return MetaCacheSizeEstimate.complete( + MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(table))); + } + + private static String unsupportedReason(Table table) { + if (table == null) { + return "unsupported_paimon_table:null"; + } + if (unwrap(table) == null) { + return "unsupported_paimon_table:" + table.getClass().getName(); + } + return null; + } + + /** The concrete FileStoreTable behind any known wrapper chain, or null. */ + private static FileStoreTable unwrap(Table table) { + if (table instanceof DelegatedFileStoreTable) { + return unwrap(((DelegatedFileStoreTable) table).wrapped()); + } + return table instanceof FileStoreTable ? (FileStoreTable) table : null; + } + + /** Uses TableSchema cardinalities only and deliberately never calls FileStoreTable.store(). */ + private static long estimateTable(Table table) { + long bytes = 0L; + Table current = table; + while (true) { + if (current instanceof FallbackReadFileStoreTable) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, WRAPPER_WEIGHT); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, estimateTable(((FallbackReadFileStoreTable) current).other())); + current = ((FallbackReadFileStoreTable) current).wrapped(); + continue; + } + if (current instanceof DelegatedFileStoreTable) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, WRAPPER_WEIGHT); + current = ((DelegatedFileStoreTable) current).wrapped(); + continue; + } + break; + } + if (!(current instanceof FileStoreTable)) { + return bytes; + } + FileStoreTable fileStoreTable = (FileStoreTable) current; + TableSchema schema = fileStoreTable.schema(); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, TABLE_BASE_WEIGHT); Review Comment: [P2] Account Paimon operational credentials in the cache weight This returns a complete table weight after charging only the fixed base plus name/location/schema/options, but each cached table owner also strongly retains its Paimon `FileIO`. For REST catalogs Doris later reads `RESTTokenFileIO.validToken().token()` from that exact handle in `PaimonVendedCredentialsProvider`; those credential/configuration keys and values can grow or rotate independently of the schema and can even materialize after admission. Both the base-table and snapshot entries can therefore retain payload beyond their entry/catalog/global reservations while still reporting a complete estimate. Please account this exposed FileIO/token graph with bounded, skew-sensitive work (or reject handles whose mutable operational state cannot be bounded), and cover fixed-metadata token growth plus a post-admission refresh. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
