924060929 commented on code in PR #67726:
URL: https://github.com/apache/doris/pull/67726#discussion_r3986747756
##########
fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java:
##########
@@ -33,15 +35,48 @@
*/
public final class CatalogMetaCache implements AutoCloseable {
private final ScopedMetaCacheRegistry registry;
+ private final MetaCacheBudgetManager budgetManager;
+ private final long catalogId;
+ private final String engine;
+ private final OptionalLong catalogMaxWeight;
+ private final boolean managed;
private final Set<String> names = ConcurrentHashMap.newKeySet();
+ private final Map<String, MetaCache<?, ?>> entries = new
ConcurrentHashMap<>();
private final AtomicBoolean closed = new AtomicBoolean(false);
public CatalogMetaCache() {
- this(new ScopedMetaCacheRegistry());
+ this(new ScopedMetaCacheRegistry(), new
MetaCacheBudgetManager(OptionalLong.empty()),
Review Comment:
[P1] Please prevent production callers from silently opting out of
governance here. This public no-arg constructor creates a private unbounded
budget manager and marks the owner unmanaged, but `AdbcConnector` and
`MaxComputeDorisConnector` both use it for their schema/partition caches.
Consequently `external_meta_cache_max_weight`, the per-catalog budget, and
unified cache statistics do not cover those caches; if an entry max-weight is
configured later, initialization also fails because those definitions have no
estimator. Make the standalone path explicitly test-only, and migrate both
production connectors to a managed owner with complete estimators.
##########
fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCacheSizeEstimator.java:
##########
@@ -0,0 +1,733 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.connector.iceberg;
+
+import org.apache.doris.connector.cache.ConnectorTableKey;
+import org.apache.doris.connector.cache.JvmSizeUtils;
+import org.apache.doris.connector.cache.MetaCacheSizeEstimate;
+import org.apache.doris.connector.cache.ReflectiveObjectSizeEstimator;
+import
org.apache.doris.connector.iceberg.IcebergPartitionCache.CachedPartitions;
+import org.apache.doris.connector.iceberg.IcebergPartitionCache.Key;
+import
org.apache.doris.connector.iceberg.IcebergPartitionUtils.IcebergRawPartition;
+import org.apache.doris.connector.iceberg.IcebergTableCache.TableOwner;
+import org.apache.doris.connector.spi.ConnectorPartitionInfo;
+import org.apache.doris.connector.spi.mvcc.ConnectorMvccPartition;
+import org.apache.doris.connector.spi.mvcc.ConnectorMvccPartitionView;
+
+import org.apache.iceberg.BlobMetadata;
+import org.apache.iceberg.ContentFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.HistoryEntry;
+import org.apache.iceberg.MetadataUpdate;
+import org.apache.iceberg.PartitionField;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.SortField;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.StatisticsFile;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.UnboundPartitionSpec;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.IdentityHashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/** Type-specific retained-heap estimators for the large Iceberg connector
cache values. */
+final class IcebergCacheSizeEstimator {
+ private static final String[] CONTENT_FILE_FIELD_NAMES = {
+ "content", "file_path", "file_format", "partition",
"record_count", "file_size_in_bytes",
+ "column_sizes", "value_counts", "null_value_counts",
"nan_value_counts", "lower_bounds",
+ "upper_bounds", "key_metadata", "split_offsets", "equality_ids",
"sort_order_id", "first_row_id",
+ "referenced_data_file", "content_offset", "content_size_in_bytes"
+ };
+ private static final long TABLE_IDENTIFIER_SHALLOW_BYTES =
JvmSizeUtils.instanceSize(TableIdentifier.class);
+ private static final long CACHED_TABLE_SHALLOW_BYTES =
JvmSizeUtils.instanceSize(TableOwner.class);
+ private static final long TABLE_METADATA_SHALLOW_BYTES =
JvmSizeUtils.instanceSize(TableMetadata.class);
+ private static final long PARTITION_KEY_SHALLOW_BYTES =
JvmSizeUtils.instanceSize(Key.class);
+ private static final long CACHED_PARTITIONS_SHALLOW_BYTES =
JvmSizeUtils.instanceSize(CachedPartitions.class);
+ private static final long RAW_PARTITION_SHALLOW_BYTES =
JvmSizeUtils.instanceSize(IcebergRawPartition.class);
+ private static final long CONNECTOR_TABLE_KEY_SHALLOW_BYTES =
JvmSizeUtils.instanceSize(ConnectorTableKey.class);
+ private static final long CONNECTOR_PARTITION_SHALLOW_BYTES =
+ JvmSizeUtils.instanceSize(ConnectorPartitionInfo.class);
+ private static final long MVCC_PARTITION_VIEW_SHALLOW_BYTES =
+ JvmSizeUtils.instanceSize(ConnectorMvccPartitionView.class);
+ private static final long MVCC_PARTITION_SHALLOW_BYTES =
+ JvmSizeUtils.instanceSize(ConnectorMvccPartition.class);
+ private static final long MANIFEST_KEY_SHALLOW_BYTES =
JvmSizeUtils.instanceSize(IcebergManifestEntryKey.class);
+ private static final long MANIFEST_VALUE_SHALLOW_BYTES =
JvmSizeUtils.instanceSize(ManifestCacheValue.class);
+ private static final long INTEGER_SHALLOW_BYTES =
JvmSizeUtils.instanceSize(Integer.class);
+ private static final long LONG_SHALLOW_BYTES =
JvmSizeUtils.instanceSize(Long.class);
+ private static final long HASH_MAP_NODE_SHALLOW_BYTES =
classSize("java.util.HashMap$Node");
+ private static final long LINKED_HASH_MAP_ENTRY_SHALLOW_BYTES =
classSize("java.util.LinkedHashMap$Entry");
+ private static final long CONTENT_FILE_SCHEMA_BYTES =
estimateContentFileSchema();
+
+ private IcebergCacheSizeEstimator() {
+ }
+
+ /** Admission callback: the expensive table graph was sized once when
{@link TableOwner} was constructed. */
+ static MetaCacheSizeEstimate estimateTableEntry(TableIdentifier key,
TableOwner value) {
+ return value.sizeEstimate.isComplete()
+ ?
MetaCacheSizeEstimate.complete(add(estimateTableIdentifier(key),
value.sizeEstimate.getBytes()))
+ : value.sizeEstimate;
+ }
+
+ static long estimateTable(Table table) {
+ long bytes = add(CACHED_TABLE_SHALLOW_BYTES,
JvmSizeUtils.instanceSize(table.getClass()));
+ bytes = add(bytes, JvmSizeUtils.stringSize(table.name()));
+ if (!(table instanceof HasTableOperations)) {
+ bytes = add(bytes, JvmSizeUtils.stringSize(table.location()));
+ return add(bytes, estimateStringMap(table.properties()));
+ }
+
+ TableOperations operations = ((HasTableOperations) table).operations();
+ bytes = add(bytes, JvmSizeUtils.instanceSize(operations.getClass()));
+ return add(bytes, estimateTableMetadata(operations.current()));
+ }
+
+ static long estimateSerializedTableMetadata(String metadataJson) {
+ return JvmSizeUtils.stringSize(metadataJson);
+ }
+
+ static long estimatePartitionKey(Key key) {
+ return Math.max(add(PARTITION_KEY_SHALLOW_BYTES,
estimateTableIdentifier(key.id)),
+ ReflectiveObjectSizeEstimator.estimate(key));
+ }
+
+ static long estimatePartitions(List<IcebergRawPartition> partitions) {
+ long bytes = CACHED_PARTITIONS_SHALLOW_BYTES;
+ // CachedPartitions owns an unmodifiable wrapper around an exact-size
ArrayList copy.
+ bytes = add(bytes, JvmSizeUtils.instanceSize(partitions.getClass()));
+ bytes = add(bytes, JvmSizeUtils.arrayListSize(partitions.size()));
+ for (IcebergRawPartition partition : partitions) {
+ bytes = add(bytes, RAW_PARTITION_SHALLOW_BYTES);
+ bytes = add(bytes,
JvmSizeUtils.stringSize(partition.nameForWeight()));
+ bytes = add(bytes,
estimateStringList(partition.columnNamesForWeight()));
+ bytes = add(bytes,
estimateStringList(partition.valuesForWeight()));
+ bytes = add(bytes,
estimateStringList(partition.transformsForWeight()));
+ }
+ return Math.max(bytes,
ReflectiveObjectSizeEstimator.estimate(partitions));
+ }
+
+ /** Admission callback: the partition list was sized once when {@link
CachedPartitions} was constructed. */
+ static MetaCacheSizeEstimate estimatePartitionEntry(Key key,
CachedPartitions value) {
+ return value.sizeEstimate.isComplete()
+ ?
MetaCacheSizeEstimate.complete(add(estimatePartitionKey(key),
value.sizeEstimate.getBytes()))
+ : value.sizeEstimate;
+ }
+
+ static MetaCacheSizeEstimate estimatePartitionInfoViewEntry(
+ ConnectorTableKey key, List<ConnectorPartitionInfo> partitions) {
+ long bytes = estimateConnectorTableKey(key);
+ bytes = add(bytes, estimateListStructure(partitions));
+ int previousArity = -1;
+ long structureBytes = 0L;
+ for (ConnectorPartitionInfo partition : partitions) {
+ // toConnectorPartitions uses one ordered value per map entry and
an empty null-flag list.
+ // Specs may evolve, so reuse the fixed structure only while the
partition arity matches.
+ int arity = partition.getPartitionValues().size();
+ if (arity != previousArity) {
+ structureBytes =
estimateConnectorPartitionStructure(partition);
+ previousArity = arity;
+ }
+ bytes = add(bytes, structureBytes);
+ bytes = add(bytes,
JvmSizeUtils.stringSize(partition.getPartitionName()));
+ for (String column : partition.getPartitionValues().keySet()) {
+ bytes = add(bytes, JvmSizeUtils.stringSize(column));
+ }
+ // The map and ordered list share their value Strings.
+ for (String value : partition.getOrderedPartitionValues()) {
+ bytes = add(bytes, JvmSizeUtils.stringSize(value));
+ }
+ }
+ return MetaCacheSizeEstimate.complete(bytes);
+ }
+
+ static MetaCacheSizeEstimate estimateMvccPartitionViewEntry(
+ ConnectorTableKey key, ConnectorMvccPartitionView view) {
+ long bytes = add(estimateConnectorTableKey(key),
MVCC_PARTITION_VIEW_SHALLOW_BYTES);
+ bytes = add(bytes, JvmSizeUtils.instanceSize(ArrayList.class));
+ bytes = add(bytes, estimateListStructure(view.getPartitions()));
+ for (ConnectorMvccPartition partition : view.getPartitions()) {
+ bytes = add(bytes, estimateMvccPartition(partition));
+ }
+ return MetaCacheSizeEstimate.complete(bytes);
+ }
+
+ private static long estimateMvccPartition(ConnectorMvccPartition
partition) {
+ long bytes = MVCC_PARTITION_SHALLOW_BYTES;
+ bytes = add(bytes, JvmSizeUtils.stringSize(partition.getName()));
+ bytes = add(bytes,
estimateWrappedStringList(partition.getLowerBound()));
+ return add(bytes,
estimateWrappedStringList(partition.getUpperBound()));
+ }
+
+ private static long estimateConnectorTableKey(ConnectorTableKey key) {
+ long bytes = CONNECTOR_TABLE_KEY_SHALLOW_BYTES;
+ bytes = add(bytes, JvmSizeUtils.stringSize(key.getDb()));
+ return add(bytes, JvmSizeUtils.stringSize(key.getTable()));
+ }
+
+ private static long
estimateConnectorPartitionStructure(ConnectorPartitionInfo partition) {
+ long bytes = CONNECTOR_PARTITION_SHALLOW_BYTES;
+ // toConnectorPartitions owns a default-constructed LinkedHashMap
behind the SPI wrapper.
+ bytes = add(bytes,
JvmSizeUtils.instanceSize(partition.getPartitionValues().getClass()));
+ bytes = add(bytes, JvmSizeUtils.instanceSize(LinkedHashMap.class));
+ int valueCount = partition.getPartitionValues().size();
+ if (valueCount > 0) {
+ bytes = add(bytes,
JvmSizeUtils.objectArraySize(hashCapacity(valueCount)));
+ bytes = add(bytes, multiply(valueCount,
LINKED_HASH_MAP_ENTRY_SHALLOW_BYTES));
+ }
+ bytes = add(bytes, estimateMapStructure(partition.getProperties()));
+ bytes = add(bytes,
estimateWrappedReferenceList(partition.getOrderedPartitionValues()));
+ return add(bytes,
estimateWrappedReferenceList(partition.getPartitionValueNullFlags()));
+ }
+
+ private static long estimateWrappedStringList(List<String> values) {
+ long bytes = estimateWrappedReferenceList(values);
+ for (String value : values) {
+ bytes = add(bytes, JvmSizeUtils.stringSize(value));
+ }
+ return bytes;
+ }
+
+ private static long estimateWrappedReferenceList(List<?> values) {
+ return add(JvmSizeUtils.instanceSize(ArrayList.class),
estimateListStructure(values));
+ }
+
+ static long estimateManifestKey(IcebergManifestEntryKey key) {
+ return add(MANIFEST_KEY_SHALLOW_BYTES,
JvmSizeUtils.stringSize(key.getManifestPath()));
+ }
+
+ static long estimateManifestValue(ManifestCacheValue value) {
+ long bytes = MANIFEST_VALUE_SHALLOW_BYTES;
+ bytes = add(bytes, estimateContentFileList(value.getDataFiles()));
+ bytes = add(bytes, estimateContentFileList(value.getDeleteFiles()));
+ return Math.max(bytes, ReflectiveObjectSizeEstimator.estimate(value));
+ }
+
+ /** Admission callback: the manifest payload size is precomputed during
construction. */
+ static MetaCacheSizeEstimate estimateManifestEntry(IcebergManifestEntryKey
key, ManifestCacheValue value) {
+ return value.getSizeEstimate().isComplete()
+ ? MetaCacheSizeEstimate.complete(
+ add(estimateManifestKey(key),
value.getSizeEstimate().getBytes()))
+ : value.getSizeEstimate();
+ }
+
+ private static long estimateTableIdentifier(TableIdentifier identifier) {
+ long bytes = TABLE_IDENTIFIER_SHALLOW_BYTES;
+ Namespace namespace = identifier.namespace();
+ bytes = add(bytes, JvmSizeUtils.instanceSize(namespace.getClass()));
+ String[] levels = namespace.levels();
+ bytes = add(bytes, JvmSizeUtils.objectArraySize(levels.length));
+ for (String level : levels) {
+ bytes = add(bytes, JvmSizeUtils.stringSize(level));
+ }
+ return add(bytes, JvmSizeUtils.stringSize(identifier.name()));
+ }
+
+ private static long estimateTableMetadata(TableMetadata metadata) {
+ long bytes = TABLE_METADATA_SHALLOW_BYTES;
+ bytes = add(bytes,
JvmSizeUtils.stringSize(metadata.metadataFileLocation()));
+ bytes = add(bytes, JvmSizeUtils.stringSize(metadata.uuid()));
+ bytes = add(bytes, JvmSizeUtils.stringSize(metadata.location()));
+ bytes = add(bytes, estimateStringMap(metadata.properties()));
+
+ List<Schema> schemas = metadata.schemas();
+ bytes = add(bytes, estimateListStructure(schemas));
+ bytes = add(bytes, estimateIntegerIndexMap(metadata.schemasById()));
+ for (Schema schema : schemas) {
+ bytes = add(bytes, estimateSchema(schema));
+ }
+
+ List<PartitionSpec> specs = metadata.specs();
+ bytes = add(bytes, estimateListStructure(specs));
+ bytes = add(bytes, estimateIntegerIndexMap(metadata.specsById()));
+ for (PartitionSpec spec : specs) {
+ bytes = add(bytes, estimatePartitionSpec(spec));
+ }
+
+ List<SortOrder> sortOrders = metadata.sortOrders();
+ bytes = add(bytes, estimateListStructure(sortOrders));
+ bytes = add(bytes, estimateIntegerIndexMap(metadata.sortOrdersById()));
+ for (SortOrder sortOrder : sortOrders) {
+ bytes = add(bytes, estimateSortOrder(sortOrder));
+ }
+
+ List<Snapshot> snapshots = metadata.snapshots();
+ bytes = add(bytes, estimateListStructure(snapshots));
+ bytes = add(bytes, estimateLongIndexMap(snapshots));
+ List<Snapshot> embeddedManifestSnapshots = new ArrayList<>();
+ for (Snapshot snapshot : snapshots) {
+ if (snapshot.manifestListLocation() == null) {
+ embeddedManifestSnapshots.add(snapshot);
+ } else {
+ bytes = add(bytes, estimateSnapshot(snapshot));
+ }
+ }
+ if (!embeddedManifestSnapshots.isEmpty()) {
+ // Serialization materializes legacy embedded manifests. Traverse
them without sampling, sharing
+ // one visit budget/identity set across snapshots and their shared
Avro schemas. No manifest IO.
+ bytes = add(bytes,
ReflectiveObjectSizeEstimator.estimateComplete(embeddedManifestSnapshots));
+ }
+
+ bytes = add(bytes, estimateHistory(metadata.snapshotLog()));
+ bytes = add(bytes, estimateMetadataLog(metadata.previousFiles()));
+ bytes = add(bytes, estimateSnapshotRefs(metadata.refs()));
+ bytes = add(bytes,
estimateStatisticsFiles(metadata.statisticsFiles()));
+ bytes = add(bytes,
estimatePartitionStatisticsFiles(metadata.partitionStatisticsFiles()));
+ bytes = add(bytes, estimateMetadataUpdates(metadata.changes()));
+ bytes = add(bytes, estimateListStructure(metadata.encryptionKeys()));
+ for (Object key : metadata.encryptionKeys()) {
+ bytes = add(bytes,
ReflectiveObjectSizeEstimator.estimateComplete(key));
+ }
+ // TableMetadata retains a serializable snapshot supplier after the
immutable snapshot list is loaded.
+ bytes = add(bytes, JvmSizeUtils.objectArraySize(1));
+ return Math.max(bytes,
ReflectiveObjectSizeEstimator.estimate(metadata));
+ }
+
+ private static long estimateSchema(Schema schema) {
+ List<Types.NestedField> columns = schema.columns();
+ long bytes = JvmSizeUtils.instanceSize(schema.getClass());
+ bytes = add(bytes,
JvmSizeUtils.instanceSize(schema.asStruct().getClass()));
+ bytes = add(bytes, estimateListStructure(columns));
+ for (Types.NestedField field : columns) {
+ bytes = add(bytes, estimateNestedField(field));
+ }
+ bytes = add(bytes,
JvmSizeUtils.objectArraySize(schema.identifierFieldIds().size()));
+ bytes = add(bytes, estimateMapStructure(schema.getAliases()));
+ return add(bytes, estimateSchemaIndexes(columns.size()));
+ }
+
+ private static long estimateNestedField(Types.NestedField field) {
+ long bytes = JvmSizeUtils.instanceSize(field.getClass());
+ bytes = add(bytes, JvmSizeUtils.stringSize(field.name()));
+ bytes = add(bytes, JvmSizeUtils.stringSize(field.doc()));
+ // Defaults are owned variable-sized literals, not shared schema
infrastructure.
+ if (field.initialDefaultLiteral() != null) {
+ bytes = add(bytes,
ReflectiveObjectSizeEstimator.estimateComplete(field.initialDefaultLiteral()));
+ }
+ if (field.writeDefaultLiteral() != null && field.writeDefaultLiteral()
!= field.initialDefaultLiteral()) {
+ bytes = add(bytes,
ReflectiveObjectSizeEstimator.estimateComplete(field.writeDefaultLiteral()));
+ }
+ return add(bytes, estimateIcebergType(field.type()));
+ }
+
+ private static long estimateIcebergType(Type type) {
+ long bytes = JvmSizeUtils.instanceSize(type.getClass());
+ if (type.isStructType()) {
+ List<Types.NestedField> fields = type.asStructType().fields();
+ bytes = add(bytes, estimateListStructure(fields));
+ for (Types.NestedField field : fields) {
+ bytes = add(bytes, estimateNestedField(field));
+ }
+ } else if (type.isListType()) {
+ bytes = add(bytes,
estimateNestedField(type.asListType().fields().get(0)));
+ } else if (type.isMapType()) {
+ for (Types.NestedField field : type.asMapType().fields()) {
+ bytes = add(bytes, estimateNestedField(field));
+ }
+ }
+ return bytes;
+ }
+
+ private static long estimatePartitionSpec(PartitionSpec spec) {
+ List<PartitionField> fields = spec.fields();
+ long bytes = add(JvmSizeUtils.instanceSize(spec.getClass()),
JvmSizeUtils.objectArraySize(fields.size()));
+ for (PartitionField field : fields) {
+ bytes = add(bytes, JvmSizeUtils.instanceSize(field.getClass()));
+ bytes = add(bytes, JvmSizeUtils.stringSize(field.name()));
+ bytes = add(bytes,
JvmSizeUtils.instanceSize(field.transform().getClass()));
+ }
+ return bytes;
+ }
+
+ private static long estimateSortOrder(SortOrder sortOrder) {
+ List<SortField> fields = sortOrder.fields();
+ long bytes = add(JvmSizeUtils.instanceSize(sortOrder.getClass()),
JvmSizeUtils.objectArraySize(fields.size()));
+ for (SortField field : fields) {
+ bytes = add(bytes, JvmSizeUtils.instanceSize(field.getClass()));
+ bytes = add(bytes,
JvmSizeUtils.instanceSize(field.transform().getClass()));
+ }
+ return bytes;
+ }
+
+ private static long estimateSnapshot(Snapshot snapshot) {
+ long bytes = JvmSizeUtils.instanceSize(snapshot.getClass());
+ bytes = add(bytes, estimateBoxed(snapshot.parentId(),
LONG_SHALLOW_BYTES));
+ bytes = add(bytes, estimateBoxed(snapshot.schemaId(),
INTEGER_SHALLOW_BYTES));
+ bytes = add(bytes, estimateBoxed(snapshot.firstRowId(),
LONG_SHALLOW_BYTES));
+ bytes = add(bytes, estimateBoxed(snapshot.addedRows(),
LONG_SHALLOW_BYTES));
+ bytes = add(bytes, JvmSizeUtils.stringSize(snapshot.operation()));
+ bytes = add(bytes,
JvmSizeUtils.stringSize(snapshot.manifestListLocation()));
+ bytes = add(bytes, JvmSizeUtils.stringSize(snapshot.keyId()));
+ return add(bytes, estimateStringMap(snapshot.summary()));
+ }
+
+ private static long estimateHistory(List<HistoryEntry> history) {
+ long bytes = estimateListStructure(history);
+ for (HistoryEntry entry : history) {
+ bytes = add(bytes, JvmSizeUtils.instanceSize(entry.getClass()));
+ }
+ return bytes;
+ }
+
+ private static long
estimateMetadataLog(List<TableMetadata.MetadataLogEntry> entries) {
+ long bytes = estimateListStructure(entries);
+ for (TableMetadata.MetadataLogEntry entry : entries) {
+ bytes = add(bytes, JvmSizeUtils.instanceSize(entry.getClass()));
+ bytes = add(bytes, JvmSizeUtils.stringSize(entry.file()));
+ }
+ return bytes;
+ }
+
+ private static long estimateSnapshotRefs(Map<String, SnapshotRef> refs) {
+ long bytes = estimateMapStructure(refs);
+ for (Map.Entry<String, SnapshotRef> entry : refs.entrySet()) {
+ bytes = add(bytes, JvmSizeUtils.stringSize(entry.getKey()));
+ SnapshotRef ref = entry.getValue();
+ bytes = add(bytes, JvmSizeUtils.instanceSize(ref.getClass()));
+ bytes = add(bytes, estimateBoxed(ref.minSnapshotsToKeep(),
INTEGER_SHALLOW_BYTES));
+ bytes = add(bytes, estimateBoxed(ref.maxSnapshotAgeMs(),
LONG_SHALLOW_BYTES));
+ bytes = add(bytes, estimateBoxed(ref.maxRefAgeMs(),
LONG_SHALLOW_BYTES));
+ }
+ return bytes;
+ }
+
+ private static long estimateStatisticsFiles(List<StatisticsFile> files) {
+ long bytes = estimateListStructure(files);
+ for (StatisticsFile file : files) {
+ bytes = add(bytes, JvmSizeUtils.instanceSize(file.getClass()));
+ bytes = add(bytes, JvmSizeUtils.stringSize(file.path()));
+ List<BlobMetadata> blobs = file.blobMetadata();
+ bytes = add(bytes, estimateListStructure(blobs));
+ for (BlobMetadata blob : blobs) {
+ bytes = add(bytes, JvmSizeUtils.instanceSize(blob.getClass()));
+ bytes = add(bytes, JvmSizeUtils.stringSize(blob.type()));
+ bytes = add(bytes, estimateBoxedList(blob.fields(),
INTEGER_SHALLOW_BYTES));
+ bytes = add(bytes, estimateStringMap(blob.properties()));
+ }
+ }
+ return bytes;
+ }
+
+ private static long
estimatePartitionStatisticsFiles(List<PartitionStatisticsFile> files) {
+ long bytes = estimateListStructure(files);
+ for (PartitionStatisticsFile file : files) {
+ bytes = add(bytes, JvmSizeUtils.instanceSize(file.getClass()));
+ bytes = add(bytes, JvmSizeUtils.stringSize(file.path()));
+ }
+ return bytes;
+ }
+
+ private static long estimateMetadataUpdates(List<MetadataUpdate> updates) {
+ long bytes = estimateListStructure(updates);
+ for (MetadataUpdate update : updates) {
+ bytes = add(bytes, JvmSizeUtils.instanceSize(update.getClass()));
+ if (update instanceof MetadataUpdate.SetProperties) {
+ bytes = add(bytes,
estimateMapStructure(((MetadataUpdate.SetProperties) update).updated()));
+ } else if (update instanceof MetadataUpdate.AddPartitionSpec) {
+ UnboundPartitionSpec spec = ((MetadataUpdate.AddPartitionSpec)
update).spec();
+ bytes = add(bytes, JvmSizeUtils.instanceSize(spec.getClass()));
+ bytes = add(bytes, estimateListStructure(spec.fields()));
+ } else if (update instanceof MetadataUpdate.AddSortOrder) {
+ bytes = add(bytes, JvmSizeUtils.instanceSize(
+ ((MetadataUpdate.AddSortOrder)
update).sortOrder().getClass()));
+ }
+ }
+ return bytes;
+ }
+
+ private static long estimateContentFileList(List<? extends ContentFile<?>>
files) {
+ if (files.isEmpty()) {
+ return 0L;
+ }
+ long bytes = add(estimateListStructure(files),
CONTENT_FILE_SCHEMA_BYTES);
+ Set<Object> ownedObjects = java.util.Collections.newSetFromMap(new
IdentityHashMap<>());
+ for (ContentFile<?> file : files) {
+ bytes = add(bytes, estimateContentFile(file, ownedObjects));
+ }
+ return bytes;
+ }
+
+ private static long estimateContentFileSchema() {
+ long bytes = JvmSizeUtils.instanceSize(Types.StructType.class);
+ bytes = add(bytes,
JvmSizeUtils.arrayListSize(CONTENT_FILE_FIELD_NAMES.length));
+ for (String name : CONTENT_FILE_FIELD_NAMES) {
+ bytes = add(bytes,
JvmSizeUtils.instanceSize(Types.NestedField.class));
+ bytes = add(bytes, JvmSizeUtils.stringSize(name));
+ }
+ return bytes;
+ }
+
+ private static long estimateContentFile(ContentFile<?> file, Set<Object>
ownedObjects) {
+ long bytes = JvmSizeUtils.instanceSize(file.getClass());
+ if (file instanceof StructLike) {
+ bytes = add(bytes, JvmSizeUtils.intArraySize(((StructLike)
file).size()));
+ bytes = add(bytes, LONG_SHALLOW_BYTES);
+ }
+ bytes = add(bytes, estimateOwnedCharSequence(file.path(),
ownedObjects));
+ bytes = add(bytes, estimateOwnedString(file.manifestLocation(),
ownedObjects));
+ bytes = add(bytes, estimatePartition(file.partition(), ownedObjects));
+ bytes = add(bytes, estimateLongMap(file.columnSizes()));
+ bytes = add(bytes, estimateLongMap(file.valueCounts()));
+ bytes = add(bytes, estimateLongMap(file.nullValueCounts()));
+ bytes = add(bytes, estimateLongMap(file.nanValueCounts()));
+ bytes = add(bytes, estimateByteBufferMap(file.lowerBounds()));
+ bytes = add(bytes, estimateByteBufferMap(file.upperBounds()));
+ ByteBuffer keyMetadata = file.keyMetadata();
+ if (keyMetadata != null) {
+ bytes = add(bytes,
JvmSizeUtils.byteArraySize(keyMetadata.remaining()));
+ }
+ List<Long> splitOffsets = file.splitOffsets();
+ if (splitOffsets != null) {
+ bytes = add(bytes,
JvmSizeUtils.longArraySize(splitOffsets.size()));
+ }
+ List<Integer> equalityFieldIds = file.equalityFieldIds();
+ if (equalityFieldIds != null) {
+ bytes = add(bytes,
JvmSizeUtils.intArraySize(equalityFieldIds.size()));
+ }
+ bytes = add(bytes, estimateBoxed(file.pos(), LONG_SHALLOW_BYTES));
+ bytes = add(bytes, estimateBoxed(file.sortOrderId(),
INTEGER_SHALLOW_BYTES));
+ bytes = add(bytes, estimateBoxed(file.dataSequenceNumber(),
LONG_SHALLOW_BYTES));
+ bytes = add(bytes, estimateBoxed(file.fileSequenceNumber(),
LONG_SHALLOW_BYTES));
+ bytes = add(bytes, estimateBoxed(file.firstRowId(),
LONG_SHALLOW_BYTES));
+ if (file instanceof DeleteFile) {
+ DeleteFile deleteFile = (DeleteFile) file;
+ bytes = add(bytes,
estimateOwnedString(deleteFile.referencedDataFile(), ownedObjects));
+ bytes = add(bytes, estimateBoxed(deleteFile.contentOffset(),
LONG_SHALLOW_BYTES));
+ bytes = add(bytes, estimateBoxed(deleteFile.contentSizeInBytes(),
LONG_SHALLOW_BYTES));
+ }
+ return bytes;
+ }
+
+ private static long estimatePartition(StructLike partition, Set<Object>
ownedObjects) {
+ if (partition == null || !ownedObjects.add(partition)) {
+ return 0L;
+ }
+ long bytes = JvmSizeUtils.instanceSize(partition.getClass());
+ bytes = add(bytes, JvmSizeUtils.objectArraySize(partition.size()));
+ for (int i = 0; i < partition.size(); i++) {
+ bytes = add(bytes, estimateOwnedScalar(partition.get(i,
Object.class), ownedObjects));
+ }
+ return bytes;
+ }
+
+ private static long estimateOwnedScalar(Object value, Set<Object>
ownedObjects) {
+ if (value == null || !ownedObjects.add(value)) {
+ return 0L;
+ }
+ if (value instanceof CharSequence) {
+ return estimateCharSequence((CharSequence) value);
+ }
+ if (value instanceof ByteBuffer) {
+ return estimateByteBuffer((ByteBuffer) value);
+ }
+ return JvmSizeUtils.instanceSize(value.getClass());
+ }
+
+ private static long estimateOwnedCharSequence(CharSequence value,
Set<Object> ownedObjects) {
+ return value == null || !ownedObjects.add(value) ? 0L :
estimateCharSequence(value);
+ }
+
+ private static long estimateOwnedString(String value, Set<? super String>
ownedStrings) {
+ return value == null || !ownedStrings.add(value) ? 0L :
JvmSizeUtils.stringSize(value);
+ }
+
+ private static long estimateCharSequence(CharSequence value) {
+ if (value instanceof String) {
+ return JvmSizeUtils.stringSize((String) value);
+ }
+ return add(JvmSizeUtils.instanceSize(value.getClass()),
JvmSizeUtils.stringSize(value.toString()));
+ }
+
+ private static long estimateLongMap(Map<Integer, Long> values) {
+ if (values == null || values.isEmpty()) {
+ return 0L;
+ }
+ return add(estimateMapStructure(values), multiply(values.size(),
INTEGER_SHALLOW_BYTES + LONG_SHALLOW_BYTES));
+ }
+
+ private static long estimateByteBufferMap(Map<Integer, ByteBuffer> values)
{
+ if (values == null || values.isEmpty()) {
+ return 0L;
+ }
+ long bytes = add(estimateMapStructure(values), multiply(values.size(),
INTEGER_SHALLOW_BYTES));
+ for (ByteBuffer value : values.values()) {
+ bytes = add(bytes, estimateByteBuffer(value));
+ }
+ return bytes;
+ }
+
+ private static long estimateByteBuffer(ByteBuffer value) {
+ if (value == null) {
+ return 0L;
+ }
+ long bytes = JvmSizeUtils.instanceSize(value.getClass());
+ return value.hasArray() ? add(bytes,
JvmSizeUtils.byteArraySize(value.capacity())) : bytes;
Review Comment:
[P2] `hasArray() == false` does not mean the buffer retains no heap payload.
For example, `ByteBuffer.wrap(new byte[1048576]).asReadOnlyBuffer()` reports
`hasArray() == false` but still retains the 1 MiB array; JOL measures about 56
bytes shallow versus 1,048,648 bytes for the retained graph. This estimator and
the reflective fallback therefore mark a potentially unbounded underestimate as
complete. Please count the retained payload conservatively or return an
incomplete estimate, and add a read-only heap-buffer test.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
