xiangfu0 commented on code in PR #19998:
URL: https://github.com/apache/pinot/pull/19998#discussion_r4364819064
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java:
##########
@@ -101,12 +103,17 @@ private static final class ImmutableState {
private final List<String> _sortedColumns;
private final ColumnMinMaxValueGeneratorMode
_columnMinMaxValueGeneratorMode;
private final boolean _hasOpenStructColumns;
+ private final boolean _mappedSegmentMetadataEnabled;
private ImmutableState(@Nullable InstanceDataManagerConfig
instanceDataManagerConfig,
@Nullable TableConfig tableConfig, @Nullable Schema schema) {
_instanceDataManagerConfig = instanceDataManagerConfig;
_tableConfig = tableConfig;
_schema = schema;
+ PinotConfiguration instanceConfig =
+ instanceDataManagerConfig != null ?
instanceDataManagerConfig.getConfig() : null;
+ _mappedSegmentMetadataEnabled =
+ instanceConfig != null &&
instanceConfig.getProperty(MAPPED_SEGMENT_METADATA_ENABLED, true);
Review Comment:
Fixed in a03ee9d: mapping now defaults off and requires an explicit instance
setting to enable. The PR description makes full-server startup and Linux VMA
headroom checks rollout gates.
##########
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/ColumnMetadata.java:
##########
@@ -57,6 +60,31 @@ default boolean isNonNull() {
/// Returns `true` when the column is auto-generated by the default column
handler.
boolean isAutoGenerated();
+ /// OPEN_STRUCT child and sparse-manifest metadata. Older columns have no
such metadata.
+ default boolean isMaterializedChild() {
Review Comment:
Fixed in a03ee9d: preprocessing now uses the ColumnMetadata interface for
materialized children, including mapped metadata. The mapped OPEN_STRUCT reload
regression passes.
##########
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/SegmentMetadataImpl.java:
##########
@@ -483,21 +585,95 @@ public String getEndOffset() {
@Override
public TreeMap<String, ColumnMetadata> getColumnMetadataMap() {
- return _columnMetadataMap;
+ if (_mappedStore == null) {
+ return _columnMetadataMap;
+ }
+ // Compatibility API for metadata/admin callers. The serving path uses
column names and direct lookup instead.
+ TreeMap<String, ColumnMetadata> result = new TreeMap<>();
+ for (String column : getPhysicalColumnNames()) {
+ result.put(column, getColumnMetadataFor(column));
+ }
+ result.putAll(_virtualColumnMetadata);
+ return result;
+ }
+
+ /// Returns the physical column names without materializing a metadata map
or Schema on mapped segments.
+ public Set<String> getPhysicalColumnNames() {
+ return _mappedStore != null ? _physicalColumnNames :
_schema.getPhysicalColumnNames();
+ }
+
+ /// Indicates whether the immutable segment is reading its metadata from the
binary mapping.
+ public boolean isMapped() {
+ return _mappedStore != null;
+ }
+
+ @Nullable
+ @Override
+ public ColumnMetadata getColumnMetadataFor(String column) {
+ if (_mappedStore == null) {
+ return _columnMetadataMap != null ? _columnMetadataMap.get(column) :
null;
+ }
+ ColumnMetadata virtual = _virtualColumnMetadata.get(column);
+ if (virtual != null) {
+ return virtual;
+ }
+ if (_removedColumns.contains(column)) {
+ return null;
+ }
+ int ordinal = _mappedStore.ordinal(column);
+ if (ordinal < 0) {
+ return null;
+ }
+ MappedColumnMetadata metadata = _mappedColumnMetadata.get(ordinal);
+ if (metadata == null) {
+ metadata = new MappedColumnMetadata(_mappedStore, ordinal);
+ if (!_mappedColumnMetadata.compareAndSet(ordinal, null, metadata)) {
+ metadata = _mappedColumnMetadata.get(ordinal);
+ }
+ }
+ return metadata;
+ }
+
+ /// Adds a virtual column generated during serving load without changing the
on-disk physical-column directory.
+ public void addVirtualColumnMetadata(String column, ColumnMetadata metadata)
{
+ if (_mappedStore == null) {
+ _columnMetadataMap.put(column, metadata);
+ } else {
+ _virtualColumnMetadata.put(column, metadata);
+ }
+ }
+
+ /// Releases the metadata mapping. Managed serving directories close after
readers retire; other callers must
+ /// keep the mapping open while it is in use. Synchronization prevents
duplicate closes, not concurrent reads.
+ public synchronized void close()
Review Comment:
Fixed in a03ee9d: the watermark and snapshot-pass reads acquire the segment
read lock and recheck tracking before reading mapped metadata. Focused tests
cover destroyed segments and the live TTL cap.
##########
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/MappedSegmentMetadataStore.java:
##########
@@ -0,0 +1,889 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.spi.index.metadata;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.Interner;
+import com.google.common.collect.Interners;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.AtomicMoveNotSupportedException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.zip.CRC32C;
+import javax.annotation.Nullable;
+import org.apache.commons.configuration2.Configuration;
+import org.apache.commons.configuration2.PropertiesConfiguration;
+import org.apache.commons.configuration2.ex.ConfigurationException;
+import org.apache.pinot.segment.spi.ColumnMetadata;
+import org.apache.pinot.segment.spi.V1Constants;
+import org.apache.pinot.segment.spi.index.IndexService;
+import org.apache.pinot.segment.spi.index.IndexType;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.segment.spi.partition.PartitionFunction;
+import org.apache.pinot.segment.spi.store.ColumnIndexUtils;
+import org.apache.pinot.segment.spi.store.SegmentDirectoryPaths;
+import org.apache.pinot.segment.spi.utils.SegmentMetadataUtils;
+import org.apache.pinot.spi.config.table.FieldConfig.EncodingType;
+import org.apache.pinot.spi.data.ComplexFieldSpec;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.env.CommonsConfigurationUtils;
+import org.apache.pinot.spi.utils.ByteArray;
+import org.apache.pinot.spi.utils.ColumnNameInterner;
+import org.apache.pinot.spi.utils.JsonUtils;
+
+/// Versioned, file-backed metadata for a sealed segment. The source
`metadata.properties` remains the canonical
+/// representation for old readers and preprocessing.
+///
+/// The big-endian file starts with a 64-byte header containing the magic,
format version, total length, source
+/// `metadata.properties` and `index_map` length/CRC fingerprints, payload
CRC, segment-property blob reference,
+/// column count, and record-table bounds. Sorted columns occupy fixed
180-byte records, followed by variable blobs;
+/// each blob reference is an absolute 32-bit offset and length. Numeric
column statistics are read directly from the
+/// mapping. The column layout retains names and reusable [FieldSpec]
references on heap.
+///
+/// A stale or corrupt sidecar is rebuilt from the canonical source and
replaced locally, using atomic rename where
+/// supported. Change `VERSION` whenever the header, record, or blob encoding
changes; old versions are rebuilt rather
+/// than migrated.
+final class MappedSegmentMetadataStore implements Closeable {
+ static final String FILE_NAME = "segment_metadata.bin";
+ private static final String TEMP_FILE_PREFIX = ".segment_metadata.";
+ private static final String TEMP_FILE_SUFFIX = ".tmp";
+
+ static boolean isSidecarFileName(String fileName) {
+ return FILE_NAME.equals(fileName)
+ || (fileName.startsWith(TEMP_FILE_PREFIX) &&
fileName.endsWith(TEMP_FILE_SUFFIX));
+ }
+
+ private static final int MAGIC = 0x504d4431; // PMD1
+ private static final int VERSION = 3;
Review Comment:
The sidecar is now v4, and its version contract explicitly includes changes
to values derived from canonical metadata. The old-version rebuild and
eager-versus-mapped parity tests pass. This remains a manual versioning
contract; a future parser-semantic change must bump the format version.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
