xiangfu0 commented on code in PR #19998: URL: https://github.com/apache/pinot/pull/19998#discussion_r4396022717
########## pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/MappedSegmentMetadataStore.java: ########## @@ -0,0 +1,943 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.segment.spi.index.metadata; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Interner; +import com.google.common.collect.Interners; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.Closeable; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.IOException; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.zip.CRC32C; +import javax.annotation.Nullable; +import org.apache.commons.configuration2.Configuration; +import org.apache.commons.configuration2.PropertiesConfiguration; +import org.apache.commons.configuration2.ex.ConfigurationException; +import org.apache.pinot.segment.spi.ColumnMetadata; +import org.apache.pinot.segment.spi.V1Constants; +import org.apache.pinot.segment.spi.creator.SegmentVersion; +import org.apache.pinot.segment.spi.index.IndexService; +import org.apache.pinot.segment.spi.index.IndexType; +import org.apache.pinot.segment.spi.memory.PinotDataBuffer; +import org.apache.pinot.segment.spi.partition.PartitionFunction; +import org.apache.pinot.segment.spi.store.ColumnIndexUtils; +import org.apache.pinot.segment.spi.store.SegmentDirectoryPaths; +import org.apache.pinot.segment.spi.utils.SegmentMetadataUtils; +import org.apache.pinot.spi.config.table.FieldConfig.EncodingType; +import org.apache.pinot.spi.data.ComplexFieldSpec; +import org.apache.pinot.spi.data.FieldSpec; +import org.apache.pinot.spi.env.CommonsConfigurationUtils; +import org.apache.pinot.spi.utils.ByteArray; +import org.apache.pinot.spi.utils.ColumnNameInterner; +import org.apache.pinot.spi.utils.JsonUtils; +import org.apache.pinot.spi.utils.ReadMode; + +/// Versioned, file-backed metadata for a sealed segment. The source `metadata.properties` remains the canonical +/// representation for old readers and preprocessing. +/// +/// The big-endian file starts with a 64-byte header containing the magic, format version, total length, source +/// `metadata.properties` and `index_map` length/CRC fingerprints, registered index IDs, payload CRC, +/// segment-property blob reference, +/// column count, and record-table bounds. Sorted columns occupy fixed 180-byte records, followed by variable blobs; +/// each blob reference is an absolute 32-bit offset and length. Numeric column statistics are read directly from the +/// backing buffer. The column layout retains names and reusable [FieldSpec] references on heap. +/// +/// A stale or corrupt sidecar is rebuilt from the canonical source and replaced locally, using atomic rename where +/// supported. Change `VERSION` whenever the encoding or the values derived from canonical metadata change; old +/// versions are rebuilt rather than migrated. +final class MappedSegmentMetadataStore implements Closeable { + static final String FILE_NAME = "segment_metadata.bin"; + private static final String TEMP_FILE_PREFIX = ".segment_metadata."; + private static final String TEMP_FILE_SUFFIX = ".tmp"; + + static boolean isSidecarFileName(String fileName) { + return FILE_NAME.equals(fileName) + || (fileName.startsWith(TEMP_FILE_PREFIX) && fileName.endsWith(TEMP_FILE_SUFFIX)); + } + + private static final int MAGIC = 0x504d4431; // PMD1 + private static final int VERSION = 5; Review Comment: Agreed. The PR is now one initial implementation commit (578a04af15), and the sidecar starts at format version 1. The test injects unsupported version 2 and verifies a rebuild to version 1; the v5/v6 development history is removed from the test, class comment, and PR description. All 15 SegmentMetadataImplTest cases and affected style checks passed locally. Current-head CI is running. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
