richardstartin commented on a change in pull request #7595: URL: https://github.com/apache/pinot/pull/7595#discussion_r732255437
########## File path: pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueVarByteRawIndexCreator.java ########## @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.segment.local.segment.creator.impl.fwd; + +import com.google.common.annotations.VisibleForTesting; +import java.io.File; +import java.io.IOException; +import org.apache.pinot.segment.local.io.writer.impl.BaseChunkSVForwardIndexWriter; +import org.apache.pinot.segment.local.io.writer.impl.VarByteChunkSVForwardIndexWriter; +import org.apache.pinot.segment.spi.V1Constants.Indexes; +import org.apache.pinot.segment.spi.compression.ChunkCompressionType; +import org.apache.pinot.segment.spi.index.creator.ForwardIndexCreator; +import org.apache.pinot.spi.data.FieldSpec.DataType; + + +/** + * Forward index creator for raw (non-dictionary-encoded) single-value column of variable length + * data type (STRING, + * BYTES). + */ +public class MultiValueVarByteRawIndexCreator implements ForwardIndexCreator { + + private static final int DEFAULT_NUM_DOCS_PER_CHUNK = 1000; + private static final int TARGET_MAX_CHUNK_SIZE = 1024 * 1024; + + private final VarByteChunkSVForwardIndexWriter _indexWriter; + private final DataType _valueType; + + /** + * Create a var-byte raw index creator for the given column + * + * @param baseIndexDir Index directory + * @param compressionType Type of compression to use + * @param column Name of column to index + * @param totalDocs Total number of documents to index + * @param valueType Type of the values + * @param maxTotalContentLength max total content length + * @param maxElements max number of elements + */ + public MultiValueVarByteRawIndexCreator(File baseIndexDir, ChunkCompressionType compressionType, + String column, + int totalDocs, DataType valueType, int maxTotalContentLength, int maxElements) + throws IOException { + this(baseIndexDir, compressionType, column, totalDocs, valueType, maxTotalContentLength, + maxElements, false, BaseChunkSVForwardIndexWriter.DEFAULT_VERSION); + } + + /** + * Create a var-byte raw index creator for the given column + * + * @param baseIndexDir Index directory + * @param compressionType Type of compression to use + * @param column Name of column to index + * @param totalDocs Total number of documents to index + * @param valueType Type of the values + * @param maxLength max length for each entry + * @param maxElements max number of elements + * @param deriveNumDocsPerChunk true if writer should auto-derive the number of rows per + * chunk + * @param writerVersion writer format version + */ + public MultiValueVarByteRawIndexCreator(File baseIndexDir, ChunkCompressionType compressionType, + String column, int totalDocs, DataType valueType, + int maxLength, int maxElements, boolean deriveNumDocsPerChunk, int writerVersion) + throws IOException { + //we will prepend the actual content with numElements and length array containing length of each element + int totalMaxLength = Integer.BYTES + maxElements * Integer.BYTES + maxLength * maxElements; Review comment: This is currently a dangerous overestimate, we need a way to use sublinear memory for the body before we can merge this. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
