This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new f8d432013b [Index SPI] IndexType (#10191)
f8d432013b is described below
commit f8d432013bc5672fd04088106bf2efd69a0d48a9
Author: Gonzalo Ortiz Jaureguizar <[email protected]>
AuthorDate: Mon Mar 13 19:38:25 2023 +0100
[Index SPI] IndexType (#10191)
---
.../segment/index/loader/BaseIndexHandler.java | 1 +
.../segment/index/loader/IndexHandlerFactory.java | 1 +
...IndexAndDictionaryBasedForwardIndexCreator.java | 1 +
.../segment/index/loader/SegmentPreProcessor.java | 1 +
.../pinot/segment/spi/index/FieldIndexConfigs.java | 119 ++++++++++++++++++++
.../pinot/segment/spi/index/IndexCreator.java | 65 +++++++++++
.../pinot/segment/spi/index}/IndexHandler.java | 2 +-
.../BloomFilterReader.java => IndexReader.java} | 20 +---
.../spi/index/IndexReaderConstraintException.java | 47 ++++++++
...omFilterReader.java => IndexReaderFactory.java} | 26 ++---
.../apache/pinot/segment/spi/index/IndexType.java | 121 +++++++++++++++++++++
.../spi/index/reader/BloomFilterReader.java | 4 +-
.../segment/spi/index/FieldIndexConfigsTest.java} | 38 ++++---
.../apache/pinot/spi/config/table/IndexConfig.java | 34 +++---
14 files changed, 416 insertions(+), 64 deletions(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/BaseIndexHandler.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/BaseIndexHandler.java
index b4cbc7bcb8..f0e43304d0 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/BaseIndexHandler.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/BaseIndexHandler.java
@@ -24,6 +24,7 @@ import java.util.HashSet;
import java.util.Set;
import org.apache.pinot.segment.spi.ColumnMetadata;
import org.apache.pinot.segment.spi.creator.IndexCreatorProvider;
+import org.apache.pinot.segment.spi.index.IndexHandler;
import org.apache.pinot.segment.spi.store.ColumnIndexType;
import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.slf4j.Logger;
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexHandlerFactory.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexHandlerFactory.java
index 0f64475bd8..fef6b3c9f2 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexHandlerFactory.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexHandlerFactory.java
@@ -26,6 +26,7 @@ import
org.apache.pinot.segment.local.segment.index.loader.invertedindex.JsonInd
import
org.apache.pinot.segment.local.segment.index.loader.invertedindex.RangeIndexHandler;
import
org.apache.pinot.segment.local.segment.index.loader.invertedindex.TextIndexHandler;
import org.apache.pinot.segment.spi.creator.IndexCreatorProvider;
+import org.apache.pinot.segment.spi.index.IndexHandler;
import org.apache.pinot.segment.spi.store.ColumnIndexType;
import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.spi.data.Schema;
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/InvertedIndexAndDictionaryBasedForwardIndexCreator.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/InvertedIndexAndDictionaryBasedForwardIndexCreator.java
index 9080d3d56d..ea77da6a9f 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/InvertedIndexAndDictionaryBasedForwardIndexCreator.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/InvertedIndexAndDictionaryBasedForwardIndexCreator.java
@@ -32,6 +32,7 @@ import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
import org.apache.pinot.segment.spi.creator.IndexCreationContext;
import org.apache.pinot.segment.spi.creator.IndexCreatorProvider;
+import org.apache.pinot.segment.spi.index.IndexHandler;
import org.apache.pinot.segment.spi.index.creator.ForwardIndexCreator;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessor.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessor.java
index 85a86baf58..d4a496d707 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessor.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessor.java
@@ -34,6 +34,7 @@ import
org.apache.pinot.segment.local.startree.v2.builder.MultipleTreesBuilder;
import
org.apache.pinot.segment.local.startree.v2.builder.StarTreeV2BuilderConfig;
import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.creator.IndexCreatorProvider;
+import org.apache.pinot.segment.spi.index.IndexHandler;
import org.apache.pinot.segment.spi.index.IndexingOverrides;
import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2Metadata;
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/FieldIndexConfigs.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/FieldIndexConfigs.java
new file mode 100644
index 0000000000..3d192aa6a1
--- /dev/null
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/FieldIndexConfigs.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.segment.spi.index;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import org.apache.pinot.spi.config.table.IndexConfig;
+import org.apache.pinot.spi.utils.JsonUtils;
+
+
+/**
+ * FieldIndexConfigs are a map like structure that relates index types with
their configuration, providing a type safe
+ * interface.
+ */
+public class FieldIndexConfigs {
+
+ public static final FieldIndexConfigs EMPTY = new FieldIndexConfigs(new
HashMap<>());
+
+ private final Map<IndexType, IndexConfig> _configMap;
+
+ private FieldIndexConfigs(Map<IndexType, IndexConfig> configMap) {
+ _configMap = Collections.unmodifiableMap(configMap);
+ }
+
+ /**
+ * Returns the configuration associated with the given index type, which
will be null if there is no configuration for
+ * that index type.
+ */
+ public <C extends IndexConfig, I extends IndexType<C, ?, ?>> C getConfig(I
indexType) {
+ IndexConfig config = _configMap.get(indexType);
+ if (config == null) {
+ return indexType.getDefaultConfig();
+ }
+ return (C) config;
+ }
+
+ public Map<String, JsonNode> unwrapIndexes() {
+ Function<Map.Entry<IndexType, IndexConfig>, JsonNode> serializer =
+ entry -> entry.getValue().toJsonNode();
+ return _configMap.entrySet().stream()
+ .filter(e -> e.getValue() != null)
+ .collect(Collectors.toMap(entry -> entry.getKey().getId(),
serializer));
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return JsonUtils.objectToString(unwrapIndexes());
+ } catch (JsonProcessingException e) {
+ return "Unserializable value due to " + e.getMessage();
+ }
+ }
+
+ public static class Builder {
+ private final Map<IndexType, IndexConfig> _configMap;
+
+ public Builder() {
+ _configMap = new HashMap<>();
+ }
+
+ public Builder(FieldIndexConfigs other) {
+ _configMap = new HashMap<>(other._configMap);
+ }
+
+ public <C extends IndexConfig, I extends IndexType<C, ?, ?>> Builder add(I
indexType, C config) {
+ _configMap.put(indexType, config);
+ return this;
+ }
+
+ public Builder addUnsafe(IndexType<?, ?, ?> indexType, IndexConfig config)
{
+ _configMap.put(indexType, config);
+ return this;
+ }
+
+ public Builder undeclare(IndexType<?, ?, ?> indexType) {
+ _configMap.remove(indexType);
+ return this;
+ }
+
+ public FieldIndexConfigs build() {
+ return new FieldIndexConfigs(_configMap);
+ }
+ }
+
+ public static class UnrecognizedIndexException extends RuntimeException {
+ private final String _indexId;
+
+ public UnrecognizedIndexException(String indexId) {
+ super("There is no index type whose identified as " + indexId);
+ _indexId = indexId;
+ }
+
+ public String getIndexId() {
+ return _indexId;
+ }
+ }
+}
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexCreator.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexCreator.java
new file mode 100644
index 0000000000..3e2389f388
--- /dev/null
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexCreator.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.segment.spi.index;
+
+import java.io.Closeable;
+import java.io.IOException;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+
+/**
+ * The interface used to create indexes.
+ *
+ * The lifecycle for an IndexCreator is:
+ * <ol>
+ * <li>To be created.</li>
+ * <li>Zero or more calls to either {@link #add(Object, int)} or {@link
#add(Object[], int[])} (but not mix them).
+ * Calls to add methods must be done in document id order, starting from the
first document id.</li>
+ * <li>A call to {@link #seal()}</li>
+ * <li>A call to {@link #close()}</li>
+ * </ol>
+ */
+public interface IndexCreator extends Closeable {
+ /**
+ * Adds the given single value cell to the index.
+ *
+ * Rows will be added in docId order, starting with the one with docId 0.
+ *
+ * @param value The nonnull value of the cell. In case the cell was actually
null, a default value is received instead
+ * @param dictId An optional dictionary value of the cell. If there is no
dictionary, -1 is received
+ */
+ void add(@Nonnull Object value, int dictId)
+ throws IOException;
+
+ /**
+ * Adds the given multi value cell to the index
+ *
+ * Rows will be added in docId order, starting with the one with docId 0.
+ *
+ * @param values The nonnull value of the cell. In case the cell was
actually null, an empty array is received instead
+ * @param dictIds An optional array of dictionary values. If there is no
dictionary, null is received.
+ */
+ void add(@Nonnull Object[] values, @Nullable int[] dictIds)
+ throws IOException;
+
+ void seal()
+ throws IOException;
+}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexHandler.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexHandler.java
similarity index 96%
rename from
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexHandler.java
rename to
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexHandler.java
index 2d65fc5b84..fd7b05a975 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexHandler.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexHandler.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.local.segment.index.loader;
+package org.apache.pinot.segment.spi.index;
import org.apache.pinot.segment.spi.creator.IndexCreatorProvider;
import org.apache.pinot.segment.spi.store.SegmentDirectory;
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/BloomFilterReader.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexReader.java
similarity index 58%
copy from
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/BloomFilterReader.java
copy to
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexReader.java
index dccc18eb85..967c5e2b6f 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/BloomFilterReader.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexReader.java
@@ -16,25 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.spi.index.reader;
-
-import java.io.Closeable;
+package org.apache.pinot.segment.spi.index;
-/**
- * Interface for bloom filter reader.
- */
-public interface BloomFilterReader extends Closeable {
+import java.io.Closeable;
- /**
- * Returns {@code true} if the given value might have been put in this bloom
filer, {@code false} otherwise.
- */
- boolean mightContain(String value);
- /**
- * Returns {@code true} if the value with the given hash might have been put
in this bloom filer, {@code false}
- * otherwise.
- * <p>This method is provided to prevent hashing the same value multiple
times.
- */
- boolean mightContain(long hash1, long hash2);
+public interface IndexReader extends Closeable {
}
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexReaderConstraintException.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexReaderConstraintException.java
new file mode 100644
index 0000000000..105129389e
--- /dev/null
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexReaderConstraintException.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.segment.spi.index;
+
+public class IndexReaderConstraintException extends Exception {
+ public IndexReaderConstraintException() {
+ }
+
+ public IndexReaderConstraintException(String message) {
+ super(message);
+ }
+
+ public IndexReaderConstraintException(String columnName, IndexType<?, ?, ?>
type, String constraintDesc,
+ Throwable cause) {
+ this("Cannot read an index of type " + type + " on column " + columnName +
". Reason: " + constraintDesc,
+ cause);
+ }
+
+ public IndexReaderConstraintException(String columnName, IndexType<?, ?, ?>
type, String constraintDesc) {
+ this(columnName, type, constraintDesc, null);
+ }
+
+ public IndexReaderConstraintException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public IndexReaderConstraintException(Throwable cause) {
+ super(cause);
+ }
+}
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/BloomFilterReader.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexReaderFactory.java
similarity index 55%
copy from
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/BloomFilterReader.java
copy to
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexReaderFactory.java
index dccc18eb85..02be852640 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/BloomFilterReader.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexReaderFactory.java
@@ -16,25 +16,23 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.spi.index.reader;
-import java.io.Closeable;
+package org.apache.pinot.segment.spi.index;
+import java.io.IOException;
+import javax.annotation.Nullable;
+import org.apache.pinot.segment.spi.ColumnMetadata;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
-/**
- * Interface for bloom filter reader.
- */
-public interface BloomFilterReader extends Closeable {
- /**
- * Returns {@code true} if the given value might have been put in this bloom
filer, {@code false} otherwise.
- */
- boolean mightContain(String value);
+public interface IndexReaderFactory<R extends IndexReader> {
/**
- * Returns {@code true} if the value with the given hash might have been put
in this bloom filer, {@code false}
- * otherwise.
- * <p>This method is provided to prevent hashing the same value multiple
times.
+ * @throws IndexReaderConstraintException if the constraints of the index
reader are not matched. For example, some
+ * indexes may require the column to be dictionary based.
*/
- boolean mightContain(long hash1, long hash2);
+ @Nullable
+ R createIndexReader(SegmentDirectory.Reader segmentReader, FieldIndexConfigs
fieldIndexConfigs,
+ ColumnMetadata metadata)
+ throws IOException, IndexReaderConstraintException;
}
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexType.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexType.java
new file mode 100644
index 0000000000..6aeac0b1de
--- /dev/null
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexType.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.spi.index;
+
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.apache.pinot.segment.spi.ColumnMetadata;
+import org.apache.pinot.segment.spi.creator.IndexCreationContext;
+import org.apache.pinot.segment.spi.index.column.ColumnIndexContainer;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
+import org.apache.pinot.spi.config.table.IndexConfig;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.data.Schema;
+
+
+/**
+ * TODO: implement mutable indexes.
+ * @param <C> the class that represents how this object is configured.
+ * @param <IR> the {@link IndexReader} subclass that should be used to read
indexes of this type.
+ * @param <IC> the {@link IndexCreator} subclass that should be used to create
indexes of this type.
+ */
+public interface IndexType<C extends IndexConfig, IR extends IndexReader, IC
extends IndexCreator> {
+
+ /**
+ * The unique id that identifies this index type.
+ * <p>The returned value for each index should be constant across different
Pinot versions as it is used as:</p>
+ *
+ * <ul>
+ * <li>The key used when the index is registered in IndexService.</li>
+ * <li>The internal identification in v1 files and metadata persisted on
disk.</li>
+ * <li>The default toString implementation.</li>
+ * <li>The key that identifies the index config in the indexes section
inside
+ * {@link org.apache.pinot.spi.config.table.FieldConfig}, although
specific index types may choose to read other
+ * names (for example, <code>inverted_index</code> may read
<code>inverted</code> key.</li>
+ * </ul>
+ */
+ String getId();
+
+ Class<C> getIndexConfigClass();
+
+ /**
+ * The default config when it is not explicitly defined by the user.
+ */
+ C getDefaultConfig();
+
+ C getConfig(TableConfig tableConfig, Schema schema);
+
+ /**
+ * Optional method that can be implemented to ignore the index creation.
+ *
+ * Sometimes it doesn't make sense to create an index, even when the user
explicitly asked for it. For example, an
+ * inverted index shouldn't be created when the column is sorted.
+ *
+ * Apache Pinot will call this method once all index configurations have
been parsed and it is included in the
+ * {@link FieldIndexConfigs} param.
+ *
+ * This method do not need to return false when the index type itself is not
included in the {@link FieldIndexConfigs}
+ * param.
+ */
+ default boolean shouldBeCreated(IndexCreationContext context,
FieldIndexConfigs configs) {
+ return true;
+ }
+
+ /**
+ * Returns the {@link IndexCreator} that can should be used to create an
index of this type with the given context
+ * and configuration.
+ *
+ * The caller has the ownership of the creator and therefore it has to close
it.
+ * @param context The object that stores all the contextual information
related to the index creation. Like the
+ * cardinality or the total number of documents.
+ * @param indexConfig The index specific configuration that should be used.
+ */
+ IC createIndexCreator(IndexCreationContext context, C indexConfig)
+ throws Exception;
+
+ /**
+ * Returns the {@link IndexReaderFactory} that should be used to return
readers for this type.
+ */
+ IndexReaderFactory<IR> getReaderFactory();
+
+ /**
+ * This method is used to extract a compatible reader from a given
ColumnIndexContainer.
+ *
+ * Most implementations just return {@link
ColumnIndexContainer#getIndex(IndexType)}, but some may try to reuse other
+ * indexes. For example, InvertedIndexType delegates on the
ForwardIndexReader when it is sorted.
+ */
+ @Nullable
+ default IR getIndexReader(ColumnIndexContainer indexContainer) {
+ throw new UnsupportedOperationException();
+ }
+
+ String getFileExtension(ColumnMetadata columnMetadata);
+
+ /**
+ * Returns whether the index is stored as a buffer or not.
+ *
+ * Most indexes are stored as a buffer, but for example TextIndexType is
stored in a separate lucene file.
+ */
+ default boolean storedAsBuffer() {
+ return true;
+ }
+
+ IndexHandler createIndexHandler(SegmentDirectory segmentDirectory,
Map<String, FieldIndexConfigs> configsByCol,
+ @Nullable Schema schema, @Nullable TableConfig tableConfig);
+}
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/BloomFilterReader.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/BloomFilterReader.java
index dccc18eb85..5b63aa6717 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/BloomFilterReader.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/BloomFilterReader.java
@@ -18,13 +18,13 @@
*/
package org.apache.pinot.segment.spi.index.reader;
-import java.io.Closeable;
+import org.apache.pinot.segment.spi.index.IndexReader;
/**
* Interface for bloom filter reader.
*/
-public interface BloomFilterReader extends Closeable {
+public interface BloomFilterReader extends IndexReader {
/**
* Returns {@code true} if the given value might have been put in this bloom
filer, {@code false} otherwise.
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/BloomFilterReader.java
b/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/index/FieldIndexConfigsTest.java
similarity index 50%
copy from
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/BloomFilterReader.java
copy to
pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/index/FieldIndexConfigsTest.java
index dccc18eb85..804e75a916 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/BloomFilterReader.java
+++
b/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/index/FieldIndexConfigsTest.java
@@ -16,25 +16,31 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.spi.index.reader;
+package org.apache.pinot.segment.spi.index;
-import java.io.Closeable;
+import java.io.IOException;
+import org.apache.pinot.spi.config.table.IndexConfig;
+import org.apache.pinot.spi.utils.JsonUtils;
+import org.mockito.Mockito;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+import static org.testng.Assert.*;
-/**
- * Interface for bloom filter reader.
- */
-public interface BloomFilterReader extends Closeable {
- /**
- * Returns {@code true} if the given value might have been put in this bloom
filer, {@code false} otherwise.
- */
- boolean mightContain(String value);
+public class FieldIndexConfigsTest {
+
+ @Test
+ public void testToString()
+ throws IOException {
+ IndexType index1 = Mockito.mock(IndexType.class);
+ Mockito.when(index1.getId()).thenReturn("index1");
+ IndexConfig indexConf = new IndexConfig(true);
+ FieldIndexConfigs fieldIndexConfigs = new FieldIndexConfigs.Builder()
+ .add(index1, indexConf)
+ .build();
- /**
- * Returns {@code true} if the value with the given hash might have been put
in this bloom filer, {@code false}
- * otherwise.
- * <p>This method is provided to prevent hashing the same value multiple
times.
- */
- boolean mightContain(long hash1, long hash2);
+ Assert.assertEquals(fieldIndexConfigs.toString(), "{\"index1\":"
+ + JsonUtils.objectToString(indexConf) + "}");
+ }
}
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/BloomFilterReader.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/IndexConfig.java
similarity index 50%
copy from
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/BloomFilterReader.java
copy to
pinot-spi/src/main/java/org/apache/pinot/spi/config/table/IndexConfig.java
index dccc18eb85..ea73f58dab 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/BloomFilterReader.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/IndexConfig.java
@@ -16,25 +16,31 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.spi.index.reader;
+package org.apache.pinot.spi.config.table;
-import java.io.Closeable;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.pinot.spi.config.BaseJsonConfig;
/**
- * Interface for bloom filter reader.
+ * This is the base class used to configure indexes.
+ *
+ * The common logic between all indexes is that they can be enabled or
disabled.
+ *
+ * Indexes that do not require extra configuration can directly use this class.
*/
-public interface BloomFilterReader extends Closeable {
+public class IndexConfig extends BaseJsonConfig {
+ public static final IndexConfig ENABLED = new IndexConfig(true);
+ public static final IndexConfig DISABLED = new IndexConfig(false);
+ private final boolean _enabled;
- /**
- * Returns {@code true} if the given value might have been put in this bloom
filer, {@code false} otherwise.
- */
- boolean mightContain(String value);
+ @JsonCreator
+ public IndexConfig(@JsonProperty("enabled") boolean enabled) {
+ _enabled = enabled;
+ }
- /**
- * Returns {@code true} if the value with the given hash might have been put
in this bloom filer, {@code false}
- * otherwise.
- * <p>This method is provided to prevent hashing the same value multiple
times.
- */
- boolean mightContain(long hash1, long hash2);
+ public boolean isEnabled() {
+ return _enabled;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
