gortiz commented on code in PR #10184:
URL: https://github.com/apache/pinot/pull/10184#discussion_r1151630452
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/bloomfilter/BloomFilterHandler.java:
##########
@@ -264,51 +265,42 @@ private void
createBloomFilterForColumn(SegmentDirectory.Writer segmentWriter, C
if (!columnMetadata.hasDictionary()) {
// Create a temporary forward index if it is disabled and does not exist
- columnMetadata = createForwardIndexIfNeeded(segmentWriter, columnName,
indexCreatorProvider, true);
+ columnMetadata = createForwardIndexIfNeeded(segmentWriter, columnName,
true);
}
// Create new bloom filter for the column.
BloomFilterConfig bloomFilterConfig = _bloomFilterConfigs.get(columnName);
LOGGER.info("Creating new bloom filter for segment: {}, column: {} with
config: {}", segmentName, columnName,
bloomFilterConfig);
if (columnMetadata.hasDictionary()) {
- createAndSealBloomFilterForDictionaryColumn(bloomFilterCreatorProvider,
indexDir, columnMetadata,
- bloomFilterConfig, segmentWriter);
+ createAndSealBloomFilterForDictionaryColumn(indexDir, columnMetadata,
bloomFilterConfig, segmentWriter);
} else {
-
createAndSealBloomFilterForNonDictionaryColumn(bloomFilterCreatorProvider,
indexDir, columnMetadata,
- bloomFilterConfig, segmentWriter);
+ createAndSealBloomFilterForNonDictionaryColumn(indexDir, columnMetadata,
bloomFilterConfig, segmentWriter);
}
// For v3, write the generated bloom filter file into the single file and
remove it.
if (_segmentDirectory.getSegmentMetadata().getVersion() ==
SegmentVersion.v3) {
- LoaderUtils.writeIndexToV3Format(segmentWriter, columnName,
bloomFilterFile, ColumnIndexType.BLOOM_FILTER);
+ LoaderUtils.writeIndexToV3Format(segmentWriter, columnName,
bloomFilterFile, StandardIndexes.bloomFilter());
}
// Delete the marker file.
FileUtils.deleteQuietly(bloomFilterFileInProgress);
LOGGER.info("Created bloom filter for segment: {}, column: {}",
segmentName, columnName);
}
- private BaseImmutableDictionary getDictionaryReader(ColumnMetadata
columnMetadata,
- SegmentDirectory.Writer segmentWriter)
+ private Dictionary getDictionaryReader(ColumnMetadata columnMetadata,
SegmentDirectory.Writer segmentWriter)
throws IOException {
- PinotDataBuffer dictionaryBuffer =
- segmentWriter.getIndexFor(columnMetadata.getColumnName(),
ColumnIndexType.DICTIONARY);
- int cardinality = columnMetadata.getCardinality();
DataType dataType = columnMetadata.getDataType();
+
switch (dataType) {
case INT:
- return new IntDictionary(dictionaryBuffer, cardinality);
case LONG:
- return new LongDictionary(dictionaryBuffer, cardinality);
case FLOAT:
- return new FloatDictionary(dictionaryBuffer, cardinality);
case DOUBLE:
- return new DoubleDictionary(dictionaryBuffer, cardinality);
case STRING:
- return new StringDictionary(dictionaryBuffer, cardinality,
columnMetadata.getColumnMaxLength());
case BYTES:
- return new BytesDictionary(dictionaryBuffer, cardinality,
columnMetadata.getColumnMaxLength());
+ PinotDataBuffer buf =
segmentWriter.getIndexFor(columnMetadata.getColumnName(),
StandardIndexes.dictionary());
+ return DictionaryIndexType.read(buf, columnMetadata, new
DictionaryIndexConfig(false, true));
Review Comment:
I've introduced `DictionaryIndexConfig.DEFAULT_OFFHEAP`. Just `OFFHEAP`
doesn't seem to be a good name given that there is at least another axis of
configuration (whether it uses var length dictionaries or not)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
