kfaraz commented on a change in pull request #12073:
URL: https://github.com/apache/druid/pull/12073#discussion_r796302979
##########
File path:
processing/src/main/java/org/apache/druid/segment/StringDimensionIndexer.java
##########
@@ -60,20 +60,29 @@ private static String emptyToNullIfNeeded(@Nullable Object
o)
private final MultiValueHandling multiValueHandling;
private final boolean hasBitmapIndexes;
private final boolean hasSpatialIndexes;
+ private final boolean useMaxMemoryEstimates;
private volatile boolean hasMultipleValues = false;
- public StringDimensionIndexer(MultiValueHandling multiValueHandling, boolean
hasBitmapIndexes, boolean hasSpatialIndexes)
+ public StringDimensionIndexer(
+ MultiValueHandling multiValueHandling,
+ boolean hasBitmapIndexes,
+ boolean hasSpatialIndexes,
+ boolean useMaxMemoryEstimates
+ )
{
+ super(useMaxMemoryEstimates ? new DimensionDictionary<>() : new
StringDimensionDictionary());
this.multiValueHandling = multiValueHandling == null ?
MultiValueHandling.ofDefault() : multiValueHandling;
this.hasBitmapIndexes = hasBitmapIndexes;
this.hasSpatialIndexes = hasSpatialIndexes;
+ this.useMaxMemoryEstimates = useMaxMemoryEstimates;
}
@Override
- public int[] processRowValsToUnsortedEncodedKeyComponent(@Nullable Object
dimValues, boolean reportParseExceptions)
+ public EncodedKeyComponent<int[]>
processRowValsToUnsortedEncodedKeyComponent(@Nullable Object dimValues, boolean
reportParseExceptions)
{
final int[] encodedDimensionValues;
final int oldDictSize = dimLookup.size();
+ final long oldDictSizeInBytes = useMaxMemoryEstimates ? 0 :
dimLookup.sizeInBytes();
Review comment:
We need the size of the dictionary before adding the dimension values.
At the end, we take the final size of dictionary and check the diff.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
