abhishekagarwal87 commented on code in PR #15417:
URL: https://github.com/apache/druid/pull/15417#discussion_r1407496113
##########
processing/src/main/java/org/apache/druid/segment/AutoTypeColumnIndexer.java:
##########
@@ -133,6 +150,51 @@ public EncodedKeyComponent<StructuredData>
processRowValsToUnsortedEncodedKeyCom
} else if (isConstant) {
isConstant = Objects.equals(dimValues, constantValue);
}
+
+ if (castToExpressionType != null) {
+ return processCast(dimValues);
+ } else {
+ return processAuto(dimValues);
+ }
+ }
+
+ /**
+ * Process values which will all be cast to {@link #castToExpressionType}.
This method should not be used for
+ * and does not handle actual nested data structures, use {@link
#processAuto(Object)} instead.
+ */
+ private EncodedKeyComponent<StructuredData> processCast(@Nullable Object
dimValues)
+ {
+ final long oldDictSizeInBytes = globalDictionary.sizeInBytes();
+ final int oldFieldKeySize = estimatedFieldKeySize;
+ ExprEval<?> eval = ExprEval.bestEffortOf(dimValues);
+ try {
+ eval = eval.castTo(castToExpressionType);
+ FieldIndexer fieldIndexer =
fieldIndexers.get(NestedPathFinder.JSON_PATH_ROOT);
+ if (fieldIndexer == null) {
+ estimatedFieldKeySize +=
StructuredDataProcessor.estimateStringSize(NestedPathFinder.JSON_PATH_ROOT);
+ fieldIndexer = new FieldIndexer(globalDictionary);
+ fieldIndexers.put(NestedPathFinder.JSON_PATH_ROOT, fieldIndexer);
+ }
+ StructuredDataProcessor.ProcessedValue<?> rootValue =
fieldIndexer.processValue(eval);
+ long effectiveSizeBytes = rootValue.getSize();
+ // then, we add the delta of size change to the global dictionaries to
account for any new space added by the
+ // 'raw' data
+ effectiveSizeBytes += (globalDictionary.sizeInBytes() -
oldDictSizeInBytes);
+ effectiveSizeBytes += (estimatedFieldKeySize - oldFieldKeySize);
+ return new EncodedKeyComponent<>(StructuredData.wrap(eval.value()),
effectiveSizeBytes);
+ }
+ catch (IAE invalidCast) {
Review Comment:
oops. yup.
##########
processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java:
##########
@@ -584,7 +584,7 @@ IncrementalIndexRowResult toIncrementalIndexRow(InputRow
row)
wasNewDim = true;
final DimensionHandler<?, ?, ?> handler;
if (useSchemaDiscovery) {
- handler = new NestedCommonFormatColumnHandler(dimension);
+ handler = new NestedCommonFormatColumnHandler(dimension, null);
Review Comment:
oh ok. this is the `wasNewDim = true` case.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
