This is an automated email from the ASF dual-hosted git repository.
gortiz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new c297fbec3b4 Enhance JSON extraction functions to support null handling
in queries (#17867)
c297fbec3b4 is described below
commit c297fbec3b4e3b59bac956687ddb9007d82a7e87
Author: Gonzalo Ortiz Jaureguizar <[email protected]>
AuthorDate: Mon Mar 23 14:20:11 2026 +0100
Enhance JSON extraction functions to support null handling in queries
(#17867)
---
.../JsonExtractScalarTransformFunction.java | 82 +++++++++++++++++++++-
.../apache/pinot/queries/BaseJsonQueryTest.java | 4 +-
.../pinot/queries/JsonExtractScalarTest.java | 38 ++++++++++
3 files changed, 120 insertions(+), 4 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractScalarTransformFunction.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractScalarTransformFunction.java
index f171d1aed2a..5c76ed9e01c 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractScalarTransformFunction.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractScalarTransformFunction.java
@@ -30,6 +30,7 @@ import java.math.BigDecimal;
import java.util.List;
import java.util.Map;
import java.util.function.IntFunction;
+import javax.annotation.Nullable;
import org.apache.pinot.common.function.JsonPathCache;
import org.apache.pinot.core.operator.ColumnContext;
import org.apache.pinot.core.operator.blocks.ValueBlock;
@@ -38,6 +39,7 @@ import org.apache.pinot.core.util.NumberUtils;
import org.apache.pinot.core.util.NumericException;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.utils.JsonUtils;
+import org.roaringbitmap.RoaringBitmap;
/**
@@ -72,6 +74,7 @@ public class JsonExtractScalarTransformFunction extends
BaseTransformFunction {
private TransformFunction _jsonFieldTransformFunction;
private JsonPath _jsonPath;
private Object _defaultValue;
+ private boolean _defaultIsNull;
private TransformResultMetadata _resultMetadata;
@Override
@@ -80,8 +83,9 @@ public class JsonExtractScalarTransformFunction extends
BaseTransformFunction {
}
@Override
- public void init(List<TransformFunction> arguments, Map<String,
ColumnContext> columnContextMap) {
- super.init(arguments, columnContextMap);
+ public void init(List<TransformFunction> arguments, Map<String,
ColumnContext> columnContextMap,
+ boolean nullHandlingEnabled) {
+ super.init(arguments, columnContextMap, nullHandlingEnabled);
// Check that there are exactly 3 or 4 arguments
if (arguments.size() < 3 || arguments.size() > 4) {
throw new IllegalArgumentException(
@@ -110,7 +114,44 @@ public class JsonExtractScalarTransformFunction extends
BaseTransformFunction {
+ "/DOUBLE_ARRAY/STRING_ARRAY", resultsType));
}
if (arguments.size() == 4) {
- _defaultValue = dataType.convert(((LiteralTransformFunction)
arguments.get(3)).getStringLiteral());
+ LiteralTransformFunction literalTransformFun =
(LiteralTransformFunction) arguments.get(3);
+ _defaultIsNull = literalTransformFun.isNull() && _nullHandlingEnabled;
+ switch (dataType) {
+ case INT:
+ _defaultValue = literalTransformFun.getIntLiteral();
+ break;
+ case LONG:
+ _defaultValue = literalTransformFun.getLongLiteral();
+ break;
+ case FLOAT:
+ _defaultValue = literalTransformFun.getFloatLiteral();
+ break;
+ case DOUBLE:
+ _defaultValue = literalTransformFun.getDoubleLiteral();
+ break;
+ case TIMESTAMP:
+ // Use long literal so numeric millis stay exact and string
timestamps use LiteralContext parsing.
+ _defaultValue = literalTransformFun.getLongLiteral();
+ break;
+ case BOOLEAN:
+ _defaultValue = literalTransformFun.getBooleanLiteral();
+ break;
+ case BIG_DECIMAL:
+ _defaultValue = literalTransformFun.getBigDecimalLiteral();
+ break;
+ case STRING:
+ case JSON:
+ _defaultValue = literalTransformFun.getStringLiteral();
+ break;
+ case BYTES:
+ _defaultValue = literalTransformFun.getBytesLiteral();
+ break;
+ default:
+ throw new IllegalArgumentException(
+ "Unsupported results type: " + dataType + " for
jsonExtractScalar function. Supported types are: "
+ +
"INT/LONG/FLOAT/DOUBLE/BOOLEAN/BIG_DECIMAL/TIMESTAMP/STRING/JSON/BYTES"
+ );
+ }
}
_resultMetadata = new TransformResultMetadata(dataType, isSingleValue,
false);
}
@@ -120,6 +161,41 @@ public class JsonExtractScalarTransformFunction extends
BaseTransformFunction {
return _resultMetadata;
}
+ @Override
+ @Nullable
+ public RoaringBitmap getNullBitmap(ValueBlock valueBlock) {
+ if (!_defaultIsNull) {
+ return super.getNullBitmap(valueBlock);
+ }
+ RoaringBitmap bitmap = new RoaringBitmap();
+ for (TransformFunction arg : _arguments.subList(1, _arguments.size() - 1))
{
+ RoaringBitmap argBitmap = arg.getNullBitmap(valueBlock);
+ if (argBitmap != null) {
+ bitmap.or(argBitmap);
+ }
+ }
+ int numDocs = valueBlock.getNumDocs();
+ RoaringBitmap nullBitmap = new RoaringBitmap();
+ IntFunction<Object> resultExtractor = getResultExtractor(valueBlock);
+ for (int i = 0; i < numDocs; i++) {
+ Object result = null;
+ try {
+ result = resultExtractor.apply(i);
+ } catch (Exception ignored) {
+ }
+ if (result == null) {
+ nullBitmap.add(i);
+ }
+ }
+ if (!nullBitmap.isEmpty()) {
+ bitmap.or(nullBitmap);
+ }
+ if (bitmap.isEmpty()) {
+ return null;
+ }
+ return bitmap;
+ }
+
@Override
public int[] transformToIntValuesSV(ValueBlock valueBlock) {
if (_resultMetadata.getDataType().getStoredType() != DataType.INT) {
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/BaseJsonQueryTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/BaseJsonQueryTest.java
index 901e66c8b68..25990f062e2 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/BaseJsonQueryTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/BaseJsonQueryTest.java
@@ -34,6 +34,7 @@ import
org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.data.readers.GenericRow;
+import org.intellij.lang.annotations.Language;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.DataProvider;
@@ -145,6 +146,7 @@ public abstract class BaseJsonQueryTest extends
BaseQueriesTest {
records.add(createRecord(16, 16, "john doe", "{\"longVal\":
\"-9223372036854775808\" }"));
records.add(createRecord(17, 17, "john doe", "{\"longVal\": \"-100.12345\"
}"));
records.add(createRecord(18, 18, "john doe", "{\"longVal\": \"10e2\" }"));
+ records.add(createRecord(19, 19, "john doe", "{\"longVal\": null }"));
tableConfig.getIndexingConfig().setJsonIndexColumns(List.of("jsonColumn"));
SegmentGeneratorConfig segmentGeneratorConfig = new
SegmentGeneratorConfig(tableConfig, schema);
@@ -178,7 +180,7 @@ public abstract class BaseJsonQueryTest extends
BaseQueriesTest {
return _indexSegments;
}
- protected void checkResult(String query, Object[][] expectedResults) {
+ protected void checkResult(@Language("sql") String query, Object[][]
expectedResults) {
BrokerResponseNative brokerResponse =
getBrokerResponseForOptimizedQuery(query, schema());
QueriesTestUtils.testInterSegmentsResult(brokerResponse,
Arrays.asList(expectedResults));
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/JsonExtractScalarTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/JsonExtractScalarTest.java
index 5ef40e3ce75..5c7049369a0 100644
---
a/pinot-core/src/test/java/org/apache/pinot/queries/JsonExtractScalarTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/queries/JsonExtractScalarTest.java
@@ -171,4 +171,42 @@ public class JsonExtractScalarTest extends
BaseJsonQueryTest {
+ "limit 4",
new Object[][]{{15, Long.MAX_VALUE}, {16, Long.MIN_VALUE}, {17,
-100L}, {18, 1000L}});
}
+
+ @Test
+ public void testNullAsDefaultValueWithNullHandlingDisabled() {
+ checkResult(
+ "SET enableNullHandling=false;"
+ + "SELECT intColumn, jsonextractscalar(" + JSON_COLUMN + ",
'$.longVal', 'long', null) "
+ + "FROM testTable "
+ + "where intColumn >= 15 and intColumn <= 19 "
+ + "group by 1, 2 "
+ + "order by 1, 2 ",
+ new Object[][]{
+ {15, Long.MAX_VALUE},
+ {16, Long.MIN_VALUE},
+ {17, -100L},
+ {18, 1000L},
+ {19, 0L} // when enableNullHandling is false, null is treated as 0
for LONG type
+ }
+ );
+ }
+
+ @Test
+ public void testNullAsDefaultValueWithNullHandlingEnabled() {
+ checkResult(
+ "SET enableNullHandling=true;"
+ + "SELECT intColumn, jsonextractscalar(" + JSON_COLUMN + ",
'$.longVal', 'long', null) "
+ + "FROM testTable "
+ + "where intColumn >= 15 and intColumn <= 19 "
+ + "group by 1, 2 "
+ + "order by 1, 2 ",
+ new Object[][]{
+ {15, Long.MAX_VALUE},
+ {16, Long.MIN_VALUE},
+ {17, -100L},
+ {18, 1000L},
+ {19, null} // when enableNullHandling is true, null is treated as
null
+ }
+ );
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
