This is an automated email from the ASF dual-hosted git repository.
karan pushed a commit to branch 36.0.0
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/36.0.0 by this push:
new 8216487f200 fix json column isNumeric check to properly consider array
element selector types (#18948) (#18951)
8216487f200 is described below
commit 8216487f200beab720e1fafbc651a7a0ad07c2a8
Author: Clint Wylie <[email protected]>
AuthorDate: Mon Jan 26 21:37:45 2026 -0800
fix json column isNumeric check to properly consider array element selector
types (#18948) (#18951)
---
.../druid/msq/exec/MSQComplexGroupByTest.java | 6 ++---
.../nested/CompressedNestedDataComplexColumn.java | 15 ++++++++++-
.../segment/virtual/NestedFieldVirtualColumn.java | 22 ++++++++++-----
.../query/groupby/NestedGroupByArrayQueryTest.java | 30 +++++++++++++++++++++
.../druid/query/scan/NestedDataScanQueryTest.java | 2 +-
.../timeseries/NestedDataTimeseriesQueryTest.java | 31 ++++++++++++++++++++++
.../test/resources/nested-all-types-test-data.json | 2 +-
.../sql/calcite/CalciteNestedDataQueryTest.java | 2 +-
8 files changed, 97 insertions(+), 13 deletions(-)
diff --git
a/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQComplexGroupByTest.java
b/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQComplexGroupByTest.java
index 274eb0027c4..79de012a3e7 100644
---
a/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQComplexGroupByTest.java
+++
b/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQComplexGroupByTest.java
@@ -178,7 +178,7 @@ public class MSQComplexGroupByTest extends MSQTestBase
"y", 1.1,
"z", List.of(2, 4, 6)
),
- "c", List.of("a", "b"),
+ "c", List.of("a", "123"),
"v", Collections.emptyList()
)
),
@@ -313,7 +313,7 @@ public class MSQComplexGroupByTest extends MSQTestBase
"y", 1.1,
"z", List.of(2, 4, 6)
),
- "c", List.of("a", "b"),
+ "c", List.of("a", "123"),
"v", Collections.emptyList()
)
),
@@ -435,7 +435,7 @@ public class MSQComplexGroupByTest extends MSQTestBase
.setQueryContext(context)
.setExpectedResultRows(List.of(
new
Object[]{"{\"a\":600,\"b\":{\"x\":\"f\",\"y\":1.1,\"z\":[6,7,8,9]},\"c\":12.3,\"v\":\"b\"}"},
- new
Object[]{"{\"a\":200,\"b\":{\"x\":\"b\",\"y\":1.1,\"z\":[2,4,6]},\"c\":[\"a\",\"b\"],\"v\":[]}"},
+ new
Object[]{"{\"a\":200,\"b\":{\"x\":\"b\",\"y\":1.1,\"z\":[2,4,6]},\"c\":[\"a\",\"123\"],\"v\":[]}"},
new
Object[]{"{\"a\":400,\"b\":{\"x\":\"d\",\"y\":1.1,\"z\":[3,4]},\"c\":{\"a\":1},\"v\":[]}"},
new
Object[]{"{\"a\":500,\"b\":{\"x\":\"e\",\"z\":[1,2,3,4]},\"c\":\"hello\",\"v\":\"a\"}"},
new
Object[]{"{\"a\":700,\"b\":{\"x\":\"g\",\"y\":1.1,\"z\":[9,null,9,9]},\"c\":null,\"v\":[]}"},
diff --git
a/processing/src/main/java/org/apache/druid/segment/nested/CompressedNestedDataComplexColumn.java
b/processing/src/main/java/org/apache/druid/segment/nested/CompressedNestedDataComplexColumn.java
index 99dccbe81db..f452178b262 100644
---
a/processing/src/main/java/org/apache/druid/segment/nested/CompressedNestedDataComplexColumn.java
+++
b/processing/src/main/java/org/apache/druid/segment/nested/CompressedNestedDataComplexColumn.java
@@ -49,6 +49,7 @@ import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.DictionaryEncodedColumn;
import org.apache.druid.segment.column.StringEncodingStrategies;
+import org.apache.druid.segment.column.TypeSignature;
import org.apache.druid.segment.column.TypeStrategies;
import org.apache.druid.segment.column.TypeStrategy;
import org.apache.druid.segment.column.ValueType;
@@ -1043,8 +1044,20 @@ public abstract class
CompressedNestedDataComplexColumn<TKeyDictionary extends I
if (field instanceof NestedField) {
final NestedField nestedField = (NestedField) field;
return getColumnHolder(nestedField.fieldName,
nestedField.fieldIndex).getCapabilities().isNumeric();
+ } else if (field instanceof NestedArrayElement) {
+ final NestedArrayElement element = (NestedArrayElement) field;
+ final TypeSignature<ValueType> elementType = getColumnHolder(
+ element.nestedField.fieldName,
+ element.nestedField.fieldIndex
+ ).getCapabilities().getElementType();
+ if (elementType != null) {
+ return elementType.isNumeric();
+ }
+ // if element type is null, the field was not an array, so don't
consider it as numeric
+ return false;
}
- return true;
+ // a non-existent field can be considered numeric via a nil selector
+ return field == null;
}
@SuppressWarnings("unchecked")
diff --git
a/processing/src/main/java/org/apache/druid/segment/virtual/NestedFieldVirtualColumn.java
b/processing/src/main/java/org/apache/druid/segment/virtual/NestedFieldVirtualColumn.java
index 185a1a3ad9e..8ffde9a4f66 100644
---
a/processing/src/main/java/org/apache/druid/segment/virtual/NestedFieldVirtualColumn.java
+++
b/processing/src/main/java/org/apache/druid/segment/virtual/NestedFieldVirtualColumn.java
@@ -411,7 +411,17 @@ public class NestedFieldVirtualColumn implements
VirtualColumn
if (elementNumber < 0) {
throw new IAE("Cannot make array element selector, negative array
index not supported");
}
- return new ArrayElementColumnValueSelector(arraySelector, elementNumber);
+ final ColumnValueSelector<?> elementSelector = new
ArrayElementColumnValueSelector(arraySelector, elementNumber);
+ final ColumnType fieldType = (ColumnType)
arrayColumn.getLogicalType().getElementType();
+ if (fieldType != null && fieldSpec.expectedType != null &&
!fieldSpec.expectedType.equals(fieldType)) {
+ return ExpressionSelectors.castColumnValueSelector(
+ offset::getOffset,
+ elementSelector,
+ fieldType,
+ fieldSpec.expectedType
+ );
+ }
+ return elementSelector;
}
if (holder.getCapabilities().isArray() ||
ColumnType.NESTED_DATA.equals(holder.getCapabilities().toColumnType())) {
@@ -1260,11 +1270,11 @@ public class NestedFieldVirtualColumn implements
VirtualColumn
longs[i] = n.longValue();
nulls[i] = false;
} else {
- Double d = anArray[elementNumber] instanceof String
- ? Doubles.tryParse((String) anArray[elementNumber])
- : null;
- if (d != null) {
- longs[i] = d.longValue();
+ Number number = anArray[elementNumber] instanceof String
+ ? ExprEval.computeNumber((String)
anArray[elementNumber])
+ : null;
+ if (number != null) {
+ longs[i] = number.longValue();
nulls[i] = false;
} else {
longs[i] = 0L;
diff --git
a/processing/src/test/java/org/apache/druid/query/groupby/NestedGroupByArrayQueryTest.java
b/processing/src/test/java/org/apache/druid/query/groupby/NestedGroupByArrayQueryTest.java
index c9685eb5b7c..b39b850c9a3 100644
---
a/processing/src/test/java/org/apache/druid/query/groupby/NestedGroupByArrayQueryTest.java
+++
b/processing/src/test/java/org/apache/druid/query/groupby/NestedGroupByArrayQueryTest.java
@@ -497,6 +497,36 @@ public class NestedGroupByArrayQueryTest
);
}
+ @Test
+ public void testGroupByRootArrayVariantElementLong()
+ {
+ GroupByQuery groupQuery = GroupByQuery.builder()
+ .setDataSource("test_datasource")
+ .setGranularity(Granularities.ALL)
+ .setInterval(Intervals.ETERNITY)
+
.setDimensions(DefaultDimensionSpec.of("v0", ColumnType.LONG))
+ .setVirtualColumns(
+ new NestedFieldVirtualColumn(
+ "arrayVariant",
+ "$[1]",
+ "v0",
+ ColumnType.LONG
+ )
+ )
+ .setAggregatorSpecs(new
CountAggregatorFactory("count"))
+ .setContext(getContext())
+ .build();
+
+
+ runResults(
+ groupQuery,
+ ImmutableList.of(
+ new Object[]{null, 20L},
+ new Object[]{1L, 8L}
+ )
+ );
+ }
+
private void runResults(
GroupByQuery groupQuery,
List<Object[]> expectedResults
diff --git
a/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java
b/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java
index 766131ce1e2..bb225eb06a1 100644
---
a/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java
+++
b/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java
@@ -606,7 +606,7 @@ public class NestedDataScanQueryTest extends
InitializedNullHandlingTest
Assert.assertEquals(1, resultsRealtime.size());
Assert.assertEquals(resultsRealtime.size(), resultsSegments.size());
Assert.assertEquals(
- "[[1672531200000, null, null, null, 1, 51, -0.13, 1, [], [51, -35],
{a=700, b={x=g, y=1.1, z=[9, null, 9, 9]}, c=null, v=[]}, {x=400, y=[{l=[null],
m=100, n=5}, {l=[a, b, c], m=a, n=1}], z={}}, null, [a, b], null, [2, 3], null,
[null], null, [1, 0, 1], null, [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1,
y=hello, z={a=1.1, b=1234, c=[a, b, c], d=[]}}, [a, b, c], [1, 2, 3], [1.1,
2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1],
[1672531200000, , 2, null, 0, [...]
+ "[[1672531200000, null, null, null, 1, 51, -0.13, 1, [], [51, -35],
{a=700, b={x=g, y=1.1, z=[9, null, 9, 9]}, c=null, v=[]}, {x=400, y=[{l=[null],
m=100, n=5}, {l=[a, b, c], m=a, n=1}], z={}}, null, [a, b], null, [2, 3], null,
[null], null, [1, 0, 1], null, [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1,
y=hello, z={a=1.1, b=1234, c=[a, b, c], d=[]}}, [a, b, c], [1, 2, 3], [1.1,
2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1],
[1672531200000, , 2, null, 0, [...]
resultsSegments.get(0).getEvents().toString()
);
Assert.assertEquals(resultsRealtime.get(0).getEvents().toString(),
resultsSegments.get(0).getEvents().toString());
diff --git
a/processing/src/test/java/org/apache/druid/query/timeseries/NestedDataTimeseriesQueryTest.java
b/processing/src/test/java/org/apache/druid/query/timeseries/NestedDataTimeseriesQueryTest.java
index 7e3269fe5d2..34184d7774f 100644
---
a/processing/src/test/java/org/apache/druid/query/timeseries/NestedDataTimeseriesQueryTest.java
+++
b/processing/src/test/java/org/apache/druid/query/timeseries/NestedDataTimeseriesQueryTest.java
@@ -653,6 +653,37 @@ public class NestedDataTimeseriesQueryTest extends
InitializedNullHandlingTest
);
}
+ @Test
+ public void testSumArrayElement()
+ {
+ TimeseriesQuery query = Druids.newTimeseriesQueryBuilder()
+ .dataSource("test_datasource")
+
.intervals(Collections.singletonList(Intervals.ETERNITY))
+ .aggregators(
+ new CountAggregatorFactory("count"),
+ new
LongSumAggregatorFactory("sumNestedVariantNumericArrayElement", "v0")
+ )
+ .virtualColumns(
+ new NestedFieldVirtualColumn("obj",
"$.c[1]", "v0", ColumnType.LONG)
+ )
+ .context(getContext())
+ .build();
+ runResults(
+ query,
+ ImmutableList.of(
+ new Result<>(
+ DateTimes.of("2023-01-01T00:00:00.000Z"),
+ new TimeseriesResultValue(
+ ImmutableMap.<String, Object>builder()
+ .put("count", 14L)
+ .put("sumNestedVariantNumericArrayElement",
246L)
+ .build()
+ )
+ )
+ )
+ );
+ }
+
private void runResults(
TimeseriesQuery timeseriesQuery,
diff --git a/processing/src/test/resources/nested-all-types-test-data.json
b/processing/src/test/resources/nested-all-types-test-data.json
index e8d4a4ac26e..ea1bd98a7dc 100644
--- a/processing/src/test/resources/nested-all-types-test-data.json
+++ b/processing/src/test/resources/nested-all-types-test-data.json
@@ -1,5 +1,5 @@
{"timestamp": "2023-01-01T00:00:00", "str":"a", "long":1, "double":1.0,
"bool": true, "variant": 1, "variantNumeric": 1,
"variantEmptyObj":1, "variantEmtpyArray":1, "variantWithArrays": 1,
"obj":{"a": 100, "b": {"x": "a", "y": 1.1, "z": [1, 2, 3, 4]}, "c": [100], "v":
[]}, "complexObj":{"x": 1234, "y": [{"l": ["a", "b", "c"], "m": "a", "n":
1},{"l": ["a", "b", "c"], "m": "a", "n": 1}], "z": {"a": [1.1, 2.2, 3.3], "b":
true}}, "arrayString": ["a", "b"], [...]
-{"timestamp": "2023-01-01T00:00:00", "str":"", "long":2,
"bool": false, "variant": "b", "variantNumeric": 1.1,
"variantEmptyObj":"b", "variantEmtpyArray":2, "variantWithArrays": "b",
"obj":{"a": 200, "b": {"x": "b", "y": 1.1, "z": [2, 4, 6]}, "c": ["a", "b"],
"v": []}, "complexObj":{"x": 10, "y": [{"l": ["b", "b", "c"], "m": "b", "n":
2}, [1, 2, 3]], "z": {"a": [5.5], "b": false}},
"arrayString": ["a", "b", "c"] [...]
+{"timestamp": "2023-01-01T00:00:00", "str":"", "long":2,
"bool": false, "variant": "b", "variantNumeric": 1.1,
"variantEmptyObj":"b", "variantEmtpyArray":2, "variantWithArrays": "b",
"obj":{"a": 200, "b": {"x": "b", "y": 1.1, "z": [2, 4, 6]}, "c": ["a", "123"],
"v": []}, "complexObj":{"x": 10, "y": [{"l": ["b", "b", "c"], "m": "b", "n":
2}, [1, 2, 3]], "z": {"a": [5.5], "b": false}},
"arrayString": ["a", "b", "c [...]
{"timestamp": "2023-01-01T00:00:00", "str":"null", "long":3, "double":2.0,
"variant": 3.0, "variantNumeric": 1.0,
"variantEmptyObj":3.3, "variantEmtpyArray":3, "variantWithArrays": 3.0,
"obj":{"a": 300},
"complexObj":{"x": 4.4, "y": [{"l": [], "m": 100, "n": 3},{"l": ["a"]},
{"l": ["b"], "n": []}], "z": {"a": [], "b": true}},
"arrayString": ["b", "c"], [...]
{"timestamp": "2023-01-01T00:00:00", "str":"b", "long":4, "double":3.3,
"bool": true, "variant": "1",
"variantEmptyObj":{}, "variantEmtpyArray":4, "variantWithArrays": "1",
"obj":{"a": 400, "b": {"x": "d", "y": 1.1, "z": [3, 4]}, "c": {"a": 1},"v":
[]}, "complexObj":{"x": 1234, "z": {"a": [1.1, 2.2, 3.3], "b": true}},
"arrayString": ["d", "e"], [...]
{"timestamp": "2023-01-01T00:00:00", "str":"c", "long": null, "double":4.4,
"bool": true, "variant": "hello", "variantNumeric": -1000,
"variantEmptyObj":{}, "variantEmtpyArray":[], "variantWithArrays": "hello",
"obj":{"a": 500, "b": {"x": "e", "z": [1, 2, 3, 4]}, "c": "hello","v": "a"},
"complexObj":{"x": 11, "y": [], "z": {"a": [null], "b": false}},
"arrayString": null, [...]
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java
index 64c17a82f4d..1ff849b02bf 100644
---
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java
+++
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java
@@ -6326,7 +6326,7 @@ public abstract class CalciteNestedDataQueryTest extends
BaseCalciteQueryTest
"\"b\"",
"2",
"b",
-
"{\"a\":200,\"b\":{\"x\":\"b\",\"y\":1.1,\"z\":[2,4,6]},\"c\":[\"a\",\"b\"],\"v\":[]}",
+
"{\"a\":200,\"b\":{\"x\":\"b\",\"y\":1.1,\"z\":[2,4,6]},\"c\":[\"a\",\"123\"],\"v\":[]}",
"{\"x\":10,\"y\":[{\"l\":[\"b\",\"b\",\"c\"],\"m\":\"b\",\"n\":2},[1,2,3]],\"z\":{\"a\":[5.5],\"b\":false}}",
"[\"a\",\"b\",\"c\"]",
"[null,\"b\"]",
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
