This is an automated email from the ASF dual-hosted git repository.
amatya pushed a commit to branch 27.0.0
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/27.0.0 by this push:
new 9d3bb92655 [Backport 27.0.0] fix nested field virtual column array
column element vector object se… (#14736)
9d3bb92655 is described below
commit 9d3bb926557627141a053cbe7d2c3a0e4aced615
Author: AmatyaAvadhanula <[email protected]>
AuthorDate: Thu Aug 3 08:49:51 2023 +0530
[Backport 27.0.0] fix nested field virtual column array column element
vector object se… (#14736)
* fix nested field virtual column array column element vector object
selector (#14729)
Fixes a case I missed in #14688 when the return type is STRING but its
coming from a top level array typed column instead of a nested array column
while making a vector object selector.
Also while here I noticed that the internal JSON_VALUE functions for array
types were named inconsistently with the non-array functions, so I renamed
them. These are not documented so it should not be disruptive in any way, since
they are only used internally for rewrites while planning to make the correctly
virtual column.
JSON_VALUE_RETURNING_ARRAY_VARCHAR -> JSON_VALUE_ARRAY_VARCHAR
JSON_VALUE_RETURNING_ARRAY_BIGINT -> JSON_VALUE_ARRAY_BIGINT
JSON_VALUE_RETURNING_ARRAY_DOUBLE -> JSON_VALUE_ARRAY_DOUBLE
The internal non-array functions are JSON_VALUE_VARCHAR, JSON_VALUE_BIGINT,
and JSON_VALUE_DOUBLE.
* Fix IT
---------
Co-authored-by: Clint Wylie <[email protected]>
---
.../apache/druid/tests/query/ITJdbcQueryTest.java | 2 +-
.../segment/virtual/NestedFieldVirtualColumn.java | 14 ++++++-
.../timeseries/NestedDataTimeseriesQueryTest.java | 49 ++++++++++++++++++++++
.../builtin/NestedDataOperatorConversions.java | 6 +--
4 files changed, 65 insertions(+), 6 deletions(-)
diff --git
a/integration-tests/src/test/java/org/apache/druid/tests/query/ITJdbcQueryTest.java
b/integration-tests/src/test/java/org/apache/druid/tests/query/ITJdbcQueryTest.java
index 8d07f1b8d6..dbec477559 100644
---
a/integration-tests/src/test/java/org/apache/druid/tests/query/ITJdbcQueryTest.java
+++
b/integration-tests/src/test/java/org/apache/druid/tests/query/ITJdbcQueryTest.java
@@ -60,7 +60,7 @@ public class ITJdbcQueryTest
private static final String QUERY_TEMPLATE =
"SELECT \"user\", SUM(\"added\"), COUNT(*)" +
"FROM \"wikipedia\" " +
- "WHERE \"__time\" >= CURRENT_TIMESTAMP - INTERVAL '10' YEAR AND
\"language\" = %s" +
+ "WHERE \"__time\" >= CURRENT_TIMESTAMP - INTERVAL '99' YEAR AND
\"language\" = %s" +
"GROUP BY 1 ORDER BY 3 DESC LIMIT 10";
private static final String QUERY = StringUtils.format(QUERY_TEMPLATE,
"'en'");
diff --git
a/processing/src/main/java/org/apache/druid/segment/virtual/NestedFieldVirtualColumn.java
b/processing/src/main/java/org/apache/druid/segment/virtual/NestedFieldVirtualColumn.java
index 897f71c8ff..9a93c9428e 100644
---
a/processing/src/main/java/org/apache/druid/segment/virtual/NestedFieldVirtualColumn.java
+++
b/processing/src/main/java/org/apache/druid/segment/virtual/NestedFieldVirtualColumn.java
@@ -535,7 +535,11 @@ public class NestedFieldVirtualColumn implements
VirtualColumn
);
}
final VectorObjectSelector objectSelector =
complexColumn.makeVectorObjectSelector(parts, offset);
- if (leastRestrictiveType != null && leastRestrictiveType.isArray() &&
!expectedType.isArray()) {
+ if (leastRestrictiveType != null &&
+ leastRestrictiveType.isArray() &&
+ expectedType != null &&
+ !expectedType.isArray()
+ ) {
final ExpressionType elementType =
ExpressionType.fromColumnTypeStrict(leastRestrictiveType.getElementType());
final ExpressionType castTo =
ExpressionType.fromColumnTypeStrict(expectedType);
return makeVectorArrayToScalarObjectSelector(offset, objectSelector,
elementType, castTo);
@@ -575,6 +579,12 @@ public class NestedFieldVirtualColumn implements
VirtualColumn
if (parts.size() == 1 && parts.get(0) instanceof NestedPathArrayElement &&
column instanceof VariantColumn) {
final VariantColumn<?> arrayColumn = (VariantColumn<?>) column;
+ final ExpressionType elementType = ExpressionType.fromColumnTypeStrict(
+ arrayColumn.getLogicalType().isArray() ?
arrayColumn.getLogicalType().getElementType() : arrayColumn.getLogicalType()
+ );
+ final ExpressionType castTo = expectedType == null
+ ? ExpressionType.STRING
+ :
ExpressionType.fromColumnTypeStrict(expectedType);
VectorObjectSelector arraySelector =
arrayColumn.makeVectorObjectSelector(offset);
final int elementNumber = ((NestedPathArrayElement)
parts.get(0)).getIndex();
if (elementNumber < 0) {
@@ -595,7 +605,7 @@ public class NestedFieldVirtualColumn implements
VirtualColumn
if (maybeArray instanceof Object[]) {
Object[] anArray = (Object[]) maybeArray;
if (elementNumber < anArray.length) {
- elements[i] = anArray[elementNumber];
+ elements[i] = ExprEval.ofType(elementType,
anArray[elementNumber]).castTo(castTo).value();
} else {
elements[i] = null;
}
diff --git
a/processing/src/test/java/org/apache/druid/query/timeseries/NestedDataTimeseriesQueryTest.java
b/processing/src/test/java/org/apache/druid/query/timeseries/NestedDataTimeseriesQueryTest.java
index ac63221472..130654d092 100644
---
a/processing/src/test/java/org/apache/druid/query/timeseries/NestedDataTimeseriesQueryTest.java
+++
b/processing/src/test/java/org/apache/druid/query/timeseries/NestedDataTimeseriesQueryTest.java
@@ -287,6 +287,55 @@ public class NestedDataTimeseriesQueryTest extends
InitializedNullHandlingTest
);
}
+ @Test
+ public void testFilterLongArrayElementFilters()
+ {
+ TimeseriesQuery query = Druids.newTimeseriesQueryBuilder()
+ .dataSource("test_datasource")
+
.intervals(Collections.singletonList(Intervals.ETERNITY))
+ .filters(
+ new AndDimFilter(
+ NullFilter.forColumn("v0"),
+ NullFilter.forColumn("v1"),
+ NullFilter.forColumn("v2")
+ )
+ )
+ .virtualColumns(
+ new NestedFieldVirtualColumn(
+ "arrayLongNulls",
+ "$[1]",
+ "v0",
+ ColumnType.STRING
+ ),
+ new NestedFieldVirtualColumn(
+ "arrayLongNulls",
+ "$[1]",
+ "v1",
+ ColumnType.LONG
+ ),
+ new NestedFieldVirtualColumn(
+ "arrayStringNulls",
+ "$[1]",
+ "v2",
+ ColumnType.LONG
+ )
+ )
+ .aggregators(new
CountAggregatorFactory("count"))
+ .context(getContext())
+ .build();
+ runResults(
+ query,
+ ImmutableList.of(
+ new Result<>(
+ DateTimes.of("2023-01-01T00:00:00.000Z"),
+ new TimeseriesResultValue(
+ ImmutableMap.of("count", 8L)
+ )
+ )
+ )
+ );
+ }
+
@Test
public void testFilterVariantAsString()
{
diff --git
a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/NestedDataOperatorConversions.java
b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/NestedDataOperatorConversions.java
index 77a33e266c..2870221386 100644
---
a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/NestedDataOperatorConversions.java
+++
b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/NestedDataOperatorConversions.java
@@ -557,7 +557,7 @@ public class NestedDataOperatorConversions
public static class JsonValueReturningArrayBigIntOperatorConversion extends
JsonValueReturningArrayTypeOperatorConversion
{
- static final SqlFunction FUNCTION =
buildArrayFunction("JSON_VALUE_RETURNING_ARRAY_BIGINT", SqlTypeName.BIGINT);
+ static final SqlFunction FUNCTION =
buildArrayFunction("JSON_VALUE_ARRAY_BIGINT", SqlTypeName.BIGINT);
public JsonValueReturningArrayBigIntOperatorConversion()
{
@@ -567,7 +567,7 @@ public class NestedDataOperatorConversions
public static class JsonValueReturningArrayDoubleOperatorConversion extends
JsonValueReturningArrayTypeOperatorConversion
{
- static final SqlFunction FUNCTION =
buildArrayFunction("JSON_VALUE_RETURNING_ARRAY_DOUBLE", SqlTypeName.DOUBLE);
+ static final SqlFunction FUNCTION =
buildArrayFunction("JSON_VALUE_ARRAY_DOUBLE", SqlTypeName.DOUBLE);
public JsonValueReturningArrayDoubleOperatorConversion()
{
@@ -577,7 +577,7 @@ public class NestedDataOperatorConversions
public static class JsonValueReturningArrayVarcharOperatorConversion extends
JsonValueReturningArrayTypeOperatorConversion
{
- static final SqlFunction FUNCTION =
buildArrayFunction("JSON_VALUE_RETURNING_ARRAY_VARCHAR", SqlTypeName.VARCHAR);
+ static final SqlFunction FUNCTION =
buildArrayFunction("JSON_VALUE_ARRAY_VARCHAR", SqlTypeName.VARCHAR);
public JsonValueReturningArrayVarcharOperatorConversion()
{
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
