Copilot commented on code in PR #18722:
URL: https://github.com/apache/druid/pull/18722#discussion_r2510769676
##########
processing/src/test/java/org/apache/druid/query/NestedDataTestUtils.java:
##########
@@ -307,76 +160,144 @@ public static List<Segment>
createSegmentsWithConcatenatedJsonInput(
File file = selfConcatenateResourceFile(tempFolder, inputFile,
numCopies);
inputFiles.add(new LocalInputSource(file.getParentFile(),
file.getName()));
}
- return createSegments(
- tempFolder,
- closer,
- inputFiles,
- TestIndex.DEFAULT_JSON_INPUT_FORMAT,
- TIMESTAMP_SPEC,
- AUTO_DISCOVERY,
- TransformSpec.NONE,
- COUNT,
- granularity,
- rollup,
- IndexSpec.getDefault()
- );
+ return new SegmentBuilder(tempFolder, closer).inputSources(inputFiles)
+ .granularity(granularity)
+ .rollup(rollup)
+ .build();
}
- public static List<Segment> createSegmentsForJsonInput(
- TemporaryFolder tempFolder,
- Closer closer,
- String inputFile,
- IndexSpec indexSpec
- )
- throws Exception
+ public static class SegmentBuilder
{
- return createSegmentsForJsonInput(
- tempFolder,
- closer,
- inputFile,
- Granularities.NONE,
- true,
- indexSpec
- );
- }
+ private TemporaryFolder tempFolder;
+ private Closer closer;
+
+ private List<InputSource> inputSources =
+
List.of(ResourceInputSource.of(NestedDataTestUtils.class.getClassLoader(),
SIMPLE_DATA_FILE));
+ private InputFormat inputFormat = TestIndex.DEFAULT_JSON_INPUT_FORMAT;
+ private TimestampSpec timestampSpec = TIMESTAMP_SPEC;
+ private DimensionsSpec dimensionsSpec = AUTO_DISCOVERY;
+ private TransformSpec transformSpec = TransformSpec.NONE;
+ private AggregatorFactory[] aggregators = COUNT;
+ private Granularity queryGranularity = Granularities.NONE;
+ private boolean rollup = true;
+ private IndexSpec indexSpec = IndexSpec.getDefault();
+
+ /**
+ * Builder for an {@link IncrementalIndexSegment} or a list of{@link
QueryableIndexSegment}, with some defaults:
Review Comment:
Missing space between 'of' and '{@link QueryableIndexSegment}'.
```suggestion
* Builder for an {@link IncrementalIndexSegment} or a list of {@link
QueryableIndexSegment}, with some defaults:
```
##########
processing/src/main/java/org/apache/druid/segment/nested/NestedCommonFormatColumnFormatSpec.java:
##########
@@ -56,10 +58,11 @@ public static Builder
builder(NestedCommonFormatColumnFormatSpec spec)
}
/**
- * Create a {@link NestedCommonFormatColumnFormatSpec} with all fields fully
populated. Values from the supplied
- * column format spec take priority, any null values are then populated by
checking
- * {@link IndexSpec#getAutoColumnFormatSpec()}, then falling back to fields
on {@link IndexSpec} itself if applicable,
- * and finally resorting to hard coded defaults.
+ * Create a {@link NestedCommonFormatColumnFormatSpec} with all fields fully
populated. Values are populated in the following order:
+ * <li> value in the given column format spec, if non-null </li>
+ * <li> value in {@link IndexSpec#getAutoColumnFormatSpec()}, if non-null
</li>
+ * <li> fall back to equivalent fields on {@link IndexSpec} itself if
applicable </li>
+ * <li> hard coded defaults in {@link #DEFAULT}</li>
Review Comment:
The list items should use proper HTML list formatting with `<ul>` and
`</ul>` tags instead of bare `<li>` tags.
```suggestion
* <ul>
* <li> value in the given column format spec, if non-null </li>
* <li> value in {@link IndexSpec#getAutoColumnFormatSpec()}, if
non-null </li>
* <li> fall back to equivalent fields on {@link IndexSpec} itself if
applicable </li>
* <li> hard coded defaults in {@link #DEFAULT}</li>
* </ul>
```
##########
processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java:
##########
@@ -867,50 +625,32 @@ public void
testIngestAndScanSegmentsAndFilterPartialPathArrayIndex() throws Exc
}
@Test
- public void testIngestAndScanSegmentsAndFilterPartialPath() throws Exception
+ @Parameters(method = "getNestedColumnFormatSpec")
+ @TestCaseName("{0}")
+ public void testIngestAndScanSegmentsAndFilterPartialPath(
+ String name,
+ boolean auto,
+ NestedCommonFormatColumnFormatSpec spec
+ ) throws Exception
{
- Query<ScanResultValue> scanQuery = Druids.newScanQueryBuilder()
- .dataSource("test_datasource")
- .intervals(
- new
MultipleIntervalSegmentSpec(
-
Collections.singletonList(Intervals.ETERNITY)
- )
- )
- .filters(
-
NotDimFilter.of(NullFilter.forColumn("v0"))
- )
- .virtualColumns(
- new NestedFieldVirtualColumn(
- "obj",
- "v0",
- ColumnType.NESTED_DATA,
- null,
- true,
- "$.b",
- false
- )
- )
-
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
- .limit(100)
- .context(ImmutableMap.of())
- .build();
- List<Segment> segs = NestedDataTestUtils.createSegmentsForJsonInput(
- tempFolder,
- closer,
- NestedDataTestUtils.ALL_TYPES_TEST_DATA_FILE,
- Granularities.HOUR,
- true,
- IndexSpec.getDefault()
- );
-
- List<Segment> realtimeSegs = ImmutableList.of(
- NestedDataTestUtils.createIncrementalIndexForJsonInput(
- tempFolder,
- NestedDataTestUtils.ALL_TYPES_TEST_DATA_FILE,
- Granularities.NONE,
- true
- )
- );
+ DimensionsSpec dimensionsSpec =
+ DimensionsSpec.builder()
+ .setDimensions(List.of(auto
+ ? new AutoTypeColumnSchema("obj",
ColumnType.NESTED_DATA, spec)
+ : new
NestedDataColumnSchema("obj", 5, spec, DEFAULT_FORMAT)))
+ .build();
+ Query<ScanResultValue> scanQuery = queryBuilder()
+ .columns("timestamp", "str", "double", "bool", "variant",
+ "variantNumeric", "variantEmptyObj", "variantEmtpyArray",
"variantWithArrays"
Review Comment:
Corrected spelling of 'variantEmtpyArray' to 'variantEmptyArray'.
```suggestion
"variantNumeric", "variantEmptyObj", "variantEmptyArray",
"variantWithArrays"
```
##########
processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java:
##########
@@ -805,54 +578,39 @@ public void
testIngestAndScanSegmentsRealtimeSchemaDiscoveryTypeGauntlet() throw
"[[1672531200000, null, null, null, 1, 51, -0.13, 1, [], [51, -35],
{a=700, b={x=g, y=1.1, z=[9, null, 9, 9]}, c=null, v=[]}, {x=400, y=[{l=[null],
m=100, n=5}, {l=[a, b, c], m=a, n=1}], z={}}, null, [a, b], null, [2, 3], null,
[null], null, [1, 0, 1], null, [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1,
y=hello, z={a=1.1, b=1234, c=[a, b, c], d=[]}}, [a, b, c], [1, 2, 3], [1.1,
2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1],
[1672531200000, , 2, null, 0, b, 1.1, b, 2, b, {a=200, b={x=b, y=1.1, z=[2, 4,
6]}, c=[a, b], v=[]}, {x=10, y=[{l=[b, b, c], m=b, n=2}, [1, 2, 3]],
z={a=[5.5], b=false}}, [a, b, c], [null, b], [2, 3], null, [3.3, 4.4, 5.5],
[999.0, null, 5.5], [null, null, 2.2], [1, 1], [null, [null], []], [{x=3},
{x=4}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c],
d=[]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {},
{}], [{a=b, x=1, y=1.3}], 1], [1672531200000, a, 1, 1.0, 1, 1, 1, 1, 1, 1, {a
=100, b={x=a, y=1.1, z=[1, 2, 3, 4]}, c=[100], v=[]}, {x=1234, y=[{l=[a, b,
c], m=a, n=1}, {l=[a, b, c], m=a, n=1}], z={a=[1.1, 2.2, 3.3], b=true}}, [a,
b], [a, b], [1, 2, 3], [1, null, 3], [1.1, 2.2, 3.3], [1.1, 2.2, null], [a, 1,
2.2], [1, 0, 1], [[1, 2, null], [3, 4]], [{x=1}, {x=2}], null, hello, 1234,
1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c], d=[]}}, [a, b, c], [1, 2,
3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}],
1], [1672531200000, b, 4, 3.3, 1, 1, null, {}, 4, 1, {a=400, b={x=d, y=1.1,
z=[3, 4]}, c={a=1}, v=[]}, {x=1234, z={a=[1.1, 2.2, 3.3], b=true}}, [d, e], [b,
b], [1, 4], [1], [2.2, 3.3, 4.0], null, [a, b, c], [null, 0, 1], [[1, 2], [3,
4], [5, 6, 7]], [{x=null}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello,
z={a=1.1, b=1234, c=[a, b, c], d=[]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3],
[], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, c,
null, 4.4, 1, hello, -1000, {}, [], hello, {a=500, b={x=e, z=[1,
2, 3, 4]}, c=hello, v=a}, {x=11, y=[], z={a=[null], b=false}}, null, null,
[1, 2, 3], [], [1.1, 2.2, 3.3], null, null, [0], null, [{x=1000}, {y=2000}],
null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c], d=[]}},
[a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}],
[{a=b, x=1, y=1.3}], 1], [1672531200000, d, 5, 5.9, 0, null, 3.33, a, 6, null,
{a=600, b={x=f, y=1.1, z=[6, 7, 8, 9]}, c=12.3, v=b}, null, [a, b], null, null,
[null, 2, 9], null, [999.0, 5.5, null], [a, 1, 2.2], [], [[1], [1, 2, null]],
[{a=1}, {b=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234,
c=[a, b, c], d=[]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null,
null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, null, 3, 2.0,
null, 3.0, 1.0, 3.3, 3, 3.0, {a=300}, {x=4.4, y=[{l=[], m=100, n=3}, {l=[a]},
{l=[b], n=[]}], z={a=[], b=true}}, [b, c], [d, null, b], [1, 2, 3, 4], [1, 2,
3], [1.1, 3.3], [null, 2.2, null], [1, null, 1], [1, null, 1], [[1], n
ull, [1, 2, 3]], [null, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello,
z={a=1.1, b=1234, c=[a, b, c], d=[]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3],
[], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1]]",
resultsSegments.get(0).getEvents().toString()
);
- Assert.assertEquals(resultsSegments.get(0).getEvents().toString(),
resultsRealtime.get(0).getEvents().toString());
+ Assert.assertEquals(resultsRealtime.get(0).getEvents().toString(),
resultsSegments.get(0).getEvents().toString());
}
@Test
- public void testIngestAndScanSegmentsAndFilterPartialPathArrayIndex() throws
Exception
+ @Parameters(method = "getNestedColumnFormatSpec")
+ @TestCaseName("{0}")
+ public void testIngestAndScanSegmentsAndFilterPartialPathArrayIndex(
+ String name,
+ boolean auto,
+ NestedCommonFormatColumnFormatSpec spec
+ ) throws Exception
{
- Query<ScanResultValue> scanQuery = Druids.newScanQueryBuilder()
- .dataSource("test_datasource")
- .intervals(
- new
MultipleIntervalSegmentSpec(
-
Collections.singletonList(Intervals.ETERNITY)
- )
- )
- .filters(
-
NotDimFilter.of(NullFilter.forColumn("v0"))
- )
- .virtualColumns(
- new NestedFieldVirtualColumn(
- "complexObj",
- "v0",
- ColumnType.NESTED_DATA,
- null,
- true,
- "$.y[0]",
- false
- )
- )
-
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
- .limit(100)
- .context(ImmutableMap.of())
- .build();
- List<Segment> segs = NestedDataTestUtils.createSegmentsForJsonInput(
- tempFolder,
- closer,
- NestedDataTestUtils.ALL_TYPES_TEST_DATA_FILE,
- Granularities.HOUR,
- true,
- IndexSpec.getDefault()
- );
-
- List<Segment> realtimeSegs = ImmutableList.of(
- NestedDataTestUtils.createIncrementalIndexForJsonInput(
- tempFolder,
- NestedDataTestUtils.ALL_TYPES_TEST_DATA_FILE,
- Granularities.NONE,
- true
+ DimensionsSpec dimensionsSpec =
+ DimensionsSpec.builder()
+ .setDimensions(List.of(auto
+ ? new
AutoTypeColumnSchema("complexObj", ColumnType.NESTED_DATA, spec)
+ : new
NestedDataColumnSchema("complexObj", 5, spec, DEFAULT_FORMAT)))
+ .build();
+ Query<ScanResultValue> scanQuery = queryBuilder()
+ .columns("timestamp", "str", "double", "bool", "variant",
+ "variantNumeric", "variantEmptyObj", "variantEmtpyArray",
"variantWithArrays"
Review Comment:
Corrected spelling of 'variantEmtpyArray' to 'variantEmptyArray'.
```suggestion
"variantNumeric", "variantEmptyObj", "variantEmptyArray",
"variantWithArrays"
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
