This is an automated email from the ASF dual-hosted git repository.
cwylie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new d61bd7f8f1 fix bug in nested v4 format merger from refactoring (#14053)
d61bd7f8f1 is described below
commit d61bd7f8f16c3f25ba4f81c5573edb1d7c504770
Author: Clint Wylie <[email protected]>
AuthorDate: Mon Apr 10 20:38:58 2023 -0700
fix bug in nested v4 format merger from refactoring (#14053)
---
.../segment/QueryableIndexIndexableAdapter.java | 5 +++-
.../apache/druid/query/NestedDataTestUtils.java | 12 ++++----
.../org/apache/druid/segment/IndexBuilder.java | 34 +++++++++++++++++-----
3 files changed, 37 insertions(+), 14 deletions(-)
diff --git
a/processing/src/main/java/org/apache/druid/segment/QueryableIndexIndexableAdapter.java
b/processing/src/main/java/org/apache/druid/segment/QueryableIndexIndexableAdapter.java
index 28e26c02de..cf7b352dbf 100644
---
a/processing/src/main/java/org/apache/druid/segment/QueryableIndexIndexableAdapter.java
+++
b/processing/src/main/java/org/apache/druid/segment/QueryableIndexIndexableAdapter.java
@@ -35,6 +35,7 @@ import org.apache.druid.segment.data.CloseableIndexed;
import org.apache.druid.segment.data.ImmutableBitmapValues;
import org.apache.druid.segment.data.IndexedIterable;
import org.apache.druid.segment.nested.NestedCommonFormatColumn;
+import org.apache.druid.segment.nested.NestedDataComplexTypeSerde;
import org.apache.druid.segment.nested.SortedValueDictionary;
import org.apache.druid.segment.selector.settable.SettableColumnValueSelector;
import
org.apache.druid.segment.selector.settable.SettableLongColumnValueSelector;
@@ -174,7 +175,9 @@ public class QueryableIndexIndexableAdapter implements
IndexableAdapter
if (columnHolder == null) {
return null;
}
- if (!(columnHolder.getColumnFormat() instanceof
NestedCommonFormatColumn.Format)) {
+ final ColumnFormat format = columnHolder.getColumnFormat();
+ if (!(format instanceof NestedCommonFormatColumn.Format
+ || format instanceof
NestedDataComplexTypeSerde.NestedColumnFormatV4)) {
return null;
}
diff --git
a/processing/src/test/java/org/apache/druid/query/NestedDataTestUtils.java
b/processing/src/test/java/org/apache/druid/query/NestedDataTestUtils.java
index c6b77c211c..9a43c77b9c 100644
--- a/processing/src/test/java/org/apache/druid/query/NestedDataTestUtils.java
+++ b/processing/src/test/java/org/apache/druid/query/NestedDataTestUtils.java
@@ -40,10 +40,10 @@ import
org.apache.druid.java.util.common.parsers.JSONPathSpec;
import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.CountAggregatorFactory;
import org.apache.druid.query.expression.TestExprMacroTable;
-import org.apache.druid.segment.AutoTypeColumnSchema;
import org.apache.druid.segment.IncrementalIndexSegment;
import org.apache.druid.segment.IndexBuilder;
import org.apache.druid.segment.IndexSpec;
+import org.apache.druid.segment.NestedDataDimensionSchema;
import org.apache.druid.segment.QueryableIndexSegment;
import org.apache.druid.segment.Segment;
import org.apache.druid.segment.TestHelper;
@@ -96,11 +96,11 @@ public class NestedDataTestUtils
DimensionsSpec.builder()
.setDimensions(
Arrays.asList(
- new AutoTypeColumnSchema("dim"),
- new AutoTypeColumnSchema("nest_json"),
- new AutoTypeColumnSchema("nester_json"),
- new AutoTypeColumnSchema("variant_json"),
- new AutoTypeColumnSchema("list_json")
+ new NestedDataDimensionSchema("dim"),
+ new NestedDataDimensionSchema("nest_json"),
+ new NestedDataDimensionSchema("nester_json"),
+ new NestedDataDimensionSchema("variant_json"),
+ new NestedDataDimensionSchema("list_json")
)
)
.build();
diff --git
a/processing/src/test/java/org/apache/druid/segment/IndexBuilder.java
b/processing/src/test/java/org/apache/druid/segment/IndexBuilder.java
index 90a4411c87..95a863dbb0 100644
--- a/processing/src/test/java/org/apache/druid/segment/IndexBuilder.java
+++ b/processing/src/test/java/org/apache/druid/segment/IndexBuilder.java
@@ -53,6 +53,7 @@ import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
@@ -222,15 +223,34 @@ public class IndexBuilder
Preconditions.checkNotNull(indexMerger, "indexMerger");
Preconditions.checkNotNull(tmpDir, "tmpDir");
try (final IncrementalIndex incrementalIndex = buildIncrementalIndex()) {
+ List<IndexableAdapter> adapters = Collections.singletonList(
+ new QueryableIndexIndexableAdapter(
+ indexIO.loadIndex(
+ indexMerger.persist(
+ incrementalIndex,
+ new File(
+ tmpDir,
+ StringUtils.format("testIndex-%s",
ThreadLocalRandom.current().nextInt(Integer.MAX_VALUE))
+ ),
+ indexSpec,
+ null
+ )
+ )
+ )
+ );
+ // Do a 'merge' of the persisted segment even though there is only one;
this time it will be reading from the
+ // queryable index instead of the incremental index, which also mimics
the behavior of real ingestion tasks
+ // which persist incremental indexes as intermediate segments and then
merges all the intermediate segments to
+ // publish
return indexIO.loadIndex(
- indexMerger.persist(
- incrementalIndex,
- new File(
- tmpDir,
- StringUtils.format("testIndex-%s",
ThreadLocalRandom.current().nextInt(Integer.MAX_VALUE))
- ),
+ indexMerger.merge(
+ adapters,
+ schema.isRollup(),
+ schema.getMetrics(),
+ tmpDir,
+ schema.getDimensionsSpec(),
indexSpec,
- null
+ Integer.MAX_VALUE
)
);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
