This is an automated email from the ASF dual-hosted git repository.
Jackie-Jiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new f46b0a9fa08 Resolve absent MAP/OPEN_STRUCT keys to a shared
default-column-style NullDataSource (#19439)
f46b0a9fa08 is described below
commit f46b0a9fa0842cd4c5d6a3abffe5b39830c4a52c
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Fri Sep 4 15:56:04 2026 -0700
Resolve absent MAP/OPEN_STRUCT keys to a shared default-column-style
NullDataSource (#19439)
---
.../core/operator/blocks/ProjectionBlock.java | 11 +-
.../core/operator/filter/MapFilterOperator.java | 106 ++-------
.../transform/function/ItemTransformFunction.java | 38 +--
.../pinot/core/plan/AggregationPlanNode.java | 5 +-
.../filter/MapFilterOperatorOpenStructTest.java | 80 ++++---
.../ItemTransformFunctionNullBitmapTest.java | 74 +++---
.../impl/openstruct/OpenStructColumnSplitter.java | 4 +
.../segment/index/datasource/NullDataSource.java | 57 +++++
.../local/segment/index/map/BaseMapDataSource.java | 38 ++-
.../local/segment/index/map/NullDataSource.java | 255 ---------------------
.../openstruct/ImmutableOpenStructDataSource.java | 32 +--
.../openstruct/MutableOpenStructDataSource.java | 16 +-
.../index/openstruct/MutableOpenStructIndex.java | 10 +-
.../index/openstruct/OpenStructNullDataSource.java | 195 ----------------
.../index/openstruct/SparseKeyDataSource.java | 7 +-
.../BaseConstantValueVirtualColumnProvider.java | 2 +
.../DefaultNullValueVirtualColumnProviderTest.java | 148 +++++++++---
.../index/datasource/NullDataSourceTest.java | 135 +++++++++++
.../segment/index/map/BaseMapDataSourceTest.java | 71 ++++++
.../ImmutableOpenStructDataSourceTest.java | 34 ++-
.../MutableOpenStructDataSourceTest.java | 32 ++-
.../openstruct/OpenStructNullDataSourceTest.java | 133 -----------
.../segment/spi/datasource/MapDataSource.java | 3 +-
.../spi/datasource/OpenStructDataSource.java | 52 ++---
24 files changed, 614 insertions(+), 924 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/ProjectionBlock.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/ProjectionBlock.java
index 0a9bcd66c30..c70929348c4 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/ProjectionBlock.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/ProjectionBlock.java
@@ -23,8 +23,6 @@ import
org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.core.common.BlockValSet;
import org.apache.pinot.core.common.DataBlockCache;
import org.apache.pinot.core.operator.docvalsets.ProjectionBlockValSet;
-import org.apache.pinot.segment.local.segment.index.map.NullDataSource;
-import
org.apache.pinot.segment.local.segment.index.openstruct.OpenStructNullDataSource;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.segment.spi.datasource.MapDataSource;
import org.apache.pinot.segment.spi.datasource.OpenStructDataSource;
@@ -88,15 +86,8 @@ public class ProjectionBlock implements ValueBlock {
DataSource keyDataSource;
if (columnDataSource instanceof MapDataSource) {
keyDataSource = ((MapDataSource)
columnDataSource).getDataSource(paths[1]);
- if (keyDataSource == null) {
- keyDataSource = new NullDataSource(paths[1]);
- }
} else if (columnDataSource instanceof OpenStructDataSource) {
- OpenStructDataSource osDs = (OpenStructDataSource) columnDataSource;
- keyDataSource = osDs.getDataSource(paths[1]);
- if (keyDataSource == null) {
- keyDataSource = OpenStructNullDataSource.forAbsentKey(osDs, paths[1]);
- }
+ keyDataSource = ((OpenStructDataSource)
columnDataSource).getDataSource(paths[1]);
} else {
throw new IllegalStateException("Path-based access requires MAP or
OPEN_STRUCT column: " + paths[0]);
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/MapFilterOperator.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/MapFilterOperator.java
index 29889e6c069..9c62e3d2466 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/MapFilterOperator.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/MapFilterOperator.java
@@ -37,13 +37,11 @@ import
org.apache.pinot.core.operator.ExplainAttributeBuilder;
import org.apache.pinot.core.operator.filter.predicate.PredicateEvaluator;
import
org.apache.pinot.core.operator.filter.predicate.PredicateEvaluatorProvider;
import org.apache.pinot.core.query.request.context.QueryContext;
-import
org.apache.pinot.segment.local.segment.index.openstruct.OpenStructNullDataSource;
import org.apache.pinot.segment.spi.IndexSegment;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.segment.spi.datasource.OpenStructDataSource;
import org.apache.pinot.segment.spi.index.IndexService;
import org.apache.pinot.segment.spi.index.IndexType;
-import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.JsonIndexReader;
import org.apache.pinot.segment.spi.index.reader.NullValueVectorReader;
import org.apache.pinot.spi.data.FieldSpec;
@@ -52,7 +50,8 @@ import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
/// Filter operator for Map/OPEN_STRUCT matching. Dispatches in priority order:
/// 1. Per-key materialized index (OPEN_STRUCT columns with per-key
inverted/range/bloom indexes)
-/// 2. Sparse virtual scan / manifest short-circuit (unmaterialized keys with
a sparse blob tier)
+/// 2. Sparse virtual scan (unmaterialized keys with a sparse blob tier); a
key absent from the segment resolves to an
+/// all-null source and takes the per-key path
/// 3. JSON index (JsonMatchFilterOperator)
/// 4. Expression scan fallback (ExpressionFilterOperator)
public class MapFilterOperator extends BaseFilterOperator {
@@ -107,92 +106,24 @@ public class MapFilterOperator extends BaseFilterOperator
{
return null;
}
OpenStructDataSource osDs = (OpenStructDataSource) columnDs;
-
- if (osDs.isMaterialized(_keyName)) {
- DataSource keyDs = osDs.getDataSource(_keyName);
- return buildPerKeyFilterOperator(keyDs, queryContext, numDocs);
- }
-
- // Key not materialized but the segment is fully materialized — definitive
absence.
- if (osDs.isFullyMaterialized()) {
- return buildAbsentKeyFilterOperator(osDs, queryContext, numDocs);
- }
-
- // Sparse tier: virtual source from blob, or null if manifest proves
absence.
- DataSource sparseKeyDs = osDs.getDataSource(_keyName);
- if (sparseKeyDs == null) {
- return buildAbsentKeyFilterOperator(osDs, queryContext, numDocs);
- }
- BaseFilterOperator jsonFastPath = trySparseJsonIndex(osDs, sparseKeyDs,
queryContext, numDocs);
- if (jsonFastPath != null) {
- return jsonFastPath;
- }
- return buildPerKeyFilterOperator(sparseKeyDs, queryContext, numDocs);
- }
-
- /// Builds the filter for a key that is definitively absent from a fully
materialized segment.
- /// Every document answers the predicate identically, so it is folded to a
match-all / match-none
- /// once instead of scanning an all-null column.
- ///
- /// With null handling on, three-valued logic makes every value predicate
UNKNOWN and nothing
- /// matches. With it off the key reads as its type's default null value, so
the predicate is
- /// evaluated against that single value — notably NOT_EQ / NOT_IN then match
every document.
- ///
- /// The value comes from {@link OpenStructNullDataSource#forAbsentKey}, the
same source the
- /// projection path feeds to `item()`, so filter and projection cannot
disagree. For a key with no
- /// declared child spec that factory falls back to STRING, which means a
numeric range over an
- /// undeclared key compares lexicographically — the same answer `item()`
yields for it.
- @Nullable
- private BaseFilterOperator buildAbsentKeyFilterOperator(OpenStructDataSource
osDs, QueryContext queryContext,
- int numDocs) {
- Predicate.Type type = _predicate.getType();
- if (type == Predicate.Type.IS_NULL) {
- return new MatchAllFilterOperator(numDocs);
- }
- if (type == Predicate.Type.IS_NOT_NULL) {
- return EmptyFilterOperator.getInstance();
- }
- Predicate rewritten = rewritePredicateForKey(_predicate);
- if (rewritten == null) {
- return null;
- }
- if (queryContext.isNullHandlingEnabled()) {
- return EmptyFilterOperator.getInstance();
- }
- DataSource keyDs = OpenStructNullDataSource.forAbsentKey(osDs, _keyName);
- PredicateEvaluator evaluator =
PredicateEvaluatorProvider.getPredicateEvaluator(rewritten, keyDs,
queryContext);
- Boolean matches = matchesDefaultNullValue(evaluator,
keyDs.getForwardIndex());
- if (matches == null) {
- return null;
- }
- return matches ? new MatchAllFilterOperator(numDocs) :
EmptyFilterOperator.getInstance();
- }
-
- /// Applies the evaluator to the default null value every document of an
absent key reads as.
- /// Returns `null` for a stored type the all-null forward index cannot
serve, so the caller falls
- /// through to the expression path rather than failing the query.
- @Nullable
- private static Boolean matchesDefaultNullValue(PredicateEvaluator evaluator,
ForwardIndexReader<?> forwardIndex) {
- switch (forwardIndex.getStoredType()) {
- case INT:
- return evaluator.applySV(forwardIndex.getInt(0, null));
- case LONG:
- return evaluator.applySV(forwardIndex.getLong(0, null));
- case FLOAT:
- return evaluator.applySV(forwardIndex.getFloat(0, null));
- case DOUBLE:
- return evaluator.applySV(forwardIndex.getDouble(0, null));
- case BIG_DECIMAL:
- return evaluator.applySV(forwardIndex.getBigDecimal(0, null));
- case STRING:
- return evaluator.applySV(forwardIndex.getString(0, null));
- case BYTES:
- return evaluator.applySV(forwardIndex.getBytes(0, null));
- default:
- return null;
+ DataSource keyDs = osDs.getDataSource(_keyName);
+ // Among unmaterialized keys only a sparse virtual source lacks a
dictionary; an absent key folds through its
+ // single-entry dictionary, which is cheaper on the per-key path than
consulting the blob's JSON index
+ if (!osDs.isMaterialized(_keyName) && keyDs.getDictionary() == null) {
+ BaseFilterOperator jsonFastPath = trySparseJsonIndex(osDs, keyDs,
queryContext, numDocs);
+ if (jsonFastPath != null) {
+ return jsonFastPath;
+ }
}
+ return buildPerKeyFilterOperator(keyDs, queryContext, numDocs);
}
+ /// Builds the filter over the key's data source. A key absent from the
segment resolves to an all-null source whose
+ /// single-entry dictionary folds every value predicate to always-true or
always-false, so no document is scanned:
+ /// with null handling on every document is null, so IS_NULL matches
everything and nothing matches a value
+ /// predicate; with it off the key reads as its default null value, so
notably NOT_EQ / NOT_IN match every document.
+ /// An undeclared key is typed STRING, so a numeric range over it compares
lexicographically, the same answer `item()`
+ /// yields for it.
@Nullable
private BaseFilterOperator buildPerKeyFilterOperator(DataSource keyDs,
QueryContext queryContext, int numDocs) {
switch (_predicate.getType()) {
@@ -258,7 +189,8 @@ public class MapFilterOperator extends BaseFilterOperator {
default:
return null;
}
- if (values.contains(FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_STRING)) {
+ // Docs without the key read as the key's default null value, which the
JSON index cannot see
+ if (values.contains((String)
sparseKeyDs.getDataSourceMetadata().getFieldSpec().getDefaultNullValue())) {
return null;
}
if (negated && queryContext.isNullHandlingEnabled()) {
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ItemTransformFunction.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ItemTransformFunction.java
index f19ea36f8d4..86722c286f9 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ItemTransformFunction.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ItemTransformFunction.java
@@ -25,8 +25,6 @@ import javax.annotation.Nullable;
import org.apache.pinot.core.operator.ColumnContext;
import org.apache.pinot.core.operator.blocks.ValueBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
-import org.apache.pinot.segment.local.segment.index.map.NullDataSource;
-import
org.apache.pinot.segment.local.segment.index.openstruct.OpenStructNullDataSource;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.segment.spi.datasource.DataSourceMetadata;
import org.apache.pinot.segment.spi.datasource.MapDataSource;
@@ -64,21 +62,11 @@ public class ItemTransformFunction extends
BaseTransformFunction {
DataSource dataSource = columnContextMap.get(column).getDataSource();
Preconditions.checkState(dataSource instanceof MapDataSource || dataSource
instanceof OpenStructDataSource,
"Column: %s must be a MAP or OPEN_STRUCT column", column);
- DataSource valueDataSource;
- if (dataSource instanceof MapDataSource) {
- valueDataSource = ((MapDataSource) dataSource).getDataSource(key);
- if (valueDataSource == null) {
- valueDataSource = new NullDataSource(key);
- }
- _perKeyNullsAvailable = false;
- } else {
- OpenStructDataSource osDs = (OpenStructDataSource) dataSource;
- valueDataSource = osDs.getDataSource(key);
- if (valueDataSource == null) {
- valueDataSource = OpenStructNullDataSource.forAbsentKey(osDs, key);
- }
- _perKeyNullsAvailable = true;
- }
+ DataSource valueDataSource = dataSource instanceof MapDataSource
+ ? ((MapDataSource) dataSource).getDataSource(key) :
((OpenStructDataSource) dataSource).getDataSource(key);
+ // Per-key nulls are exact whenever the key's source tracks them: every
OPEN_STRUCT key does, a MAP key only when
+ // it is absent from the segment
+ _perKeyNullsAvailable = valueDataSource.getNullValueVector() != null;
// Only expose the dictionary when the forward index is dict-encoded. A
column can have a dictionary alongside
// a RAW forward index (e.g. dict + inverted/range), in which case
transformToDictIdsSV would fail because
// BlockValueSet.getDictionaryIdsSV requires a dict-encoded forward index.
@@ -105,16 +93,12 @@ public class ItemTransformFunction extends
BaseTransformFunction {
return _dictionary;
}
- /// Null semantics differ between the two backing column types:
- ///
- /// - OPEN_STRUCT keeps a per-key presence bitmap (materialized into a
- /// {@link
org.apache.pinot.segment.spi.index.reader.NullValueVectorReader} on both the
mutable and sealed paths),
- /// so the per-key value set knows exactly which docs lack the key. Use it.
- /// - MAP has no per-key null information — an absent key resolves to a
- /// {@link NullDataSource} that carries only a forward index, so the
per-key value set would report "no nulls" for
- /// a key that is missing from every doc. Fall back to {@link
BaseTransformFunction#getNullBitmap} which ORs the
- /// argument bitmaps, yielding the MAP column's own null bitmap: a
conservative over-estimate that downstream
- /// null handling narrows further.
+ /// Uses the per-key null bitmap when the key's data source tracks nulls,
which is exact: every OPEN_STRUCT key does
+ /// (the per-key presence bitmap is materialized into a null value vector on
both the mutable and sealed paths), and
+ /// so does a key absent from a MAP column, whose all-null source marks
every document null. A key present in a MAP
+ /// column carries no per-key null information, so fall back to
[BaseTransformFunction#getNullBitmap] which ORs the
+ /// argument bitmaps, yielding the MAP column's own null bitmap: a
conservative over-estimate that downstream null
+ /// handling narrows further.
@Nullable
@Override
public RoaringBitmap getNullBitmap(ValueBlock valueBlock) {
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/plan/AggregationPlanNode.java
b/pinot-core/src/main/java/org/apache/pinot/core/plan/AggregationPlanNode.java
index b3fc9244bc0..81aa6490ff5 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/plan/AggregationPlanNode.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/plan/AggregationPlanNode.java
@@ -304,15 +304,14 @@ public class AggregationPlanNode implements PlanNode {
DataSource columnDs = segment.getDataSource(columnName, schema);
if (columnDs instanceof OpenStructDataSource) {
OpenStructDataSource osDs = (OpenStructDataSource) columnDs;
- DataSource keyDs = osDs.isMaterialized(key) ? osDs.getDataSource(key) :
null;
- if (keyDs == null) {
+ if (!osDs.isMaterialized(key)) {
return null;
}
// A consuming key column's dictionary can contain a phantom entry (the
reserved default,
// never observed in any doc); dictionary-based aggregation over it
would diverge from the
// sealed segment. The segment layer owns that invariant — force the
scan path when the
// dictionary is not exact.
- return osDs.isKeyDictionaryExact(key) ? keyDs : null;
+ return osDs.isKeyDictionaryExact(key) ? osDs.getDataSource(key) : null;
}
return null;
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/operator/filter/MapFilterOperatorOpenStructTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/operator/filter/MapFilterOperatorOpenStructTest.java
index 247bbc0b451..cea361a37c1 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/operator/filter/MapFilterOperatorOpenStructTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/operator/filter/MapFilterOperatorOpenStructTest.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import java.util.function.Function;
import javax.annotation.Nullable;
import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.common.request.context.FilterContext;
@@ -39,6 +40,7 @@ import org.apache.pinot.core.common.BlockDocIdIterator;
import org.apache.pinot.core.common.BlockDocIdSet;
import org.apache.pinot.core.operator.transform.function.ItemTransformFunction;
import org.apache.pinot.core.query.request.context.QueryContext;
+import org.apache.pinot.segment.local.segment.index.datasource.NullDataSource;
import
org.apache.pinot.segment.local.segment.index.openstruct.FakeStringForwardIndex;
import
org.apache.pinot.segment.local.segment.index.openstruct.OpenStructSparseBlobReader;
import
org.apache.pinot.segment.local.segment.index.openstruct.SparseKeyDataSource;
@@ -54,13 +56,16 @@ import
org.apache.pinot.segment.spi.index.reader.NullValueVectorReader;
import org.apache.pinot.spi.data.ComplexFieldSpec;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.roaringbitmap.buffer.MutableRoaringBitmap;
import org.testng.annotations.Test;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.*;
-import static org.testng.Assert.*;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
public class MapFilterOperatorOpenStructTest {
@@ -70,7 +75,7 @@ public class MapFilterOperatorOpenStructTest {
private static ExpressionContext itemExpr(String column, String key) {
ExpressionContext colArg = ExpressionContext.forIdentifier(column);
- ExpressionContext keyArg =
ExpressionContext.forLiteral(FieldSpec.DataType.STRING, key);
+ ExpressionContext keyArg = ExpressionContext.forLiteral(DataType.STRING,
key);
FunctionContext fn = new FunctionContext(FunctionContext.Type.TRANSFORM,
ItemTransformFunction.FUNCTION_NAME, Arrays.asList(colArg, keyArg));
return ExpressionContext.forFunction(fn);
@@ -109,27 +114,35 @@ public class MapFilterOperatorOpenStructTest {
when(qc.isNullHandlingEnabled()).thenReturn(nullHandlingEnabled);
when(qc.isIndexUseAllowed(any(DataSource.class), any())).thenReturn(true);
when(qc.isIndexUseAllowed(anyString(), any())).thenReturn(true);
+ // The dictionary-based IN evaluator caches the sorted values on the query
context
+ //noinspection unchecked
+ when(qc.getOrComputeSharedValue(any(), any(), any())).thenAnswer(
+ inv -> ((Function<Object, Object>)
inv.getArgument(2)).apply(inv.getArgument(1)));
return qc;
}
- /// OPEN_STRUCT source that is fully materialized but does not hold {@code
key}. Stubs the field
- /// spec and doc count that {@code OpenStructNullDataSource.forAbsentKey}
reads.
+ /// OPEN_STRUCT source that is fully materialized but does not hold `key`,
so the key resolves to an all-null source.
private static OpenStructDataSource mockFullyMaterializedAbsentKey(String
key) {
return mockFullyMaterializedAbsentKey(key, Map.of());
}
- /// OPEN_STRUCT source that is fully materialized but does not hold {@code
key}. Stubs the field
- /// spec and doc count that {@code OpenStructNullDataSource.forAbsentKey}
reads. {@code children}
- /// carries the declared child specs — pass an empty map for an undeclared
key.
+ /// OPEN_STRUCT source that is fully materialized but does not hold `key`,
so the key resolves to an all-null source
+ /// typed by the real `getValueFieldSpec` rule over `children`, the declared
child specs; pass an empty map for an
+ /// undeclared key.
private static OpenStructDataSource mockFullyMaterializedAbsentKey(String
key, Map<String, FieldSpec> children) {
- OpenStructDataSource osDs = mock(OpenStructDataSource.class);
+ OpenStructDataSource osDs = mockOpenStructSource(children);
when(osDs.isMaterialized(key)).thenReturn(false);
when(osDs.isFullyMaterialized()).thenReturn(true);
- when(osDs.getFieldSpec()).thenReturn(
- new ComplexFieldSpec(COLUMN, FieldSpec.DataType.OPEN_STRUCT, true,
children));
- DataSourceMetadata osMeta = mock(DataSourceMetadata.class);
- when(osMeta.getNumDocs()).thenReturn(NUM_DOCS);
- when(osDs.getDataSourceMetadata()).thenReturn(osMeta);
+ NullDataSource absentKeyDs = new
NullDataSource(osDs.getValueFieldSpec(key), NUM_DOCS);
+ when(osDs.getDataSource(key)).thenReturn(absentKeyDs);
+ return osDs;
+ }
+
+ /// OPEN_STRUCT source mock whose key field specs come from the real
`getValueFieldSpec` rule over `children`.
+ private static OpenStructDataSource mockOpenStructSource(Map<String,
FieldSpec> children) {
+ OpenStructDataSource osDs = mock(OpenStructDataSource.class);
+ when(osDs.getFieldSpec()).thenReturn(new ComplexFieldSpec(COLUMN,
DataType.OPEN_STRUCT, true, children));
+ when(osDs.getValueFieldSpec(anyString())).thenCallRealMethod();
return osDs;
}
@@ -153,22 +166,17 @@ public class MapFilterOperatorOpenStructTest {
Map<String, FieldSpec> children, String[] blobs) {
OpenStructSparseBlobReader blob = new OpenStructSparseBlobReader(
new FakeStringForwardIndex(blobs),
FakeStringForwardIndex.nullVector(blobs), NUM_DOCS);
- OpenStructDataSource osDs = mock(OpenStructDataSource.class);
+ OpenStructDataSource osDs = mockOpenStructSource(children);
when(osDs.isMaterialized(anyString())).thenReturn(false);
when(osDs.isFullyMaterialized()).thenReturn(false);
- when(osDs.getFieldSpec()).thenReturn(
- new ComplexFieldSpec(COLUMN, FieldSpec.DataType.OPEN_STRUCT, true,
children));
DataSourceMetadata osMeta = mock(DataSourceMetadata.class);
when(osMeta.getNumDocs()).thenReturn(NUM_DOCS);
when(osDs.getDataSourceMetadata()).thenReturn(osMeta);
when(osDs.getDataSource(anyString())).thenAnswer(inv -> {
String key = inv.getArgument(0);
+ FieldSpec childSpec = osDs.getValueFieldSpec(key);
if (manifest != null && !manifest.contains(key)) {
- return null;
- }
- FieldSpec childSpec = children.get(key);
- if (childSpec == null) {
- childSpec = new DimensionFieldSpec(key, FieldSpec.DataType.STRING,
true);
+ return new NullDataSource(childSpec, NUM_DOCS);
}
return new SparseKeyDataSource(childSpec, blob);
});
@@ -194,7 +202,7 @@ public class MapFilterOperatorOpenStructTest {
when(osDs.getDataSource(KEY)).thenReturn(keyDs);
DataSourceMetadata meta = mock(DataSourceMetadata.class);
- when(meta.getDataType()).thenReturn(FieldSpec.DataType.STRING);
+ when(meta.getDataType()).thenReturn(DataType.STRING);
when(meta.isSorted()).thenReturn(false);
when(meta.isSingleValue()).thenReturn(true);
when(keyDs.getDataSourceMetadata()).thenReturn(meta);
@@ -285,7 +293,7 @@ public class MapFilterOperatorOpenStructTest {
IndexSegment segment = mockSegment(osDs);
Predicate predicate = new RangePredicate(itemExpr(COLUMN, "missing_key"),
false, "100", false,
- RangePredicate.UNBOUNDED, FieldSpec.DataType.LONG);
+ RangePredicate.UNBOUNDED, DataType.LONG);
MapFilterOperator op = new MapFilterOperator(segment, predicate,
mockQueryContext(), NUM_DOCS);
assertEquals(countMatches(op), NUM_DOCS);
@@ -297,11 +305,11 @@ public class MapFilterOperatorOpenStructTest {
@Test
public void testAbsentDeclaredKeyRangeMatchesNothing() {
OpenStructDataSource osDs = mockFullyMaterializedAbsentKey("missing_key",
- Map.of("missing_key", new DimensionFieldSpec("missing_key",
FieldSpec.DataType.LONG, true)));
+ Map.of("missing_key", new DimensionFieldSpec("missing_key",
DataType.LONG, true)));
IndexSegment segment = mockSegment(osDs);
Predicate predicate = new RangePredicate(itemExpr(COLUMN, "missing_key"),
false, "100", false,
- RangePredicate.UNBOUNDED, FieldSpec.DataType.LONG);
+ RangePredicate.UNBOUNDED, DataType.LONG);
MapFilterOperator op = new MapFilterOperator(segment, predicate,
mockQueryContext(), NUM_DOCS);
assertTrue(op.canOptimizeCount());
@@ -373,7 +381,7 @@ public class MapFilterOperatorOpenStructTest {
assertFalse(op.toExplainString().contains("delegateTo:per_key_index"));
} catch (Exception e) {
// The per-key path still had to decline before the expression fallback
was attempted.
- verify(osDs).isFullyMaterialized();
+ verify(osDs).getDataSource("missing_key");
}
}
@@ -470,12 +478,12 @@ public class MapFilterOperatorOpenStructTest {
blobs[i] = i % 2 == 0 ? "{\"latencyMs\":" + i + "}" : null;
}
Map<String, FieldSpec> children =
- Map.of("latencyMs", new DimensionFieldSpec("latencyMs",
FieldSpec.DataType.LONG, true));
+ Map.of("latencyMs", new DimensionFieldSpec("latencyMs", DataType.LONG,
true));
OpenStructDataSource osDs = mockSparseSegmentSource(List.of("latencyMs"),
children, blobs);
IndexSegment segment = mockSegment(osDs);
Predicate range = new RangePredicate(itemExpr(COLUMN, "latencyMs"), false,
"50", false,
- RangePredicate.UNBOUNDED, FieldSpec.DataType.LONG);
+ RangePredicate.UNBOUNDED, DataType.LONG);
MapFilterOperator op = new MapFilterOperator(segment, range,
mockQueryContext(), NUM_DOCS);
assertTrue(op.toExplainString().contains("delegateTo:per_key_index"));
assertEquals(countMatches(op), 24);
@@ -495,7 +503,7 @@ public class MapFilterOperatorOpenStructTest {
assertFalse(op.toExplainString().contains("delegateTo:per_key_index"));
} catch (Exception e) {
// Per-key path declined; expression fallback attempted but may fail on
mock internals.
- verify(osDs).isFullyMaterialized();
+ verify(osDs).getDataSource("region");
}
}
@@ -543,7 +551,7 @@ public class MapFilterOperatorOpenStructTest {
when(osDs.getDataSource(KEY)).thenReturn(keyDs);
DataSourceMetadata meta = mock(DataSourceMetadata.class);
- when(meta.getDataType()).thenReturn(FieldSpec.DataType.STRING);
+ when(meta.getDataType()).thenReturn(DataType.STRING);
when(meta.isSorted()).thenReturn(false);
when(meta.isSingleValue()).thenReturn(true);
when(keyDs.getDataSourceMetadata()).thenReturn(meta);
@@ -648,13 +656,23 @@ public class MapFilterOperatorOpenStructTest {
longBlobs[i] = i % 2 == 0 ? "{\"latencyMs\":" + i + "}" : null;
}
Map<String, FieldSpec> children =
- Map.of("latencyMs", new DimensionFieldSpec("latencyMs",
FieldSpec.DataType.LONG, true));
+ Map.of("latencyMs", new DimensionFieldSpec("latencyMs", DataType.LONG,
true));
OpenStructDataSource c = withSparseJsonIndex(
mockSparseSegmentSource(List.of("latencyMs"), children, longBlobs),
jsonIndex);
MapFilterOperator opC = new MapFilterOperator(mockSegment(c),
makeEqPredicate(COLUMN, "latencyMs", "42"), mockQueryContext(),
NUM_DOCS);
assertTrue(opC.toExplainString().contains("delegateTo:per_key_index"));
+ // (d) EQ against the custom default null value declared on a key the
manifest excludes: the key folds through its
+ // dictionary, which holds that default, instead of consulting the
JSON index
+ Map<String, FieldSpec> customDefault =
+ Map.of("missing", new DimensionFieldSpec("missing", DataType.STRING,
true, "N/A"));
+ OpenStructDataSource d =
withSparseJsonIndex(mockSparseSegmentSource(List.of("region"), customDefault),
jsonIndex);
+ MapFilterOperator opD = new MapFilterOperator(mockSegment(d),
+ makeEqPredicate(COLUMN, "missing", "N/A"), mockQueryContext(),
NUM_DOCS);
+ assertTrue(opD.toExplainString().contains("delegateTo:per_key_index"));
+ assertEquals(countMatches(opD), NUM_DOCS);
+
verify(jsonIndex, never()).getMatchingDocIds(any(FilterContext.class));
}
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ItemTransformFunctionNullBitmapTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ItemTransformFunctionNullBitmapTest.java
index a497b0b3960..a9d7331623b 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ItemTransformFunctionNullBitmapTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ItemTransformFunctionNullBitmapTest.java
@@ -23,14 +23,15 @@ import java.util.Map;
import org.apache.pinot.core.common.BlockValSet;
import org.apache.pinot.core.operator.ColumnContext;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
+import org.apache.pinot.segment.local.segment.index.datasource.NullDataSource;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.segment.spi.datasource.DataSourceMetadata;
import org.apache.pinot.segment.spi.datasource.MapDataSource;
import org.apache.pinot.segment.spi.datasource.OpenStructDataSource;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
-import org.apache.pinot.spi.data.ComplexFieldSpec;
+import org.apache.pinot.segment.spi.index.reader.NullValueVectorReader;
import org.apache.pinot.spi.data.DimensionFieldSpec;
-import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.roaringbitmap.RoaringBitmap;
@@ -39,25 +40,23 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-import static org.testng.Assert.*;
+import static org.mockito.Mockito.*;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
-/// Covers {@link ItemTransformFunction#getNullBitmap} for both backing column
types.
+/// Covers [ItemTransformFunction#getNullBitmap] for both backing column types.
///
-/// <p>OPEN_STRUCT keeps a per-key presence bitmap, so {@code getNullBitmap}
reads the per-key value set and must return
-/// block-local indices (projected by
-/// {@link org.apache.pinot.core.operator.docvalsets.ProjectionBlockValSet})
rather than raw segment-level doc IDs.
+/// OPEN_STRUCT keeps a per-key presence bitmap, so `getNullBitmap` reads the
per-key value set and must return
+/// block-local indices (projected by
[org.apache.pinot.core.operator.docvalsets.ProjectionBlockValSet]) rather than
raw
+/// segment-level doc IDs.
///
-/// <p>MAP has no per-key null information, so {@code getNullBitmap} must fall
back to
-/// {@link BaseTransformFunction#getNullBitmap} — the OR of the argument
bitmaps, which yields the MAP column's own
-/// null bitmap. Reading the per-key value set for a MAP column would report
"no nulls" for a key that is absent from
-/// every doc, because an absent MAP key resolves to a
-/// {@link org.apache.pinot.segment.local.segment.index.map.NullDataSource}
carrying only a forward index.
+/// MAP has no per-key null information for a key present in the column, so
`getNullBitmap` must fall back to
+/// [BaseTransformFunction#getNullBitmap], the OR of the argument bitmaps,
which yields the MAP column's own null
+/// bitmap. A key absent from the column resolves to an all-null
+/// [org.apache.pinot.segment.local.segment.index.datasource.NullDataSource],
whose null value vector is exact, so the
+/// per-key value set is used for it.
public class ItemTransformFunctionNullBitmapTest {
private static final String COLUMN = "myMap";
private static final String KEY = "foo";
@@ -78,9 +77,9 @@ public class ItemTransformFunctionNullBitmapTest {
@Mock
private DataSource _keyDataSource;
@Mock
- private DataSourceMetadata _keyMetadata;
+ private NullValueVectorReader _keyNullValueVector;
@Mock
- private DataSourceMetadata _parentMetadata;
+ private DataSourceMetadata _keyMetadata;
@Mock
private IdentifierTransformFunction _identifierArg;
@Mock
@@ -94,7 +93,7 @@ public class ItemTransformFunctionNullBitmapTest {
when(forwardIndex.isDictionaryEncoded()).thenReturn(false);
doReturn(forwardIndex).when(_keyDataSource).getForwardIndex();
when(_keyDataSource.getDataSourceMetadata()).thenReturn(_keyMetadata);
- when(_keyMetadata.getDataType()).thenReturn(FieldSpec.DataType.STRING);
+ when(_keyMetadata.getDataType()).thenReturn(DataType.STRING);
when(_keyMetadata.isSingleValue()).thenReturn(true);
when(_identifierArg.getColumnName()).thenReturn(COLUMN);
@@ -118,6 +117,7 @@ public class ItemTransformFunctionNullBitmapTest {
@Test
public void testOpenStructReturnsBlockLocalPerKeyIndices() {
when(_openStructDataSource.getDataSource(KEY)).thenReturn(_keyDataSource);
+ when(_keyDataSource.getNullValueVector()).thenReturn(_keyNullValueVector);
RoaringBitmap projectedBitmap = RoaringBitmap.bitmapOf(0, 2);
when(_perKeyBlockValSet.getNullBitmap()).thenReturn(projectedBitmap);
@@ -133,6 +133,7 @@ public class ItemTransformFunctionNullBitmapTest {
@Test
public void testOpenStructIgnoresParentColumnBitmap() {
when(_openStructDataSource.getDataSource(KEY)).thenReturn(_keyDataSource);
+ when(_keyDataSource.getNullValueVector()).thenReturn(_keyNullValueVector);
when(_perKeyBlockValSet.getNullBitmap()).thenReturn(RoaringBitmap.bitmapOf(0,
2));
RoaringBitmap result =
initFunction(_openStructDataSource).getNullBitmap(_projectionBlock);
@@ -144,22 +145,19 @@ public class ItemTransformFunctionNullBitmapTest {
@Test
public void testOpenStructReturnsNullWhenNoNulls() {
when(_openStructDataSource.getDataSource(KEY)).thenReturn(_keyDataSource);
+ when(_keyDataSource.getNullValueVector()).thenReturn(_keyNullValueVector);
when(_perKeyBlockValSet.getNullBitmap()).thenReturn(null);
assertNull(initFunction(_openStructDataSource).getNullBitmap(_projectionBlock));
}
/// A key absent from the OPEN_STRUCT segment resolves to an all-null
- /// {@link
org.apache.pinot.segment.local.segment.index.openstruct.OpenStructNullDataSource},
which still exposes
- /// per-key nulls — so the per-key value set stays authoritative.
+ ///
[org.apache.pinot.segment.local.segment.index.datasource.NullDataSource], which
still exposes per-key nulls, so
+ /// the per-key value set stays authoritative.
@Test
public void testOpenStructAbsentKeyStillUsesPerKeyBitmap() {
- when(_openStructDataSource.getDataSource(KEY)).thenReturn(null);
- when(_openStructDataSource.getFieldSpec()).thenReturn(
- new ComplexFieldSpec(COLUMN, FieldSpec.DataType.OPEN_STRUCT, true,
- Map.of(KEY, new DimensionFieldSpec(KEY, FieldSpec.DataType.STRING,
true))));
-
when(_openStructDataSource.getDataSourceMetadata()).thenReturn(_parentMetadata);
- when(_parentMetadata.getNumDocs()).thenReturn(NUM_DOCS);
+ when(_openStructDataSource.getDataSource(KEY)).thenReturn(
+ new NullDataSource(new DimensionFieldSpec(KEY, DataType.STRING, true),
NUM_DOCS));
RoaringBitmap allNull = RoaringBitmap.bitmapOf(0, 1, 2, 3, 4);
when(_perKeyBlockValSet.getNullBitmap()).thenReturn(allNull);
@@ -171,7 +169,7 @@ public class ItemTransformFunctionNullBitmapTest {
}
//
---------------------------------------------------------------------------------------------
- // MAP — no per-key null info, fall back to the MAP column's own
(conservative) bitmap
+ // MAP — a present key has no per-key null info (fall back to the MAP
column's bitmap), an absent key is all-null
//
---------------------------------------------------------------------------------------------
/// The per-key value set reports "no nulls" (MAP keeps no per-key null
vector), but the MAP column itself is null
@@ -190,18 +188,20 @@ public class ItemTransformFunctionNullBitmapTest {
}
/// A MAP key absent from the segment resolves to a
- /// {@link org.apache.pinot.segment.local.segment.index.map.NullDataSource},
which carries no null value vector.
- /// The fallback must still surface the MAP column's own nulls rather than
reporting none.
+ ///
[org.apache.pinot.segment.local.segment.index.datasource.NullDataSource], whose
null value vector marks every
+ /// document null, so the per-key value set is authoritative and the MAP
column's own bitmap must not widen it.
@Test
- public void testMapAbsentKeyFallsBackToParentColumnBitmap() {
- when(_mapDataSource.getDataSource(KEY)).thenReturn(null);
- when(_perKeyBlockValSet.getNullBitmap()).thenReturn(null);
-
when(_identifierArg.getNullBitmap(_projectionBlock)).thenReturn(RoaringBitmap.bitmapOf(2));
+ public void testMapAbsentKeyUsesPerKeyBitmap() {
+ when(_mapDataSource.getDataSource(KEY)).thenReturn(
+ new NullDataSource(new DimensionFieldSpec(KEY, DataType.STRING, true),
NUM_DOCS));
+
+ RoaringBitmap allNull = RoaringBitmap.bitmapOf(0, 1, 2, 3, 4);
+ when(_perKeyBlockValSet.getNullBitmap()).thenReturn(allNull);
RoaringBitmap result =
initFunction(_mapDataSource).getNullBitmap(_projectionBlock);
- assertNotNull(result);
- assertEquals(result, RoaringBitmap.bitmapOf(2));
+ assertEquals(result, allNull);
+ verify(_identifierArg, never()).getNullBitmap(any());
}
@Test
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/openstruct/OpenStructColumnSplitter.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/openstruct/OpenStructColumnSplitter.java
index 01565e06605..cfa82d50f33 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/openstruct/OpenStructColumnSplitter.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/openstruct/OpenStructColumnSplitter.java
@@ -376,6 +376,10 @@ public class OpenStructColumnSplitter implements
ColumnarOpenStructIndexCreator
RoaringBitmap presence = _presenceBitmaps.get(key);
List<Object> values = _values.get(key);
+ // TODO: Honor the declared child field spec (field type,
single/multi-value and custom default null value) instead
+ // of synthesizing a single-value dimension of the stored type, so a
document without the key reads the same as
+ // through OpenStructDataSource.getValueFieldSpec, which returns the
declared spec for a key absent from the
+ // segment. See https://github.com/apache/pinot/issues/19466
// Synthetic field spec for the materialized child. Its natural Pinot
dimension null value is the value
// stored for absent docs, so column metadata stays consistent with
on-disk content.
DimensionFieldSpec childFieldSpec = new
DimensionFieldSpec(materializedCol, storedType, true);
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/datasource/NullDataSource.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/datasource/NullDataSource.java
new file mode 100644
index 00000000000..583fee40ee3
--- /dev/null
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/datasource/NullDataSource.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.index.datasource;
+
+import
org.apache.pinot.segment.local.segment.index.column.DefaultNullValueVirtualColumnProvider;
+import
org.apache.pinot.segment.local.segment.index.readers.AllNullValueVectorReader;
+import
org.apache.pinot.segment.local.segment.virtualcolumn.VirtualColumnContext;
+import org.apache.pinot.segment.spi.index.reader.NullValueVectorReader;
+import org.apache.pinot.spi.data.FieldSpec;
+
+
+/// Data source for a column that holds no value in any document, such as a
key that is absent from a complex column.
+///
+/// The column is built the same way as a default column, i.e. a schema column
that is missing from the segment: a
+/// single-entry dictionary holding the field's default null value, a constant
sorted forward index for a single-value
+/// field or constant multi-value forward and inverted indexes otherwise, and
metadata reporting a cardinality of one
+/// with the default null value as both min and max. On top of that every
document is marked null in the null value
+/// vector, so with null handling enabled the column reads as null rather than
as the default value.
+///
+/// Immutable and safe to share across threads: every reader is stateless, and
the null value vector materializes its
+/// bitmap lazily under a benign race.
+public class NullDataSource extends ImmutableDataSource {
+ private static final AllNullColumnProvider PROVIDER = new
AllNullColumnProvider();
+
+ public NullDataSource(FieldSpec fieldSpec, int numDocs) {
+ this(new VirtualColumnContext(fieldSpec, numDocs));
+ }
+
+ private NullDataSource(VirtualColumnContext context) {
+ super(PROVIDER.buildMetadata(context),
PROVIDER.buildColumnIndexContainer(context));
+ }
+
+ /// Default column provider that additionally marks every document null.
+ private static class AllNullColumnProvider extends
DefaultNullValueVirtualColumnProvider {
+
+ @Override
+ public NullValueVectorReader buildNullValueVector(VirtualColumnContext
context) {
+ return new AllNullValueVectorReader(context.getTotalDocCount());
+ }
+ }
+}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/map/BaseMapDataSource.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/map/BaseMapDataSource.java
index 102154185e3..c1b51b807ce 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/map/BaseMapDataSource.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/map/BaseMapDataSource.java
@@ -25,7 +25,7 @@ import java.util.Set;
import javax.annotation.Nullable;
import org.apache.pinot.segment.local.segment.index.datasource.BaseDataSource;
import
org.apache.pinot.segment.local.segment.index.datasource.ImmutableDataSource;
-import org.apache.pinot.segment.spi.ColumnMetadata;
+import org.apache.pinot.segment.local.segment.index.datasource.NullDataSource;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.segment.spi.datasource.DataSourceMetadata;
import org.apache.pinot.segment.spi.datasource.MapDataSource;
@@ -35,13 +35,9 @@ import
org.apache.pinot.segment.spi.index.column.ColumnIndexContainer;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.segment.spi.index.reader.MapIndexReader;
import org.apache.pinot.spi.data.ComplexFieldSpec;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
public abstract class BaseMapDataSource extends BaseDataSource implements
MapDataSource {
- private static final Logger LOGGER =
LoggerFactory.getLogger(BaseMapDataSource.class);
-
protected final Map<String, DataSource> _keyDataSources;
public BaseMapDataSource(DataSourceMetadata dataSourceMetadata,
ColumnIndexContainer indexContainer) {
@@ -60,30 +56,22 @@ public abstract class BaseMapDataSource extends
BaseDataSource implements MapDat
return null;
}
- /// Get the Data Source representation of a single key within this map
column.
- ///
- /// @param key to get the DataSource for
- /// @return DataSource for the key
+ /// Returns the data source for a single key within this map column. A key
absent from the map resolves to an
+ /// all-null [NullDataSource] typed as the map's value field.
public DataSource getDataSource(String key) {
- if (_keyDataSources.containsKey(key)) {
- return _keyDataSources.get(key);
+ DataSource dataSource = _keyDataSources.get(key);
+ if (dataSource != null) {
+ return dataSource;
}
- Map<IndexType, IndexReader> indexes = getMapIndexReader().getIndexes(key);
-
+ MapIndexReader mapIndexReader = getMapIndexReader();
+ Map<IndexType, IndexReader> indexes = mapIndexReader.getIndexes(key);
if (indexes == null) {
- // The key does not exist in the map
- return new NullDataSource(key);
+ return new NullDataSource(getFieldSpec().getValueFieldSpec(),
getDataSourceMetadata().getNumDocs());
}
-
- try (ColumnIndexContainer indexContainer = new
ColumnIndexContainer.FromMap(indexes)) {
- ColumnMetadata keyMeta = getMapIndexReader().getColumnMetadata(key);
- ImmutableDataSource dataSource = new ImmutableDataSource(keyMeta,
indexContainer);
- _keyDataSources.put(key, dataSource);
- return dataSource;
- } catch (Exception ex) {
- LOGGER.error("Caught exception while creating key data source for key:
{}", key, ex);
- }
- return null;
+ dataSource =
+ new ImmutableDataSource(mapIndexReader.getColumnMetadata(key), new
ColumnIndexContainer.FromMap(indexes));
+ _keyDataSources.put(key, dataSource);
+ return dataSource;
}
public abstract MapIndexReader getMapIndexReader();
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/map/NullDataSource.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/map/NullDataSource.java
deleted file mode 100644
index caedc153591..00000000000
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/map/NullDataSource.java
+++ /dev/null
@@ -1,255 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.segment.local.segment.index.map;
-
-import java.util.Map;
-import java.util.Set;
-import javax.annotation.Nullable;
-import org.apache.pinot.segment.spi.datasource.DataSource;
-import org.apache.pinot.segment.spi.datasource.DataSourceMetadata;
-import org.apache.pinot.segment.spi.index.IndexReader;
-import org.apache.pinot.segment.spi.index.IndexType;
-import org.apache.pinot.segment.spi.index.StandardIndexes;
-import org.apache.pinot.segment.spi.index.column.ColumnIndexContainer;
-import org.apache.pinot.segment.spi.index.reader.BloomFilterReader;
-import org.apache.pinot.segment.spi.index.reader.Dictionary;
-import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
-import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
-import org.apache.pinot.segment.spi.index.reader.H3IndexReader;
-import org.apache.pinot.segment.spi.index.reader.InvertedIndexReader;
-import org.apache.pinot.segment.spi.index.reader.JsonIndexReader;
-import org.apache.pinot.segment.spi.index.reader.NullValueVectorReader;
-import org.apache.pinot.segment.spi.index.reader.RangeIndexReader;
-import org.apache.pinot.segment.spi.index.reader.TextIndexReader;
-import org.apache.pinot.segment.spi.index.reader.VectorIndexReader;
-import org.apache.pinot.segment.spi.partition.PartitionFunction;
-import org.apache.pinot.spi.data.DimensionFieldSpec;
-import org.apache.pinot.spi.data.FieldSpec;
-
-
-/// If a key does not exist in a Map Column, then the Map Data Source will
return this NulLDataSource.
-/// The NullDataSource represents an INT column where every document has the
Default Null Value. Semantically,
-/// this means that if a key is not in a Map column, then the value will
always resolve to "Null".
-public class NullDataSource implements DataSource {
- private final NullDataSourceMetadata _md;
- private final ColumnIndexContainer _indexes;
-
- public NullDataSource(String name) {
- _md = new NullDataSourceMetadata(name);
- _indexes = new
ColumnIndexContainer.FromMap(Map.of(StandardIndexes.forward(), new
NullForwardIndex()));
- }
-
- @Override
- public DataSourceMetadata getDataSourceMetadata() {
- return _md;
- }
-
- @Override
- public ColumnIndexContainer getIndexContainer() {
- return _indexes;
- }
-
- @Override
- public <R extends IndexReader> R getIndex(IndexType<?, R, ?> type) {
- return type.getIndexReader(_indexes);
- }
-
- @Override
- public ForwardIndexReader<?> getForwardIndex() {
- return getIndex(StandardIndexes.forward());
- }
-
- @Nullable
- @Override
- public Dictionary getDictionary() {
- return getIndex(StandardIndexes.dictionary());
- }
-
- @Nullable
- @Override
- public InvertedIndexReader<?> getInvertedIndex() {
- return getIndex(StandardIndexes.inverted());
- }
-
- @Nullable
- @Override
- public RangeIndexReader<?> getRangeIndex() {
- return getIndex(StandardIndexes.range());
- }
-
- @Nullable
- @Override
- public TextIndexReader getTextIndex() {
- return getIndex(StandardIndexes.text());
- }
-
- @Nullable
- @Override
- public TextIndexReader getFSTIndex() {
- return getIndex(StandardIndexes.fst());
- }
-
- @Nullable
- @Override
- public TextIndexReader getIFSTIndex() {
- return getIndex(StandardIndexes.ifst());
- }
-
- @Nullable
- @Override
- public JsonIndexReader getJsonIndex() {
- return getIndex(StandardIndexes.json());
- }
-
- @Nullable
- @Override
- public H3IndexReader getH3Index() {
- return getIndex(StandardIndexes.h3());
- }
-
- @Nullable
- @Override
- public BloomFilterReader getBloomFilter() {
- return getIndex(StandardIndexes.bloomFilter());
- }
-
- @Nullable
- @Override
- public NullValueVectorReader getNullValueVector() {
- return getIndex(StandardIndexes.nullValueVector());
- }
-
- @Nullable
- @Override
- public VectorIndexReader getVectorIndex() {
- return getIndex(StandardIndexes.vector());
- }
-
- public static class NullDataSourceMetadata implements DataSourceMetadata {
- String _name;
-
- NullDataSourceMetadata(String name) {
- _name = name;
- }
-
- @Override
- public FieldSpec getFieldSpec() {
- return new DimensionFieldSpec(_name, FieldSpec.DataType.INT, true);
- }
-
- @Override
- public boolean isSorted() {
- return false;
- }
-
- @Override
- public int getNumDocs() {
- return 0;
- }
-
- @Override
- public int getNumValues() {
- return 0;
- }
-
- @Override
- public int getMaxNumValuesPerMVEntry() {
- return 0;
- }
-
- @Nullable
- @Override
- public Comparable getMinValue() {
- return FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_INT;
- }
-
- @Nullable
- @Override
- public Comparable getMaxValue() {
- return FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_INT;
- }
-
- @Nullable
- @Override
- public PartitionFunction getPartitionFunction() {
- return null;
- }
-
- @Nullable
- @Override
- public Set<Integer> getPartitions() {
- return null;
- }
-
- @Override
- public int getCardinality() {
- return 1;
- }
- }
-
- public class NullForwardIndex implements
ForwardIndexReader<ForwardIndexReaderContext> {
- NullForwardIndex() {
- }
-
- @Override
- public boolean isDictionaryEncoded() {
- return false;
- }
-
- @Override
- public boolean isSingleValue() {
- return false;
- }
-
- @Override
- public FieldSpec.DataType getStoredType() {
- return FieldSpec.DataType.INT;
- }
-
- @Override
- public int getInt(int docId, ForwardIndexReaderContext context) {
- return FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_INT;
- }
-
- @Override
- public long getLong(int docId, ForwardIndexReaderContext context) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public float getFloat(int docId, ForwardIndexReaderContext context) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public double getDouble(int docId, ForwardIndexReaderContext context) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public String getString(int docId, ForwardIndexReaderContext context) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void close() {
- }
- }
-}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/ImmutableOpenStructDataSource.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/ImmutableOpenStructDataSource.java
index fbc8c10dc64..6972604027b 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/ImmutableOpenStructDataSource.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/ImmutableOpenStructDataSource.java
@@ -25,6 +25,7 @@ import java.util.concurrent.ConcurrentHashMap;
import javax.annotation.Nullable;
import org.apache.pinot.segment.local.segment.index.datasource.BaseDataSource;
import
org.apache.pinot.segment.local.segment.index.datasource.ImmutableDataSource;
+import org.apache.pinot.segment.local.segment.index.datasource.NullDataSource;
import org.apache.pinot.segment.spi.Constants;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.segment.spi.datasource.DataSourceMetadata;
@@ -34,13 +35,12 @@ import
org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.JsonIndexReader;
import org.apache.pinot.segment.spi.partition.PartitionFunction;
import org.apache.pinot.spi.data.ComplexFieldSpec;
-import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
-/// Per-key {@link DataSource} accessor for sealed OPEN_STRUCT segments. Dense
keys get
-/// materialized DataSources; sparse keys get virtual [SparseKeyDataSource]s
backed by the
-/// shared blob parser. Manifest-absent keys return null (definitively absent).
+/// Per-key [DataSource] accessor for sealed OPEN_STRUCT segments. Dense keys
get materialized DataSources; sparse keys
+/// get virtual [SparseKeyDataSource]s backed by the shared blob parser; keys
absent from the segment (no sparse blob,
+/// or not listed in the sparse manifest) resolve to an all-null
[NullDataSource].
public class ImmutableOpenStructDataSource extends BaseDataSource implements
OpenStructDataSource {
private final ComplexFieldSpec _fieldSpec;
private final Map<String, DataSource> _perKeyDataSources;
@@ -92,27 +92,17 @@ public class ImmutableOpenStructDataSource extends
BaseDataSource implements Ope
}
@Override
- @Nullable
public DataSource getDataSource(String key) {
DataSource ds = _perKeyDataSources.get(key);
if (ds != null) {
return ds;
}
- if (_sparseBlobReader == null) {
- return null;
+ if (_sparseBlobReader == null || (_sparseKeys != null &&
!_sparseKeys.contains(key))) {
+ // Definitively absent: no sparse blob, or the sparse manifest does not
list the key
+ return new NullDataSource(getValueFieldSpec(key),
getDataSourceMetadata().getNumDocs());
}
- if (_sparseKeys != null && !_sparseKeys.contains(key)) {
- return null;
- }
- return _sparseKeyDataSourceCache.computeIfAbsent(key,
this::buildSparseKeyDataSource);
- }
-
- private DataSource buildSparseKeyDataSource(String key) {
- FieldSpec childSpec = _fieldSpec.getChildFieldSpec(key);
- if (childSpec == null) {
- childSpec = new DimensionFieldSpec(key, FieldSpec.DataType.STRING, true);
- }
- return new SparseKeyDataSource(childSpec, _sparseBlobReader);
+ return _sparseKeyDataSourceCache.computeIfAbsent(key,
+ k -> new SparseKeyDataSource(getValueFieldSpec(k), _sparseBlobReader));
}
@Override
@@ -133,10 +123,8 @@ public class ImmutableOpenStructDataSource extends
BaseDataSource implements Ope
}
@Override
- @Nullable
public DataSourceMetadata getDataSourceMetadata(String key) {
- DataSource ds = getDataSource(key);
- return ds != null ? ds.getDataSourceMetadata() : null;
+ return getDataSource(key).getDataSourceMetadata();
}
@Override
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/MutableOpenStructDataSource.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/MutableOpenStructDataSource.java
index 497e0967649..1f05cd8e8af 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/MutableOpenStructDataSource.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/MutableOpenStructDataSource.java
@@ -24,6 +24,7 @@ import java.util.concurrent.ConcurrentHashMap;
import javax.annotation.Nullable;
import org.apache.pinot.segment.local.segment.index.datasource.BaseDataSource;
import
org.apache.pinot.segment.local.segment.index.datasource.ImmutableDataSource;
+import org.apache.pinot.segment.local.segment.index.datasource.NullDataSource;
import org.apache.pinot.segment.spi.ColumnMetadata;
import org.apache.pinot.segment.spi.Constants;
import org.apache.pinot.segment.spi.datasource.DataSource;
@@ -69,12 +70,12 @@ public class MutableOpenStructDataSource extends
BaseDataSource implements OpenS
}
@Override
- @Nullable
public DataSource getDataSource(String key) {
- // Live lookup, outside the memo: a key not yet observed may still be
created by the ingestion thread.
+ // Live lookup, outside the memo: a key not yet observed may still be
created by the ingestion thread, so an
+ // absent key resolves to a fresh all-null source instead of being memoised
MutableKeyColumn col = _index.getKeyColumn(key);
if (col == null) {
- return null;
+ return new NullDataSource(getValueFieldSpec(key), _numDocs);
}
return _perKeyDataSourceCache.computeIfAbsent(key, k -> {
Map<IndexType, IndexReader> indexes = new
HashMap<>(_index.getIndexes(k));
@@ -118,19 +119,14 @@ public class MutableOpenStructDataSource extends
BaseDataSource implements OpenS
public Map<String, DataSource> getDataSources() {
Map<String, DataSource> result = new HashMap<>();
for (String key : _index.getKeys()) {
- DataSource ds = getDataSource(key);
- if (ds != null) {
- result.put(key, ds);
- }
+ result.put(key, getDataSource(key));
}
return result;
}
@Override
- @Nullable
public DataSourceMetadata getDataSourceMetadata(String key) {
- DataSource ds = getDataSource(key);
- return ds != null ? ds.getDataSourceMetadata() : null;
+ return getDataSource(key).getDataSourceMetadata();
}
@Override
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/MutableOpenStructIndex.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/MutableOpenStructIndex.java
index 23bb33795a1..b1c81dac7b2 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/MutableOpenStructIndex.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/MutableOpenStructIndex.java
@@ -196,11 +196,11 @@ public class MutableOpenStructIndex implements
OpenStructIndexReader<ForwardInde
/// publishes it via volatile copy-on-write.
private MutableKeyColumn allocateKeyColumn(String key, DataType storedType) {
String allocationContext = _openStructColumn + "$" + key;
- // Use the standard dimension default for the resolved stored type,
regardless of any child
- // spec's own default: OpenStructColumnSplitter#writeDenseKeyColumn (the
sealed build path)
- // always derives the absent-doc default from a throwaway
DimensionFieldSpec(key, storedType,
- // true) rather than the real child spec, so mirroring that exactly is
what keeps a doc's
- // resolved value identical before and after seal.
+ // TODO: Use the declared child field spec's default null value, as
OpenStructDataSource.getValueFieldSpec does for
+ // a key absent from the segment. The standard dimension default of the
stored type is kept for now because
+ // OpenStructColumnSplitter#writeDenseKeyColumn (the sealed build path)
derives the absent-doc default the same
+ // way, and a doc's resolved value must stay identical before and after
seal.
+ // See https://github.com/apache/pinot/issues/19466
Object defaultNullValue =
FieldSpec.getDefaultNullValue(FieldSpec.FieldType.DIMENSION, storedType, null);
MutableKeyColumn newCol =
new MutableKeyColumn(key, storedType, defaultNullValue,
_memoryManager, _capacity, allocationContext);
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/OpenStructNullDataSource.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/OpenStructNullDataSource.java
deleted file mode 100644
index 1b36dceff4d..00000000000
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/OpenStructNullDataSource.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.segment.local.segment.index.openstruct;
-
-import java.math.BigDecimal;
-import java.util.Map;
-import java.util.Set;
-import javax.annotation.Nullable;
-import org.apache.pinot.segment.local.segment.index.datasource.BaseDataSource;
-import
org.apache.pinot.segment.local.segment.index.readers.AllNullValueVectorReader;
-import org.apache.pinot.segment.spi.datasource.DataSourceMetadata;
-import org.apache.pinot.segment.spi.datasource.OpenStructDataSource;
-import org.apache.pinot.segment.spi.index.StandardIndexes;
-import org.apache.pinot.segment.spi.index.column.ColumnIndexContainer;
-import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
-import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
-import org.apache.pinot.segment.spi.partition.PartitionFunction;
-import org.apache.pinot.spi.data.DimensionFieldSpec;
-import org.apache.pinot.spi.data.FieldSpec;
-import org.apache.pinot.spi.data.FieldSpec.DataType;
-
-
-/// Typed all-null {@link org.apache.pinot.segment.spi.datasource.DataSource}
for an OPEN_STRUCT key
-/// that is absent from a segment.
-///
-/// Unlike the MAP column's {@code NullDataSource} (which is hardcoded as INT
with numDocs=0),
-/// this class uses the correct {@link DataType} from the OPEN_STRUCT child
{@link FieldSpec} and
-/// reports the segment's actual doc count. Every document is null: the
forward index returns the
-/// type-correct default null value, and the {@link NullValueVectorReader}
returns a full-segment
-/// bitmap.
-///
-/// Extends {@link BaseDataSource}: every other index accessor (dictionary,
inverted, range, text,
-/// bloom filter, vector, etc.) resolves to {@code null} automatically since
the index container
-/// only holds a forward index and a null value vector.
-public class OpenStructNullDataSource extends BaseDataSource {
- public OpenStructNullDataSource(FieldSpec fieldSpec, int numDocs) {
- super(new AllNullMetadata(fieldSpec, numDocs), new
ColumnIndexContainer.FromMap(Map.of(
- StandardIndexes.forward(), new
TypedNullForwardIndex(fieldSpec.getDataType().getStoredType()),
- StandardIndexes.nullValueVector(), new
AllNullValueVectorReader(numDocs))));
- }
-
- /// Creates an all-null DataSource for a key absent from this OPEN_STRUCT
segment.
- /// Resolves the key's type from the child FieldSpec, falling back to STRING.
- public static OpenStructNullDataSource forAbsentKey(OpenStructDataSource
osDs, String key) {
- FieldSpec childSpec = osDs.getFieldSpec().getChildFieldSpec(key);
- if (childSpec == null) {
- childSpec = new DimensionFieldSpec(key, FieldSpec.DataType.STRING, true);
- }
- int numDocs = osDs.getDataSourceMetadata().getNumDocs();
- return new OpenStructNullDataSource(childSpec, numDocs);
- }
-
- /// Forward index that returns the type-correct default null value for every
document.
- static class TypedNullForwardIndex implements
ForwardIndexReader<ForwardIndexReaderContext> {
- private final DataType _storedType;
-
- TypedNullForwardIndex(DataType storedType) {
- _storedType = storedType;
- }
-
- @Override
- public boolean isDictionaryEncoded() {
- return false;
- }
-
- @Override
- public boolean isSingleValue() {
- return true;
- }
-
- @Override
- public DataType getStoredType() {
- return _storedType;
- }
-
- @Override
- public int getInt(int docId, ForwardIndexReaderContext context) {
- return FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_INT;
- }
-
- @Override
- public long getLong(int docId, ForwardIndexReaderContext context) {
- return FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_LONG;
- }
-
- @Override
- public float getFloat(int docId, ForwardIndexReaderContext context) {
- return FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_FLOAT;
- }
-
- @Override
- public double getDouble(int docId, ForwardIndexReaderContext context) {
- return FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_DOUBLE;
- }
-
- @Override
- public BigDecimal getBigDecimal(int docId, ForwardIndexReaderContext
context) {
- return FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_BIG_DECIMAL;
- }
-
- @Override
- public String getString(int docId, ForwardIndexReaderContext context) {
- return FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_STRING;
- }
-
- @Override
- public byte[] getBytes(int docId, ForwardIndexReaderContext context) {
- return FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_BYTES;
- }
-
- @Override
- public void close() {
- }
- }
-
- private static class AllNullMetadata implements DataSourceMetadata {
- private final FieldSpec _fieldSpec;
- private final int _numDocs;
-
- AllNullMetadata(FieldSpec fieldSpec, int numDocs) {
- _fieldSpec = fieldSpec;
- _numDocs = numDocs;
- }
-
- @Override
- public FieldSpec getFieldSpec() {
- return _fieldSpec;
- }
-
- @Override
- public boolean isSorted() {
- return false;
- }
-
- @Override
- public int getNumDocs() {
- return _numDocs;
- }
-
- @Override
- public int getNumValues() {
- return _numDocs;
- }
-
- @Override
- public int getMaxNumValuesPerMVEntry() {
- return 0;
- }
-
- @Override
- public int getCardinality() {
- return 1;
- }
-
- @Nullable
- @Override
- public Comparable getMinValue() {
- return null;
- }
-
- @Nullable
- @Override
- public Comparable getMaxValue() {
- return null;
- }
-
- @Nullable
- @Override
- public PartitionFunction getPartitionFunction() {
- return null;
- }
-
- @Nullable
- @Override
- public Set<Integer> getPartitions() {
- return null;
- }
- }
-}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/SparseKeyDataSource.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/SparseKeyDataSource.java
index c83c2500264..6ed39cef11c 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/SparseKeyDataSource.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/SparseKeyDataSource.java
@@ -103,9 +103,10 @@ public class SparseKeyDataSource extends BaseDataSource {
return node == null ? defaultValue : map.apply(node);
}
- // TODO: these hardcode the type defaults, so a child with a
schema-configured defaultNullValue
- // reads back differently here than on the dense path
(OpenStructColumnSplitter uses the spec).
- // Fixing it means threading the spec in and updating MapFilterOperator's
sentinel check to match.
+ // TODO: Read a document without the key as the child field spec's default
null value, as
+ // OpenStructDataSource.getValueFieldSpec does for a key absent from the
segment, instead of the standard type
+ // default hardcoded below. MapFilterOperator already refuses the sparse
JSON fast path for the spec's default.
+ // See https://github.com/apache/pinot/issues/19466
@Override
public int getInt(int docId, ForwardIndexReaderContext context) {
return orDefault(docId, context, JsonNode::asInt,
FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_INT);
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/virtualcolumn/BaseConstantValueVirtualColumnProvider.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/virtualcolumn/BaseConstantValueVirtualColumnProvider.java
index d4d7c9fcc10..d733ef17689 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/virtualcolumn/BaseConstantValueVirtualColumnProvider.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/virtualcolumn/BaseConstantValueVirtualColumnProvider.java
@@ -166,6 +166,8 @@ public abstract class
BaseConstantValueVirtualColumnProvider implements VirtualC
checkValue(context, value);
ColumnMetadataImpl.Builder builder = new
ColumnMetadataImpl.Builder().setFieldSpec(fieldSpec)
.setTotalDocs(context.getTotalDocCount())
+ // Every document holds exactly one value, also for a multi-value
field (a single-element entry)
+ .setTotalNumberOfEntries(context.getTotalDocCount())
.setCardinality(1)
.setHasDictionary(true);
if (fieldSpec.isSingleValueField()) {
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/column/DefaultNullValueVirtualColumnProviderTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/column/DefaultNullValueVirtualColumnProviderTest.java
index 3e7cc400dee..031d23efdcc 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/column/DefaultNullValueVirtualColumnProviderTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/column/DefaultNullValueVirtualColumnProviderTest.java
@@ -56,64 +56,142 @@ public class DefaultNullValueVirtualColumnProviderTest {
@Test
public void testBuildMetadata() {
assertEquals(new DefaultNullValueVirtualColumnProvider().buildMetadata(new
VirtualColumnContext(SV_INT, 1)),
- new
ColumnMetadataImpl.Builder().setFieldSpec(SV_INT).setTotalDocs(1).setCardinality(1).setSorted(true)
- .setHasDictionary(true).setMinValue((int)
SV_INT.getDefaultNullValue())
- .setMaxValue((int) SV_INT.getDefaultNullValue()).build());
+ new ColumnMetadataImpl.Builder().setFieldSpec(SV_INT)
+ .setTotalDocs(1)
+ .setTotalNumberOfEntries(1)
+ .setCardinality(1)
+ .setSorted(true)
+ .setHasDictionary(true)
+ .setMinValue((int) SV_INT.getDefaultNullValue())
+ .setMaxValue((int) SV_INT.getDefaultNullValue())
+ .build());
assertEquals(new DefaultNullValueVirtualColumnProvider().buildMetadata(new
VirtualColumnContext(SV_LONG, 1)),
- new
ColumnMetadataImpl.Builder().setFieldSpec(SV_LONG).setTotalDocs(1).setCardinality(1).setSorted(true)
- .setHasDictionary(true).setMinValue((long)
SV_LONG.getDefaultNullValue())
- .setMaxValue((long) SV_LONG.getDefaultNullValue()).build());
+ new ColumnMetadataImpl.Builder().setFieldSpec(SV_LONG)
+ .setTotalDocs(1)
+ .setTotalNumberOfEntries(1)
+ .setCardinality(1)
+ .setSorted(true)
+ .setHasDictionary(true)
+ .setMinValue((long) SV_LONG.getDefaultNullValue())
+ .setMaxValue((long) SV_LONG.getDefaultNullValue())
+ .build());
assertEquals(new DefaultNullValueVirtualColumnProvider().buildMetadata(new
VirtualColumnContext(SV_FLOAT, 1)),
- new
ColumnMetadataImpl.Builder().setFieldSpec(SV_FLOAT).setTotalDocs(1).setCardinality(1).setSorted(true)
- .setHasDictionary(true).setMinValue((float)
SV_FLOAT.getDefaultNullValue())
- .setMaxValue((float) SV_FLOAT.getDefaultNullValue()).build());
+ new ColumnMetadataImpl.Builder().setFieldSpec(SV_FLOAT)
+ .setTotalDocs(1)
+ .setTotalNumberOfEntries(1)
+ .setCardinality(1)
+ .setSorted(true)
+ .setHasDictionary(true)
+ .setMinValue((float) SV_FLOAT.getDefaultNullValue())
+ .setMaxValue((float) SV_FLOAT.getDefaultNullValue())
+ .build());
assertEquals(new DefaultNullValueVirtualColumnProvider().buildMetadata(new
VirtualColumnContext(SV_DOUBLE, 1)),
- new
ColumnMetadataImpl.Builder().setFieldSpec(SV_DOUBLE).setTotalDocs(1).setCardinality(1).setSorted(true)
- .setHasDictionary(true).setMinValue((double)
SV_DOUBLE.getDefaultNullValue())
- .setMaxValue((double) SV_DOUBLE.getDefaultNullValue()).build());
+ new ColumnMetadataImpl.Builder().setFieldSpec(SV_DOUBLE)
+ .setTotalDocs(1)
+ .setTotalNumberOfEntries(1)
+ .setCardinality(1)
+ .setSorted(true)
+ .setHasDictionary(true)
+ .setMinValue((double) SV_DOUBLE.getDefaultNullValue())
+ .setMaxValue((double) SV_DOUBLE.getDefaultNullValue())
+ .build());
assertEquals(new DefaultNullValueVirtualColumnProvider().buildMetadata(new
VirtualColumnContext(SV_STRING, 1)),
- new
ColumnMetadataImpl.Builder().setFieldSpec(SV_STRING).setTotalDocs(1).setCardinality(1).setSorted(true)
- .setHasDictionary(true).setMinValue((String)
SV_STRING.getDefaultNullValue())
- .setMaxValue((String) SV_STRING.getDefaultNullValue()).build());
+ new ColumnMetadataImpl.Builder().setFieldSpec(SV_STRING)
+ .setTotalDocs(1)
+ .setTotalNumberOfEntries(1)
+ .setCardinality(1)
+ .setSorted(true)
+ .setHasDictionary(true)
+ .setMinValue((String) SV_STRING.getDefaultNullValue())
+ .setMaxValue((String) SV_STRING.getDefaultNullValue())
+ .build());
assertEquals(
new DefaultNullValueVirtualColumnProvider().buildMetadata(new
VirtualColumnContext(SV_STRING_WITH_DEFAULT, 1)),
- new
ColumnMetadataImpl.Builder().setFieldSpec(SV_STRING_WITH_DEFAULT).setTotalDocs(1).setCardinality(1)
-
.setSorted(true).setHasDictionary(true).setMinValue("default").setMaxValue("default").build());
+ new ColumnMetadataImpl.Builder().setFieldSpec(SV_STRING_WITH_DEFAULT)
+ .setTotalDocs(1)
+ .setTotalNumberOfEntries(1)
+ .setCardinality(1)
+ .setSorted(true)
+ .setHasDictionary(true)
+ .setMinValue("default")
+ .setMaxValue("default")
+ .build());
assertEquals(new DefaultNullValueVirtualColumnProvider().buildMetadata(new
VirtualColumnContext(SV_BYTES, 1)),
- new
ColumnMetadataImpl.Builder().setFieldSpec(SV_BYTES).setTotalDocs(1).setCardinality(1).setSorted(true)
- .setHasDictionary(true).setMinValue(new ByteArray((byte[])
SV_BYTES.getDefaultNullValue()))
- .setMaxValue(new ByteArray((byte[])
SV_BYTES.getDefaultNullValue())).build());
+ new ColumnMetadataImpl.Builder().setFieldSpec(SV_BYTES)
+ .setTotalDocs(1)
+ .setTotalNumberOfEntries(1)
+ .setCardinality(1)
+ .setSorted(true)
+ .setHasDictionary(true)
+ .setMinValue(new ByteArray((byte[])
SV_BYTES.getDefaultNullValue()))
+ .setMaxValue(new ByteArray((byte[])
SV_BYTES.getDefaultNullValue()))
+ .build());
assertEquals(new DefaultNullValueVirtualColumnProvider().buildMetadata(new
VirtualColumnContext(MV_INT, 1)),
- new
ColumnMetadataImpl.Builder().setFieldSpec(MV_INT).setTotalDocs(1).setCardinality(1).setSorted(false)
-
.setHasDictionary(true).setMaxNumberOfMultiValues(1).setMinValue((int)
MV_INT.getDefaultNullValue())
- .setMaxValue((int) MV_INT.getDefaultNullValue()).build());
+ new ColumnMetadataImpl.Builder().setFieldSpec(MV_INT)
+ .setTotalDocs(1)
+ .setTotalNumberOfEntries(1)
+ .setCardinality(1)
+ .setSorted(false)
+ .setHasDictionary(true)
+ .setMaxNumberOfMultiValues(1)
+ .setMinValue((int) MV_INT.getDefaultNullValue())
+ .setMaxValue((int) MV_INT.getDefaultNullValue())
+ .build());
assertEquals(new DefaultNullValueVirtualColumnProvider().buildMetadata(new
VirtualColumnContext(MV_LONG, 1)),
- new
ColumnMetadataImpl.Builder().setFieldSpec(MV_LONG).setTotalDocs(1).setCardinality(1).setSorted(false)
-
.setHasDictionary(true).setMaxNumberOfMultiValues(1).setMinValue((long)
MV_LONG.getDefaultNullValue())
- .setMaxValue((long) MV_LONG.getDefaultNullValue()).build());
+ new ColumnMetadataImpl.Builder().setFieldSpec(MV_LONG)
+ .setTotalDocs(1)
+ .setTotalNumberOfEntries(1)
+ .setCardinality(1)
+ .setSorted(false)
+ .setHasDictionary(true)
+ .setMaxNumberOfMultiValues(1)
+ .setMinValue((long) MV_LONG.getDefaultNullValue())
+ .setMaxValue((long) MV_LONG.getDefaultNullValue())
+ .build());
assertEquals(new DefaultNullValueVirtualColumnProvider().buildMetadata(new
VirtualColumnContext(MV_FLOAT, 1)),
- new
ColumnMetadataImpl.Builder().setFieldSpec(MV_FLOAT).setTotalDocs(1).setCardinality(1).setSorted(false)
-
.setHasDictionary(true).setMaxNumberOfMultiValues(1).setMinValue((float)
MV_FLOAT.getDefaultNullValue())
- .setMaxValue((float) MV_FLOAT.getDefaultNullValue()).build());
+ new ColumnMetadataImpl.Builder().setFieldSpec(MV_FLOAT)
+ .setTotalDocs(1)
+ .setTotalNumberOfEntries(1)
+ .setCardinality(1)
+ .setSorted(false)
+ .setHasDictionary(true)
+ .setMaxNumberOfMultiValues(1)
+ .setMinValue((float) MV_FLOAT.getDefaultNullValue())
+ .setMaxValue((float) MV_FLOAT.getDefaultNullValue())
+ .build());
assertEquals(new DefaultNullValueVirtualColumnProvider().buildMetadata(new
VirtualColumnContext(MV_DOUBLE, 1)),
- new
ColumnMetadataImpl.Builder().setFieldSpec(MV_DOUBLE).setTotalDocs(1).setCardinality(1).setSorted(false)
-
.setHasDictionary(true).setMaxNumberOfMultiValues(1).setMinValue((double)
MV_DOUBLE.getDefaultNullValue())
- .setMaxValue((double) MV_DOUBLE.getDefaultNullValue()).build());
+ new ColumnMetadataImpl.Builder().setFieldSpec(MV_DOUBLE)
+ .setTotalDocs(1)
+ .setTotalNumberOfEntries(1)
+ .setCardinality(1)
+ .setSorted(false)
+ .setHasDictionary(true)
+ .setMaxNumberOfMultiValues(1)
+ .setMinValue((double) MV_DOUBLE.getDefaultNullValue())
+ .setMaxValue((double) MV_DOUBLE.getDefaultNullValue())
+ .build());
assertEquals(new DefaultNullValueVirtualColumnProvider().buildMetadata(new
VirtualColumnContext(MV_STRING, 1)),
- new
ColumnMetadataImpl.Builder().setFieldSpec(MV_STRING).setTotalDocs(1).setCardinality(1).setSorted(false)
-
.setHasDictionary(true).setMaxNumberOfMultiValues(1).setMinValue((String)
MV_STRING.getDefaultNullValue())
- .setMaxValue((String) MV_STRING.getDefaultNullValue()).build());
+ new ColumnMetadataImpl.Builder().setFieldSpec(MV_STRING)
+ .setTotalDocs(1)
+ .setTotalNumberOfEntries(1)
+ .setCardinality(1)
+ .setSorted(false)
+ .setHasDictionary(true)
+ .setMaxNumberOfMultiValues(1)
+ .setMinValue((String) MV_STRING.getDefaultNullValue())
+ .setMaxValue((String) MV_STRING.getDefaultNullValue())
+ .build());
}
@Test
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/datasource/NullDataSourceTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/datasource/NullDataSourceTest.java
new file mode 100644
index 00000000000..1181d70c70f
--- /dev/null
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/datasource/NullDataSourceTest.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.index.datasource;
+
+import org.apache.pinot.segment.spi.datasource.DataSourceMetadata;
+import org.apache.pinot.segment.spi.index.reader.Dictionary;
+import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
+import org.apache.pinot.segment.spi.index.reader.NullValueVectorReader;
+import org.apache.pinot.segment.spi.index.reader.SortedIndexReader;
+import org.apache.pinot.spi.data.DimensionFieldSpec;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.*;
+
+
+/// Covers [NullDataSource], the data source for a column holding no value in
any document. Asserts it is built
+/// like a default column — a single-entry dictionary over the field's default
null value, constant sorted or
+/// multi-value readers, and metadata with cardinality one — and that every
document is additionally marked null.
+public class NullDataSourceTest {
+ private static final int NUM_DOCS = 100;
+
+ @Test
+ public void testBuiltLikeDefaultColumn() {
+ NullDataSource dataSource = new NullDataSource(new DimensionFieldSpec("k",
DataType.LONG, true), NUM_DOCS);
+
+ DataSourceMetadata metadata = dataSource.getDataSourceMetadata();
+ assertEquals(metadata.getDataType(), DataType.LONG);
+ assertTrue(metadata.isSingleValue());
+ assertTrue(metadata.isSorted());
+ assertEquals(metadata.getNumDocs(), NUM_DOCS);
+ assertEquals(metadata.getNumValues(), NUM_DOCS);
+ assertEquals(metadata.getMaxNumValuesPerMVEntry(), -1);
+ assertEquals(metadata.getCardinality(), 1);
+ assertEquals(metadata.getMinValue(),
FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_LONG);
+ assertEquals(metadata.getMaxValue(),
FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_LONG);
+
+ Dictionary dictionary = dataSource.getDictionary();
+ assertNotNull(dictionary);
+ assertEquals(dictionary.length(), 1);
+ assertEquals(dictionary.getLongValue(0), (long)
FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_LONG);
+
+ ForwardIndexReader<?> forwardIndex = dataSource.getForwardIndex();
+ assertTrue(forwardIndex.isDictionaryEncoded());
+ assertTrue(forwardIndex.isSingleValue());
+ assertEquals(forwardIndex.getDictId(NUM_DOCS - 1, null), 0);
+
+ SortedIndexReader<?> sortedIndex = (SortedIndexReader<?>)
dataSource.getInvertedIndex();
+ assertEquals(sortedIndex.getDocIds(0).getLeft(), 0);
+ assertEquals(sortedIndex.getDocIds(0).getRight(), NUM_DOCS - 1);
+
+ assertNull(dataSource.getRangeIndex());
+ assertNull(dataSource.getJsonIndex());
+ assertNull(dataSource.getBloomFilter());
+ assertNull(dataSource.getTextIndex());
+ assertNull(dataSource.getVectorIndex());
+ }
+
+ @Test
+ public void testEveryDocumentIsNull() {
+ NullDataSource dataSource = new NullDataSource(new DimensionFieldSpec("k",
DataType.INT, true), 50);
+ NullValueVectorReader nullValueVector = dataSource.getNullValueVector();
+ assertNotNull(nullValueVector);
+ for (int i = 0; i < 50; i++) {
+ assertTrue(nullValueVector.isNull(i));
+ }
+ assertEquals(nullValueVector.getNullBitmap().getCardinality(), 50);
+
+ dataSource = new NullDataSource(new DimensionFieldSpec("k", DataType.INT,
true), 0);
+
assertEquals(dataSource.getNullValueVector().getNullBitmap().getCardinality(),
0);
+ }
+
+ @Test
+ public void testDictionaryHoldsDefaultNullValue() {
+ assertEquals(dictionaryValue(DataType.INT),
FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_INT);
+ assertEquals(dictionaryValue(DataType.LONG),
FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_LONG);
+ assertEquals(dictionaryValue(DataType.FLOAT),
FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_FLOAT);
+ assertEquals(dictionaryValue(DataType.DOUBLE),
FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_DOUBLE);
+ assertEquals(dictionaryValue(DataType.BIG_DECIMAL),
FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_BIG_DECIMAL);
+ assertEquals(dictionaryValue(DataType.BOOLEAN),
FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_BOOLEAN);
+ assertEquals(dictionaryValue(DataType.TIMESTAMP),
FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_TIMESTAMP);
+ assertEquals(dictionaryValue(DataType.STRING),
FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_STRING);
+ assertEquals(dictionaryValue(DataType.JSON),
FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_JSON);
+ assertEquals(dictionaryValue(DataType.BYTES),
FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_BYTES);
+ }
+
+ @Test
+ public void testCustomDefaultNullValue() {
+ NullDataSource dataSource = new NullDataSource(new DimensionFieldSpec("k",
DataType.STRING, true, "N/A"), NUM_DOCS);
+ assertEquals(dataSource.getDictionary().get(0), "N/A");
+ assertEquals(dataSource.getDataSourceMetadata().getMinValue(), "N/A");
+ assertEquals(dataSource.getDataSourceMetadata().getMaxValue(), "N/A");
+ }
+
+ @Test
+ public void testMultiValue() {
+ NullDataSource dataSource = new NullDataSource(new DimensionFieldSpec("k",
DataType.INT, false), NUM_DOCS);
+
+ DataSourceMetadata metadata = dataSource.getDataSourceMetadata();
+ assertFalse(metadata.isSingleValue());
+ assertFalse(metadata.isSorted());
+ assertEquals(metadata.getNumValues(), NUM_DOCS);
+ assertEquals(metadata.getMaxNumValuesPerMVEntry(), 1);
+
+ ForwardIndexReader<?> forwardIndex = dataSource.getForwardIndex();
+ assertTrue(forwardIndex.isDictionaryEncoded());
+ assertFalse(forwardIndex.isSingleValue());
+ assertEquals(forwardIndex.getNumValuesMV(0, null), 1);
+ assertEquals(forwardIndex.getDictIdMV(0, null), new int[]{0});
+ ImmutableRoaringBitmap docIds = (ImmutableRoaringBitmap)
dataSource.getInvertedIndex().getDocIds(0);
+ assertEquals(docIds.getCardinality(), NUM_DOCS);
+ }
+
+ private static Object dictionaryValue(DataType dataType) {
+ return new NullDataSource(new DimensionFieldSpec("k", dataType, true),
NUM_DOCS).getDictionary().get(0);
+ }
+}
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/map/BaseMapDataSourceTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/map/BaseMapDataSourceTest.java
new file mode 100644
index 00000000000..409893bbb25
--- /dev/null
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/map/BaseMapDataSourceTest.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.index.map;
+
+import java.util.Map;
+import org.apache.pinot.segment.local.segment.index.datasource.NullDataSource;
+import org.apache.pinot.segment.spi.datasource.DataSource;
+import org.apache.pinot.segment.spi.datasource.DataSourceMetadata;
+import org.apache.pinot.segment.spi.index.column.ColumnIndexContainer;
+import org.apache.pinot.segment.spi.index.reader.MapIndexReader;
+import org.apache.pinot.spi.data.ComplexFieldSpec;
+import org.apache.pinot.spi.data.DimensionFieldSpec;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.testng.annotations.Test;
+
+import static org.apache.pinot.spi.data.ComplexFieldSpec.KEY_FIELD;
+import static org.apache.pinot.spi.data.ComplexFieldSpec.VALUE_FIELD;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+
+/// Covers [BaseMapDataSource#getDataSource] on the branch taken when the map
index reports the key absent: the
+/// key must resolve to an all-null [NullDataSource] typed as the map's value
field and spanning the doc count.
+public class BaseMapDataSourceTest {
+ private static final int NUM_DOCS = 100;
+
+ @Test
+ public void testAbsentKeyResolvesToAllNullSourceOfValueType() {
+ ComplexFieldSpec mapFieldSpec = new ComplexFieldSpec("m", DataType.MAP,
true, Map.of(
+ KEY_FIELD, new DimensionFieldSpec(KEY_FIELD, DataType.STRING, true),
+ VALUE_FIELD, new DimensionFieldSpec(VALUE_FIELD, DataType.LONG, true)
+ ));
+ DataSourceMetadata mapMetadata = mock(DataSourceMetadata.class);
+ when(mapMetadata.getFieldSpec()).thenReturn(mapFieldSpec);
+ when(mapMetadata.getNumDocs()).thenReturn(NUM_DOCS);
+ MapIndexReader<?> mapIndexReader = mock(MapIndexReader.class);
+ when(mapIndexReader.getIndexes("absent")).thenReturn(null);
+ BaseMapDataSource mapDataSource = new BaseMapDataSource(mapMetadata,
ColumnIndexContainer.Empty.INSTANCE) {
+ @Override
+ public MapIndexReader<?> getMapIndexReader() {
+ return mapIndexReader;
+ }
+ };
+
+ DataSource keyDataSource = mapDataSource.getDataSource("absent");
+ assertTrue(keyDataSource instanceof NullDataSource);
+ DataSourceMetadata metadata = keyDataSource.getDataSourceMetadata();
+ assertEquals(metadata.getDataType(), DataType.LONG);
+ assertEquals(metadata.getNumDocs(), NUM_DOCS);
+ assertEquals(keyDataSource.getDictionary().get(0),
FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_LONG);
+ }
+}
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/openstruct/ImmutableOpenStructDataSourceTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/openstruct/ImmutableOpenStructDataSourceTest.java
index 8fcd0cf8090..4572ad11207 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/openstruct/ImmutableOpenStructDataSourceTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/openstruct/ImmutableOpenStructDataSourceTest.java
@@ -20,6 +20,7 @@ package
org.apache.pinot.segment.local.segment.index.openstruct;
import java.util.List;
import java.util.Map;
+import org.apache.pinot.segment.local.segment.index.datasource.NullDataSource;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.segment.spi.datasource.DataSourceMetadata;
import org.apache.pinot.segment.spi.index.column.ColumnIndexContainer;
@@ -31,12 +32,7 @@ import org.testng.annotations.Test;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertNull;
-import static org.testng.Assert.assertSame;
-import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.*;
public class ImmutableOpenStructDataSourceTest {
@@ -62,8 +58,26 @@ public class ImmutableOpenStructDataSourceTest {
null);
assertSame(ds.getDataSource("clicks"), clicksDs);
- // absent key with mock sparse (no real forward index) returns null
- assertNull(ds.getDataSource("unknown"));
+ // Absent key with mock sparse (no real forward index) resolves to an
all-null STRING source
+ DataSource unknownDs = ds.getDataSource("unknown");
+ assertTrue(unknownDs instanceof NullDataSource);
+ assertEquals(unknownDs.getDataSourceMetadata().getDataType(),
DataType.STRING);
+ }
+
+ /// A key absent from the whole segment is described by its declared child
field spec, including a custom default
+ /// null value.
+ @Test
+ public void testAbsentDeclaredKeyReadsDeclaredDefault() {
+ ComplexFieldSpec spec = new ComplexFieldSpec("event",
DataType.OPEN_STRUCT, true,
+ Map.of("score", new DimensionFieldSpec("score", DataType.STRING, true,
"N/A")));
+ ImmutableOpenStructDataSource ds = new ImmutableOpenStructDataSource(spec,
Map.of(), null, 7, null);
+
+ DataSource scoreDs = ds.getDataSource("score");
+ assertTrue(scoreDs instanceof NullDataSource);
+ assertEquals(scoreDs.getDataSourceMetadata().getDataType(),
DataType.STRING);
+ assertEquals(scoreDs.getDataSourceMetadata().getNumDocs(), 7);
+ assertEquals(scoreDs.getDictionary().get(0), "N/A");
+
assertEquals(scoreDs.getNullValueVector().getNullBitmap().getCardinality(), 7);
}
@Test
@@ -153,7 +167,7 @@ public class ImmutableOpenStructDataSourceTest {
null);
assertSame(ds.getDataSourceMetadata("clicks"), clicksMeta);
- assertNull(ds.getDataSourceMetadata("absent"));
+ assertEquals(ds.getDataSourceMetadata("absent").getDataType(),
DataType.STRING);
}
@Test
@@ -235,7 +249,7 @@ public class ImmutableOpenStructDataSourceTest {
assertTrue(regionDs instanceof SparseKeyDataSource);
assertEquals(regionDs.getForwardIndex().getString(0, null), "us");
assertSame(ds.getDataSource("region"), regionDs);
- assertNull(ds.getDataSource("nope"));
+ assertTrue(ds.getDataSource("nope") instanceof NullDataSource);
assertFalse(ds.isMaterialized("region"));
assertFalse(ds.isFullyMaterialized());
}
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/openstruct/MutableOpenStructDataSourceTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/openstruct/MutableOpenStructDataSourceTest.java
index 271490e850b..aa2c67b5410 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/openstruct/MutableOpenStructDataSourceTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/openstruct/MutableOpenStructDataSourceTest.java
@@ -21,6 +21,7 @@ package
org.apache.pinot.segment.local.segment.index.openstruct;
import java.util.HashMap;
import java.util.Map;
import org.apache.pinot.segment.local.io.writer.impl.DirectMemoryManager;
+import org.apache.pinot.segment.local.segment.index.datasource.NullDataSource;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.memory.PinotDataBufferMemoryManager;
@@ -36,12 +37,7 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertNull;
-import static org.testng.Assert.assertSame;
-import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.*;
public class MutableOpenStructDataSourceTest {
@@ -83,11 +79,29 @@ public class MutableOpenStructDataSourceTest {
try (MutableOpenStructIndex idx = new MutableOpenStructIndex("metrics",
"testTable_REALTIME", spec(),
OpenStructIndexConfig.DEFAULT, _mm, 100)) {
MutableOpenStructDataSource ds = new MutableOpenStructDataSource(spec(),
idx, 0);
- assertNull(ds.getDataSource("missing"));
+ assertTrue(ds.getDataSource("missing") instanceof NullDataSource);
assertFalse(ds.isMaterialized("missing"));
}
}
+ /// A key absent from the consuming segment is described by its declared
child field spec, including a custom
+ /// default null value.
+ @Test
+ public void testAbsentDeclaredKeyReadsDeclaredDefault()
+ throws Exception {
+ ComplexFieldSpec spec = new ComplexFieldSpec("metrics",
DataType.OPEN_STRUCT, true,
+ Map.of("score", new DimensionFieldSpec("score", DataType.STRING, true,
"N/A")));
+ try (MutableOpenStructIndex idx = new MutableOpenStructIndex("metrics",
"testTable_REALTIME", spec,
+ OpenStructIndexConfig.DEFAULT, _mm, 100)) {
+ MutableOpenStructDataSource ds = new MutableOpenStructDataSource(spec,
idx, 3);
+ DataSource scoreDs = ds.getDataSource("score");
+ assertTrue(scoreDs instanceof NullDataSource);
+ assertEquals(scoreDs.getDataSourceMetadata().getDataType(),
DataType.STRING);
+ assertEquals(scoreDs.getDataSourceMetadata().getNumDocs(), 3);
+ assertEquals(scoreDs.getDictionary().get(0), "N/A");
+ }
+ }
+
@Test
public void testGetDataSourcesReturnsAllKeys()
throws Exception {
@@ -155,9 +169,9 @@ public class MutableOpenStructDataSourceTest {
try (MutableOpenStructIndex idx = new MutableOpenStructIndex("metrics",
"testTable_REALTIME", spec(),
OpenStructIndexConfig.DEFAULT, _mm, 100)) {
MutableOpenStructDataSource ds = new MutableOpenStructDataSource(spec(),
idx, 1);
- assertNull(ds.getDataSource("clicks"));
+ assertTrue(ds.getDataSource("clicks") instanceof NullDataSource);
idx.index(0, Map.of("clicks", 5L));
- assertNotNull(ds.getDataSource("clicks"));
+ assertFalse(ds.getDataSource("clicks") instanceof NullDataSource);
}
}
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/openstruct/OpenStructNullDataSourceTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/openstruct/OpenStructNullDataSourceTest.java
deleted file mode 100644
index 7dcd20d7ef2..00000000000
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/openstruct/OpenStructNullDataSourceTest.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.segment.local.segment.index.openstruct;
-
-import org.apache.pinot.segment.spi.datasource.DataSourceMetadata;
-import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
-import org.apache.pinot.segment.spi.index.reader.NullValueVectorReader;
-import org.apache.pinot.spi.data.DimensionFieldSpec;
-import org.apache.pinot.spi.data.FieldSpec;
-import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
-import org.testng.annotations.Test;
-
-import static org.testng.Assert.*;
-
-
-public class OpenStructNullDataSourceTest {
-
- @Test
- public void testMetadataReportsCorrectTypeAndNumDocs() {
- FieldSpec spec = new DimensionFieldSpec("region",
FieldSpec.DataType.STRING, true);
- OpenStructNullDataSource ds = new OpenStructNullDataSource(spec, 100);
- DataSourceMetadata metadata = ds.getDataSourceMetadata();
-
- assertEquals(metadata.getFieldSpec().getDataType(),
FieldSpec.DataType.STRING);
- assertEquals(metadata.getNumDocs(), 100);
- assertEquals(metadata.getCardinality(), 1);
- }
-
- @Test
- public void testNullValueVectorAllDocsNull() {
- OpenStructNullDataSource ds = new OpenStructNullDataSource(
- new DimensionFieldSpec("k", FieldSpec.DataType.INT, true), 50);
- NullValueVectorReader nullReader = ds.getNullValueVector();
-
- assertNotNull(nullReader);
- for (int i = 0; i < 50; i++) {
- assertTrue(nullReader.isNull(i));
- }
- ImmutableRoaringBitmap bitmap = nullReader.getNullBitmap();
- assertEquals(bitmap.getCardinality(), 50);
- }
-
- @Test
- public void testNullValueVectorZeroDocs() {
- OpenStructNullDataSource ds = new OpenStructNullDataSource(
- new DimensionFieldSpec("k", FieldSpec.DataType.INT, true), 0);
- NullValueVectorReader nullReader = ds.getNullValueVector();
- assertEquals(nullReader.getNullBitmap().getCardinality(), 0);
- }
-
- @Test
- public void testForwardIndexReturnsDefaultNullValues() {
- // INT
- verifyForwardIndex(FieldSpec.DataType.INT, 10, (fwd) -> {
- assertEquals(fwd.getInt(0, null),
FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_INT);
- assertEquals(fwd.getStoredType(), FieldSpec.DataType.INT);
- });
- // LONG
- verifyForwardIndex(FieldSpec.DataType.LONG, 10, (fwd) -> {
- assertEquals(fwd.getLong(0, null), (long)
FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_LONG);
- });
- // FLOAT
- verifyForwardIndex(FieldSpec.DataType.FLOAT, 10, (fwd) -> {
- assertEquals(fwd.getFloat(0, null),
FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_FLOAT);
- });
- // DOUBLE
- verifyForwardIndex(FieldSpec.DataType.DOUBLE, 10, (fwd) -> {
- assertEquals(fwd.getDouble(0, null),
FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_DOUBLE);
- });
- // STRING
- verifyForwardIndex(FieldSpec.DataType.STRING, 10, (fwd) -> {
- assertEquals(fwd.getString(0, null),
FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_STRING);
- assertEquals(fwd.getStoredType(), FieldSpec.DataType.STRING);
- });
- // BYTES
- verifyForwardIndex(FieldSpec.DataType.BYTES, 10, (fwd) -> {
- assertEquals(fwd.getBytes(0, null),
FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_BYTES);
- });
- // BIG_DECIMAL
- verifyForwardIndex(FieldSpec.DataType.BIG_DECIMAL, 10, (fwd) -> {
- assertEquals(fwd.getBigDecimal(0, null),
FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_BIG_DECIMAL);
- });
- }
-
- @Test
- public void testForwardIndexIsSingleValueNotDictEncoded() {
- OpenStructNullDataSource ds = new OpenStructNullDataSource(
- new DimensionFieldSpec("k", FieldSpec.DataType.STRING, true), 5);
- ForwardIndexReader<?> fwd = ds.getForwardIndex();
- assertTrue(fwd.isSingleValue());
- assertFalse(fwd.isDictionaryEncoded());
- }
-
- @Test
- public void testNoDictionaryOrSecondaryIndexes() {
- OpenStructNullDataSource ds = new OpenStructNullDataSource(
- new DimensionFieldSpec("k", FieldSpec.DataType.STRING, true), 5);
- assertNull(ds.getDictionary());
- assertNull(ds.getInvertedIndex());
- assertNull(ds.getRangeIndex());
- assertNull(ds.getJsonIndex());
- assertNull(ds.getBloomFilter());
- assertNull(ds.getTextIndex());
- assertNull(ds.getVectorIndex());
- }
-
- @FunctionalInterface
- private interface ForwardIndexAssertion {
- void check(ForwardIndexReader<?> fwd);
- }
-
- private void verifyForwardIndex(FieldSpec.DataType type, int numDocs,
ForwardIndexAssertion assertion) {
- OpenStructNullDataSource ds = new OpenStructNullDataSource(
- new DimensionFieldSpec("k", type, true), numDocs);
- assertion.check(ds.getForwardIndex());
- }
-}
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/datasource/MapDataSource.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/datasource/MapDataSource.java
index 292043450ab..4458fd5f3d8 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/datasource/MapDataSource.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/datasource/MapDataSource.java
@@ -30,7 +30,8 @@ public interface MapDataSource extends DataSource {
/// Returns the map FieldSpec.
ComplexFieldSpec.MapFieldSpec getFieldSpec();
- /// Returns the DataSource for the given map key's values.
+ /// Returns the DataSource for the given map key's values. A key absent from
the map resolves to an all-null
+ /// DataSource typed as the map's value field, so the result is never `null`.
DataSource getDataSource(String key);
/// Returns DataSources for all keys present in this segment.
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/datasource/OpenStructDataSource.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/datasource/OpenStructDataSource.java
index 5879402ebb7..3be2f7f35aa 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/datasource/OpenStructDataSource.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/datasource/OpenStructDataSource.java
@@ -23,6 +23,9 @@ import javax.annotation.Nullable;
import org.apache.pinot.segment.spi.index.column.ColumnIndexContainer;
import org.apache.pinot.segment.spi.index.reader.JsonIndexReader;
import org.apache.pinot.spi.data.ComplexFieldSpec;
+import org.apache.pinot.spi.data.DimensionFieldSpec;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
/// DataSource for an OPEN_STRUCT column. Provides per-key DataSources that
can be used for
@@ -33,21 +36,23 @@ public interface OpenStructDataSource extends DataSource {
/// Returns the OPEN_STRUCT ComplexFieldSpec.
ComplexFieldSpec getFieldSpec();
- /// Returns the DataSource for the given key's values. The DataSource's
value type is the
- /// per-key declared type (from `childFieldSpecs`) when present, otherwise
STRING.
- ///
- /// Three possible outcomes:
+ /// Returns the DataSource for the given key's values, typed per
[#getValueFieldSpec(String)]. Never `null`:
/// - **Materialized key** → the dense per-key DataSource (has dictionary /
inverted index).
- /// - **Sparse key** → a virtual blob-backed DataSource that parses the
sparse JSON column
- /// per doc and coerces values to the resolved stored type. Returned when
the key is inside
- /// the sparse manifest, or when there is no manifest (pre-manifest
segments treat every
- /// unmaterialized key as potentially sparse).
- /// - **Definitively absent** → `null`. Returned when the segment is fully
materialized (no
- /// sparse column), or when the sparse manifest exists and does not list
this key. Callers
- /// should synthesize a typed all-null source (see
`OpenStructNullDataSource.forAbsentKey`).
- @Nullable
+ /// - **Sparse key** → a virtual blob-backed DataSource that parses the
sparse JSON column per doc and coerces
+ /// values to the resolved stored type. Returned when the key is inside
the sparse manifest, or when there is no
+ /// manifest (pre-manifest segments treat every unmaterialized key as
potentially sparse).
+ /// - **Definitively absent** → an all-null DataSource built like a default
column: every document is null, and
+ /// reads as the key's default null value with null handling off. Returned
when the segment is fully materialized
+ /// (no sparse column), or when the sparse manifest exists and does not
list this key.
DataSource getDataSource(String key);
+ /// Returns the field spec describing the given key's values: the declared
child field spec when present, otherwise a
+ /// single-value STRING dimension named after the key.
+ default FieldSpec getValueFieldSpec(String key) {
+ FieldSpec childFieldSpec = getFieldSpec().getChildFieldSpec(key);
+ return childFieldSpec != null ? childFieldSpec : new
DimensionFieldSpec(key, DataType.STRING, true);
+ }
+
/// Returns whether the given key has a materialized per-key index in this
segment. Exact,
/// O(1) lookup into the materialized key set. Virtual sparse-backed sources
do not count as
/// materialized.
@@ -57,32 +62,27 @@ public interface OpenStructDataSource extends DataSource {
/// a non-materialized key resolves to a typed all-null source, so the scan
sees NULL at every
/// document.
///
- /// A `false` return combined with a non-null [#getDataSource(String)] means
the key exists
- /// in the sparse tier. A `false` return with a null getDataSource means the
key is definitively
- /// absent.
+ /// A `false` return means the key is either in the sparse tier or
definitively absent; [#getDataSource(String)]
+ /// returns the matching source either way.
boolean isMaterialized(String key);
/// Returns whether every key in this segment is materialized — i.e., there
is no sparse
/// blob and the materialized key set is exhaustive.
///
- /// When `true`, a `false` return from [#isMaterialized(String)] is a
definitive "absent"
- /// and callers can treat the key as present-but-all-null — e.g. evaluate
predicates against a
- /// typed all-null DataSource, which yields the correct answer under both
null-handling modes
- /// (an absent key reads as its type default with null handling off, and as
NULL with it on).
+ /// When `true`, a `false` return from [#isMaterialized(String)] is a
definitive "absent", and
+ /// [#getDataSource(String)] returns an all-null DataSource for the key,
which yields the correct answer under both
+ /// null-handling modes (an absent key reads as its type default with null
handling off, and as NULL with it on).
boolean isFullyMaterialized();
/// Returns DataSources for all keys present in this segment.
Map<String, DataSource> getDataSources();
- /// Returns the DataSourceMetadata for the given key, or `null` when
[#getDataSource(String)]
- /// returns null for this key. Includes metadata from virtual sparse-backed
sources.
- @Nullable
+ /// Returns the DataSourceMetadata for the given key, i.e. that of
[#getDataSource(String)].
DataSourceMetadata getDataSourceMetadata(String key);
- /// Returns the ColumnIndexContainer for the given key, or `null` when the
key has no
- /// materialized DataSource. Virtual sparse-backed sources do not expose a
ColumnIndexContainer
- /// (they use BaseDataSource, not ImmutableDataSource), so this returns null
for sparse keys
- /// even when [#getDataSource(String)] is non-null.
+ /// Returns the ColumnIndexContainer for the given key, or `null` for a
sparse key: virtual sparse-backed sources do
+ /// not expose a ColumnIndexContainer (they use BaseDataSource, not
ImmutableDataSource). A key absent from the
+ /// segment resolves to an all-null DataSource and returns its container.
@Nullable
ColumnIndexContainer getIndexContainer(String key);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
