This is an automated email from the ASF dual-hosted git repository.
Jackie-Jiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new ab3fd2b214b Support backfilling null value vector from the default
null value (#19072)
ab3fd2b214b is described below
commit ab3fd2b214bf393b1feeb3b8ad8e3e250e31881a
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Sat Jul 25 23:23:57 2026 -0700
Support backfilling null value vector from the default null value (#19072)
---
.../segment/creator/impl/BaseSegmentCreator.java | 8 +
.../impl/nullvalue/NullValueVectorCreator.java | 37 +-
.../index/nullvalue/NullValueIndexType.java | 93 ++--
.../index/nullvalue/NullValueVectorHandler.java | 372 +++++++++++++++
.../local/utils/NullValueTransformerUtils.java | 33 +-
.../segment/local/utils/TableConfigUtils.java | 30 ++
.../index/nullvalue/NullValueIndexTypeTest.java | 105 ++++-
.../nullvalue/NullValueVectorHandlerTest.java | 511 +++++++++++++++++++++
.../segment/local/utils/TableConfigUtilsTest.java | 65 +++
.../apache/pinot/segment/spi/ColumnMetadata.java | 9 +
.../org/apache/pinot/segment/spi/V1Constants.java | 6 +
.../pinot/segment/spi/index/StandardIndexes.java | 5 +-
.../spi/index/metadata/ColumnMetadataImpl.java | 31 +-
.../spi/config/table/NullValueVectorConfig.java | 66 +++
14 files changed, 1300 insertions(+), 71 deletions(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/BaseSegmentCreator.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/BaseSegmentCreator.java
index c5b1b5c2b71..d79f9c2c738 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/BaseSegmentCreator.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/BaseSegmentCreator.java
@@ -559,6 +559,14 @@ public abstract class BaseSegmentCreator implements
SegmentCreator {
columnIndexCreators.getIndexConfigs().getConfig(StandardIndexes.forward());
addColumnMetadataInfo(properties, column, columnStatistics, _totalDocs,
_schema.getFieldSpecFor(column),
hasDictionary, dictionaryElementSize, fwdConfig.getEncodingType(),
false);
+ // When null handling is enabled for a column but it has no null values,
NullValueVectorCreator.seal() writes no
+ // bitmap file. Record a metadata flag for that case so such a column is
distinguishable from one that never had
+ // null handling (both lack a bitmap file), which is what the
reload-time backfill relies on. Columns that do
+ // have null values are identified by the bitmap file itself and need no
flag.
+ NullValueVectorCreator nullValueVectorCreator =
columnIndexCreators.getNullValueVectorCreator();
+ if (nullValueVectorCreator != null &&
nullValueVectorCreator.isNonNull()) {
+ properties.setProperty(getKeyFor(column, IS_NON_NULL),
String.valueOf(true));
+ }
}
if (_config.isCompressionStatsEnabled()) {
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/nullvalue/NullValueVectorCreator.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/nullvalue/NullValueVectorCreator.java
index ee5bb527238..1db2b2d8a55 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/nullvalue/NullValueVectorCreator.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/nullvalue/NullValueVectorCreator.java
@@ -43,6 +43,9 @@ import org.roaringbitmap.RoaringBitmapWriter;
public class NullValueVectorCreator implements IndexCreator {
private final RoaringBitmapWriter<RoaringBitmap> _bitmapWriter;
private final File _nullValueVectorFile;
+ private boolean _hasNulls;
+ // Materialized from the writer on first access; see getNullBitmap() for the
contract
+ private RoaringBitmap _nullBitmap;
@Override
public void add(Object value, int dictId)
@@ -62,23 +65,47 @@ public class NullValueVectorCreator implements IndexCreator
{
}
public void setNull(int docId) {
+ // Enforces the contract documented on getNullBitmap(). Kept as an assert
so the check is free in production while
+ // still catching a misordered caller in tests, where assertions are
enabled.
+ assert _nullBitmap == null : "setNull() called after the null bitmap was
materialized";
_bitmapWriter.add(docId);
+ _hasNulls = true;
+ }
+
+ /// Returns `true` when no doc has been marked null, i.e. [#seal] writes no
bitmap file.
+ public boolean isNonNull() {
+ return !_hasNulls;
+ }
+
+ /// Returns the number of docs marked null. Subject to the same contract as
[#getNullBitmap].
+ public int getNumNulls() {
+ return _hasNulls ? getNullBitmap().getCardinality() : 0;
}
public void seal()
throws IOException {
- // Create null value vector file only if the bitmap is not empty
- RoaringBitmap nullBitmap = _bitmapWriter.get();
- if (!nullBitmap.isEmpty()) {
+ // Create null value vector file only if at least one doc was marked null
+ if (_hasNulls) {
try (DataOutputStream outputStream = new DataOutputStream(new
FileOutputStream(_nullValueVectorFile))) {
- nullBitmap.serialize(outputStream);
+ getNullBitmap().serialize(outputStream);
}
}
}
+ /// Returns the bitmap of null doc ids.
+ ///
+ /// Must be called only once every [#setNull] call has been made:
materializing the bitmap flushes the writer, and the
+ /// result is cached here so that repeated calls (e.g. [#getNumNulls]
followed by [#seal]) flush at most once. Doc ids
+ /// marked after the first call are therefore not guaranteed to be reflected.
+ ///
+ /// No explicit `runOptimize` is needed: the writer run-length encodes each
container as it is appended
+ /// (`runCompress` defaults to `true`), which is what keeps a clustered or
all-null vector compact.
@VisibleForTesting
RoaringBitmap getNullBitmap() {
- return _bitmapWriter.get();
+ if (_nullBitmap == null) {
+ _nullBitmap = _bitmapWriter.get();
+ }
+ return _nullBitmap;
}
@Override
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/nullvalue/NullValueIndexType.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/nullvalue/NullValueIndexType.java
index 9bafad83bb9..6d7b14d37e7 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/nullvalue/NullValueIndexType.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/nullvalue/NullValueIndexType.java
@@ -19,6 +19,7 @@
package org.apache.pinot.segment.local.segment.index.nullvalue;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import java.io.File;
import java.io.IOException;
@@ -34,6 +35,7 @@ import
org.apache.pinot.segment.spi.creator.IndexCreationContext;
import org.apache.pinot.segment.spi.index.AbstractIndexType;
import org.apache.pinot.segment.spi.index.ColumnConfigDeserializer;
import org.apache.pinot.segment.spi.index.FieldIndexConfigs;
+import org.apache.pinot.segment.spi.index.IndexConfigDeserializer;
import org.apache.pinot.segment.spi.index.IndexHandler;
import org.apache.pinot.segment.spi.index.IndexReaderFactory;
import org.apache.pinot.segment.spi.index.IndexType;
@@ -41,35 +43,37 @@ import org.apache.pinot.segment.spi.index.StandardIndexes;
import org.apache.pinot.segment.spi.index.reader.NullValueVectorReader;
import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.store.SegmentDirectory;
-import org.apache.pinot.spi.config.table.IndexConfig;
+import org.apache.pinot.spi.config.table.NullValueVectorConfig;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.data.Schema;
-public class NullValueIndexType extends AbstractIndexType<IndexConfig,
NullValueVectorReader, NullValueVectorCreator> {
+public class NullValueIndexType
+ extends AbstractIndexType<NullValueVectorConfig, NullValueVectorReader,
NullValueVectorCreator> {
public static final String INDEX_DISPLAY_NAME = "null";
- private static final List<String> EXTENSIONS =
- List.of(V1Constants.Indexes.NULLVALUE_VECTOR_FILE_EXTENSION);
+ private static final NullValueVectorConfig DEFAULT_CONFIG = new
NullValueVectorConfig(false, false);
+ private static final List<String> EXTENSIONS =
List.of(V1Constants.Indexes.NULLVALUE_VECTOR_FILE_EXTENSION);
protected NullValueIndexType() {
super(StandardIndexes.NULL_VALUE_VECTOR_ID);
}
@Override
- public Class<IndexConfig> getIndexConfigClass() {
- return IndexConfig.class;
+ public Class<NullValueVectorConfig> getIndexConfigClass() {
+ return NullValueVectorConfig.class;
}
@Override
- public NullValueVectorCreator createIndexCreator(IndexCreationContext
context, IndexConfig indexConfig)
+ public NullValueVectorCreator createIndexCreator(IndexCreationContext
context, NullValueVectorConfig indexConfig)
throws Exception {
return new NullValueVectorCreator(context.getIndexDir(),
context.getFieldSpec().getName());
}
@Override
- public IndexConfig getDefaultConfig() {
- return IndexConfig.ENABLED;
+ public NullValueVectorConfig getDefaultConfig() {
+ return DEFAULT_CONFIG;
}
@Override
@@ -77,25 +81,27 @@ public class NullValueIndexType extends
AbstractIndexType<IndexConfig, NullValue
return INDEX_DISPLAY_NAME;
}
+ /// Resolves the per-column null value vector config. Unlike a normal index,
the `enabled` state is not set directly
+ /// by the user but derived from null handling (column-based
[FieldSpec#isNullable] when the schema opts into
+ /// column-based null handling, otherwise the table-level
`nullHandlingEnabled` flag). The user-facing part is the
+ /// `backfill` flag, read from the column's `indexes` config. The two are
merged here rather than treated as
+ /// exclusive alternatives (which is what the default
[#createDeserializerForLegacyConfigs] composition would do).
@Override
- public ColumnConfigDeserializer<IndexConfig>
createDeserializerForLegacyConfigs() {
+ protected ColumnConfigDeserializer<NullValueVectorConfig>
createDeserializer() {
+ ColumnConfigDeserializer<NullValueVectorConfig> fromIndexes =
+ IndexConfigDeserializer.fromIndexes(getPrettyName(),
getIndexConfigClass());
return (TableConfig tableConfig, Schema schema) -> {
+ Map<String, NullValueVectorConfig> fromIndexesMap =
fromIndexes.deserialize(tableConfig, schema);
Collection<FieldSpec> allFieldSpecs = schema.getAllFieldSpecs();
- Map<String, IndexConfig> configMap =
Maps.newHashMapWithExpectedSize(allFieldSpecs.size());
-
+ Map<String, NullValueVectorConfig> configMap =
Maps.newHashMapWithExpectedSize(allFieldSpecs.size());
boolean columnBasedNullHandlingEnabled =
schema.isEnableColumnBasedNullHandling();
boolean nullHandlingEnabled =
tableConfig.getIndexingConfig().isNullHandlingEnabled();
-
for (FieldSpec fieldSpec : allFieldSpecs) {
- IndexConfig indexConfig;
- boolean enabled;
- if (columnBasedNullHandlingEnabled) {
- enabled = fieldSpec.isNullable();
- } else {
- enabled = nullHandlingEnabled;
- }
- indexConfig = enabled ? IndexConfig.ENABLED : IndexConfig.DISABLED;
- configMap.put(fieldSpec.getName(), indexConfig);
+ String column = fieldSpec.getName();
+ boolean enabled = columnBasedNullHandlingEnabled ?
fieldSpec.isNullable() : nullHandlingEnabled;
+ NullValueVectorConfig fromIndex = fromIndexesMap.get(column);
+ boolean backfill = fromIndex != null && fromIndex.isBackfill();
+ configMap.put(column, new NullValueVectorConfig(!enabled, backfill));
}
return configMap;
};
@@ -110,20 +116,53 @@ public class NullValueIndexType extends
AbstractIndexType<IndexConfig, NullValue
return ReaderFactory.INSTANCE;
}
+ @Override
+ public void validate(FieldIndexConfigs indexConfigs, FieldSpec fieldSpec,
TableConfig tableConfig) {
+ if (indexConfigs.getConfig(this).isBackfill()) {
+ // Backfill reconstructs nulls by comparing each stored value against
the column's default null value, which is
+ // only meaningful for scalar stored types. MAP (and other complex
types) are not supported because:
+ // - the default null value for a MAP is an empty map — an ordinary
value rather than a rare sentinel — so
+ // treating every empty map as null would be far too lossy to be
safe; and
+ // - an OPEN_STRUCT-backed MAP is materialized into child columns with
no single scannable parent forward
+ // index, so there is nothing coherent to scan for the parent column.
+ // TODO: Revisit MAP/complex backfill if complex-type null handling
matures and a safe (non-occurring) sentinel
+ // default null value becomes available.
+ DataType storedType = fieldSpec.getDataType().getStoredType();
+ Preconditions.checkState(isBackfillSupported(storedType),
+ "Null value vector backfill is not supported for column: %s of type:
%s", fieldSpec.getName(),
+ fieldSpec.getDataType());
+ }
+ }
+
+ private static boolean isBackfillSupported(DataType storedType) {
+ switch (storedType) {
+ case INT:
+ case LONG:
+ case FLOAT:
+ case DOUBLE:
+ case BIG_DECIMAL:
+ case STRING:
+ case BYTES:
+ return true;
+ default:
+ return false;
+ }
+ }
+
@Override
public IndexHandler createIndexHandler(SegmentDirectory segmentDirectory,
Map<String, FieldIndexConfigs> configsByCol,
Schema schema, TableConfig tableConfig) {
- return IndexHandler.NoOp.INSTANCE;
+ return new NullValueVectorHandler(segmentDirectory, configsByCol,
tableConfig, schema);
}
@Override
- public boolean requiresDictionary(FieldSpec fieldSpec, IndexConfig
indexConfig) {
+ public boolean requiresDictionary(FieldSpec fieldSpec, NullValueVectorConfig
indexConfig) {
// The null value vector is a bitmap of doc IDs whose value is null; no
dictionary involvement.
return false;
}
@Override
- public boolean shouldInvalidateOnDictionaryChange(FieldSpec fieldSpec,
IndexConfig indexConfig) {
+ public boolean shouldInvalidateOnDictionaryChange(FieldSpec fieldSpec,
NullValueVectorConfig indexConfig) {
// The null value vector is keyed by doc ID and independent of the
column's value representation.
return false;
}
@@ -144,8 +183,8 @@ public class NullValueIndexType extends
AbstractIndexType<IndexConfig, NullValue
@Override
public NullValueVectorReader createIndexReader(SegmentDirectory.Reader
segmentReader,
FieldIndexConfigs fieldIndexConfigs, ColumnMetadata metadata)
- throws IOException {
- IndexType<IndexConfig, NullValueVectorReader, ?> indexType =
StandardIndexes.nullValueVector();
+ throws IOException {
+ IndexType<NullValueVectorConfig, NullValueVectorReader, ?> indexType =
StandardIndexes.nullValueVector();
if (fieldIndexConfigs.getConfig(indexType).isDisabled()) {
return null;
}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/nullvalue/NullValueVectorHandler.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/nullvalue/NullValueVectorHandler.java
new file mode 100644
index 00000000000..94a382b09af
--- /dev/null
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/nullvalue/NullValueVectorHandler.java
@@ -0,0 +1,372 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.index.nullvalue;
+
+import java.io.File;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.commons.io.FileUtils;
+import
org.apache.pinot.segment.local.segment.creator.impl.nullvalue.NullValueVectorCreator;
+import
org.apache.pinot.segment.local.segment.index.dictionary.DictionaryIndexType;
+import org.apache.pinot.segment.local.segment.index.loader.BaseIndexHandler;
+import org.apache.pinot.segment.local.segment.index.loader.LoaderUtils;
+import org.apache.pinot.segment.local.segment.readers.PinotSegmentColumnReader;
+import org.apache.pinot.segment.spi.ColumnMetadata;
+import org.apache.pinot.segment.spi.V1Constants;
+import org.apache.pinot.segment.spi.creator.SegmentVersion;
+import org.apache.pinot.segment.spi.index.FieldIndexConfigs;
+import org.apache.pinot.segment.spi.index.IndexReaderFactory;
+import org.apache.pinot.segment.spi.index.StandardIndexes;
+import org.apache.pinot.segment.spi.index.reader.Dictionary;
+import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
+import org.apache.pinot.segment.spi.utils.SegmentMetadataUtils;
+import org.apache.pinot.spi.config.table.NullValueVectorConfig;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.data.Schema;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static
org.apache.pinot.segment.spi.V1Constants.MetadataKeys.Column.IS_NON_NULL;
+import static
org.apache.pinot.segment.spi.V1Constants.MetadataKeys.Column.getKeyFor;
+
+
+/// Backfills a null value vector for opted-in columns that don't already have
one, by treating every value equal to the
+/// column's default null value as null.
+///
+/// Backfill is per-column opt-in via the null value vector index config
([NullValueVectorConfig#isBackfill], set under
+/// the column's `indexes` config), because it is a lossy reconstruction: a
genuine value equal to the default null
+/// value is also marked null. It should be enabled only for columns whose
default null value is a sentinel that does
+/// not occur in the data (e.g. dimension `MIN_VALUE`), not for columns whose
default legitimately occurs in the
+/// data — e.g. metrics (`0`), `BOOLEAN` (`false`), or `TIMESTAMP` (epoch
`0`). Among opted-in columns, the handler
+/// targets those that:
+/// - have null handling enabled (this is the
[NullValueVectorConfig#isEnabled] state, derived from column-based
+/// [org.apache.pinot.spi.data.FieldSpec#isNullable] or the table-level
`nullHandlingEnabled` flag);
+/// - have not already been null-handled — i.e. neither the
[ColumnMetadata#isNonNull] metadata flag is
+/// set nor a null value vector file exists (segments ingested before null
handling was turned on);
+/// - have a forward index to scan and a supported scalar stored type.
+///
+/// The backfill mirrors how ingestion records nulls: a null single-value
entry is stored as the default null value, and
+/// a null multi-value entry is stored as a single-element array holding the
default null value. A doc is therefore
+/// marked null when its stored single value equals the default null value, or
when its multi-value entry is a
+/// single-element array whose sole element equals the default null value.
+///
+/// A bitmap file is written only when the column has at least one null,
exactly like segment creation. The no-null
+/// case is recorded solely via the
[V1Constants.MetadataKeys.Column#IS_NON_NULL] metadata flag, so the
+/// column is not perpetually re-eligible for backfill and behaves
symmetrically with segment creation.
+///
+/// This is a lossy reconstruction: a genuine value that happens to equal the
default null value is also marked null.
+/// That trade-off is accepted by opting into the feature.
+@SuppressWarnings({"rawtypes"})
+public class NullValueVectorHandler extends BaseIndexHandler {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(NullValueVectorHandler.class);
+
+ private final Set<String> _backfillColumns;
+
+ public NullValueVectorHandler(SegmentDirectory segmentDirectory, Map<String,
FieldIndexConfigs> fieldIndexConfigs,
+ TableConfig tableConfig, Schema schema) {
+ super(segmentDirectory, fieldIndexConfigs, tableConfig, schema);
+ // Collect the columns whose resolved null value vector config has both
null handling enabled and backfill opted
+ // in. `enabled` is derived from null handling and `backfill` from the
column's `indexes` config (see
+ // NullValueIndexType.createDeserializer).
+ _backfillColumns = new HashSet<>();
+ for (Map.Entry<String, FieldIndexConfigs> entry :
_fieldIndexConfigs.entrySet()) {
+ NullValueVectorConfig config =
entry.getValue().getConfig(StandardIndexes.nullValueVector());
+ if (config.isEnabled() && config.isBackfill()) {
+ _backfillColumns.add(entry.getKey());
+ }
+ }
+ }
+
+ @Override
+ public boolean needUpdateIndices(SegmentDirectory.Reader segmentReader) {
+ if (_backfillColumns.isEmpty()) {
+ return false;
+ }
+ return !getColumnsToBackfill(segmentReader).isEmpty();
+ }
+
+ @Override
+ public void updateIndices(SegmentDirectory.Writer segmentWriter)
+ throws Exception {
+ if (_backfillColumns.isEmpty()) {
+ return;
+ }
+ List<ColumnMetadata> columnsToBackfill =
getColumnsToBackfill(segmentWriter);
+ if (columnsToBackfill.isEmpty()) {
+ return;
+ }
+ Map<String, String> metadataUpdates = new HashMap<>();
+ for (ColumnMetadata columnMetadata : columnsToBackfill) {
+ boolean isNonNull = backfillColumn(segmentWriter, columnMetadata);
+ if (isNonNull) {
+ // The column has no null values, so no bitmap file was written.
Record the metadata flag so it is not
+ // re-scanned on subsequent reloads, keeping the behavior symmetric
with segment creation (see
+ // BaseSegmentCreator). Columns that do have null values are
identified by the bitmap file and need no flag.
+ metadataUpdates.put(getKeyFor(columnMetadata.getColumnName(),
IS_NON_NULL), String.valueOf(true));
+ }
+ }
+ if (!metadataUpdates.isEmpty()) {
+ SegmentMetadataUtils.updateMetadataProperties(_segmentDirectory,
metadataUpdates);
+ }
+ }
+
+ /// Returns the eligible columns among the opted-in set: the column has not
already been null-handled (neither the
+ /// [ColumnMetadata#isNonNull] metadata flag is set nor a null value vector
file exists) and has a forward index to
+ /// scan. Null handling being enabled and the stored type being
backfill-supported are already guaranteed — the
+ /// former when building `_backfillColumns`, the latter by
[NullValueIndexType#validate] at config time.
+ private List<ColumnMetadata> getColumnsToBackfill(SegmentDirectory.Reader
segmentReader) {
+ List<ColumnMetadata> columns = new ArrayList<>();
+ for (String column : _backfillColumns) {
+ ColumnMetadata columnMetadata =
_segmentDirectory.getSegmentMetadata().getColumnMetadataFor(column);
+ if (columnMetadata == null) {
+ continue;
+ }
+ if (columnMetadata.isNonNull() || segmentReader.hasIndexFor(column,
StandardIndexes.nullValueVector())) {
+ // Null handling was already applied to this column: either the
metadata flag is set (column-based signal,
+ // covers the no-null case), or a null value vector file exists
(covers older segments without the flag).
+ continue;
+ }
+ if (!segmentReader.hasIndexFor(column, StandardIndexes.forward())) {
+ // No forward index to scan, so skip the column. Note this is not the
same as "the forward index is disabled in
+ // the config": when a reload disables it, ForwardIndexHandler defers
the deletion to postUpdateIndicesCleanup
+ // (after all handlers) so the index is still readable here and the
column is backfilled normally. This branch
+ // covers a column whose forward index was already removed by an
earlier reload.
+ continue;
+ }
+ columns.add(columnMetadata);
+ }
+ return columns;
+ }
+
+ /// Scans the column and writes its null value vector bitmap file when it
has at least one null value. Returns `true`
+ /// when the column has no null values (bitmap skipped), in which case the
caller records the metadata flag instead.
+ private boolean backfillColumn(SegmentDirectory.Writer segmentWriter,
ColumnMetadata columnMetadata)
+ throws Exception {
+ String segmentName = _segmentDirectory.getSegmentMetadata().getName();
+ String columnName = columnMetadata.getColumnName();
+ File indexDir = _segmentDirectory.getSegmentMetadata().getIndexDir();
+ File inProgress = new File(indexDir, columnName +
V1Constants.Indexes.NULLVALUE_VECTOR_FILE_EXTENSION
+ + ".inprogress");
+ File nullValueVectorFile = new File(indexDir, columnName +
V1Constants.Indexes.NULLVALUE_VECTOR_FILE_EXTENSION);
+
+ if (!inProgress.exists()) {
+ // Marker file does not exist, which means last run ended normally.
+ FileUtils.touch(inProgress);
+ } else {
+ // Marker file exists, which means last run gets interrupted. Remove the
leftover null value vector if any.
+ FileUtils.deleteQuietly(nullValueVectorFile);
+ }
+
+ LOGGER.info("Backfilling null value vector for segment: {}, column: {}",
segmentName, columnName);
+ IndexReaderFactory<ForwardIndexReader> forwardReaderFactory =
StandardIndexes.forward().getReaderFactory();
+ int numDocs = columnMetadata.getTotalDocs();
+ boolean isNonNull;
+ int numNulls;
+ // Reuse the segment-creation creator so the bitmap is written exactly as
it is at creation time: skipped when
+ // empty, and run-length encoded by the writer.
+ try (ForwardIndexReader forwardIndexReader =
forwardReaderFactory.createIndexReader(segmentWriter,
+ _fieldIndexConfigs.get(columnName), columnMetadata);
+ Dictionary dictionary =
+ columnMetadata.hasDictionary() ?
DictionaryIndexType.read(segmentWriter, columnMetadata) : null;
+ PinotSegmentColumnReader columnReader = new
PinotSegmentColumnReader(columnName, forwardIndexReader, dictionary,
+ null, columnMetadata.getMaxNumberOfMultiValues());
+ NullValueVectorCreator creator = new NullValueVectorCreator(indexDir,
columnName)) {
+ Object defaultNullValue =
columnMetadata.getFieldSpec().getDefaultNullValue();
+ if (columnMetadata.isSingleValue()) {
+ markNullSingleValue(columnReader, defaultNullValue, numDocs, creator);
+ } else {
+ markNullMultiValue(columnReader, defaultNullValue, numDocs, creator);
+ }
+ creator.seal();
+ isNonNull = creator.isNonNull();
+ numNulls = creator.getNumNulls();
+ }
+
+ // seal() writes the bitmap file only when the column has null values. The
no-null case is recorded via the column
+ // metadata flag set in updateIndices instead, so there is no empty bitmap
file to fold in.
+ if (!isNonNull && _segmentDirectory.getSegmentMetadata().getVersion() ==
SegmentVersion.v3) {
+ // For v3, fold the generated file into the single index file and remove
it.
+ LoaderUtils.writeIndexToV3Format(segmentWriter, columnName,
nullValueVectorFile,
+ StandardIndexes.nullValueVector());
+ }
+ LOGGER.info("Backfilled null value vector for segment: {}, column: {},
numNulls: {}", segmentName, columnName,
+ numNulls);
+
+ FileUtils.deleteQuietly(inProgress);
+ return isNonNull;
+ }
+
+ private static void markNullSingleValue(PinotSegmentColumnReader
columnReader, Object defaultNullValue, int numDocs,
+ NullValueVectorCreator creator) {
+ switch (columnReader.getValueType()) {
+ case INT: {
+ int nullValue = (Integer) defaultNullValue;
+ for (int docId = 0; docId < numDocs; docId++) {
+ if (columnReader.getInt(docId) == nullValue) {
+ creator.setNull(docId);
+ }
+ }
+ break;
+ }
+ case LONG: {
+ long nullValue = (Long) defaultNullValue;
+ for (int docId = 0; docId < numDocs; docId++) {
+ if (columnReader.getLong(docId) == nullValue) {
+ creator.setNull(docId);
+ }
+ }
+ break;
+ }
+ case FLOAT: {
+ float nullValue = (Float) defaultNullValue;
+ for (int docId = 0; docId < numDocs; docId++) {
+ if (Float.compare(columnReader.getFloat(docId), nullValue) == 0) {
+ creator.setNull(docId);
+ }
+ }
+ break;
+ }
+ case DOUBLE: {
+ double nullValue = (Double) defaultNullValue;
+ for (int docId = 0; docId < numDocs; docId++) {
+ if (Double.compare(columnReader.getDouble(docId), nullValue) == 0) {
+ creator.setNull(docId);
+ }
+ }
+ break;
+ }
+ case BIG_DECIMAL: {
+ BigDecimal nullValue = (BigDecimal) defaultNullValue;
+ for (int docId = 0; docId < numDocs; docId++) {
+ if (columnReader.getBigDecimal(docId).compareTo(nullValue) == 0) {
+ creator.setNull(docId);
+ }
+ }
+ break;
+ }
+ case STRING: {
+ String nullValue = (String) defaultNullValue;
+ for (int docId = 0; docId < numDocs; docId++) {
+ if (nullValue.equals(columnReader.getString(docId))) {
+ creator.setNull(docId);
+ }
+ }
+ break;
+ }
+ case BYTES: {
+ byte[] nullValue = (byte[]) defaultNullValue;
+ for (int docId = 0; docId < numDocs; docId++) {
+ if (Arrays.equals(columnReader.getBytes(docId), nullValue)) {
+ creator.setNull(docId);
+ }
+ }
+ break;
+ }
+ default:
+ throw new IllegalStateException(
+ "Unsupported stored type for null value vector backfill: " +
columnReader.getValueType());
+ }
+ }
+
+ private static void markNullMultiValue(PinotSegmentColumnReader
columnReader, Object defaultNullValue, int numDocs,
+ NullValueVectorCreator creator) {
+ switch (columnReader.getValueType()) {
+ case INT: {
+ int nullValue = (Integer) defaultNullValue;
+ for (int docId = 0; docId < numDocs; docId++) {
+ int[] values = columnReader.getIntMV(docId);
+ if (values.length == 1 && values[0] == nullValue) {
+ creator.setNull(docId);
+ }
+ }
+ break;
+ }
+ case LONG: {
+ long nullValue = (Long) defaultNullValue;
+ for (int docId = 0; docId < numDocs; docId++) {
+ long[] values = columnReader.getLongMV(docId);
+ if (values.length == 1 && values[0] == nullValue) {
+ creator.setNull(docId);
+ }
+ }
+ break;
+ }
+ case FLOAT: {
+ float nullValue = (Float) defaultNullValue;
+ for (int docId = 0; docId < numDocs; docId++) {
+ float[] values = columnReader.getFloatMV(docId);
+ if (values.length == 1 && Float.compare(values[0], nullValue) == 0) {
+ creator.setNull(docId);
+ }
+ }
+ break;
+ }
+ case DOUBLE: {
+ double nullValue = (Double) defaultNullValue;
+ for (int docId = 0; docId < numDocs; docId++) {
+ double[] values = columnReader.getDoubleMV(docId);
+ if (values.length == 1 && Double.compare(values[0], nullValue) == 0)
{
+ creator.setNull(docId);
+ }
+ }
+ break;
+ }
+ case BIG_DECIMAL: {
+ BigDecimal nullValue = (BigDecimal) defaultNullValue;
+ for (int docId = 0; docId < numDocs; docId++) {
+ BigDecimal[] values = columnReader.getBigDecimalMV(docId);
+ if (values.length == 1 && values[0].compareTo(nullValue) == 0) {
+ creator.setNull(docId);
+ }
+ }
+ break;
+ }
+ case STRING: {
+ String nullValue = (String) defaultNullValue;
+ for (int docId = 0; docId < numDocs; docId++) {
+ String[] values = columnReader.getStringMV(docId);
+ if (values.length == 1 && nullValue.equals(values[0])) {
+ creator.setNull(docId);
+ }
+ }
+ break;
+ }
+ case BYTES: {
+ byte[] nullValue = (byte[]) defaultNullValue;
+ for (int docId = 0; docId < numDocs; docId++) {
+ byte[][] values = columnReader.getBytesMV(docId);
+ if (values.length == 1 && Arrays.equals(values[0], nullValue)) {
+ creator.setNull(docId);
+ }
+ }
+ break;
+ }
+ default:
+ throw new IllegalStateException(
+ "Unsupported stored type for null value vector backfill: " +
columnReader.getValueType());
+ }
+ }
+}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/NullValueTransformerUtils.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/NullValueTransformerUtils.java
index 85d68f45a38..5acfff9c35b 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/NullValueTransformerUtils.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/NullValueTransformerUtils.java
@@ -87,28 +87,37 @@ public class NullValueTransformerUtils {
Preconditions.checkState(timeColumnSpec != null, "Failed to find time
field: %s from schema: %s", timeColumnName,
schema.getSchemaName());
- String defaultTimeString = timeColumnSpec.getDefaultNullValueString();
- DateTimeFormatSpec dateTimeFormatSpec = timeColumnSpec.getFormatSpec();
-
- // Try to use the default time from the field spec if it's valid
- try {
- long defaultTimeMs =
dateTimeFormatSpec.fromFormatToMillis(defaultTimeString);
- if (TimeUtils.timeValueInValidRange(defaultTimeMs)) {
- return timeColumnSpec.getDefaultNullValue();
- }
- } catch (Exception e) {
- // Ignore and fall through to use current time
+ // Use the default time from the field spec if it's valid
+ if (isDefaultTimeValueInValidRange(timeColumnSpec)) {
+ return timeColumnSpec.getDefaultNullValue();
}
// Use current time if default time is not valid
+ DateTimeFormatSpec dateTimeFormatSpec = timeColumnSpec.getFormatSpec();
String currentTimeString =
dateTimeFormatSpec.fromMillisToFormat(System.currentTimeMillis());
Object currentTime =
timeColumnSpec.getDataType().convert(currentTimeString);
LOGGER.info(
"Default time: {} does not comply with format: {}, using current time:
{} as the default time for table: {}",
- defaultTimeString, timeColumnSpec.getFormat(), currentTime,
tableConfig.getTableName());
+ timeColumnSpec.getDefaultNullValueString(),
timeColumnSpec.getFormat(), currentTime,
+ tableConfig.getTableName());
return currentTime;
}
+ /// Returns `true` when the time column's default null value is within the
valid time range, i.e. ingestion stores it
+ /// as-is for null rows instead of substituting the current time.
+ ///
+ /// Callers that need to reason about what was actually written for a null
time value must consult this: when it
+ /// returns `false`, the stored value is the ingestion-time current time,
which does not match the default null value
+ /// recorded in the segment metadata.
+ public static boolean isDefaultTimeValueInValidRange(DateTimeFieldSpec
timeColumnSpec) {
+ try {
+ return TimeUtils.timeValueInValidRange(
+
timeColumnSpec.getFormatSpec().fromFormatToMillis(timeColumnSpec.getDefaultNullValueString()));
+ } catch (Exception e) {
+ return false;
+ }
+ }
+
/**
* Transforms a value by replacing null with the default null value.
*
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java
index 0c5fd6bf283..31fbf48dc6c 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java
@@ -1692,6 +1692,10 @@ public final class TableConfigUtils {
}
}
+ // Null value vector backfill on the time column needs the whole table
config (for the time column name), so it
+ // cannot be validated by NullValueIndexType.validate which only sees one
field at a time.
+ validateNullValueVectorBackfillForTimeColumn(tableConfig, schema,
indexConfigsMap);
+
validateMultiColumnTextIndex(indexingConfig.getMultiColumnTextIndexConfig());
// OPEN_STRUCT materialized child columns use a reserved separator '$' in
their name. When any
@@ -1758,6 +1762,32 @@ public final class TableConfigUtils {
}
}
+ /// Rejects a null value vector backfill opt-in on the time column when its
default null value is outside the valid
+ /// time range.
+ ///
+ /// In that case ingestion substitutes the ingestion-time current time for
null rows (see
+ /// [NullValueTransformerUtils#isDefaultTimeValueInValidRange]) while the
segment metadata records the field spec's
+ /// default null value. A backfill scan compares against the recorded
default, so it would never match: the column
+ /// would silently keep its nulls unmarked and be recorded as containing no
nulls. A time column with an explicit
+ /// in-range default null value is stored as-is and is therefore allowed.
+ private static void validateNullValueVectorBackfillForTimeColumn(TableConfig
tableConfig, Schema schema,
+ Map<String, FieldIndexConfigs> indexConfigsMap) {
+ String timeColumnName =
tableConfig.getValidationConfig().getTimeColumnName();
+ if (StringUtils.isEmpty(timeColumnName)) {
+ return;
+ }
+ FieldIndexConfigs indexConfigs = indexConfigsMap.get(timeColumnName);
+ if (indexConfigs == null ||
!indexConfigs.getConfig(StandardIndexes.nullValueVector()).isBackfill()) {
+ return;
+ }
+ DateTimeFieldSpec timeColumnSpec =
schema.getSpecForTimeColumn(timeColumnName);
+ Preconditions.checkState(timeColumnSpec != null, "Failed to find time
column: %s in schema", timeColumnName);
+
Preconditions.checkState(NullValueTransformerUtils.isDefaultTimeValueInValidRange(timeColumnSpec),
+ "Null value vector backfill is not supported for time column: %s with
default null value: %s outside the valid "
+ + "time range, because ingestion stores the current time for null
values instead of the default null value",
+ timeColumnName, timeColumnSpec.getDefaultNullValueString());
+ }
+
private static void validateMultiColumnTextIndex(MultiColumnTextIndexConfig
multiColTextIndex) {
if (multiColTextIndex == null) {
return;
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/nullvalue/NullValueIndexTypeTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/nullvalue/NullValueIndexTypeTest.java
index cb0df4d8164..c0ce48e6f2f 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/nullvalue/NullValueIndexTypeTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/nullvalue/NullValueIndexTypeTest.java
@@ -18,49 +18,122 @@
*/
package org.apache.pinot.segment.local.segment.index.nullvalue;
+import java.util.List;
+import java.util.Map;
import org.apache.pinot.segment.local.segment.index.AbstractSerdeIndexContract;
+import org.apache.pinot.segment.spi.index.FieldIndexConfigs;
+import org.apache.pinot.segment.spi.index.FieldIndexConfigsUtil;
import org.apache.pinot.segment.spi.index.StandardIndexes;
-import org.apache.pinot.spi.config.table.IndexConfig;
+import org.apache.pinot.spi.config.table.FieldConfig;
+import org.apache.pinot.spi.config.table.NullValueVectorConfig;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.ComplexFieldSpec;
+import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
-import org.testng.Assert;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.JsonUtils;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertThrows;
+import static org.testng.Assert.assertTrue;
+
public class NullValueIndexTypeTest {
+ private static final NullValueVectorConfig DISABLED = new
NullValueVectorConfig(true, false);
+ private static final NullValueVectorConfig ENABLED = new
NullValueVectorConfig(false, false);
@DataProvider(name = "provideCases")
public Object[][] provideCases() {
return new Object[][]{
// This is the semantic table, assuming a null bitmap buffer exists in
the segment
// enableColumnBasedNullHandling | table nullable | column nullable |
expected index config
- new Object[]{false, false, false, IndexConfig.DISABLED}, new Object[]{
- false, false, true, IndexConfig.DISABLED
- }, new Object[]{false, true, false, IndexConfig.ENABLED}, new Object[]{
- false, true, true, IndexConfig.ENABLED
- },
-
- new Object[]{true, false, false, IndexConfig.DISABLED}, new
Object[]{true, false, true, IndexConfig.ENABLED},
- new Object[]{true, true, false, IndexConfig.DISABLED}, new
Object[]{true, true, true, IndexConfig.ENABLED}
+ new Object[]{false, false, false, DISABLED},
+ new Object[]{false, false, true, DISABLED},
+ new Object[]{false, true, false, ENABLED},
+ new Object[]{false, true, true, ENABLED},
+ new Object[]{true, false, false, DISABLED},
+ new Object[]{true, false, true, ENABLED},
+ new Object[]{true, true, false, DISABLED},
+ new Object[]{true, true, true, ENABLED}
};
}
- public static class ConfTest extends AbstractSerdeIndexContract {
+ @Test
+ public void testConfigSerde()
+ throws Exception {
+ // The config itself round-trips (it is persisted as part of the table
config).
+ NullValueVectorConfig config = new NullValueVectorConfig(false, true);
+ NullValueVectorConfig deserialized =
+ JsonUtils.stringToObject(JsonUtils.objectToString(config),
NullValueVectorConfig.class);
+ assertEquals(deserialized, config);
+ assertTrue(deserialized.isBackfill());
+ assertTrue(deserialized.isEnabled());
- protected void assertEquals(IndexConfig expected) {
- Assert.assertEquals(getActualConfig("dimStr",
StandardIndexes.nullValueVector()), expected);
- }
+ // The backfill opt-in survives a full table config round-trip and still
resolves per column.
+ Schema schema = new Schema.SchemaBuilder().setSchemaName("testTable")
+ .addSingleValueDimension("intCol", DataType.INT)
+ .addSingleValueDimension("otherCol", DataType.INT)
+ .build();
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable")
+ .setNullHandlingEnabled(true)
+ .setFieldConfigList(List.of(new FieldConfig.Builder("intCol")
+ .withIndexes(JsonUtils.stringToJsonNode("{\"null\": {\"backfill\":
true}}"))
+ .build()))
+ .build();
+ TableConfig reloaded =
JsonUtils.stringToObject(tableConfig.toJsonString(), TableConfig.class);
+
+ Map<String, FieldIndexConfigs> configsByCol =
+ FieldIndexConfigsUtil.createIndexConfigsByColName(reloaded, schema);
+ NullValueVectorConfig intColConfig =
configsByCol.get("intCol").getConfig(StandardIndexes.nullValueVector());
+ assertTrue(intColConfig.isEnabled());
+ assertTrue(intColConfig.isBackfill());
+ // A column without the opt-in is still enabled by table-level null
handling, but not backfilled.
+ NullValueVectorConfig otherColConfig =
configsByCol.get("otherCol").getConfig(StandardIndexes.nullValueVector());
+ assertTrue(otherColConfig.isEnabled());
+ assertFalse(otherColConfig.isBackfill());
+ }
+
+ @Test
+ public void testBackfillValidation() {
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ FieldIndexConfigs backfillOn = new FieldIndexConfigs.Builder()
+ .add(StandardIndexes.nullValueVector(), new
NullValueVectorConfig(false, true))
+ .build();
+
+ // Scalar column opted into backfill is allowed.
+ StandardIndexes.nullValueVector()
+ .validate(backfillOn, new DimensionFieldSpec("scalar", DataType.INT,
true), tableConfig);
+
+ // MAP column opted into backfill is rejected at config time.
+ FieldSpec mapFieldSpec = new ComplexFieldSpec("map", DataType.MAP, true,
Map.of());
+ assertThrows(IllegalStateException.class,
+ () -> StandardIndexes.nullValueVector().validate(backfillOn,
mapFieldSpec, tableConfig));
+
+ // MAP column without the backfill opt-in is fine (validation is a no-op).
+ FieldIndexConfigs backfillOff = new FieldIndexConfigs.Builder()
+ .add(StandardIndexes.nullValueVector(), new
NullValueVectorConfig(false, false))
+ .build();
+ StandardIndexes.nullValueVector().validate(backfillOff, mapFieldSpec,
tableConfig);
+ }
+
+ public static class ConfTest extends AbstractSerdeIndexContract {
@Test(dataProvider = "provideCases", dataProviderClass =
NullValueIndexTypeTest.class)
public void isEnabledWhenNullable(boolean enableColumnBasedNullHandling,
boolean tableNullable,
- boolean fieldNullable, IndexConfig expected) {
+ boolean fieldNullable, NullValueVectorConfig expected) {
_schema.setEnableColumnBasedNullHandling(enableColumnBasedNullHandling);
_tableConfig.getIndexingConfig().setNullHandlingEnabled(tableNullable);
FieldSpec fieldSpec = _schema.getFieldSpecFor("dimStr");
fieldSpec.setNullable(fieldNullable);
- assertEquals(expected);
+ assertEquals(getActualConfig("dimStr",
StandardIndexes.nullValueVector()), expected);
}
}
}
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/nullvalue/NullValueVectorHandlerTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/nullvalue/NullValueVectorHandlerTest.java
new file mode 100644
index 00000000000..34b31600277
--- /dev/null
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/nullvalue/NullValueVectorHandlerTest.java
@@ -0,0 +1,511 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.index.nullvalue;
+
+import java.io.File;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.commons.io.FileUtils;
+import
org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader;
+import
org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
+import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
+import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
+import org.apache.pinot.segment.local.segment.readers.PinotSegmentColumnReader;
+import org.apache.pinot.segment.local.segment.store.SegmentLocalFSDirectory;
+import org.apache.pinot.segment.spi.ImmutableSegment;
+import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
+import org.apache.pinot.segment.spi.store.SegmentDirectory;
+import org.apache.pinot.spi.config.table.FieldConfig;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.data.readers.GenericRow;
+import org.apache.pinot.spi.utils.JsonUtils;
+import org.apache.pinot.spi.utils.ReadMode;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.*;
+
+
+public class NullValueVectorHandlerTest {
+ private static final File TEMP_DIR = new File(FileUtils.getTempDirectory(),
"NullValueVectorHandlerTest");
+ private static final String RAW_TABLE_NAME = "testTable";
+ private static final String SEGMENT_NAME = "testSegment";
+
+ private static final String SV_INT_COLUMN = "svInt";
+ private static final String SV_STRING_COLUMN = "svString";
+ private static final String MV_INT_COLUMN = "mvInt";
+ private static final String SV_LONG_COLUMN = "svLong";
+ // Opted out of backfill even though it also holds the sentinel default
value, to verify selectivity.
+ private static final String SV_INT_NO_OPT_IN_COLUMN = "svIntNoOptIn";
+
+ private static final Schema SCHEMA = new
Schema.SchemaBuilder().setSchemaName(RAW_TABLE_NAME)
+ .addSingleValueDimension(SV_INT_COLUMN, DataType.INT)
+ .addSingleValueDimension(SV_STRING_COLUMN, DataType.STRING)
+ .addMultiValueDimension(MV_INT_COLUMN, DataType.INT)
+ .addSingleValueDimension(SV_LONG_COLUMN, DataType.LONG)
+ .addSingleValueDimension(SV_INT_NO_OPT_IN_COLUMN, DataType.INT)
+ .build();
+
+ // Rows carry the default null values inline (as a pre-null-handling segment
would), so the segment is built without a
+ // null value vector. Dimension defaults: INT -> Integer.MIN_VALUE, STRING
-> "null". A null multi-value entry is
+ // stored as a single-element array holding the default null value. svLong
holds no default null value.
+ private static final List<GenericRow> ROWS = List.of(
+ createRow(1, "a", new Object[]{10, 20}, 100L, 7),
+ createRow(Integer.MIN_VALUE, "null", new Object[]{Integer.MIN_VALUE},
200L, Integer.MIN_VALUE),
+ createRow(3, "null", new Object[]{Integer.MIN_VALUE, 5}, 300L, 9)
+ );
+
+ private static GenericRow createRow(int svInt, String svString, Object[]
mvInt, long svLong, int svIntNoOptIn) {
+ GenericRow row = new GenericRow();
+ row.putValue(SV_INT_COLUMN, svInt);
+ row.putValue(SV_STRING_COLUMN, svString);
+ row.putValue(MV_INT_COLUMN, mvInt);
+ row.putValue(SV_LONG_COLUMN, svLong);
+ row.putValue(SV_INT_NO_OPT_IN_COLUMN, svIntNoOptIn);
+ return row;
+ }
+
+ /// Builds a row for the creation-path test. A `null` svInt marks that cell
as null so ingestion records it in the
+ /// null value vector; all other columns are non-null.
+ private static GenericRow creationRow(Integer svInt, long svLong) {
+ GenericRow row = new GenericRow();
+ row.putValue(SV_INT_COLUMN, svInt);
+ row.putValue(SV_STRING_COLUMN, "s");
+ row.putValue(MV_INT_COLUMN, new Object[]{1, 2});
+ row.putValue(SV_LONG_COLUMN, svLong);
+ row.putValue(SV_INT_NO_OPT_IN_COLUMN, 5);
+ return row;
+ }
+
+ @BeforeMethod
+ public void setUp()
+ throws Exception {
+ FileUtils.deleteQuietly(TEMP_DIR);
+ // Build the segment without null handling, so no null value vector is
created.
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).build();
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig,
SCHEMA);
+ config.setOutDir(TEMP_DIR.getAbsolutePath());
+ config.setSegmentName(SEGMENT_NAME);
+ SegmentIndexCreationDriverImpl driver = new
SegmentIndexCreationDriverImpl();
+ driver.init(config, new GenericRowRecordReader(ROWS));
+ driver.build();
+ }
+
+ @AfterMethod
+ public void tearDown() {
+ FileUtils.deleteQuietly(TEMP_DIR);
+ }
+
+ private static FieldConfig backfillOptIn(String column)
+ throws Exception {
+ // Opt into backfill via the null value vector index config:
indexes.null.backfill = true.
+ return new
FieldConfig.Builder(column).withIndexes(JsonUtils.stringToJsonNode("{\"null\":
{\"backfill\": true}}"))
+ .build();
+ }
+
+ @Test
+ public void testBackfillNullVectorForOptedInColumns()
+ throws Exception {
+ // Opt in every column except SV_INT_NO_OPT_IN_COLUMN.
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME)
+ .setNullHandlingEnabled(true)
+ .setFieldConfigList(
+ List.of(backfillOptIn(SV_INT_COLUMN),
backfillOptIn(SV_STRING_COLUMN), backfillOptIn(MV_INT_COLUMN),
+ backfillOptIn(SV_LONG_COLUMN)))
+ .build();
+
+ ImmutableSegment segment =
+ ImmutableSegmentLoader.load(new File(TEMP_DIR, SEGMENT_NAME), new
IndexLoadingConfig(tableConfig, SCHEMA));
+ try {
+ // svInt: only the Integer.MIN_VALUE row is null.
+ assertNullDocIds(segment, SV_INT_COLUMN, false, true, false);
+ // svString: both "null" rows are null.
+ assertNullDocIds(segment, SV_STRING_COLUMN, false, true, true);
+ // mvInt: only the single-element {MIN_VALUE} row is null; multi-element
rows are not.
+ assertNullDocIds(segment, MV_INT_COLUMN, false, true, false);
+ // svLong: no value equals the default null value, so no bitmap file is
written (matching segment creation).
+ assertNullDocIds(segment, SV_LONG_COLUMN, false, false, false);
+
+ // Columns with null values carry a bitmap file and are not flagged
non-null; the null-free column is flagged.
+ for (String column : List.of(SV_INT_COLUMN, SV_STRING_COLUMN,
MV_INT_COLUMN)) {
+ assertNotNull(segment.getDataSource(column).getNullValueVector(),
+ "Column with null values should have a bitmap file: " + column);
+
assertFalse(segment.getSegmentMetadata().getColumnMetadataFor(column).isNonNull(),
+ "Column with null values should not be flagged non-null: " +
column);
+ }
+
assertTrue(segment.getSegmentMetadata().getColumnMetadataFor(SV_LONG_COLUMN).isNonNull(),
+ "Null-free column should be flagged non-null (idempotency signal)");
+
+ // The un-opted column holds the sentinel default value too, but must be
left untouched.
+ assertNullDocIds(segment, SV_INT_NO_OPT_IN_COLUMN, false, false, false);
+
assertNull(segment.getDataSource(SV_INT_NO_OPT_IN_COLUMN).getNullValueVector(),
+ "Column not opted into backfill must not get a null value vector");
+
assertFalse(segment.getSegmentMetadata().getColumnMetadataFor(SV_INT_NO_OPT_IN_COLUMN).isNonNull(),
+ "Column not opted into backfill must not be flagged");
+ } finally {
+ segment.destroy();
+ }
+ }
+
+ /// A non-default value per data type, in the form ingestion accepts. Covers
every backfill-supported stored type
+ /// (`INT`, `LONG`, `FLOAT`, `DOUBLE`, `BIG_DECIMAL`, `STRING`, `BYTES`)
plus every logical type that maps onto one of
+ /// them (`BOOLEAN`->`INT`, `TIMESTAMP`->`LONG`, `JSON`->`STRING`,
`UUID`->`BYTES`), so both the per-type comparison
+ /// branches and the default-null-value round-trip through segment metadata
are exercised.
+ private static Map<DataType, Object> nonDefaultValues() {
+ Map<DataType, Object> values = new LinkedHashMap<>();
+ values.put(DataType.INT, 1);
+ values.put(DataType.LONG, 1L);
+ values.put(DataType.FLOAT, 1.5f);
+ values.put(DataType.DOUBLE, 2.5d);
+ values.put(DataType.BIG_DECIMAL, BigDecimal.ONE);
+ values.put(DataType.BOOLEAN, true);
+ values.put(DataType.TIMESTAMP, 1000L);
+ values.put(DataType.STRING, "a");
+ values.put(DataType.JSON, "{\"a\":1}");
+ values.put(DataType.BYTES, new byte[]{1, 2});
+ values.put(DataType.UUID, "11111111-2222-3333-4444-555555555555");
+ return values;
+ }
+
+ /// Data types allowed for a metric field. Their default null values differ
from the dimension defaults for
+ /// `INT`/`LONG`/`FLOAT`/`DOUBLE` (`0`/`0.0` instead of
`MIN_VALUE`/`-Infinity`), so they are covered separately.
+ private static final List<DataType> METRIC_TYPES =
+ List.of(DataType.INT, DataType.LONG, DataType.FLOAT, DataType.DOUBLE,
DataType.BIG_DECIMAL, DataType.BYTES);
+
+ /// `JSON` is single-value only;
`SchemaUtils.validateMultiValueCompatibility` rejects an MV JSON column.
+ private static final Set<DataType> MV_UNSUPPORTED_TYPES =
Set.of(DataType.JSON);
+
+ private static String svColumn(DataType dataType) {
+ return "sv" + dataType.name();
+ }
+
+ private static String mvColumn(DataType dataType) {
+ return "mv" + dataType.name();
+ }
+
+ private static String metricColumn(DataType dataType) {
+ return "me" + dataType.name();
+ }
+
+ @Test
+ public void testBackfillAllSupportedStoredTypes()
+ throws Exception {
+ // One SV and one MV dimension column per data type, plus one metric
column per metric-allowed type (metric
+ // defaults differ), so every per-type comparison branch is exercised
against every default null value. Row 0 holds
+ // a non-default value; row 1 leaves every cell null so the ingestion
pipeline substitutes the column's default null
+ // value (and, for MV, the single-element array holding it) exactly as it
would for a pre-null-handling segment.
+ // Row 1's MV entries are therefore single-element, so the element
comparison (not just the length check) is
+ // exercised in both directions.
+ Map<DataType, Object> nonDefaultValues = nonDefaultValues();
+ Schema.SchemaBuilder schemaBuilder = new
Schema.SchemaBuilder().setSchemaName(RAW_TABLE_NAME);
+ for (DataType dataType : nonDefaultValues.keySet()) {
+ schemaBuilder.addSingleValueDimension(svColumn(dataType), dataType);
+ if (!MV_UNSUPPORTED_TYPES.contains(dataType)) {
+ schemaBuilder.addMultiValueDimension(mvColumn(dataType), dataType);
+ }
+ }
+ // Metric columns are single-value only, and carry different default null
values than dimensions.
+ for (DataType dataType : METRIC_TYPES) {
+ schemaBuilder.addMetric(metricColumn(dataType), dataType);
+ }
+ Schema schema = schemaBuilder.build();
+
+ GenericRow nonNullRow = new GenericRow();
+ GenericRow nullRow = new GenericRow();
+ List<FieldConfig> fieldConfigs = new ArrayList<>();
+ for (Map.Entry<DataType, Object> entry : nonDefaultValues.entrySet()) {
+ DataType dataType = entry.getKey();
+ Object nonDefaultValue = entry.getValue();
+ String svColumn = svColumn(dataType);
+ nonNullRow.putValue(svColumn, nonDefaultValue);
+ nullRow.putValue(svColumn, null);
+ fieldConfigs.add(backfillOptIn(svColumn));
+ if (!MV_UNSUPPORTED_TYPES.contains(dataType)) {
+ String mvColumn = mvColumn(dataType);
+ nonNullRow.putValue(mvColumn, new Object[]{nonDefaultValue});
+ nullRow.putValue(mvColumn, null);
+ fieldConfigs.add(backfillOptIn(mvColumn));
+ }
+ }
+ for (DataType dataType : METRIC_TYPES) {
+ String metricColumn = metricColumn(dataType);
+ nonNullRow.putValue(metricColumn, nonDefaultValues.get(dataType));
+ nullRow.putValue(metricColumn, null);
+ fieldConfigs.add(backfillOptIn(metricColumn));
+ }
+
+ // Build without null handling, so no null value vector exists.
+ String segmentName = "allTypesSegment";
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(
+ new
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).build(),
schema);
+ config.setOutDir(TEMP_DIR.getAbsolutePath());
+ config.setSegmentName(segmentName);
+ SegmentIndexCreationDriverImpl driver = new
SegmentIndexCreationDriverImpl();
+ driver.init(config, new GenericRowRecordReader(List.of(nonNullRow,
nullRow)));
+ driver.build();
+
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME)
+ .setNullHandlingEnabled(true)
+ .setFieldConfigList(fieldConfigs)
+ .build();
+ ImmutableSegment segment =
+ ImmutableSegmentLoader.load(new File(TEMP_DIR, segmentName), new
IndexLoadingConfig(tableConfig, schema));
+ try {
+ for (DataType dataType : nonDefaultValues.keySet()) {
+ assertNullDocIds(segment, svColumn(dataType), false, true);
+ if (!MV_UNSUPPORTED_TYPES.contains(dataType)) {
+ assertNullDocIds(segment, mvColumn(dataType), false, true);
+ }
+ }
+ for (DataType dataType : METRIC_TYPES) {
+ assertNullDocIds(segment, metricColumn(dataType), false, true);
+ }
+ } finally {
+ segment.destroy();
+ }
+ }
+
+ @Test
+ public void testBackfillWithConfiguredDefaultNullValues()
+ throws Exception {
+ // Columns with an explicit (non-default) default null value in the schema
— the recommended way to use backfill,
+ // since the sentinel can be chosen so it does not occur in the data. This
takes a different resolution path than
+ // the built-in defaults: the configured value is persisted as the
column's DEFAULT_NULL_VALUE and parsed back via
+ // DataType.convert when the segment metadata is read. One column per
distinct comparison mechanism, plus MV.
+ Schema schema = new Schema.SchemaBuilder().setSchemaName(RAW_TABLE_NAME)
+ .addSingleValueDimension("cfgInt", DataType.INT, -1)
+ .addMultiValueDimension("cfgIntMv", DataType.INT, -1)
+ .addSingleValueDimension("cfgDouble", DataType.DOUBLE, -1.5d)
+ // The configured scale (-1.50) differs from the stored scale (-1.5,
trailing zeros stripped at ingestion), so
+ // this only matches because the comparison uses BigDecimal.compareTo
rather than equals.
+ .addSingleValueDimension("cfgBigDecimal", DataType.BIG_DECIMAL, new
BigDecimal("-1.50"))
+ .addSingleValueDimension("cfgString", DataType.STRING, "N/A")
+ .addSingleValueDimension("cfgBytes", DataType.BYTES, new byte[]{9})
+ .addSingleValueDimension("cfgTimestamp", DataType.TIMESTAMP,
1600000000000L)
+ .build();
+ List<String> columns =
+ List.of("cfgInt", "cfgIntMv", "cfgDouble", "cfgBigDecimal",
"cfgString", "cfgBytes", "cfgTimestamp");
+
+ GenericRow nonNullRow = new GenericRow();
+ nonNullRow.putValue("cfgInt", 1);
+ nonNullRow.putValue("cfgIntMv", new Object[]{1});
+ nonNullRow.putValue("cfgDouble", 2.5d);
+ nonNullRow.putValue("cfgBigDecimal", BigDecimal.ONE);
+ nonNullRow.putValue("cfgString", "a");
+ nonNullRow.putValue("cfgBytes", new byte[]{1, 2});
+ nonNullRow.putValue("cfgTimestamp", 1700000000000L);
+ // Leave every cell null so ingestion substitutes each column's configured
default null value.
+ GenericRow nullRow = new GenericRow();
+ for (String column : columns) {
+ nullRow.putValue(column, null);
+ }
+
+ String segmentName = "configuredDefaultsSegment";
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(
+ new
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).build(),
schema);
+ config.setOutDir(TEMP_DIR.getAbsolutePath());
+ config.setSegmentName(segmentName);
+ SegmentIndexCreationDriverImpl driver = new
SegmentIndexCreationDriverImpl();
+ driver.init(config, new GenericRowRecordReader(List.of(nonNullRow,
nullRow)));
+ driver.build();
+
+ List<FieldConfig> fieldConfigs = new ArrayList<>();
+ for (String column : columns) {
+ fieldConfigs.add(backfillOptIn(column));
+ }
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME)
+ .setNullHandlingEnabled(true)
+ .setFieldConfigList(fieldConfigs)
+ .build();
+ ImmutableSegment segment =
+ ImmutableSegmentLoader.load(new File(TEMP_DIR, segmentName), new
IndexLoadingConfig(tableConfig, schema));
+ try {
+ for (String column : columns) {
+ assertNullDocIds(segment, column, false, true);
+ }
+ } finally {
+ segment.destroy();
+ }
+ }
+
+ @Test
+ public void testBackfillOnTimeColumn()
+ throws Exception {
+ // The time column is the one column whose stored null value is not always
its default: ingestion substitutes the
+ // current time when the configured default is outside the valid time
range (see NullValueTransformerUtils), which
+ // is why TableConfigUtils rejects a backfill opt-in in that case. Here
the default is deliberately in range, so
+ // ingestion stores it as-is and backfill can match it -- the only
time-column configuration backfill supports.
+ String timeColumn = "ts";
+ long inRangeDefault = 1600000000000L;
+ Schema schema = new Schema.SchemaBuilder().setSchemaName(RAW_TABLE_NAME)
+ .addDateTime(timeColumn, DataType.LONG, "1:MILLISECONDS:EPOCH",
"1:MILLISECONDS", inRangeDefault, null)
+ .build();
+
+ GenericRow nonNullRow = new GenericRow();
+ nonNullRow.putValue(timeColumn, 1700000000000L);
+ GenericRow nullRow = new GenericRow();
+ nullRow.putValue(timeColumn, null);
+
+ // Build without null handling, so no null value vector exists and the
null row stores the configured default.
+ String segmentName = "timeColumnSegment";
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(
+ new TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME)
+ .setTimeColumnName(timeColumn)
+ .build(), schema);
+ config.setOutDir(TEMP_DIR.getAbsolutePath());
+ config.setSegmentName(segmentName);
+ SegmentIndexCreationDriverImpl driver = new
SegmentIndexCreationDriverImpl();
+ driver.init(config, new GenericRowRecordReader(List.of(nonNullRow,
nullRow)));
+ driver.build();
+
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME)
+ .setTimeColumnName(timeColumn)
+ .setNullHandlingEnabled(true)
+ .setFieldConfigList(List.of(backfillOptIn(timeColumn)))
+ .build();
+ ImmutableSegment segment =
+ ImmutableSegmentLoader.load(new File(TEMP_DIR, segmentName), new
IndexLoadingConfig(tableConfig, schema));
+ try {
+ assertNullDocIds(segment, timeColumn, false, true);
+ } finally {
+ segment.destroy();
+ }
+ }
+
+ @Test
+ public void testNonNullFlagWrittenAtCreation()
+ throws Exception {
+ // Build a fresh segment WITH null handling enabled (no backfill opt-in):
svInt has a null value, the others do not.
+ String segmentName = "creationSegment";
+ TableConfig tableConfig =
+ new
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).setNullHandlingEnabled(true).build();
+ List<GenericRow> rows = List.of(creationRow(1, 100L), creationRow(null,
200L), creationRow(3, 300L));
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig,
SCHEMA);
+ config.setOutDir(TEMP_DIR.getAbsolutePath());
+ config.setSegmentName(segmentName);
+ SegmentIndexCreationDriverImpl driver = new
SegmentIndexCreationDriverImpl();
+ driver.init(config, new GenericRowRecordReader(rows));
+ driver.build();
+
+ ImmutableSegment segment =
+ ImmutableSegmentLoader.load(new File(TEMP_DIR, segmentName), new
IndexLoadingConfig(tableConfig, SCHEMA));
+ try {
+ // svInt has a null -> a bitmap file is written at creation and the
column is not flagged non-null.
+ assertNotNull(segment.getDataSource(SV_INT_COLUMN).getNullValueVector());
+
assertTrue(segment.getDataSource(SV_INT_COLUMN).getNullValueVector().isNull(1));
+
assertFalse(segment.getSegmentMetadata().getColumnMetadataFor(SV_INT_COLUMN).isNonNull());
+ // svLong has no nulls -> no bitmap file, and the column is flagged
non-null at creation.
+ assertNull(segment.getDataSource(SV_LONG_COLUMN).getNullValueVector());
+
assertTrue(segment.getSegmentMetadata().getColumnMetadataFor(SV_LONG_COLUMN).isNonNull());
+ } finally {
+ segment.destroy();
+ }
+ }
+
+ @Test
+ public void testBackfillIsIdempotent()
+ throws Exception {
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME)
+ .setNullHandlingEnabled(true)
+ .setFieldConfigList(List.of(backfillOptIn(SV_INT_COLUMN),
backfillOptIn(SV_LONG_COLUMN)))
+ .build();
+ IndexLoadingConfig indexLoadingConfig = new
IndexLoadingConfig(tableConfig, SCHEMA);
+ File segmentDir = new File(TEMP_DIR, SEGMENT_NAME);
+
+ // First reload backfills: svInt gets a bitmap file (has the sentinel
value), svLong gets the non-null flag.
+ ImmutableSegmentLoader.load(segmentDir, indexLoadingConfig).destroy();
+
+ // A second reload must find nothing left to backfill for either column.
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(segmentDir, ReadMode.mmap);
+ SegmentDirectory.Reader reader = segmentDirectory.createReader()) {
+ NullValueVectorHandler handler = new
NullValueVectorHandler(segmentDirectory,
+ indexLoadingConfig.getFieldIndexConfigByColName(), tableConfig,
SCHEMA);
+ assertFalse(handler.needUpdateIndices(reader));
+ }
+ }
+
+ @Test
+ public void testColumnMissingFromSegmentIsSkipped()
+ throws Exception {
+ // A column present in the schema but not yet in the segment (normally
materialized by DefaultColumnHandler before
+ // the index handlers run) must be skipped rather than fail when the
handler is invoked on its own.
+ String missingColumn = "svIntMissing";
+ Schema schema = new Schema.SchemaBuilder().setSchemaName(RAW_TABLE_NAME)
+ .addSingleValueDimension(SV_INT_COLUMN, DataType.INT)
+ .addSingleValueDimension(SV_STRING_COLUMN, DataType.STRING)
+ .addMultiValueDimension(MV_INT_COLUMN, DataType.INT)
+ .addSingleValueDimension(SV_LONG_COLUMN, DataType.LONG)
+ .addSingleValueDimension(SV_INT_NO_OPT_IN_COLUMN, DataType.INT)
+ .addSingleValueDimension(missingColumn, DataType.INT)
+ .build();
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME)
+ .setNullHandlingEnabled(true)
+ .setFieldConfigList(List.of(backfillOptIn(missingColumn)))
+ .build();
+ IndexLoadingConfig indexLoadingConfig = new
IndexLoadingConfig(tableConfig, schema);
+
+ try (SegmentDirectory segmentDirectory = new SegmentLocalFSDirectory(new
File(TEMP_DIR, SEGMENT_NAME),
+ ReadMode.mmap); SegmentDirectory.Reader reader =
segmentDirectory.createReader()) {
+ NullValueVectorHandler handler = new
NullValueVectorHandler(segmentDirectory,
+ indexLoadingConfig.getFieldIndexConfigByColName(), tableConfig,
schema);
+ assertFalse(handler.needUpdateIndices(reader),
+ "A column missing from the segment must not be reported as needing a
backfill");
+ }
+ }
+
+ @Test
+ public void testNoBackfillWithoutOptIn()
+ throws Exception {
+ // Null handling on, but no column opts into backfill: no null value
vector should be generated.
+ TableConfig tableConfig =
+ new
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).setNullHandlingEnabled(true).build();
+
+ ImmutableSegment segment =
+ ImmutableSegmentLoader.load(new File(TEMP_DIR, SEGMENT_NAME), new
IndexLoadingConfig(tableConfig, SCHEMA));
+ try {
+ assertNullDocIds(segment, SV_INT_COLUMN, false, false, false);
+ assertNullDocIds(segment, SV_STRING_COLUMN, false, false, false);
+ assertNullDocIds(segment, MV_INT_COLUMN, false, false, false);
+ assertNull(segment.getDataSource(SV_INT_COLUMN).getNullValueVector(),
+ "No null value vector should be generated without opt-in");
+ } finally {
+ segment.destroy();
+ }
+ }
+
+ private static void assertNullDocIds(ImmutableSegment segment, String
column, boolean... expectedNull)
+ throws Exception {
+ try (PinotSegmentColumnReader columnReader = new
PinotSegmentColumnReader(segment, column)) {
+ assertEquals(segment.getSegmentMetadata().getTotalDocs(),
expectedNull.length);
+ for (int docId = 0; docId < expectedNull.length; docId++) {
+ assertEquals(columnReader.isNull(docId), expectedNull[docId],
+ "Unexpected null status for column: " + column + ", docId: " +
docId);
+ }
+ }
+ }
+}
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/TableConfigUtilsTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/TableConfigUtilsTest.java
index 3bd12121812..178b7cb8a7d 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/TableConfigUtilsTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/TableConfigUtilsTest.java
@@ -64,6 +64,7 @@ import
org.apache.pinot.spi.config.table.ingestion.IngestionConfig;
import org.apache.pinot.spi.config.table.ingestion.SourceFieldConfig;
import org.apache.pinot.spi.config.table.ingestion.StreamIngestionConfig;
import org.apache.pinot.spi.config.table.ingestion.TransformConfig;
+import org.apache.pinot.spi.data.ComplexFieldSpec;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.data.MetricFieldSpec;
@@ -1840,6 +1841,70 @@ public class TableConfigUtilsTest {
}
}
+ @Test
+ public void testValidateNullValueVectorBackfill()
+ throws Exception {
+ // Valid: opting a scalar column into null value vector backfill
(indexes.null.backfill) passes validation.
+ Schema scalarSchema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
+ .addSingleValueDimension("intCol", DataType.INT)
+ .build();
+ FieldConfig scalarFieldConfig = new FieldConfig.Builder("intCol")
+ .withIndexes(JsonUtils.stringToJsonNode("{\"null\": {\"backfill\":
true}}"))
+ .build();
+ TableConfig scalarTableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME)
+ .setNullHandlingEnabled(true)
+ .setFieldConfigList(List.of(scalarFieldConfig))
+ .build();
+ TableConfigUtils.validate(scalarTableConfig, scalarSchema);
+
+ // Invalid: opting a MAP column into backfill is rejected with a clear
message.
+ Schema mapSchema = new
Schema.SchemaBuilder().setSchemaName(TABLE_NAME).build();
+ mapSchema.addField(new ComplexFieldSpec("mapCol", DataType.MAP, true,
Map.of(
+ "key", new DimensionFieldSpec("key", DataType.STRING, true),
+ "value", new DimensionFieldSpec("value", DataType.INT, true)
+ )));
+ FieldConfig mapFieldConfig = new FieldConfig.Builder("mapCol")
+ .withIndexes(JsonUtils.stringToJsonNode("{\"null\": {\"backfill\":
true}}"))
+ .build();
+ TableConfig mapTableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME)
+ .setNullHandlingEnabled(true)
+ .setFieldConfigList(List.of(mapFieldConfig))
+ .build();
+ IllegalStateException e =
+ expectThrows(IllegalStateException.class, () ->
TableConfigUtils.validate(mapTableConfig, mapSchema));
+ assertTrue(e.getMessage().contains("Null value vector backfill is not
supported"),
+ "Unexpected validation failure: " + e.getMessage());
+ }
+
+ @Test
+ public void testValidateNullValueVectorBackfillOnTimeColumn()
+ throws Exception {
+ FieldConfig timeFieldConfig = new FieldConfig.Builder(TIME_COLUMN)
+ .withIndexes(JsonUtils.stringToJsonNode("{\"null\": {\"backfill\":
true}}"))
+ .build();
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME)
+ .setTimeColumnName(TIME_COLUMN)
+ .setNullHandlingEnabled(true)
+ .setFieldConfigList(List.of(timeFieldConfig))
+ .build();
+
+ // Invalid: the default default null value (Long.MIN_VALUE) is outside the
valid time range, so ingestion stores the
+ // current time for null values and a backfill scan could never match it.
+ Schema defaultDefaultSchema = new
Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
+ .addDateTime(TIME_COLUMN, DataType.LONG, "1:MILLISECONDS:EPOCH",
"1:MILLISECONDS")
+ .build();
+ IllegalStateException e =
+ expectThrows(IllegalStateException.class, () ->
TableConfigUtils.validate(tableConfig, defaultDefaultSchema));
+ assertTrue(e.getMessage().contains("Null value vector backfill is not
supported for time column"),
+ "Unexpected validation failure: " + e.getMessage());
+
+ // Valid: an explicit in-range default null value is stored as-is for null
values, so backfill can match it.
+ Schema inRangeDefaultSchema = new
Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
+ .addDateTime(TIME_COLUMN, DataType.LONG, "1:MILLISECONDS:EPOCH",
"1:MILLISECONDS", 1600000000000L, null)
+ .build();
+ TableConfigUtils.validate(tableConfig, inRangeDefaultSchema);
+ }
+
@Test
public void testValidateBFOnBoolean() {
Schema schema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/ColumnMetadata.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/ColumnMetadata.java
index d2d762a0856..8344cf33cb4 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/ColumnMetadata.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/ColumnMetadata.java
@@ -39,6 +39,15 @@ public interface ColumnMetadata extends ColumnShape {
/// Returns the forward-index encoding for this column.
EncodingType getForwardIndexEncoding();
+ /// Returns `true` when this column is known to contain no null values: null
handling is enabled and no nulls were
+ /// found, so no null value vector bitmap file is written. This is the only
signal that distinguishes such a column
+ /// from one that was never null-handled (both lack a bitmap file). A column
that does have null values carries a
+ /// bitmap file instead and is not flagged. Defaults to `false` for segments
created before this flag was introduced
+ /// (`false` therefore does not assert the presence of nulls).
+ default boolean isNonNull() {
+ return false;
+ }
+
/// Returns `true` when both min and max value are invalid, and there is no
need to regenerate them.
boolean isMinMaxValueInvalid();
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
index fb67e32b36f..53464d338d6 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
@@ -161,6 +161,12 @@ public class V1Constants {
public static final String FORWARD_INDEX_ENCODING =
"forwardIndexEncoding";
// Mandatory, treated as `false` when missing for backward compatibility
public static final String IS_SORTED = "isSorted";
+ // Optional, treated as `false` when missing for backward compatibility.
Set to `true` only when null handling is
+ // enabled for the column but it has no null values, so the null value
vector bitmap file is skipped. This
+ // distinguishes "null handling applied, no nulls" (flag set, no bitmap
file) from "null handling never applied"
+ // (no flag, no bitmap file) — which absence of the bitmap file alone
cannot. Columns that do have null values
+ // are identified by the presence of the bitmap file and are not flagged.
+ public static final String IS_NON_NULL = "isNonNull";
// Optional
public static final String MIN_VALUE = "minValue";
// Optional
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/StandardIndexes.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/StandardIndexes.java
index 8df197c4bb2..8a63762623e 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/StandardIndexes.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/StandardIndexes.java
@@ -45,6 +45,7 @@ import
org.apache.pinot.segment.spi.index.reader.VectorIndexReader;
import org.apache.pinot.spi.config.table.BloomFilterConfig;
import org.apache.pinot.spi.config.table.IndexConfig;
import org.apache.pinot.spi.config.table.JsonIndexConfig;
+import org.apache.pinot.spi.config.table.NullValueVectorConfig;
import org.apache.pinot.spi.config.table.OpenStructIndexConfig;
@@ -97,8 +98,8 @@ public class StandardIndexes {
IndexService.getInstance().get(DICTIONARY_ID);
}
- public static IndexType<IndexConfig, NullValueVectorReader, ?>
nullValueVector() {
- return (IndexType<IndexConfig, NullValueVectorReader, ?>)
+ public static IndexType<NullValueVectorConfig, NullValueVectorReader, ?>
nullValueVector() {
+ return (IndexType<NullValueVectorConfig, NullValueVectorReader, ?>)
IndexService.getInstance().get(NULL_VALUE_VECTOR_ID);
}
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/ColumnMetadataImpl.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/ColumnMetadataImpl.java
index 0755f9a9b6c..af2c3f5f8df 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/ColumnMetadataImpl.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/ColumnMetadataImpl.java
@@ -64,6 +64,7 @@ public class ColumnMetadataImpl implements ColumnMetadata {
private final boolean _hasDictionary;
private final EncodingType _forwardIndexEncoding;
private final boolean _sorted;
+ private final boolean _nonNull;
private final Comparable _minValue;
private final Comparable _maxValue;
private final boolean _minMaxValueInvalid;
@@ -90,7 +91,7 @@ public class ColumnMetadataImpl implements ColumnMetadata {
private final LongArrayList _indexTypeSizeList = new LongArrayList(2);
private ColumnMetadataImpl(FieldSpec fieldSpec, int totalDocs, int
cardinality, boolean hasDictionary,
- @Nullable EncodingType forwardIndexEncoding, boolean sorted, @Nullable
Comparable minValue,
+ @Nullable EncodingType forwardIndexEncoding, boolean sorted, boolean
nonNull, @Nullable Comparable minValue,
@Nullable Comparable maxValue,
boolean minMaxValueInvalid, int lengthOfShortestElement, int
lengthOfLongestElement, boolean isAscii,
int totalNumberOfEntries, int maxNumberOfMultiValues, int
maxRowLengthInBytes, int bitsPerElement,
@@ -102,6 +103,7 @@ public class ColumnMetadataImpl implements ColumnMetadata {
_hasDictionary = hasDictionary;
_forwardIndexEncoding = forwardIndexEncoding;
_sorted = sorted;
+ _nonNull = nonNull;
_minValue = minValue;
_maxValue = maxValue;
_minMaxValueInvalid = minMaxValueInvalid;
@@ -149,6 +151,11 @@ public class ColumnMetadataImpl implements ColumnMetadata {
return _sorted;
}
+ @Override
+ public boolean isNonNull() {
+ return _nonNull;
+ }
+
@Nullable
@Override
public Comparable<?> getMinValue() {
@@ -302,7 +309,7 @@ public class ColumnMetadataImpl implements ColumnMetadata {
&& _cardinality == that._cardinality
&& _hasDictionary == that._hasDictionary
&& _forwardIndexEncoding == that._forwardIndexEncoding
- && _sorted == that._sorted
+ && _sorted == that._sorted && _nonNull == that._nonNull
&& _minMaxValueInvalid == that._minMaxValueInvalid
&& _lengthOfShortestElement == that._lengthOfShortestElement
&& _lengthOfLongestElement == that._lengthOfLongestElement
@@ -324,7 +331,7 @@ public class ColumnMetadataImpl implements ColumnMetadata {
@Override
public int hashCode() {
- return Objects.hash(_fieldSpec, _totalDocs, _cardinality, _hasDictionary,
_forwardIndexEncoding, _sorted,
+ return Objects.hash(_fieldSpec, _totalDocs, _cardinality, _hasDictionary,
_forwardIndexEncoding, _sorted, _nonNull,
_minValue, _maxValue, _minMaxValueInvalid, _lengthOfShortestElement,
_lengthOfLongestElement, _isAscii,
_totalNumberOfEntries, _maxNumberOfMultiValues, _maxRowLengthInBytes,
_bitsPerElement, _partitionFunction,
_partitions, _autoGenerated, _parentColumn, _compressionMetadata,
_indexTypeSizeList);
@@ -338,7 +345,7 @@ public class ColumnMetadataImpl implements ColumnMetadata {
+ ", _cardinality=" + _cardinality
+ ", _hasDictionary=" + _hasDictionary
+ ", _forwardIndexEncoding=" + _forwardIndexEncoding
- + ", _sorted=" + _sorted
+ + ", _sorted=" + _sorted + ", _nonNull=" + _nonNull
+ ", _minValue=" + _minValue
+ ", _maxValue=" + _maxValue
+ ", _minMaxValueInvalid=" + _minMaxValueInvalid
@@ -369,6 +376,7 @@ public class ColumnMetadataImpl implements ColumnMetadata {
.setForwardIndexEncoding(
config.getEnum(Column.getKeyFor(column,
Column.FORWARD_INDEX_ENCODING), EncodingType.class, null))
.setSorted(config.getBoolean(Column.getKeyFor(column,
Column.IS_SORTED), false))
+ .setNonNull(config.getBoolean(Column.getKeyFor(column,
Column.IS_NON_NULL), false))
.setLengthOfShortestElement(
config.getInt(Column.getKeyFor(column,
Column.LENGTH_OF_SHORTEST_ELEMENT), UNAVAILABLE))
.setLengthOfLongestElement(
@@ -378,8 +386,7 @@ public class ColumnMetadataImpl implements ColumnMetadata {
.setTotalNumberOfEntries(config.getInt(Column.getKeyFor(column,
Column.TOTAL_NUMBER_OF_ENTRIES), UNAVAILABLE))
.setMaxNumberOfMultiValues(
config.getInt(Column.getKeyFor(column,
Column.MAX_MULTI_VALUE_ELEMENTS), UNAVAILABLE))
- .setMaxRowLengthInBytes(
- config.getInt(Column.getKeyFor(column,
Column.MAX_ROW_LENGTH_IN_BYTES), UNAVAILABLE))
+ .setMaxRowLengthInBytes(config.getInt(Column.getKeyFor(column,
Column.MAX_ROW_LENGTH_IN_BYTES), UNAVAILABLE))
.setBitsPerElement(config.getInt(Column.getKeyFor(column,
Column.BITS_PER_ELEMENT), UNAVAILABLE))
.setAutoGenerated(config.getBoolean(Column.getKeyFor(column,
Column.IS_AUTO_GENERATED), false))
.setParentColumn(config.getString(Column.getKeyFor(column,
Column.PARENT_COLUMN), null));
@@ -613,6 +620,7 @@ public class ColumnMetadataImpl implements ColumnMetadata {
private boolean _hasDictionary;
private EncodingType _forwardIndexEncoding;
private boolean _sorted;
+ private boolean _nonNull;
private Comparable<?> _minValue;
private Comparable<?> _maxValue;
private boolean _minMaxValueInvalid;
@@ -662,6 +670,11 @@ public class ColumnMetadataImpl implements ColumnMetadata {
return this;
}
+ public Builder setNonNull(boolean nonNull) {
+ _nonNull = nonNull;
+ return this;
+ }
+
public Builder setMinValue(Comparable<?> minValue) {
_minValue = minValue;
return this;
@@ -793,9 +806,9 @@ public class ColumnMetadataImpl implements ColumnMetadata {
}
return new ColumnMetadataImpl(_fieldSpec, _totalDocs, _cardinality,
_hasDictionary, _forwardIndexEncoding,
- _sorted, _minValue, _maxValue, _minMaxValueInvalid,
_lengthOfShortestElement, _lengthOfLongestElement,
- _isAscii, _totalNumberOfEntries, _maxNumberOfMultiValues,
_maxRowLengthInBytes, _bitsPerElement,
- _partitionFunction, _partitions, _autoGenerated, _parentColumn,
+ _sorted, _nonNull, _minValue, _maxValue, _minMaxValueInvalid,
_lengthOfShortestElement,
+ _lengthOfLongestElement, _isAscii, _totalNumberOfEntries,
_maxNumberOfMultiValues, _maxRowLengthInBytes,
+ _bitsPerElement, _partitionFunction, _partitions, _autoGenerated,
_parentColumn,
CompressionMetadata.create(_uncompressedValueSizeInBytes,
_forwardIndexChunkCompressionType,
_dictionaryUncompressedValueSizeInBytes));
}
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/NullValueVectorConfig.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/NullValueVectorConfig.java
new file mode 100644
index 00000000000..8b6a6d796f2
--- /dev/null
+++
b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/NullValueVectorConfig.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.spi.config.table;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.Objects;
+
+
+/// Configuration for the null value vector index of a column.
+///
+/// The `enabled` state is derived from null handling (schema column-based
nullability or the table-level
+/// `nullHandlingEnabled` flag) rather than set directly, so users typically
only configure `backfill`.
+///
+/// When `backfill` is `true`, segment reload generates a null value vector
for the column if it has null handling
+/// enabled but no null value vector yet, by treating every value equal to the
column's default null value as null. This
+/// is a lossy reconstruction (a genuine value equal to the default is also
marked null), so it is per-column opt-in —
+/// enable it only for columns whose default null value is a sentinel that
does not occur in the data (e.g. a
+/// dimension's `MIN_VALUE`), not for columns whose default is a value that
legitimately occurs in the data — e.g.
+/// metrics (`0`), `BOOLEAN` (`false`), or `TIMESTAMP` (epoch `0`).
+public class NullValueVectorConfig extends IndexConfig {
+ private final boolean _backfill;
+
+ @JsonCreator
+ public NullValueVectorConfig(@JsonProperty("disabled") Boolean disabled,
+ @JsonProperty("backfill") boolean backfill) {
+ super(disabled);
+ _backfill = backfill;
+ }
+
+ public boolean isBackfill() {
+ return _backfill;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!super.equals(o)) {
+ return false;
+ }
+ return _backfill == ((NullValueVectorConfig) o)._backfill;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), _backfill);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
