This is an automated email from the ASF dual-hosted git repository.
manishswaminathan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 9f248c52aff Add InMemoryColumnReader for testing and update
PinotSegmentColumnReaderFactory to support configurable default value readers
(#17415)
9f248c52aff is described below
commit 9f248c52aff01ff6740dfbab5a72d6b185ba1e2c
Author: Krishan Goyal <[email protected]>
AuthorDate: Mon Dec 29 14:39:40 2025 +0530
Add InMemoryColumnReader for testing and update
PinotSegmentColumnReaderFactory to support configurable default value readers
(#17415)
* Add InMemoryColumnReader for testing and update
PinotSegmentColumnReaderFactory to support configurable default value readers
* Remove default in PinotSegmentColumnarDataSource
---
.../readers/PinotSegmentColumnReaderFactory.java | 31 ++-
.../readers/PinotSegmentColumnarDataSource.java | 6 +-
.../segment/readers/InMemoryColumnReader.java | 258 +++++++++++++++++++++
3 files changed, 288 insertions(+), 7 deletions(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/PinotSegmentColumnReaderFactory.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/PinotSegmentColumnReaderFactory.java
index 2b40444a52a..b6eb206c31d 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/PinotSegmentColumnReaderFactory.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/PinotSegmentColumnReaderFactory.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
+import javax.annotation.Nullable;
import org.apache.pinot.segment.spi.IndexSegment;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
@@ -49,6 +50,7 @@ public class PinotSegmentColumnReaderFactory implements
ColumnReaderFactory {
private Schema _targetSchema;
private Set<String> _colsToRead;
private Map<String, ColumnReader> _columnReaders;
+ private final boolean _initializeDefaultValueReaders;
/**
* Create a PinotSegmentColumnReaderFactory.
@@ -56,8 +58,22 @@ public class PinotSegmentColumnReaderFactory implements
ColumnReaderFactory {
* @param indexSegment Source segment to read from
*/
public PinotSegmentColumnReaderFactory(IndexSegment indexSegment) {
+ this(indexSegment, true);
+ }
+
+ /**
+ * Create a PinotSegmentColumnReaderFactory.
+ *
+ * @param indexSegment Source segment to read from
+ * @param initializeDefaultValueReaders Whether to initialize default value
readers for missing columns
+ * TODO - Ideally this factory shouldn't initialize default value
readers.
+ * The clients of this factory should decide whether to
create default value readers or not.
+ * This parameter is kept for backward compatibility and
will be removed in future.
+ */
+ public PinotSegmentColumnReaderFactory(IndexSegment indexSegment, boolean
initializeDefaultValueReaders) {
_indexSegment = indexSegment;
_columnReaders = new HashMap<>();
+ _initializeDefaultValueReaders = initializeDefaultValueReaders;
}
@Override
@@ -103,21 +119,24 @@ public class PinotSegmentColumnReaderFactory implements
ColumnReaderFactory {
* Internal method to create a column reader for the specified column.
* This method is called during initialization to create all readers.
*/
+ @Nullable
private ColumnReader createColumnReader(String columnName, FieldSpec
targetFieldSpec) {
if (targetFieldSpec.isVirtualColumn()) {
throw new IllegalStateException("Target field spec is a virtual
column.");
}
- ColumnReader columnReader;
+ ColumnReader columnReader = null;
if (hasColumn(columnName)) {
// Column exists in source segment - create a segment column reader
LOGGER.debug("Creating segment column reader for existing column: {}",
columnName);
columnReader = new PinotSegmentColumnReaderImpl(_indexSegment,
columnName);
} else {
- // New column - create a default value reader
- LOGGER.debug("Creating default value reader for new column: {}",
columnName);
- columnReader = new DefaultValueColumnReader(columnName, getNumDocs(),
targetFieldSpec);
+ if (_initializeDefaultValueReaders) {
+ // New column - create a default value reader
+ LOGGER.debug("Creating default value reader for new column: {}",
columnName);
+ columnReader = new DefaultValueColumnReader(columnName, getNumDocs(),
targetFieldSpec);
+ }
}
return columnReader;
@@ -146,7 +165,9 @@ public class PinotSegmentColumnReaderFactory implements
ColumnReaderFactory {
continue;
}
ColumnReader reader = createColumnReader(columnName, fieldSpec);
- allReaders.put(columnName, reader);
+ if (reader != null) {
+ allReaders.put(columnName, reader);
+ }
}
return allReaders;
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/PinotSegmentColumnarDataSource.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/PinotSegmentColumnarDataSource.java
index 8b3306f6946..960dc7ebc4b 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/PinotSegmentColumnarDataSource.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/PinotSegmentColumnarDataSource.java
@@ -32,10 +32,12 @@ public class PinotSegmentColumnarDataSource implements
ColumnarDataSource {
private final IndexSegment _indexSegment;
private final int _totalDocs;
+ private final boolean _initializeDefaultValueReaders;
- public PinotSegmentColumnarDataSource(IndexSegment indexSegment) {
+ public PinotSegmentColumnarDataSource(IndexSegment indexSegment, boolean
initializeDefaultValueReaders) {
_indexSegment = indexSegment;
_totalDocs = indexSegment.getSegmentMetadata().getTotalDocs();
+ _initializeDefaultValueReaders = initializeDefaultValueReaders;
}
@Override
@@ -45,7 +47,7 @@ public class PinotSegmentColumnarDataSource implements
ColumnarDataSource {
@Override
public ColumnReaderFactory createColumnReaderFactory() {
- return new PinotSegmentColumnReaderFactory(_indexSegment);
+ return new PinotSegmentColumnReaderFactory(_indexSegment,
_initializeDefaultValueReaders);
}
@Override
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/readers/InMemoryColumnReader.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/readers/InMemoryColumnReader.java
new file mode 100644
index 00000000000..adda7e52286
--- /dev/null
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/readers/InMemoryColumnReader.java
@@ -0,0 +1,258 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.readers;
+
+import javax.annotation.Nullable;
+import org.apache.pinot.spi.data.readers.ColumnReader;
+import org.apache.pinot.spi.data.readers.MultiValueResult;
+
+
+/**
+ * Mock ColumnReader for testing with configurable data.
+ */
+public class InMemoryColumnReader implements ColumnReader {
+ private final String _columnName;
+ private final Object[] _values;
+ private final boolean[] _isNull;
+ private final boolean _isSingleValue;
+ private final Class<?> _dataType;
+ private int _currentIndex = 0;
+
+ public InMemoryColumnReader(String columnName, Object[] values, boolean[]
isNull, boolean isSingleValue,
+ Class<?> dataType) {
+ _columnName = columnName;
+ _values = values;
+ _isNull = isNull;
+ _isSingleValue = isSingleValue;
+ _dataType = dataType;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return _currentIndex < _values.length;
+ }
+
+ @Nullable
+ @Override
+ public Object next() {
+ if (!hasNext()) {
+ throw new IllegalStateException("No more values");
+ }
+ Object value = _values[_currentIndex];
+ _currentIndex++;
+ return value;
+ }
+
+ @Override
+ public boolean isNextNull() {
+ return hasNext() && _isNull[_currentIndex];
+ }
+
+ @Override
+ public void skipNext() {
+ if (hasNext()) {
+ _currentIndex++;
+ }
+ }
+
+ @Override
+ public boolean isSingleValue() {
+ return _isSingleValue;
+ }
+
+ @Override
+ public boolean isInt() {
+ return _dataType == Integer.class;
+ }
+
+ @Override
+ public boolean isLong() {
+ return _dataType == Long.class;
+ }
+
+ @Override
+ public boolean isFloat() {
+ return _dataType == Float.class;
+ }
+
+ @Override
+ public boolean isDouble() {
+ return _dataType == Double.class;
+ }
+
+ @Override
+ public boolean isString() {
+ return _dataType == String.class;
+ }
+
+ @Override
+ public boolean isBytes() {
+ return _dataType == byte[].class;
+ }
+
+ @Override
+ public int nextInt() {
+ return (Integer) next();
+ }
+
+ @Override
+ public long nextLong() {
+ return (Long) next();
+ }
+
+ @Override
+ public float nextFloat() {
+ return (Float) next();
+ }
+
+ @Override
+ public double nextDouble() {
+ return (Double) next();
+ }
+
+ @Override
+ public String nextString() {
+ return (String) next();
+ }
+
+ @Override
+ public byte[] nextBytes() {
+ return (byte[]) next();
+ }
+
+ @Override
+ public MultiValueResult<int[]> nextIntMV() {
+ return MultiValueResult.of((int[]) next(), null);
+ }
+
+ @Override
+ public MultiValueResult<long[]> nextLongMV() {
+ return MultiValueResult.of((long[]) next(), null);
+ }
+
+ @Override
+ public MultiValueResult<float[]> nextFloatMV() {
+ return MultiValueResult.of((float[]) next(), null);
+ }
+
+ @Override
+ public MultiValueResult<double[]> nextDoubleMV() {
+ return MultiValueResult.of((double[]) next(), null);
+ }
+
+ @Override
+ public String[] nextStringMV() {
+ return (String[]) next();
+ }
+
+ @Override
+ public byte[][] nextBytesMV() {
+ return (byte[][]) next();
+ }
+
+ @Override
+ public void rewind() {
+ _currentIndex = 0;
+ }
+
+ @Override
+ public String getColumnName() {
+ return _columnName;
+ }
+
+ @Override
+ public int getTotalDocs() {
+ return _values.length;
+ }
+
+ @Override
+ public boolean isNull(int docId) {
+ return _isNull[docId];
+ }
+
+ @Override
+ public int getInt(int docId) {
+ return (Integer) _values[docId];
+ }
+
+ @Override
+ public long getLong(int docId) {
+ return (Long) _values[docId];
+ }
+
+ @Override
+ public float getFloat(int docId) {
+ return (Float) _values[docId];
+ }
+
+ @Override
+ public double getDouble(int docId) {
+ return (Double) _values[docId];
+ }
+
+ @Override
+ public String getString(int docId) {
+ return (String) _values[docId];
+ }
+
+ @Override
+ public byte[] getBytes(int docId) {
+ return (byte[]) _values[docId];
+ }
+
+ @Override
+ public Object getValue(int docId) {
+ return _values[docId];
+ }
+
+ @Override
+ public MultiValueResult<int[]> getIntMV(int docId) {
+ return MultiValueResult.of((int[]) _values[docId], null);
+ }
+
+ @Override
+ public MultiValueResult<long[]> getLongMV(int docId) {
+ return MultiValueResult.of((long[]) _values[docId], null);
+ }
+
+ @Override
+ public MultiValueResult<float[]> getFloatMV(int docId) {
+ return MultiValueResult.of((float[]) _values[docId], null);
+ }
+
+ @Override
+ public MultiValueResult<double[]> getDoubleMV(int docId) {
+ return MultiValueResult.of((double[]) _values[docId], null);
+ }
+
+ @Override
+ public String[] getStringMV(int docId) {
+ return (String[]) _values[docId];
+ }
+
+ @Override
+ public byte[][] getBytesMV(int docId) {
+ return (byte[][]) _values[docId];
+ }
+
+ @Override
+ public void close() {
+ // No-op
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
