This is an automated email from the ASF dual-hosted git repository.
Jackie-Jiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new e55c18172a5 Add explicit target data type constructor to
DataTypeColumnTransformer (#18954)
e55c18172a5 is described below
commit e55c18172a5ddfc313dc79420d86268c167bec66
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Thu Jul 9 23:49:56 2026 -0700
Add explicit target data type constructor to DataTypeColumnTransformer
(#18954)
---
.../DataTypeColumnTransformer.java | 22 ++++++++++++++++------
.../DataTypeColumnTransformerTest.java | 11 +++++++++++
2 files changed, 27 insertions(+), 6 deletions(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/columntransformer/DataTypeColumnTransformer.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/columntransformer/DataTypeColumnTransformer.java
index f108aeb7926..5214a97d2c8 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/columntransformer/DataTypeColumnTransformer.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/columntransformer/DataTypeColumnTransformer.java
@@ -30,8 +30,13 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+/// The `DataTypeColumnTransformer` is the column-major counterpart of the
row-major `DataTypeTransformer`. It converts
+/// the values read from a [ColumnReader] to a target [PinotDataType]. It has
two usages:
+/// - Schema columns: constructed from a [FieldSpec], it converts the values
to the data type defined for the column in
+/// the [org.apache.pinot.spi.data.Schema].
+/// - Source fields: constructed from an explicit [PinotDataType] (see
`IngestionConfig.sourceFieldConfigs`), it fixes
+/// the data type of a source field before other transformers (such as the
expression transformer) consume it.
public class DataTypeColumnTransformer implements ColumnTransformer {
-
private static final Logger LOGGER =
LoggerFactory.getLogger(DataTypeColumnTransformer.class);
private final PinotDataType _destDataType;
@@ -39,12 +44,17 @@ public class DataTypeColumnTransformer implements
ColumnTransformer {
private final boolean _continueOnError;
private final ThrottledLogger _throttledLogger;
- /**
- * @param fieldSpec - The field spec for the column being created in Pinot.
- * @param columnReader - The column reader to read the source data.
- */
+ /// Creates a transformer that converts the values read from `columnReader`
to the data type defined for the column
+ /// in the [org.apache.pinot.spi.data.Schema] (derived from the given
[FieldSpec]).
public DataTypeColumnTransformer(TableConfig tableConfig, FieldSpec
fieldSpec, ColumnReader columnReader) {
- _destDataType = PinotDataType.getPinotDataTypeForIngestion(fieldSpec);
+ this(tableConfig, PinotDataType.getPinotDataTypeForIngestion(fieldSpec),
columnReader);
+ }
+
+ /// Creates a transformer that converts the values read from `columnReader`
to the given [PinotDataType]. This is
+ /// useful for fixing the data type of a source field before other
transformers (such as the expression transformer)
+ /// consume it.
+ public DataTypeColumnTransformer(TableConfig tableConfig, PinotDataType
destDataType, ColumnReader columnReader) {
+ _destDataType = destDataType;
_columnReader = columnReader;
IngestionConfig ingestionConfig = tableConfig.getIngestionConfig();
_continueOnError = ingestionConfig != null &&
ingestionConfig.isContinueOnError();
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/columntransformer/DataTypeColumnTransformerTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/columntransformer/DataTypeColumnTransformerTest.java
index aa1c02598ca..c493c024928 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/columntransformer/DataTypeColumnTransformerTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/columntransformer/DataTypeColumnTransformerTest.java
@@ -600,4 +600,15 @@ public class DataTypeColumnTransformerTest {
Object result = transformer.transform("not_a_number");
assertNull(result, "Invalid conversion should return null when
continueOnError=true");
}
+
+ @Test
+ public void testExplicitDestDataTypeConstructor() {
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ ColumnReader reader = mockReader(PinotDataType.STRING);
+ // The explicit target type drives the conversion, independent of any
field spec.
+ DataTypeColumnTransformer transformer = new
DataTypeColumnTransformer(tableConfig, PinotDataType.LONG, reader);
+
+ assertFalse(transformer.isNoOp(), "STRING source, LONG dest should not be
a no-op");
+ assertEquals(transformer.transform("12345"), 12345L);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
