This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new fac02cf4df When extracting root-level field name for complex type
handling, use the whole delimiter (#11005)
fac02cf4df is described below
commit fac02cf4df857fbf5f3d09ab2c3eedba0b265ce4
Author: Rekha Seethamraju <[email protected]>
AuthorDate: Fri Jun 30 15:41:19 2023 -0700
When extracting root-level field name for complex type handling, use the
whole delimiter (#11005)
---
.../apache/pinot/segment/local/utils/IngestionUtils.java | 2 +-
.../pinot/segment/local/utils/IngestionUtilsTest.java | 16 ++++++++++++++++
2 files changed, 17 insertions(+), 1 deletion(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/IngestionUtils.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/IngestionUtils.java
index 72aadb70cb..1d73c4cfd5 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/IngestionUtils.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/IngestionUtils.java
@@ -328,7 +328,7 @@ public final class IngestionUtils {
String delimiter = complexTypeConfig.getDelimiter() == null ?
ComplexTypeTransformer.DEFAULT_DELIMITER
: complexTypeConfig.getDelimiter();
for (String field : fieldsToRead) {
- result.add(StringUtils.split(field, delimiter)[0]);
+ result.add(StringUtils.splitByWholeSeparator(field, delimiter)[0]);
}
return result;
}
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/IngestionUtilsTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/IngestionUtilsTest.java
index 9b54600e93..817ce0362a 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/IngestionUtilsTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/IngestionUtilsTest.java
@@ -233,4 +233,20 @@ public class IngestionUtilsTest {
Assert.assertEquals(fields.size(), 1);
Assert.assertTrue(fields.containsAll(Sets.newHashSet("s1")));
}
+
+ @Test
+ public void testComplexTypeConfig() {
+ IngestionConfig ingestionConfig = new IngestionConfig();
+ ComplexTypeConfig complexTypeConfig = new ComplexTypeConfig(null, "__",
+ ComplexTypeConfig.CollectionNotUnnestedToJson.NON_PRIMITIVE, null);
+ Schema schema = new Schema();
+
+ ingestionConfig.setComplexTypeConfig(complexTypeConfig);
+ schema.addField(new DimensionFieldSpec("a_b__c_d",
FieldSpec.DataType.STRING, true));
+ schema.addField(new DimensionFieldSpec("f_d", FieldSpec.DataType.STRING,
false));
+ schema.addField(new DimensionFieldSpec("ab__cd",
FieldSpec.DataType.STRING, true));
+ Set<String> fields =
IngestionUtils.getFieldsForRecordExtractor(ingestionConfig, schema);
+ Assert.assertEquals(fields.size(), 3);
+ Assert.assertTrue(fields.containsAll(Sets.newHashSet("a_b", "f_d", "ab")));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
