openinx commented on a change in pull request #1197:
URL: https://github.com/apache/iceberg/pull/1197#discussion_r454774011
##########
File path: data/src/main/java/org/apache/iceberg/data/orc/GenericOrcWriter.java
##########
@@ -434,175 +144,12 @@ public void addValue(int rowId, BigDecimal data,
ColumnVector output) {
@Override
@SuppressWarnings("unchecked")
- public void addValue(int rowId, Record data, ColumnVector output) {
- if (data == null) {
- output.noNulls = false;
- output.isNull[rowId] = true;
- } else {
- output.isNull[rowId] = false;
- StructColumnVector cv = (StructColumnVector) output;
- for (int c = 0; c < children.length; ++c) {
- children[c].addValue(rowId, data.get(c, children[c].getJavaClass()),
cv.fields[c]);
- }
- }
- }
- }
-
- static class ListConverter implements Converter<List> {
- private final Converter children;
-
- ListConverter(TypeDescription schema) {
- this.children = buildConverter(schema.getChildren().get(0));
- }
-
- @Override
- public Class<List> getJavaClass() {
- return List.class;
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public void addValue(int rowId, List data, ColumnVector output) {
- if (data == null) {
- output.noNulls = false;
- output.isNull[rowId] = true;
- } else {
- output.isNull[rowId] = false;
- List<Object> value = (List<Object>) data;
- ListColumnVector cv = (ListColumnVector) output;
- // record the length and start of the list elements
- cv.lengths[rowId] = value.size();
- cv.offsets[rowId] = cv.childCount;
- cv.childCount += cv.lengths[rowId];
- // make sure the child is big enough
- cv.child.ensureSize(cv.childCount, true);
- // Add each element
- for (int e = 0; e < cv.lengths[rowId]; ++e) {
- children.addValue((int) (e + cv.offsets[rowId]), value.get(e),
cv.child);
- }
+ public void nonNullWrite(int rowId, Record data, ColumnVector output) {
+ StructColumnVector cv = (StructColumnVector) output;
+ for (int c = 0; c < writers.size(); ++c) {
+ OrcValueWriter child = writers.get(c);
+ child.write(rowId, data.get(c, child.getJavaClass()), cv.fields[c]);
}
}
}
-
- static class MapConverter implements Converter<Map> {
- private final Converter keyConverter;
- private final Converter valueConverter;
-
- MapConverter(TypeDescription schema) {
- this.keyConverter = buildConverter(schema.getChildren().get(0));
- this.valueConverter = buildConverter(schema.getChildren().get(1));
- }
-
- @Override
- public Class<Map> getJavaClass() {
- return Map.class;
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public void addValue(int rowId, Map data, ColumnVector output) {
- if (data == null) {
- output.noNulls = false;
- output.isNull[rowId] = true;
- } else {
- output.isNull[rowId] = false;
- Map<Object, Object> map = (Map<Object, Object>) data;
- List<Object> keys = Lists.newArrayListWithExpectedSize(map.size());
- List<Object> values = Lists.newArrayListWithExpectedSize(map.size());
- for (Map.Entry<?, ?> entry : map.entrySet()) {
- keys.add(entry.getKey());
- values.add(entry.getValue());
- }
- MapColumnVector cv = (MapColumnVector) output;
- // record the length and start of the list elements
- cv.lengths[rowId] = map.size();
- cv.offsets[rowId] = cv.childCount;
- cv.childCount += cv.lengths[rowId];
- // make sure the child is big enough
- cv.keys.ensureSize(cv.childCount, true);
- cv.values.ensureSize(cv.childCount, true);
- // Add each element
- for (int e = 0; e < cv.lengths[rowId]; ++e) {
- int pos = (int) (e + cv.offsets[rowId]);
- keyConverter.addValue(pos, keys.get(e), cv.keys);
- valueConverter.addValue(pos, values.get(e), cv.values);
- }
- }
- }
- }
-
- private static Converter buildConverter(TypeDescription schema) {
- switch (schema.getCategory()) {
- case BOOLEAN:
- return new BooleanConverter();
- case BYTE:
- return new ByteConverter();
- case SHORT:
- return new ShortConverter();
- case DATE:
- return new DateConverter();
- case INT:
- return new IntConverter();
- case LONG:
- String longAttributeValue =
schema.getAttributeValue(ORCSchemaUtil.ICEBERG_LONG_TYPE_ATTRIBUTE);
- ORCSchemaUtil.LongType longType = longAttributeValue == null ?
ORCSchemaUtil.LongType.LONG :
- ORCSchemaUtil.LongType.valueOf(longAttributeValue);
- switch (longType) {
- case TIME:
- return new TimeConverter();
- case LONG:
- return new LongConverter();
- default:
- throw new IllegalStateException("Unhandled Long type found in ORC
type attribute: " + longType);
- }
- case FLOAT:
- return new FloatConverter();
- case DOUBLE:
- return new DoubleConverter();
- case BINARY:
- String binaryAttributeValue =
schema.getAttributeValue(ORCSchemaUtil.ICEBERG_BINARY_TYPE_ATTRIBUTE);
- ORCSchemaUtil.BinaryType binaryType = binaryAttributeValue == null ?
ORCSchemaUtil.BinaryType.BINARY :
- ORCSchemaUtil.BinaryType.valueOf(binaryAttributeValue);
- switch (binaryType) {
- case UUID:
- return new UUIDConverter();
- case FIXED:
- return new FixedConverter();
Review comment:
Seems the `TypeID#javaClass` defines the literal type, means the type
to compare or serialize/deserialize. Actually, I'd prefer to pass the
`LocalDateTime` object to the comparator and do the `LocalDateTime` to `Long`
conversion when comparing. Then we hidden the literal types between different
types, and the upper layer won't to wrap again and again.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
