clintropolis commented on a change in pull request #7138: 'core' ORC extension URL: https://github.com/apache/incubator-druid/pull/7138#discussion_r273285621
########## File path: extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcStructConverter.java ########## @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.orc; + +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.io.BooleanWritable; +import org.apache.hadoop.io.ByteWritable; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.DoubleWritable; +import org.apache.hadoop.io.FloatWritable; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.ShortWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.WritableComparable; +import org.apache.orc.TypeDescription; +import org.apache.orc.mapred.OrcList; +import org.apache.orc.mapred.OrcMap; +import org.apache.orc.mapred.OrcStruct; +import org.apache.orc.mapred.OrcTimestamp; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class OrcStructConverter +{ + private static Map<Object, Object> convertMap( + TypeDescription fieldDescription, + OrcMap<? extends WritableComparable, ? extends WritableComparable> map, + boolean binaryAsString + ) + { + Map<Object, Object> converted = new HashMap<>(); + TypeDescription keyDescription = fieldDescription.getChildren().get(0); + TypeDescription valueDescription = fieldDescription.getChildren().get(1); + for (WritableComparable key : map.navigableKeySet()) { + Object newKey = convertPrimitive(keyDescription, key, binaryAsString); + if (valueDescription.getCategory().isPrimitive()) { + converted.put(newKey, convertPrimitive(valueDescription, map.get(key), binaryAsString)); + } else { + converted.put(newKey, map.get(key)); + } + } + return converted; + } + + private static Object convertPrimitive(TypeDescription fieldDescription, WritableComparable field, boolean binaryAsString) + { + /* + ORC TYPE WRITABLE TYPE + binary org.apache.hadoop.io.BytesWritable + bigint org.apache.hadoop.io.LongWritable + boolean org.apache.hadoop.io.BooleanWritable + char org.apache.hadoop.io.Text + date org.apache.hadoop.hive.serde2.io.DateWritable + decimal org.apache.hadoop.hive.serde2.io.HiveDecimalWritable + double org.apache.hadoop.io.DoubleWritable + float org.apache.hadoop.io.FloatWritable + int org.apache.hadoop.io.IntWritable + smallint org.apache.hadoop.io.ShortWritable + string org.apache.hadoop.io.Text + timestamp org.apache.orc.mapred.OrcTimestamp + tinyint org.apache.hadoop.io.ByteWritable + varchar org.apache.hadoop.io.Text + */ + switch (fieldDescription.getCategory()) { + case STRING: + case CHAR: + case VARCHAR: + return ((Text) field).toString(); + case BOOLEAN: + return ((BooleanWritable) field).get(); + case BYTE: + return ((ByteWritable) field).get(); + case SHORT: + return ((ShortWritable) field).get(); + case INT: + return ((IntWritable) field).get(); + case LONG: + return ((LongWritable) field).get(); + case FLOAT: + return ((FloatWritable) field).get(); + case DOUBLE: + return ((DoubleWritable) field).get(); + case DECIMAL: + return ((HiveDecimalWritable) field).getHiveDecimal().doubleValue(); + case TIMESTAMP: + return ((OrcTimestamp) field).getTime(); + case DATE: + return DateTimes.utc(((DateWritable) field).get().getTime()); + case BINARY: + byte[] bytes = ((BytesWritable) field).getBytes(); + if (binaryAsString) { + return StringUtils.fromUtf8(bytes); + } else { + return bytes; + } + default: + return null; + } + } + + private boolean binaryAsString; + + OrcStructConverter(boolean binaryAsString) + { + this.binaryAsString = binaryAsString; + } + + /** + * Convert a orc struct field as though it were a map. Complex types will be transformed + * into java lists and maps when possible ({@link OrcStructConverter#convertList} and + * {@link OrcStructConverter#convertMap}), and + * primitive types will be extracted into an ingestion friendly state (e.g. 'int' and 'long'). Finally, + * if a field is not present, this method will return null. + * + * Note: "Union" types are not currently supported and will be returned as null + */ + @Nullable + Object convertField(OrcStruct struct, String fieldName) + { + TypeDescription schema = struct.getSchema(); + int fieldIndex = schema.getFieldNames().indexOf(fieldName); Review comment: I've added a cache for field names to index of the root `OrcStruct`, but it can't easily be used for json path expressions because the `JsonProvider` stuff doesn't have enough information to know if an `OrcStruct` it is treating as a `Map` for the transformation is the root level or not, and comparing columns to the cache seems to go full circle to where it started maybe. Better than nothing? ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
