This is an automated email from the ASF dual-hosted git repository.
htowaileb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new ed439bce85 [ASTERIXDB-3440][EXT]: Handle legacy parquet schemas
ed439bce85 is described below
commit ed439bce8574893cce95cfe787d23ee30e9390d0
Author: Hussain Towaileb <[email protected]>
AuthorDate: Thu Jun 13 18:32:09 2024 +0300
[ASTERIXDB-3440][EXT]: Handle legacy parquet schemas
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- This change handles multiple cases of "repeated"
converter that is not following the latest parquet
standards.
Change-Id: I829df08c9ad667562194fbee027f63d8b2b31d4b
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18391
Reviewed-by: Hussain Towaileb <[email protected]>
Tested-by: Hussain Towaileb <[email protected]>
Integration-Tests: Hussain Towaileb <[email protected]>
---
.../parquet/AsterixTypeToParquetTypeVisitor.java | 2 +-
.../converter/AbstractComplexConverter.java | 185 +++++++++++++++++++++
.../converter/nested/AbstractComplexConverter.java | 174 -------------------
.../parquet/converter/nested/ArrayConverter.java | 4 +-
.../parquet/converter/nested/ObjectConverter.java | 4 +-
...Converter.java => ObjectRepeatedConverter.java} | 125 ++++++++++----
.../converter/nested/RepeatedConverter.java | 3 +-
.../converter/primitve/BinaryConverter.java | 2 +-
.../parquet/converter/primitve/DateConverter.java | 2 +-
.../converter/primitve/DecimalConverter.java | 2 +-
.../primitve/GenericPrimitiveConverter.java | 6 +-
...verter.java => IClosableRepeatedConverter.java} | 24 +--
.../converter/primitve/JsonStringConverter.java | 2 +-
.../primitve/PrimitiveConverterProvider.java | 2 +-
.../PrimitiveRepeatedConverter.java} | 112 ++++++++-----
.../parquet/converter/primitve/TimeConverter.java | 2 +-
.../converter/primitve/TimestampConverter.java | 2 +-
.../parquet/converter/primitve/UUIDConverter.java | 2 +-
.../primitve/UnsignedIntegerConverter.java | 2 +-
19 files changed, 380 insertions(+), 277 deletions(-)
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/AsterixTypeToParquetTypeVisitor.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/AsterixTypeToParquetTypeVisitor.java
index be9690e910..254280d766 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/AsterixTypeToParquetTypeVisitor.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/AsterixTypeToParquetTypeVisitor.java
@@ -26,8 +26,8 @@ import java.util.Map;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
+import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.AbstractComplexConverter;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.ParquetConverterContext;
-import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.nested.AbstractComplexConverter;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.primitve.DecimalConverter;
import org.apache.asterix.external.util.ExternalDataConstants.ParquetOptions;
import org.apache.asterix.om.types.ARecordType;
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/AbstractComplexConverter.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/AbstractComplexConverter.java
new file mode 100644
index 0000000000..abf2870a99
--- /dev/null
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/AbstractComplexConverter.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.record.reader.hdfs.parquet.converter;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import
org.apache.asterix.external.input.record.reader.hdfs.parquet.AsterixTypeToParquetTypeVisitor;
+import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.nested.ArrayConverter;
+import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.nested.ObjectConverter;
+import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.nested.ObjectRepeatedConverter;
+import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.nested.RepeatedConverter;
+import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.primitve.IClosableRepeatedConverter;
+import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.primitve.PrimitiveRepeatedConverter;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.data.std.api.IMutableValueStorage;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.parquet.io.api.Converter;
+import org.apache.parquet.io.api.GroupConverter;
+import org.apache.parquet.io.api.PrimitiveConverter;
+import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.Type;
+import org.apache.parquet.schema.Type.Repetition;
+
+public abstract class AbstractComplexConverter extends GroupConverter
+ implements IFieldValue, IClosableRepeatedConverter {
+ protected AbstractComplexConverter parent;
+ protected IValueReference fieldName;
+ protected String stringFieldName;
+ private int index;
+ protected Converter[] converters;
+ protected ParquetConverterContext context;
+ protected IMutableValueStorage tempStorage;
+
+ protected AbstractComplexConverter(AbstractComplexConverter parent, int
index, GroupType parquetType,
+ ParquetConverterContext context) throws IOException {
+ this(parent, null, index, parquetType, context);
+ }
+
+ protected AbstractComplexConverter(AbstractComplexConverter parent, String
stringFieldName, int index,
+ GroupType parquetType, ParquetConverterContext context) throws
IOException {
+ this.parent = parent;
+ this.stringFieldName = stringFieldName;
+ this.fieldName = context.getSerializedFieldName(stringFieldName);
+ this.index = index;
+ this.context = context;
+ converters = new Converter[parquetType.getFieldCount()];
+ for (int i = 0; i < parquetType.getFieldCount(); i++) {
+ final Type type = parquetType.getType(i);
+
+ LogicalTypeAnnotation typeAnnotation =
type.getLogicalTypeAnnotation();
+ if (type.isPrimitive() && type.getRepetition() !=
Repetition.REPEATED) {
+ converters[i] = createAtomicConverter(parquetType, i);
+ } else if
(LogicalTypeAnnotation.listType().equals(typeAnnotation)) {
+ converters[i] = createArrayConverter(parquetType, i);
+ } else if (LogicalTypeAnnotation.mapType().equals(typeAnnotation)
+ ||
LogicalTypeAnnotation.MapKeyValueTypeAnnotation.getInstance().equals(typeAnnotation))
{
+ converters[i] = createArrayConverter(parquetType, i);
+ } else if (isRepeated(parquetType, i)) {
+ continue;
+ } else {
+ converters[i] = createObjectConverter(parquetType, i);
+ }
+ }
+ }
+
+ /**
+ * Add child value (the caller is the child itself)
+ *
+ * @param value Child value
+ */
+ public abstract void addValue(IFieldValue value);
+
+ protected abstract PrimitiveConverter createAtomicConverter(GroupType
type, int index);
+
+ protected abstract AbstractComplexConverter createArrayConverter(GroupType
type, int index);
+
+ protected abstract AbstractComplexConverter
createObjectConverter(GroupType type, int index);
+
+ private boolean isRepeated(GroupType parquetType, int index) throws
IOException {
+ Type repeatedType = parquetType.getType(index);
+ if (repeatedType.getRepetition() != Repetition.REPEATED) {
+ return false;
+ }
+
+ // primitive type
+ if (repeatedType.isPrimitive()) {
+ if (this.getTypeTag().isListType()) {
+ // legacy 2-levels list, list -> repeated primitive type
+ converters[index] = createAtomicConverter(parquetType, index);
+ } else {
+ // primitive outside ArrayConverter, this is a legacy list of
the provided type on its own
+ ATypeTag typeTag =
AsterixTypeToParquetTypeVisitor.mapType(repeatedType, context, null);
+ converters[index] =
+ new PrimitiveRepeatedConverter(typeTag, this,
repeatedType.asPrimitiveType(), index, context);
+ }
+ } else {
+ GroupType groupRepeatedType = repeatedType.asGroupType();
+ String name = groupRepeatedType.getName();
+
+ // if contained inside a list, then this is a repeated objects for
the list
+ if (this.getTypeTag().isListType()) {
+ if (groupRepeatedType.getFieldCount() > 1 ||
"array".equals(name) || "key_value".equals(name)) {
+ // The name "array" and "key_value" are reserved names to
represent array of objects
+ // "key_value" are for MAP type
+ converters[index] = new ObjectConverter(this, index,
groupRepeatedType, context);
+ } else {
+ // implementation following standards
+ converters[index] = new RepeatedConverter(this, index,
groupRepeatedType, context);
+ }
+ } else {
+ // repeated
+ converters[index] = new ObjectRepeatedConverter(this, name,
index, groupRepeatedType, context);
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public String getStringFieldName() {
+ return stringFieldName;
+ }
+
+ @Override
+ public IValueReference getFieldName() {
+ return fieldName;
+ }
+
+ @Override
+ public int getIndex() {
+ return index;
+ }
+
+ @Override
+ public Converter getConverter(int fieldIndex) {
+ return converters[fieldIndex];
+ }
+
+ public DataOutput getDataOutput() {
+ tempStorage.reset();
+ return tempStorage.getDataOutput();
+ }
+
+ public IMutableValueStorage getValue() {
+ return tempStorage;
+ }
+
+ protected DataOutput getParentDataOutput() {
+ return parent.getDataOutput();
+ }
+
+ protected void addThisValueToParent() {
+ if (parent == null) {
+ //root
+ return;
+ }
+ parent.addValue(this);
+ }
+
+ /**
+ * This closes non-standard repeated converters, check implementers of
{@link IClosableRepeatedConverter)
+ */
+ protected void closeDirectRepeatedChildren() {
+ for (Converter converter : converters) {
+ ((IClosableRepeatedConverter) converter).internalEnd();
+ }
+ }
+}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/AbstractComplexConverter.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/AbstractComplexConverter.java
deleted file mode 100644
index 24ec4c936b..0000000000
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/AbstractComplexConverter.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.nested;
-
-import java.io.DataOutput;
-import java.io.IOException;
-
-import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.IFieldValue;
-import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.ParquetConverterContext;
-import org.apache.hyracks.data.std.api.IMutableValueStorage;
-import org.apache.hyracks.data.std.api.IValueReference;
-import org.apache.parquet.io.api.Converter;
-import org.apache.parquet.io.api.GroupConverter;
-import org.apache.parquet.io.api.PrimitiveConverter;
-import org.apache.parquet.schema.GroupType;
-import org.apache.parquet.schema.LogicalTypeAnnotation;
-import org.apache.parquet.schema.Type;
-import org.apache.parquet.schema.Type.Repetition;
-
-public abstract class AbstractComplexConverter extends GroupConverter
implements IFieldValue {
- protected final AbstractComplexConverter parent;
- private final IValueReference fieldName;
- private final String stringFieldName;
- private final int index;
- private final Converter[] converters;
- protected final ParquetConverterContext context;
- protected IMutableValueStorage tempStorage;
-
- AbstractComplexConverter(AbstractComplexConverter parent, int index,
GroupType parquetType,
- ParquetConverterContext context) throws IOException {
- this(parent, null, index, parquetType, context);
- }
-
- AbstractComplexConverter(AbstractComplexConverter parent, String
stringFieldName, int index, GroupType parquetType,
- ParquetConverterContext context) throws IOException {
- this.parent = parent;
- this.stringFieldName = stringFieldName;
- this.fieldName = context.getSerializedFieldName(stringFieldName);
- this.index = index;
- this.context = context;
- converters = new Converter[parquetType.getFieldCount()];
- for (int i = 0; i < parquetType.getFieldCount(); i++) {
- final Type type = parquetType.getType(i);
- if (type.isPrimitive()) {
- converters[i] = createAtomicConverter(parquetType, i);
- } else if
(LogicalTypeAnnotation.listType().equals(type.getLogicalTypeAnnotation())) {
- converters[i] = createArrayConverter(parquetType, i);
- } else if (type.getRepetition() == Repetition.REPEATED) {
- converters[i] = createRepeatedConverter(parquetType, i);
- } else if (type.getLogicalTypeAnnotation() ==
LogicalTypeAnnotation.mapType()) {
- converters[i] = createArrayConverter(parquetType, i);
- } else {
- converters[i] = createObjectConverter(parquetType, i);
- }
- }
- }
-
- /**
- * Add child value (the caller is the child itself)
- *
- * @param value Child value
- */
- public abstract void addValue(IFieldValue value);
-
- protected abstract PrimitiveConverter createAtomicConverter(GroupType
type, int index);
-
- protected abstract AbstractComplexConverter createArrayConverter(GroupType
type, int index);
-
- protected abstract AbstractComplexConverter
createObjectConverter(GroupType type, int index);
-
- /**
- * Parquet file created by (old) Avro writer treat repeated values
differently from files created by Spark.
- * Example:
- * Let us consider the object <pre>{"urls":[{"display_url": "string",
"expanded_url": "string"}]}</pre>
- *
- * @formatter:off
- *
- * In Avro:
- * optional group urls (LIST) {
- * repeated group array {
- * optional binary display_url (UTF8);
- * optional binary expanded_url (UTF8);
- * }
- * }
- *
- * In Spark:
- * optional group urls (LIST) {
- * repeated group list {
- * // Similar to JSON, the object fields are placed in an inner group
- * optional group item {
- * optional binary display_url (UTF8);
- * optional binary expanded_url (UTF8);
- * }
- * }
- * }
- *
- * Map type:
- * required group mapField (MAP) {
- * repeated group key_value {
- * required int32 key;
- * required int32 value;
- * }
- * }
- *
- * @formatter:on
- */
- protected AbstractComplexConverter createRepeatedConverter(GroupType type,
int index) throws IOException {
- GroupType repeatedType = type.getType(index).asGroupType();
- String name = repeatedType.getName();
- if (repeatedType.getFieldCount() > 1 || "array".equals(name) ||
"key_value".equals(name)) {
- //The name "array" and "key_value" are reserved names to represent
array of objects
- //"key_value" are for MAP type
- return new ObjectConverter(this, index, repeatedType, context);
- }
- return new RepeatedConverter(this, index, repeatedType, context);
- }
-
- @Override
- public String getStringFieldName() {
- return stringFieldName;
- }
-
- @Override
- public IValueReference getFieldName() {
- return fieldName;
- }
-
- @Override
- public int getIndex() {
- return index;
- }
-
- @Override
- public Converter getConverter(int fieldIndex) {
- return converters[fieldIndex];
- }
-
- public DataOutput getDataOutput() {
- tempStorage.reset();
- return tempStorage.getDataOutput();
- }
-
- public IMutableValueStorage getValue() {
- return tempStorage;
- }
-
- protected DataOutput getParentDataOutput() {
- return parent.getDataOutput();
- }
-
- protected void addThisValueToParent() {
- if (parent == null) {
- //root
- return;
- }
- parent.addValue(this);
- }
-}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/ArrayConverter.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/ArrayConverter.java
index 89647e0090..0fed70b687 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/ArrayConverter.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/ArrayConverter.java
@@ -21,6 +21,7 @@ package
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.n
import java.io.IOException;
import org.apache.asterix.builders.IAsterixListBuilder;
+import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.AbstractComplexConverter;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.IFieldValue;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.ParquetConverterContext;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.primitve.PrimitiveConverterProvider;
@@ -31,7 +32,7 @@ import org.apache.parquet.io.api.PrimitiveConverter;
import org.apache.parquet.schema.GroupType;
import org.apache.parquet.schema.PrimitiveType;
-class ArrayConverter extends AbstractComplexConverter {
+public class ArrayConverter extends AbstractComplexConverter {
private IAsterixListBuilder builder;
public ArrayConverter(AbstractComplexConverter parent, int index,
GroupType parquetType,
@@ -52,6 +53,7 @@ class ArrayConverter extends AbstractComplexConverter {
@Override
public void end() {
+ closeDirectRepeatedChildren();
try {
builder.write(getParentDataOutput(), true);
} catch (IOException e) {
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/ObjectConverter.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/ObjectConverter.java
index 542318b77d..6b63a7b668 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/ObjectConverter.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/ObjectConverter.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.asterix.builders.IARecordBuilder;
import
org.apache.asterix.external.input.filter.embedder.IExternalFilterValueEmbedder;
+import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.AbstractComplexConverter;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.IFieldValue;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.ParquetConverterContext;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.primitve.PrimitiveConverterProvider;
@@ -34,7 +35,7 @@ import org.apache.parquet.io.api.PrimitiveConverter;
import org.apache.parquet.schema.GroupType;
import org.apache.parquet.schema.PrimitiveType;
-class ObjectConverter extends AbstractComplexConverter {
+public class ObjectConverter extends AbstractComplexConverter {
private IARecordBuilder builder;
/**
* {@link IExternalFilterValueEmbedder} decides whether the object should
be ignored entirely
@@ -66,6 +67,7 @@ class ObjectConverter extends AbstractComplexConverter {
@Override
public void end() {
+ closeDirectRepeatedChildren();
if (!ignore) {
writeToParent();
context.getValueEmbedder().exitObject();
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/ObjectConverter.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/ObjectRepeatedConverter.java
similarity index 64%
copy from
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/ObjectConverter.java
copy to
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/ObjectRepeatedConverter.java
index 542318b77d..ef944ed9c5 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/ObjectConverter.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/ObjectRepeatedConverter.java
@@ -18,43 +18,75 @@
*/
package
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.nested;
+import java.io.DataOutput;
import java.io.IOException;
import org.apache.asterix.builders.IARecordBuilder;
+import org.apache.asterix.builders.IAsterixListBuilder;
import
org.apache.asterix.external.input.filter.embedder.IExternalFilterValueEmbedder;
+import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.AbstractComplexConverter;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.IFieldValue;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.ParquetConverterContext;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.primitve.PrimitiveConverterProvider;
import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IMutableValueStorage;
import org.apache.hyracks.data.std.api.IValueReference;
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
import org.apache.parquet.io.api.PrimitiveConverter;
import org.apache.parquet.schema.GroupType;
import org.apache.parquet.schema.PrimitiveType;
-class ObjectConverter extends AbstractComplexConverter {
- private IARecordBuilder builder;
+/**
+ * Handles the following non-standard parquet schema scenario:
+ * a group repeated list that is not contained in a LIST structure, for example
+ *
+ * {
+ * "my_group_list": [{"date": "123", "author": "abc"}, {"date": "456",
"author": "xyz"}]
+ * }
+ *
+ * Represented as:
+ * message Product {
+ * repeated group name=my_group_list {
+ * required binary name=date (STRING);
+ * required binary name=author (STRING);
+ * }
+ * }
+ *
+ * Instead of:
+ * message arrow_schema {
+ * required group myGroupArray (LIST) {
+ * repeated group list {
+ * optional group {
+ * optional binary hello (STRING);
+ * optional binary foo (STRING);
+ * }
+ * }
+ * }
+ * }
+ *
+ * In this case, this is a list and the type of the repeated is the type of
the elements in the list
+ */
+public class ObjectRepeatedConverter extends AbstractComplexConverter {
+ private IAsterixListBuilder listBuilder;
+ private IMutableValueStorage listStorage;
+
+ private IARecordBuilder recordBuilder;
/**
* {@link IExternalFilterValueEmbedder} decides whether the object should
be ignored entirely
*/
private boolean ignore = false;
- public ObjectConverter(AbstractComplexConverter parent, int index,
GroupType parquetType,
- ParquetConverterContext context) throws IOException {
- super(parent, index, parquetType, context);
- }
-
- public ObjectConverter(AbstractComplexConverter parent, String
stringFieldName, int index, GroupType parquetType,
- ParquetConverterContext context) throws IOException {
+ public ObjectRepeatedConverter(AbstractComplexConverter parent, String
stringFieldName, int index,
+ GroupType parquetType, ParquetConverterContext context) throws
IOException {
super(parent, stringFieldName, index, parquetType, context);
}
@Override
public void start() {
tempStorage = context.enterObject();
- builder =
context.getObjectBuilder(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE);
+ recordBuilder =
context.getObjectBuilder(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE);
IExternalFilterValueEmbedder valueEmbedder =
context.getValueEmbedder();
if (isRoot()) {
valueEmbedder.reset();
@@ -66,20 +98,65 @@ class ObjectConverter extends AbstractComplexConverter {
@Override
public void end() {
+ closeDirectRepeatedChildren();
if (!ignore) {
- writeToParent();
+ writeToList();
context.getValueEmbedder().exitObject();
}
-
- context.exitObject(tempStorage, null, builder);
+ context.exitObject(tempStorage, null, recordBuilder);
tempStorage = null;
- builder = null;
+ recordBuilder = null;
ignore = false;
}
+ private void writeToList() {
+ try {
+ finalizeEmbedding();
+ recordBuilder.write(getListDataOutput(), true);
+ addValueToList();
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ private void addValueToList() {
+ try {
+ listBuilder.addItem(listStorage);
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ public DataOutput getListDataOutput() {
+ if (listStorage == null) {
+ internalStart();
+ }
+ listStorage.reset();
+ return listStorage.getDataOutput();
+ }
+
@Override
public ATypeTag getTypeTag() {
- return ATypeTag.OBJECT;
+ return ATypeTag.ARRAY;
+ }
+
+ private void internalStart() {
+ listStorage = context.enterCollection();
+ listBuilder =
context.getCollectionBuilder(DefaultOpenFieldType.NESTED_OPEN_AORDERED_LIST_TYPE);
+ }
+
+ @Override
+ public void internalEnd() {
+ closeDirectRepeatedChildren();
+ try {
+ listBuilder.write(parent.getDataOutput(), true);
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ parent.addValue(this);
+ context.exitCollection(listStorage, listBuilder);
+ listStorage = null;
+ listBuilder = null;
}
@Override
@@ -92,9 +169,9 @@ class ObjectConverter extends AbstractComplexConverter {
IValueReference fieldName = value.getFieldName();
try {
if (valueEmbedder.shouldEmbed(value.getStringFieldName(),
value.getTypeTag())) {
- builder.addField(fieldName, valueEmbedder.getEmbeddedValue());
+ recordBuilder.addField(fieldName,
valueEmbedder.getEmbeddedValue());
} else {
- builder.addField(fieldName, getValue());
+ recordBuilder.addField(fieldName, getValue());
}
} catch (HyracksDataException e) {
throw new IllegalStateException(e);
@@ -143,23 +220,13 @@ class ObjectConverter extends AbstractComplexConverter {
boolean embed = valueEmbedder.shouldEmbed(getStringFieldName(),
ATypeTag.OBJECT);
if (embed) {
((ArrayBackedValueStorage)
parent.getValue()).set(valueEmbedder.getEmbeddedValue());
- addThisValueToParent();
+ addValueToList();
} else {
valueEmbedder.enterObject();
}
return embed;
}
- private void writeToParent() {
- try {
- finalizeEmbedding();
- builder.write(getParentDataOutput(), true);
- addThisValueToParent();
- } catch (IOException e) {
- throw new IllegalStateException(e);
- }
- }
-
private void finalizeEmbedding() throws IOException {
IExternalFilterValueEmbedder valueEmbedder =
context.getValueEmbedder();
if (valueEmbedder.isMissingEmbeddedValues()) {
@@ -168,7 +235,7 @@ class ObjectConverter extends AbstractComplexConverter {
String embeddedFieldName = embeddedFieldNames[i];
if (valueEmbedder.isMissing(embeddedFieldName)) {
IValueReference embeddedValue =
valueEmbedder.getEmbeddedValue();
-
builder.addField(context.getSerializedFieldName(embeddedFieldName),
embeddedValue);
+
recordBuilder.addField(context.getSerializedFieldName(embeddedFieldName),
embeddedValue);
}
}
}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/RepeatedConverter.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/RepeatedConverter.java
index 3936ad7921..87d0ae49e7 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/RepeatedConverter.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/RepeatedConverter.java
@@ -21,6 +21,7 @@ package
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.n
import java.io.DataOutput;
import java.io.IOException;
+import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.AbstractComplexConverter;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.IFieldValue;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.ParquetConverterContext;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.primitve.PrimitiveConverterProvider;
@@ -29,7 +30,7 @@ import org.apache.parquet.io.api.PrimitiveConverter;
import org.apache.parquet.schema.GroupType;
import org.apache.parquet.schema.PrimitiveType;
-class RepeatedConverter extends AbstractComplexConverter {
+public class RepeatedConverter extends AbstractComplexConverter {
public RepeatedConverter(AbstractComplexConverter parent, int index,
GroupType parquetType,
ParquetConverterContext context) throws IOException {
super(parent, index, parquetType, context);
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/BinaryConverter.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/BinaryConverter.java
index 4f371c83e6..c5e71209a9 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/BinaryConverter.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/BinaryConverter.java
@@ -20,8 +20,8 @@ package
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.p
import java.io.IOException;
+import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.AbstractComplexConverter;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.ParquetConverterContext;
-import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.nested.AbstractComplexConverter;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.parquet.io.api.Binary;
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/DateConverter.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/DateConverter.java
index 07a5f79308..4fd267cf32 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/DateConverter.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/DateConverter.java
@@ -20,8 +20,8 @@ package
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.p
import java.io.IOException;
+import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.AbstractComplexConverter;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.ParquetConverterContext;
-import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.nested.AbstractComplexConverter;
import org.apache.asterix.om.types.ATypeTag;
class DateConverter extends GenericPrimitiveConverter {
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/DecimalConverter.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/DecimalConverter.java
index 81fb36dd5e..343876c02e 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/DecimalConverter.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/DecimalConverter.java
@@ -23,8 +23,8 @@ import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.ByteBuffer;
+import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.AbstractComplexConverter;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.ParquetConverterContext;
-import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.nested.AbstractComplexConverter;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.parquet.io.api.Binary;
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/GenericPrimitiveConverter.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/GenericPrimitiveConverter.java
index 20f82f960e..e70831353b 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/GenericPrimitiveConverter.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/GenericPrimitiveConverter.java
@@ -20,16 +20,16 @@ package
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.p
import java.io.IOException;
+import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.AbstractComplexConverter;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.IFieldValue;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.ParquetConverterContext;
-import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.nested.AbstractComplexConverter;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.hyracks.data.std.api.IValueReference;
import org.apache.parquet.io.api.Binary;
import org.apache.parquet.io.api.PrimitiveConverter;
-public class GenericPrimitiveConverter extends PrimitiveConverter implements
IFieldValue {
- private final ATypeTag typeTag;
+public class GenericPrimitiveConverter extends PrimitiveConverter implements
IFieldValue, IClosableRepeatedConverter {
+ protected ATypeTag typeTag;
protected final AbstractComplexConverter parent;
protected final String stringFieldName;
protected final IValueReference fieldName;
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/DateConverter.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/IClosableRepeatedConverter.java
similarity index 55%
copy from
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/DateConverter.java
copy to
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/IClosableRepeatedConverter.java
index 07a5f79308..b23bbf91bc 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/DateConverter.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/IClosableRepeatedConverter.java
@@ -18,21 +18,15 @@
*/
package
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.primitve;
-import java.io.IOException;
-
-import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.ParquetConverterContext;
-import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.nested.AbstractComplexConverter;
-import org.apache.asterix.om.types.ATypeTag;
-
-class DateConverter extends GenericPrimitiveConverter {
- DateConverter(AbstractComplexConverter parent, String stringFieldName, int
index, ParquetConverterContext context)
- throws IOException {
- super(ATypeTag.DATE, parent, stringFieldName, index, context);
- }
+/**
+ * Special case interface to handle scenarios that are not following the
latest parquet standards. See implementors
+ * of this interface methods for more details.
+ */
+public interface IClosableRepeatedConverter {
- @Override
- public void addInt(int value) {
- context.serializeDate(value, parent.getDataOutput());
- parent.addValue(this);
+ /**
+ * Calls any necessary end operations for repeated types
+ */
+ default void internalEnd() {
}
}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/JsonStringConverter.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/JsonStringConverter.java
index 8e4c5569d6..7b106e82f0 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/JsonStringConverter.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/JsonStringConverter.java
@@ -21,8 +21,8 @@ package
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.p
import java.io.DataOutput;
import java.io.IOException;
+import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.AbstractComplexConverter;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.ParquetConverterContext;
-import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.nested.AbstractComplexConverter;
import org.apache.asterix.external.parser.JSONDataParser;
import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
import org.apache.asterix.om.types.ATypeTag;
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/PrimitiveConverterProvider.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/PrimitiveConverterProvider.java
index 0f36d11d62..0e16c21745 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/PrimitiveConverterProvider.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/PrimitiveConverterProvider.java
@@ -21,8 +21,8 @@ package
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.p
import java.io.IOException;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.AsterixTypeToParquetTypeVisitor;
+import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.AbstractComplexConverter;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.ParquetConverterContext;
-import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.nested.AbstractComplexConverter;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.parquet.io.api.PrimitiveConverter;
import org.apache.parquet.schema.LogicalTypeAnnotation;
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/ArrayConverter.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/PrimitiveRepeatedConverter.java
similarity index 50%
copy from
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/ArrayConverter.java
copy to
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/PrimitiveRepeatedConverter.java
index 89647e0090..a832484a2a 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/nested/ArrayConverter.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/PrimitiveRepeatedConverter.java
@@ -16,60 +16,82 @@
* specific language governing permissions and limitations
* under the License.
*/
-package
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.nested;
+package
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.primitve;
+import java.io.DataOutput;
import java.io.IOException;
import org.apache.asterix.builders.IAsterixListBuilder;
-import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.IFieldValue;
+import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.AbstractComplexConverter;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.ParquetConverterContext;
-import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.primitve.PrimitiveConverterProvider;
import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.parquet.io.api.PrimitiveConverter;
-import org.apache.parquet.schema.GroupType;
+import org.apache.hyracks.data.std.api.IMutableValueStorage;
+import org.apache.parquet.io.api.Binary;
import org.apache.parquet.schema.PrimitiveType;
-class ArrayConverter extends AbstractComplexConverter {
+/**
+ * Handles the following non-standard parquet schema scenario:
+ * a primitive repeated list that is not contained in a LIST structure, for
example
+ *
+ * {
+ * "my_primitive_array": ["item1", "item2"]
+ * }
+ *
+ * Represented as:
+ * message schema {
+ * repeated binary name=my_primitive_array (STRING);
+ * }
+ *
+ * Instead of:
+ * message schema {
+ * required group my_primitive_array (LIST) {
+ * repeated group list {
+ * optional binary (STRING);
+ * }
+ * }
+ * }
+ *
+ * In this case, this is a list and the type of the repeated is the type of
the elements in the list
+ */
+public class PrimitiveRepeatedConverter extends GenericPrimitiveConverter {
private IAsterixListBuilder builder;
- public ArrayConverter(AbstractComplexConverter parent, int index,
GroupType parquetType,
- ParquetConverterContext context) throws IOException {
- super(parent, index, parquetType, context);
- }
+ protected IMutableValueStorage tempStorage;
- public ArrayConverter(AbstractComplexConverter parent, String
stringFieldName, int index, GroupType parquetType,
+ public PrimitiveRepeatedConverter(ATypeTag typeTag,
AbstractComplexConverter parent, PrimitiveType type, int index,
ParquetConverterContext context) throws IOException {
- super(parent, stringFieldName, index, parquetType, context);
+ super(typeTag, parent, type.getName(), index, context);
}
- @Override
- public void start() {
+ private void internalStart() {
tempStorage = context.enterCollection();
builder =
context.getCollectionBuilder(DefaultOpenFieldType.NESTED_OPEN_AORDERED_LIST_TYPE);
}
@Override
- public void end() {
+ public void internalEnd() {
try {
- builder.write(getParentDataOutput(), true);
+ builder.write(parent.getDataOutput(), true);
} catch (IOException e) {
throw new IllegalStateException(e);
}
- addThisValueToParent();
+ parent.addValue(this);
context.exitCollection(tempStorage, builder);
tempStorage = null;
builder = null;
}
- @Override
- public ATypeTag getTypeTag() {
- return ATypeTag.ARRAY;
+ public DataOutput getDataOutput() {
+ if (tempStorage == null) {
+ internalStart();
+ }
+ tempStorage.reset();
+ return tempStorage.getDataOutput();
}
- @Override
- public void addValue(IFieldValue value) {
+ public void addValue() {
try {
builder.addItem(tempStorage);
} catch (HyracksDataException e) {
@@ -78,32 +100,36 @@ class ArrayConverter extends AbstractComplexConverter {
}
@Override
- protected PrimitiveConverter createAtomicConverter(GroupType type, int
index) {
- try {
- PrimitiveType primitiveType =
type.getType(index).asPrimitiveType();
- return
PrimitiveConverterProvider.createPrimitiveConverter(primitiveType, this, index,
context);
- } catch (IOException e) {
- throw new IllegalStateException(e);
- }
+ public void addBinary(Binary value) {
+ context.serializeString(value, getDataOutput());
+ addValue();
}
@Override
- protected ArrayConverter createArrayConverter(GroupType type, int index) {
- try {
- GroupType arrayType = type.getType(index).asGroupType();
- return new ArrayConverter(this, index, arrayType, context);
- } catch (IOException e) {
- throw new IllegalStateException(e);
- }
+ public void addBoolean(boolean value) {
+ context.serializeBoolean(value, getDataOutput());
+ addValue();
}
@Override
- protected ObjectConverter createObjectConverter(GroupType type, int index)
{
- try {
- GroupType objectType = type.getType(index).asGroupType();
- return new ObjectConverter(this, index, objectType, context);
- } catch (IOException e) {
- throw new IllegalStateException(e);
- }
+ public void addFloat(float value) {
+ addDouble(value);
+ }
+
+ @Override
+ public void addDouble(double value) {
+ context.serializeDouble(value, getDataOutput());
+ addValue();
+ }
+
+ @Override
+ public void addInt(int value) {
+ addLong(value);
+ }
+
+ @Override
+ public void addLong(long value) {
+ context.serializeInt64(value, getDataOutput());
+ addValue();
}
}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/TimeConverter.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/TimeConverter.java
index c78d6e76a3..e854a644f5 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/TimeConverter.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/TimeConverter.java
@@ -21,8 +21,8 @@ package
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.p
import java.io.IOException;
import java.util.concurrent.TimeUnit;
+import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.AbstractComplexConverter;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.ParquetConverterContext;
-import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.nested.AbstractComplexConverter;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.parquet.schema.LogicalTypeAnnotation;
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/TimestampConverter.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/TimestampConverter.java
index c36de372db..4d0a61dfc1 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/TimestampConverter.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/TimestampConverter.java
@@ -22,8 +22,8 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
+import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.AbstractComplexConverter;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.ParquetConverterContext;
-import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.nested.AbstractComplexConverter;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.parquet.io.api.Binary;
import org.apache.parquet.schema.LogicalTypeAnnotation;
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/UUIDConverter.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/UUIDConverter.java
index 23667ca71b..4384359b78 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/UUIDConverter.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/UUIDConverter.java
@@ -20,8 +20,8 @@ package
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.p
import java.io.IOException;
+import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.AbstractComplexConverter;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.ParquetConverterContext;
-import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.nested.AbstractComplexConverter;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.parquet.io.api.Binary;
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/UnsignedIntegerConverter.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/UnsignedIntegerConverter.java
index 5bb4ad7eb5..64b0ad7488 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/UnsignedIntegerConverter.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/UnsignedIntegerConverter.java
@@ -21,8 +21,8 @@ package
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.p
import java.io.IOException;
import org.apache.asterix.common.exceptions.ErrorCode;
+import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.AbstractComplexConverter;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.ParquetConverterContext;
-import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.nested.AbstractComplexConverter;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.hyracks.api.exceptions.Warning;
