Repository: drill Updated Branches: refs/heads/master 168fa904b -> 30ad950e5
DRILL-2220: Complex reader unable to read FIXED_LEN_BYTE_ARRAY types in parquet file Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/30ad950e Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/30ad950e Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/30ad950e Branch: refs/heads/master Commit: 30ad950e577b8c087fe8cae33fe189f30193c078 Parents: 168fa90 Author: adeneche <[email protected]> Authored: Thu Mar 5 12:27:53 2015 -0800 Committer: Steven Phillips <[email protected]> Committed: Thu Apr 2 11:22:41 2015 -0700 ---------------------------------------------------------------------- .../codegen/templates/AbstractFieldWriter.java | 9 ++- .../src/main/codegen/templates/BaseWriter.java | 4 ++ .../src/main/codegen/templates/MapWriters.java | 15 ++++- .../parquet2/DrillParquetGroupConverter.java | 19 +++++- .../store/parquet2/TestDrillParquetReader.java | 61 +++++++++++++++++++ .../resources/parquet2/decimal28_38.parquet | Bin 0 -> 708 bytes 6 files changed, 104 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/30ad950e/exec/java-exec/src/main/codegen/templates/AbstractFieldWriter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/AbstractFieldWriter.java b/exec/java-exec/src/main/codegen/templates/AbstractFieldWriter.java index 145130b..1b5dad1 100644 --- a/exec/java-exec/src/main/codegen/templates/AbstractFieldWriter.java +++ b/exec/java-exec/src/main/codegen/templates/AbstractFieldWriter.java @@ -26,6 +26,7 @@ package org.apache.drill.exec.vector.complex.impl; <#include "/@includes/vv_imports.ftl" /> +/* This class is generated using freemarker and the AbstractFieldWriter.java template */ @SuppressWarnings("unused") abstract class AbstractFieldWriter extends AbstractBaseWriter implements FieldWriter{ @@ -82,7 +83,13 @@ abstract class AbstractFieldWriter extends AbstractBaseWriter implements FieldWr <#if lowerName == "int" ><#assign lowerName = "integer" /></#if> <#assign upperName = minor.class?upper_case /> <#assign capName = minor.class?cap_first /> - public ${capName}Writer ${lowerName}(String name){ + <#if minor.class?starts_with("Decimal") > + public ${capName}Writer ${lowerName}(String name, int scale, int precision) { + fail("${capName}"); + return null; + } + </#if> + public ${capName}Writer ${lowerName}(String name) { fail("${capName}"); return null; } http://git-wip-us.apache.org/repos/asf/drill/blob/30ad950e/exec/java-exec/src/main/codegen/templates/BaseWriter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/BaseWriter.java b/exec/java-exec/src/main/codegen/templates/BaseWriter.java index 979d4ac..2ce4c3c 100644 --- a/exec/java-exec/src/main/codegen/templates/BaseWriter.java +++ b/exec/java-exec/src/main/codegen/templates/BaseWriter.java @@ -26,6 +26,7 @@ package org.apache.drill.exec.vector.complex.writer; <#include "/@includes/vv_imports.ftl" /> +/* This class is generated using freemarker and the BaseWriter.java template */ @SuppressWarnings("unused") public interface BaseWriter extends Positionable{ FieldWriter getParent(); @@ -44,6 +45,9 @@ public interface BaseWriter extends Positionable{ <#if lowerName == "int" ><#assign lowerName = "integer" /></#if> <#assign upperName = minor.class?upper_case /> <#assign capName = minor.class?cap_first /> + <#if minor.class?starts_with("Decimal") > + ${capName}Writer ${lowerName}(String name, int scale, int precision); + </#if> ${capName}Writer ${lowerName}(String name); </#list></#list> http://git-wip-us.apache.org/repos/asf/drill/blob/30ad950e/exec/java-exec/src/main/codegen/templates/MapWriters.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/MapWriters.java b/exec/java-exec/src/main/codegen/templates/MapWriters.java index b8d5365..4dab990 100644 --- a/exec/java-exec/src/main/codegen/templates/MapWriters.java +++ b/exec/java-exec/src/main/codegen/templates/MapWriters.java @@ -34,6 +34,7 @@ package org.apache.drill.exec.vector.complex.impl; <#include "/@includes/vv_imports.ftl" /> import java.util.Map; +import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.exec.expr.holders.RepeatedMapHolder; import org.apache.drill.exec.vector.AllocationHelper; import org.apache.drill.exec.vector.complex.reader.FieldReader; @@ -164,9 +165,21 @@ public class ${mode}MapWriter extends AbstractFieldWriter{ <#assign capName = minor.class?cap_first /> <#assign vectName = capName /> <#assign vectName = "Nullable${capName}" /> - + + <#if minor.class?starts_with("Decimal") > + public ${minor.class}Writer ${lowerName}(String name){ + // returns existing writer + FieldWriter writer = fields.get(name); + assert writer != null; + return writer; + } + + public ${minor.class}Writer ${lowerName}(String name, int scale, int precision){ + final MajorType ${upperName}_TYPE = Types.withScaleAndPrecision(MinorType.${upperName}, DataMode.OPTIONAL, scale, precision); + <#else> private static final MajorType ${upperName}_TYPE = Types.optional(MinorType.${upperName}); public ${minor.class}Writer ${lowerName}(String name){ + </#if> FieldWriter writer = fields.get(name); if(writer == null){ ${vectName}Vector vector = container.addOrGet(name, ${upperName}_TYPE, ${vectName}Vector.class); http://git-wip-us.apache.org/repos/asf/drill/blob/30ad950e/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java index 1ec24e0..389c1f6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java @@ -69,6 +69,7 @@ import parquet.io.api.PrimitiveConverter; import parquet.schema.DecimalMetadata; import parquet.schema.GroupType; import parquet.schema.MessageType; +import parquet.schema.OriginalType; import parquet.schema.PrimitiveType; import parquet.schema.Type; import parquet.schema.Type.Repetition; @@ -216,13 +217,14 @@ public class DrillParquetGroupConverter extends GroupConverter { VarCharWriter writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).varChar() : mapWriter.varChar(name); return new DrillVarCharConverter(writer, mutator.getManagedBuffer()); } + //TODO not sure if BINARY/DECIMAL is actually supported case DECIMAL: { DecimalMetadata metadata = type.getDecimalMetadata(); if (metadata.getPrecision() <= 28) { - Decimal28SparseWriter writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).decimal28Sparse() : mapWriter.decimal28Sparse(name); + Decimal28SparseWriter writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).decimal28Sparse() : mapWriter.decimal28Sparse(name, metadata.getScale(), metadata.getPrecision()); return new DrillBinaryToDecimal28Converter(writer, metadata.getPrecision(), metadata.getScale(), mutator.getManagedBuffer()); } else { - Decimal38SparseWriter writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).decimal38Sparse() : mapWriter.decimal38Sparse(name); + Decimal38SparseWriter writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).decimal38Sparse() : mapWriter.decimal38Sparse(name, metadata.getScale(), metadata.getPrecision()); return new DrillBinaryToDecimal38Converter(writer, metadata.getPrecision(), metadata.getScale(), mutator.getManagedBuffer()); } } @@ -231,6 +233,19 @@ public class DrillParquetGroupConverter extends GroupConverter { } } } + case FIXED_LEN_BYTE_ARRAY: + if (type.getOriginalType() == OriginalType.DECIMAL) { + DecimalMetadata metadata = type.getDecimalMetadata(); + if (metadata.getPrecision() <= 28) { + Decimal28SparseWriter writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).decimal28Sparse() : mapWriter.decimal28Sparse(name, metadata.getScale(), metadata.getPrecision()); + return new DrillBinaryToDecimal28Converter(writer, metadata.getPrecision(), metadata.getScale(), mutator.getManagedBuffer()); + } else { + Decimal38SparseWriter writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).decimal38Sparse() : mapWriter.decimal38Sparse(name, metadata.getScale(), metadata.getPrecision()); + return new DrillBinaryToDecimal38Converter(writer, metadata.getPrecision(), metadata.getScale(), mutator.getManagedBuffer()); + } + } else { + throw new UnsupportedOperationException("Unsupported type " + type.getOriginalType()); + } default: throw new UnsupportedOperationException("Unsupported type: " + type.getPrimitiveTypeName()); } http://git-wip-us.apache.org/repos/asf/drill/blob/30ad950e/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java new file mode 100644 index 0000000..782191f --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.parquet2; + +import org.apache.drill.BaseTestQuery; +import org.junit.Test; + +import java.math.BigDecimal; + +public class TestDrillParquetReader extends BaseTestQuery { + + private void testColumn(String columnName) throws Exception { + testNoResult("alter session set `store.parquet.use_new_reader` = true"); + + BigDecimal result = new BigDecimal("1.20000000"); + + testBuilder() + .ordered() + .sqlQuery("select %s from cp.`parquet2/decimal28_38.parquet`", columnName) + .baselineColumns(columnName) + .baselineValues(result) + .go(); + + testNoResult("alter session set `store.parquet.use_new_reader` = false"); + } + + @Test + public void testRequiredDecimal28() throws Exception { + testColumn("d28_req"); + } + + @Test + public void testRequiredDecimal38() throws Exception { + testColumn("d38_req"); + } + + @Test + public void testOptionalDecimal28() throws Exception { + testColumn("d28_opt"); + } + + @Test + public void testOptionalDecimal38() throws Exception { + testColumn("d38_opt"); + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/30ad950e/exec/java-exec/src/test/resources/parquet2/decimal28_38.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet2/decimal28_38.parquet b/exec/java-exec/src/test/resources/parquet2/decimal28_38.parquet new file mode 100644 index 0000000..0dbcf97 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet2/decimal28_38.parquet differ
