Repository: parquet-mr Updated Branches: refs/heads/master 70f28810a -> a703ee75c
PARQUET-969: Update parquet-tools to convert Decimal datatype to BigD⦠Update parquet-tools so that decimal datatypes in parquet files are converted to their actual number representation when cat'ing to stdout. Currently they are output in binary format. Author: dsfcode <[email protected]> Closes #412 from dsfcode/master and squashes the following commits: 7f05509 [dsfcode] PARQUET-969: Update parquet-tools to convert Decimal datatype to BigDecimal Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/a703ee75 Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/a703ee75 Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/a703ee75 Branch: refs/heads/master Commit: a703ee75c40e0207f6831c4d48e1c7e62f160305 Parents: 70f2881 Author: dsfcode <[email protected]> Authored: Fri May 12 14:40:29 2017 -0700 Committer: Julien Le Dem <[email protected]> Committed: Fri May 12 14:40:29 2017 -0700 ---------------------------------------------------------------------- .../tools/read/SimpleRecordConverter.java | 20 +++++++ .../tools/read/TestSimplePrimitiveRecord.java | 58 ++++++++++++++++++++ 2 files changed, 78 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/a703ee75/parquet-tools/src/main/java/org/apache/parquet/tools/read/SimpleRecordConverter.java ---------------------------------------------------------------------- diff --git a/parquet-tools/src/main/java/org/apache/parquet/tools/read/SimpleRecordConverter.java b/parquet-tools/src/main/java/org/apache/parquet/tools/read/SimpleRecordConverter.java index 5f78444..29bb44f 100644 --- a/parquet-tools/src/main/java/org/apache/parquet/tools/read/SimpleRecordConverter.java +++ b/parquet-tools/src/main/java/org/apache/parquet/tools/read/SimpleRecordConverter.java @@ -22,6 +22,9 @@ import java.nio.CharBuffer; import java.nio.charset.Charset; import java.nio.charset.CharsetDecoder; +import java.math.BigDecimal; +import java.math.BigInteger; + import org.apache.parquet.io.api.Binary; import org.apache.parquet.io.api.Converter; import org.apache.parquet.io.api.GroupConverter; @@ -70,6 +73,9 @@ public class SimpleRecordConverter extends GroupConverter { case UTF8: return new StringConverter(field.getName()); case MAP_KEY_VALUE: break; case ENUM: break; + case DECIMAL: + int scale = field.asPrimitiveType().getDecimalMetadata().getScale(); + return new DecimalConverter(field.getName(), scale); } } @@ -155,5 +161,19 @@ public class SimpleRecordConverter extends GroupConverter { record.add(name, value.toStringUsingUTF8()); } } + + private class DecimalConverter extends SimplePrimitiveConverter { + private final int scale; + + public DecimalConverter(String name, int scale) { + super(name); + this.scale = scale; + } + + @Override + public void addBinary(Binary value) { + record.add(name, new BigDecimal(new BigInteger(value.getBytes()), scale)); + } + } } http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/a703ee75/parquet-tools/src/test/java/org/apache/parquet/tools/read/TestSimplePrimitiveRecord.java ---------------------------------------------------------------------- diff --git a/parquet-tools/src/test/java/org/apache/parquet/tools/read/TestSimplePrimitiveRecord.java b/parquet-tools/src/test/java/org/apache/parquet/tools/read/TestSimplePrimitiveRecord.java new file mode 100644 index 0000000..de27fcb --- /dev/null +++ b/parquet-tools/src/test/java/org/apache/parquet/tools/read/TestSimplePrimitiveRecord.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.parquet.tools.read; + +import org.junit.Assert; +import org.junit.Test; +import java.math.BigDecimal; + +public class TestSimplePrimitiveRecord { + + class TestRecord { + private int x; + private int y; + + public TestRecord(int x, int y) { + this.x = x; + this.y = y; + } + + @Override + public String toString() { + return "TestRecord {" + x + "," + y + "}"; + } + } + + @Test + public void testBinary() { + SimpleMapRecord r = new SimpleMapRecord(); + Assert.assertEquals("null", r.keyToString(null)); + Assert.assertEquals("true", r.keyToString(true)); + Assert.assertEquals("a", r.keyToString('a')); + Assert.assertEquals("3.0", r.keyToString(3.0)); + Assert.assertEquals("4.0", r.keyToString(4.0f)); + Assert.assertEquals("100", r.keyToString(100)); + Assert.assertEquals("37", r.keyToString(37l)); + Assert.assertEquals("-1", r.keyToString((short) -1)); + Assert.assertEquals("test", r.keyToString("test")); + Assert.assertEquals("123.123", r.keyToString(new BigDecimal("123.123"))); + } +} +
