Repository: parquet-mr Updated Branches: refs/heads/master dc08bb8ea -> 82b8ecc32
PARQUET-484: Warn when Decimal is stored as INT64 while could be stored as INT32 Below is documented in [LogicalTypes.md](https://github.com/Parquet/parquet-format/blob/master/LogicalTypes.md#decimal): > int32: for 1 <= precision <= 9 > int64: for 1 <= precision <= 18; precision < 10 will produce a warning This PR implements the `precision < 10 will produce a warning` part. @rdblue @liancheng would mind taking a look at this when you have time? It's a fairly small addition; cheers. Author: Liwei Lin <[email protected]> Author: proflin <[email protected]> Closes #316 from lw-lin/P-484-2 and squashes the following commits: 207e509 [Liwei Lin] Address comments b227484 [proflin] PARQUET-484: Warn when Decimal is stored as INT64 while could be stored as INT32 Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/82b8ecc3 Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/82b8ecc3 Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/82b8ecc3 Branch: refs/heads/master Commit: 82b8ecc3275d7c3578a6531ac3f1da3ffada9dcc Parents: dc08bb8 Author: Liwei Lin <[email protected]> Authored: Tue Apr 19 09:17:01 2016 -0700 Committer: Ryan Blue <[email protected]> Committed: Tue Apr 19 09:17:01 2016 -0700 ---------------------------------------------------------------------- .../src/main/java/org/apache/parquet/schema/Types.java | 9 +++++++++ 1 file changed, 9 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/82b8ecc3/parquet-column/src/main/java/org/apache/parquet/schema/Types.java ---------------------------------------------------------------------- diff --git a/parquet-column/src/main/java/org/apache/parquet/schema/Types.java b/parquet-column/src/main/java/org/apache/parquet/schema/Types.java index 6d61b50..d3cc97b 100644 --- a/parquet-column/src/main/java/org/apache/parquet/schema/Types.java +++ b/parquet-column/src/main/java/org/apache/parquet/schema/Types.java @@ -25,6 +25,8 @@ import java.util.List; import org.apache.parquet.Preconditions; import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; import org.apache.parquet.schema.Type.ID; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This class provides fluent builders that produce Parquet schema Types. @@ -305,8 +307,11 @@ public class Types { public abstract static class BasePrimitiveBuilder<P, THIS extends BasePrimitiveBuilder<P, THIS>> extends Builder<THIS, P> { + private static final Logger LOGGER = LoggerFactory.getLogger(BasePrimitiveBuilder.class); private static final long MAX_PRECISION_INT32 = maxPrecision(4); private static final long MAX_PRECISION_INT64 = maxPrecision(8); + private static final String LOGICAL_TYPES_DOC_URL = + "https://github.com/apache/parquet-format/blob/master/LogicalTypes.md";; private final PrimitiveTypeName primitiveType; private int length = NOT_SET; private int precision = NOT_SET; @@ -406,6 +411,10 @@ public class Types { meta.getPrecision() <= MAX_PRECISION_INT64, "INT64 cannot store " + meta.getPrecision() + " digits " + "(max " + MAX_PRECISION_INT64 + ")"); + if (meta.getPrecision() <= MAX_PRECISION_INT32) { + LOGGER.warn("Decimal with {} digits is stored in an INT64, but fits in an INT32. See {}.", + precision, LOGICAL_TYPES_DOC_URL); + } } else if (primitiveType == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) { Preconditions.checkState( meta.getPrecision() <= maxPrecision(length),
