This is an automated email from the ASF dual-hosted git repository.
shangxinli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-mr.git
The following commit(s) were added to refs/heads/master by this push:
new 0819356a9 PARQUET-2167: Fix CLI serializing of footer with date fields
(#980)
0819356a9 is described below
commit 0819356a9dafd2ca07c5eab68e2bffeddc3bd3d9
Author: Bryan Keller <[email protected]>
AuthorDate: Mon Jul 25 19:55:49 2022 -0700
PARQUET-2167: Fix CLI serializing of footer with date fields (#980)
---
parquet-cli/pom.xml | 5 ++++
.../parquet/cli/commands/ShowFooterCommand.java | 2 ++
.../org/apache/parquet/cli/commands/FileTest.java | 1 +
.../parquet/cli/commands/ParquetFileTest.java | 28 ++++++++++++++--------
pom.xml | 1 +
5 files changed, 27 insertions(+), 10 deletions(-)
diff --git a/parquet-cli/pom.xml b/parquet-cli/pom.xml
index 5346f2439..f81979348 100644
--- a/parquet-cli/pom.xml
+++ b/parquet-cli/pom.xml
@@ -105,6 +105,11 @@
<artifactId>jackson-annotations</artifactId>
<version>${jackson.version}</version>
</dependency>
+ <dependency>
+ <groupId>${jackson.datatype.groupId}</groupId>
+ <artifactId>jackson-datatype-jsr310</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
<dependency>
<groupId>com.beust</groupId>
<artifactId>jcommander</artifactId>
diff --git
a/parquet-cli/src/main/java/org/apache/parquet/cli/commands/ShowFooterCommand.java
b/parquet-cli/src/main/java/org/apache/parquet/cli/commands/ShowFooterCommand.java
index a5a5c1fc0..b96ee2aaf 100644
---
a/parquet-cli/src/main/java/org/apache/parquet/cli/commands/ShowFooterCommand.java
+++
b/parquet-cli/src/main/java/org/apache/parquet/cli/commands/ShowFooterCommand.java
@@ -23,6 +23,7 @@ import static
org.apache.parquet.bytes.BytesUtils.readIntLittleEndian;
import static org.apache.parquet.hadoop.ParquetFileWriter.EFMAGIC;
import static org.apache.parquet.hadoop.ParquetFileWriter.MAGIC;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
@@ -86,6 +87,7 @@ public class ShowFooterCommand extends BaseCommand {
ObjectMapper mapper = new ObjectMapper();
mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
mapper.configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true);
+ mapper.registerModule(new JavaTimeModule());
return mapper;
}
diff --git
a/parquet-cli/src/test/java/org/apache/parquet/cli/commands/FileTest.java
b/parquet-cli/src/test/java/org/apache/parquet/cli/commands/FileTest.java
index e145e7ccc..0253a19a0 100644
--- a/parquet-cli/src/test/java/org/apache/parquet/cli/commands/FileTest.java
+++ b/parquet-cli/src/test/java/org/apache/parquet/cli/commands/FileTest.java
@@ -35,6 +35,7 @@ public abstract class FileTest {
static final String DOUBLE_FIELD = "double_field";
static final String BINARY_FIELD = "binary_field";
static final String FIXED_LEN_BYTE_ARRAY_FIELD = "flba_field";
+ static final String DATE_FIELD = "date_field";
static final String[] COLORS = {"RED", "BLUE", "YELLOW", "GREEN", "WHITE"};
diff --git
a/parquet-cli/src/test/java/org/apache/parquet/cli/commands/ParquetFileTest.java
b/parquet-cli/src/test/java/org/apache/parquet/cli/commands/ParquetFileTest.java
index 37f27b8ea..cbcdb7e9e 100644
---
a/parquet-cli/src/test/java/org/apache/parquet/cli/commands/ParquetFileTest.java
+++
b/parquet-cli/src/test/java/org/apache/parquet/cli/commands/ParquetFileTest.java
@@ -23,12 +23,17 @@ import org.apache.hadoop.fs.Path;
import org.apache.parquet.column.ParquetProperties;
import org.apache.parquet.example.data.Group;
import org.apache.parquet.example.data.simple.SimpleGroupFactory;
+import org.apache.parquet.format.DateType;
+import org.apache.parquet.format.LogicalType;
+import org.apache.parquet.format.LogicalTypes;
import org.apache.parquet.hadoop.ParquetWriter;
import org.apache.parquet.hadoop.example.GroupWriteSupport;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Types;
import org.junit.Before;
import java.io.File;
@@ -59,15 +64,17 @@ public abstract class ParquetFileTest extends FileTest {
}
private static MessageType createSchema() {
- return new MessageType("schema",
- new PrimitiveType(REQUIRED, PrimitiveTypeName.INT32, INT32_FIELD),
- new PrimitiveType(REQUIRED, PrimitiveTypeName.INT64, INT64_FIELD),
- new PrimitiveType(REQUIRED, PrimitiveTypeName.FLOAT, FLOAT_FIELD),
- new PrimitiveType(REQUIRED, PrimitiveTypeName.DOUBLE, DOUBLE_FIELD),
- new PrimitiveType(REQUIRED, PrimitiveTypeName.BINARY, BINARY_FIELD),
- new PrimitiveType(REQUIRED, PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY,
- 12, FIXED_LEN_BYTE_ARRAY_FIELD)
- );
+ return Types.buildMessage()
+ .required(PrimitiveTypeName.INT32).named(INT32_FIELD)
+ .required(PrimitiveTypeName.INT64).named(INT64_FIELD)
+ .required(PrimitiveTypeName.FLOAT).named(FLOAT_FIELD)
+ .required(PrimitiveTypeName.DOUBLE).named(DOUBLE_FIELD)
+ .required(PrimitiveTypeName.BINARY).named(BINARY_FIELD)
+ .required(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(12)
+ .named(FIXED_LEN_BYTE_ARRAY_FIELD)
+ .required(PrimitiveTypeName.INT32).as(LogicalTypeAnnotation.dateType())
+ .named(DATE_FIELD)
+ .named("schema");
}
private void createTestParquetFile() throws IOException {
@@ -102,7 +109,8 @@ public abstract class ParquetFileTest extends FileTest {
.append(DOUBLE_FIELD, 2.0d + i)
.append(BINARY_FIELD, Binary.fromString(COLORS[i % COLORS.length]))
.append(FIXED_LEN_BYTE_ARRAY_FIELD,
- Binary.fromConstantByteArray(bytes)));
+ Binary.fromConstantByteArray(bytes))
+ .append(DATE_FIELD, i));
}
}
}
diff --git a/pom.xml b/pom.xml
index 1632831fa..2f664bf90 100644
--- a/pom.xml
+++ b/pom.xml
@@ -71,6 +71,7 @@
<maven.compiler.target>1.8</maven.compiler.target>
<github.global.server>github</github.global.server>
<jackson.groupId>com.fasterxml.jackson.core</jackson.groupId>
+
<jackson.datatype.groupId>com.fasterxml.jackson.datatype</jackson.datatype.groupId>
<jackson.package>com.fasterxml.jackson</jackson.package>
<jackson.version>2.13.2</jackson.version>
<jackson-databind.version>2.13.2.2</jackson-databind.version>
