This is an automated email from the ASF dual-hosted git repository.
gangwu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-java.git
The following commit(s) were added to refs/heads/master by this push:
new fd829959d GH-2976: Parquet CLI compression commands should accept
lowercase compression name (#2977)
fd829959d is described below
commit fd829959d60683a2ef4cfc75e93d613e77a9a55b
Author: Cheng Pan <[email protected]>
AuthorDate: Wed Aug 7 13:01:57 2024 +0800
GH-2976: Parquet CLI compression commands should accept lowercase
compression name (#2977)
---
.../parquet/cli/commands/RewriteCommand.java | 3 ++-
.../cli/commands/TransCompressionCommand.java | 3 ++-
.../parquet/cli/commands/RewriteCommandTest.java | 28 ++++++++++++++++++++++
.../cli/commands/TransCompressionCommandTest.java | 19 +++++++++++++--
4 files changed, 49 insertions(+), 4 deletions(-)
diff --git
a/parquet-cli/src/main/java/org/apache/parquet/cli/commands/RewriteCommand.java
b/parquet-cli/src/main/java/org/apache/parquet/cli/commands/RewriteCommand.java
index 68ea02214..b2365d53f 100644
---
a/parquet-cli/src/main/java/org/apache/parquet/cli/commands/RewriteCommand.java
+++
b/parquet-cli/src/main/java/org/apache/parquet/cli/commands/RewriteCommand.java
@@ -30,6 +30,7 @@ import java.util.Map;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.parquet.cli.BaseCommand;
+import org.apache.parquet.cli.util.Codecs;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
import org.apache.parquet.hadoop.rewrite.MaskMode;
import org.apache.parquet.hadoop.rewrite.ParquetRewriter;
@@ -114,7 +115,7 @@ public class RewriteCommand extends BaseCommand {
}
if (codec != null) {
- CompressionCodecName codecName = CompressionCodecName.valueOf(codec);
+ CompressionCodecName codecName = Codecs.parquetCodec(codec);
builder.transform(codecName);
}
diff --git
a/parquet-cli/src/main/java/org/apache/parquet/cli/commands/TransCompressionCommand.java
b/parquet-cli/src/main/java/org/apache/parquet/cli/commands/TransCompressionCommand.java
index 49528410b..1f0605b7e 100644
---
a/parquet-cli/src/main/java/org/apache/parquet/cli/commands/TransCompressionCommand.java
+++
b/parquet-cli/src/main/java/org/apache/parquet/cli/commands/TransCompressionCommand.java
@@ -30,6 +30,7 @@ import java.util.List;
import org.apache.hadoop.fs.Path;
import org.apache.parquet.HadoopReadOptions;
import org.apache.parquet.cli.BaseCommand;
+import org.apache.parquet.cli.util.Codecs;
import org.apache.parquet.hadoop.ParquetFileReader;
import org.apache.parquet.hadoop.ParquetFileWriter;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
@@ -76,7 +77,7 @@ public class TransCompressionCommand extends BaseCommand {
Path inPath = new Path(input);
Path outPath = new Path(output);
- CompressionCodecName codecName = CompressionCodecName.valueOf(codec);
+ CompressionCodecName codecName = Codecs.parquetCodec(codec);
ParquetMetadata metaData = ParquetFileReader.readFooter(getConf(), inPath,
NO_FILTER);
MessageType schema = metaData.getFileMetaData().getSchema();
diff --git
a/parquet-cli/src/test/java/org/apache/parquet/cli/commands/RewriteCommandTest.java
b/parquet-cli/src/test/java/org/apache/parquet/cli/commands/RewriteCommandTest.java
index 33c625f45..10f8c6176 100644
---
a/parquet-cli/src/test/java/org/apache/parquet/cli/commands/RewriteCommandTest.java
+++
b/parquet-cli/src/test/java/org/apache/parquet/cli/commands/RewriteCommandTest.java
@@ -67,4 +67,32 @@ public class RewriteCommandTest extends ParquetFileTest {
Assert.assertEquals(0, command.run());
Assert.assertTrue(output.exists());
}
+
+ @Test
+ public void testRewriteCommandWithCompression_GZIP() throws IOException {
+ File file = parquetFile();
+ RewriteCommand command = new RewriteCommand(createLogger());
+ command.inputs = Arrays.asList(file.getAbsolutePath());
+ File output = new File(getTempFolder(), "converted-1.GZIP.parquet");
+ command.output = output.getAbsolutePath();
+ command.codec = "GZIP";
+ command.setConf(new Configuration());
+
+ Assert.assertEquals(0, command.run());
+ Assert.assertTrue(output.exists());
+ }
+
+ @Test
+ public void testRewriteCommandWithCompression_gzip() throws IOException {
+ File file = parquetFile();
+ RewriteCommand command = new RewriteCommand(createLogger());
+ command.inputs = Arrays.asList(file.getAbsolutePath());
+ File output = new File(getTempFolder(), "converted-2.gzip.parquet");
+ command.output = output.getAbsolutePath();
+ command.codec = "gzip";
+ command.setConf(new Configuration());
+
+ Assert.assertEquals(0, command.run());
+ Assert.assertTrue(output.exists());
+ }
}
diff --git
a/parquet-cli/src/test/java/org/apache/parquet/cli/commands/TransCompressionCommandTest.java
b/parquet-cli/src/test/java/org/apache/parquet/cli/commands/TransCompressionCommandTest.java
index 15ad7023a..07b2f6ec3 100644
---
a/parquet-cli/src/test/java/org/apache/parquet/cli/commands/TransCompressionCommandTest.java
+++
b/parquet-cli/src/test/java/org/apache/parquet/cli/commands/TransCompressionCommandTest.java
@@ -27,12 +27,12 @@ import org.junit.Test;
public class TransCompressionCommandTest extends ParquetFileTest {
@Test
- public void testTransCompressionCommand() throws IOException {
+ public void testTransCompressionCommand_ZSTD() throws IOException {
TransCompressionCommand command = new
TransCompressionCommand(createLogger());
command.input = parquetFile().getAbsolutePath();
- File output = new File(getTempFolder(), getClass().getSimpleName() +
".converted.parquet");
+ File output = new File(getTempFolder(), getClass().getSimpleName() +
".converted-1.ZSTD.parquet");
command.output = output.getAbsolutePath();
command.codec = "ZSTD";
command.setConf(new Configuration());
@@ -40,4 +40,19 @@ public class TransCompressionCommandTest extends
ParquetFileTest {
Assert.assertEquals(0, command.run());
Assert.assertTrue(output.exists());
}
+
+ @Test
+ public void testTransCompressionCommand_zstd() throws IOException {
+ TransCompressionCommand command = new
TransCompressionCommand(createLogger());
+
+ command.input = parquetFile().getAbsolutePath();
+
+ File output = new File(getTempFolder(), getClass().getSimpleName() +
".converted-2.zstd.parquet");
+ command.output = output.getAbsolutePath();
+ command.codec = "zstd";
+ command.setConf(new Configuration());
+
+ Assert.assertEquals(0, command.run());
+ Assert.assertTrue(output.exists());
+ }
}
