This is an automated email from the ASF dual-hosted git repository.
gian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new ef91fdb Zstandard decompression support (#6224)
ef91fdb is described below
commit ef91fdbf038b1cd1f549e63f149a6829bbe6cfba
Author: QiuMM <[email protected]>
AuthorDate: Mon Aug 27 07:09:24 2018 +0800
Zstandard decompression support (#6224)
---
java-util/pom.xml | 4 ++++
.../java/io/druid/java/util/common/CompressionUtils.java | 4 ++++
.../io/druid/java/util/common/CompressionUtilsTest.java | 15 +++++++++++++++
pom.xml | 5 +++++
4 files changed, 28 insertions(+)
diff --git a/java-util/pom.xml b/java-util/pom.xml
index 696e579..569aa4f 100644
--- a/java-util/pom.xml
+++ b/java-util/pom.xml
@@ -94,6 +94,10 @@
<artifactId>xz</artifactId>
</dependency>
<dependency>
+ <groupId>com.github.luben</groupId>
+ <artifactId>zstd-jni</artifactId>
+ </dependency>
+ <dependency>
<groupId>com.jayway.jsonpath</groupId>
<artifactId>json-path</artifactId>
</dependency>
diff --git
a/java-util/src/main/java/io/druid/java/util/common/CompressionUtils.java
b/java-util/src/main/java/io/druid/java/util/common/CompressionUtils.java
index 99c6367..77ef23a 100644
--- a/java-util/src/main/java/io/druid/java/util/common/CompressionUtils.java
+++ b/java-util/src/main/java/io/druid/java/util/common/CompressionUtils.java
@@ -31,6 +31,7 @@ import io.druid.java.util.common.logger.Logger;
import
org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import
org.apache.commons.compress.compressors.snappy.FramedSnappyCompressorInputStream;
import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
+import
org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
@@ -58,6 +59,7 @@ public class CompressionUtils
private static final String XZ_SUFFIX = ".xz";
private static final String ZIP_SUFFIX = ".zip";
private static final String SNAPPY_SUFFIX = ".sz";
+ private static final String ZSTD_SUFFIX = ".zst";
/**
* Zip the contents of directory into the file indicated by outputZipFile.
Sub directories are skipped
@@ -567,6 +569,8 @@ public class CompressionUtils
return new XZCompressorInputStream(in, true);
} else if (fileName.endsWith(SNAPPY_SUFFIX)) {
return new FramedSnappyCompressorInputStream(in);
+ } else if (fileName.endsWith(ZSTD_SUFFIX)) {
+ return new ZstdCompressorInputStream(in);
} else if (fileName.endsWith(ZIP_SUFFIX)) {
// This reads the first file in the archive.
final ZipInputStream zipIn = new ZipInputStream(in,
StandardCharsets.UTF_8);
diff --git
a/java-util/src/test/java/io/druid/java/util/common/CompressionUtilsTest.java
b/java-util/src/test/java/io/druid/java/util/common/CompressionUtilsTest.java
index 26d3d79..2e02c77 100644
---
a/java-util/src/test/java/io/druid/java/util/common/CompressionUtilsTest.java
+++
b/java-util/src/test/java/io/druid/java/util/common/CompressionUtilsTest.java
@@ -28,6 +28,7 @@ import com.google.common.io.Files;
import
org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
import
org.apache.commons.compress.compressors.snappy.FramedSnappyCompressorOutputStream;
import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream;
+import
org.apache.commons.compress.compressors.zstandard.ZstdCompressorOutputStream;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
@@ -291,6 +292,20 @@ public class CompressionUtilsTest
}
@Test
+ public void testDecompressZstd() throws IOException
+ {
+ final File tmpDir = temporaryFolder.newFolder("testDecompressZstd");
+ final File zstdFile = new File(tmpDir, testFile.getName() + ".zst");
+ Assert.assertFalse(zstdFile.exists());
+ try (final OutputStream out = new ZstdCompressorOutputStream(new
FileOutputStream(zstdFile))) {
+ ByteStreams.copy(new FileInputStream(testFile), out);
+ }
+ try (final InputStream inputStream = CompressionUtils.decompress(new
FileInputStream(zstdFile), zstdFile.getName())) {
+ assertGoodDataStream(inputStream);
+ }
+ }
+
+ @Test
public void testDecompressZip() throws IOException
{
final File tmpDir = temporaryFolder.newFolder("testDecompressZip");
diff --git a/pom.xml b/pom.xml
index a7bc4fa..9774ac8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -341,6 +341,11 @@
<version>1.8</version>
</dependency>
<dependency>
+ <groupId>com.github.luben</groupId>
+ <artifactId>zstd-jni</artifactId>
+ <version>1.3.3-1</version>
+ </dependency>
+ <dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>${jackson.version}</version>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
