This is an automated email from the ASF dual-hosted git repository. bchapuis pushed a commit to branch 745-daylight in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 973395d0e2d0a610fecd14b780bf7fa64f601911 Author: Bertil Chapuis <[email protected]> AuthorDate: Sun Aug 27 21:34:30 2023 +0200 Add decompress task --- .../java/org/apache/baremaps/workflow/Task.java | 1 + .../baremaps/workflow/tasks/DecompressFile.java | 149 +++++++++++++++++++++ .../org/apache/baremaps/testing/TestFiles.java | 10 ++ .../workflow/tasks/DecompressFileTest.java | 63 +++++++++ .../workflow/tasks/ExecuteCommandTest.java | 2 +- baremaps-core/src/test/resources/archives/file.bz2 | Bin 0 -> 40 bytes baremaps-core/src/test/resources/archives/file.gz | Bin 0 -> 33 bytes .../src/test/resources/archives/file.tar.bz2 | Bin 0 -> 136 bytes .../src/test/resources/archives/file.tar.gz | Bin 0 -> 123 bytes baremaps-core/src/test/resources/archives/file.zip | Bin 0 -> 170 bytes 10 files changed, 224 insertions(+), 1 deletion(-) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java index 013df2bb..3fb601b1 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java @@ -39,6 +39,7 @@ import org.apache.baremaps.workflow.tasks.*; @JsonSubTypes.Type(value = UnzipFile.class, name = "UnzipFile"), @JsonSubTypes.Type(value = UngzipFile.class, name = "UngzipFile"), @JsonSubTypes.Type(value = DecompressBZip2.class, name = "DecompressBZip2"), + @JsonSubTypes.Type(value = DecompressFile.class, name = "DecompressFile"), @JsonSubTypes.Type(value = UpdateOpenStreetMap.class, name = "UpdateOpenStreetMap"), @JsonSubTypes.Type(value = CreateGeonamesIndex.class, name = "CreateGeonamesIndex"), @JsonSubTypes.Type(value = CreateIplocIndex.class, name = "CreateIplocIndex")}) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java new file mode 100644 index 00000000..9cfb8b3b --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java @@ -0,0 +1,149 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.baremaps.workflow.tasks; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.zip.GZIPInputStream; +import java.util.zip.ZipFile; +import org.apache.baremaps.workflow.Task; +import org.apache.baremaps.workflow.WorkflowContext; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public record DecompressFile(Path source, Path target, Compression compression) implements Task { + + public enum Compression { + zip, + targz, + tarbz2, + gzip, + bzip2 + } + + private static final Logger logger = LoggerFactory.getLogger(UngzipFile.class); + + @Override + public void execute(WorkflowContext context) throws Exception { + var sourcePath = source.toAbsolutePath(); + var targetPath = target.toAbsolutePath(); + switch (compression) { + case zip: + decompressZip(sourcePath, targetPath); + break; + case targz: + decompressTarGz(sourcePath, targetPath); + break; + case tarbz2: + decompressTarBz2(sourcePath, targetPath); + break; + case gzip: + decompressGzip(sourcePath, targetPath); + break; + case bzip2: + decompressBzip2(sourcePath, targetPath); + break; + } + + } + + public static void decompressBzip2(Path sourcePath, Path targetPath) throws IOException { + try (var zis = + new BZip2CompressorInputStream(new BufferedInputStream(Files.newInputStream(sourcePath)))) { + Files.copy(zis, targetPath, StandardCopyOption.REPLACE_EXISTING); + } + } + + public static void decompressGzip(Path sourcePath, Path targetPath) throws IOException { + try (var zis = new GZIPInputStream(new BufferedInputStream(Files.newInputStream(sourcePath)))) { + Files.copy(zis, targetPath, StandardCopyOption.REPLACE_EXISTING); + } + } + + public static void decompressTarGz(Path sourcePath, Path targetPath) throws IOException { + try ( + GZIPInputStream gzipInputStream = + new GZIPInputStream(new BufferedInputStream(Files.newInputStream(sourcePath))); + TarArchiveInputStream tarInputStream = new TarArchiveInputStream(gzipInputStream)) { + TarArchiveEntry entry; + while ((entry = (TarArchiveEntry) tarInputStream.getNextEntry()) != null) { + var path = targetPath.resolve(entry.getName()); + if (entry.isDirectory()) { + Files.createDirectories(path); + } else { + Files.createDirectories(path.getParent()); + try (BufferedOutputStream outputStream = + new BufferedOutputStream(Files.newOutputStream(path))) { + int bytesRead; + byte[] buffer = new byte[4096]; + while ((bytesRead = tarInputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + } + } + } + } + } + + public static void decompressTarBz2(Path sourcePath, Path targetPath) throws IOException { + try ( + BZip2CompressorInputStream gzipInputStream = new BZip2CompressorInputStream( + new BufferedInputStream(Files.newInputStream(sourcePath))); + TarArchiveInputStream tarInputStream = new TarArchiveInputStream(gzipInputStream)) { + TarArchiveEntry entry; + while ((entry = (TarArchiveEntry) tarInputStream.getNextEntry()) != null) { + var path = targetPath.resolve(entry.getName()); + if (entry.isDirectory()) { + Files.createDirectories(path); + } else { + Files.createDirectories(path.getParent()); + try (BufferedOutputStream outputStream = + new BufferedOutputStream(Files.newOutputStream(path))) { + int bytesRead; + byte[] buffer = new byte[4096]; + while ((bytesRead = tarInputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + } + } + } + } + } + + public static void decompressZip(Path sourcePath, Path targetPath) throws IOException { + try (var zipFile = new ZipFile(sourcePath.toFile())) { + var entries = zipFile.entries(); + while (entries.hasMoreElements()) { + var entry = entries.nextElement(); + var path = targetPath.resolve(entry.getName()); + Files.createDirectories(path.getParent()); + Files.write(path, new byte[] {}, StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING); + try (var input = new BufferedInputStream(zipFile.getInputStream(entry)); + var output = new BufferedOutputStream(new FileOutputStream(path.toFile()))) { + int nBytes = -1; + byte[] buffer = new byte[4096]; + while ((nBytes = input.read(buffer)) > 0) { + output.write(buffer, 0, nBytes); + } + } + } + } + } +} diff --git a/baremaps-core/src/test/java/org/apache/baremaps/testing/TestFiles.java b/baremaps-core/src/test/java/org/apache/baremaps/testing/TestFiles.java index 450c0ace..932115f9 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/testing/TestFiles.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/testing/TestFiles.java @@ -50,6 +50,16 @@ public class TestFiles { public static final Path STYLE_JS = resolve("style.js"); + public static final Path FILE_BZ2 = resolve("archives/file.bz2"); + + public static final Path FILE_GZ = resolve("archives/file.gz"); + + public static final Path FILE_TAR_BZ2 = resolve("archives/file.tar.bz2"); + + public static final Path FILE_TAR_GZ = resolve("archives/file.tar.gz"); + + public static final Path FILE_ZIP = resolve("archives/file.zip"); + public static Path resolve(String resource) { Path cwd = Path.of("").toAbsolutePath(); Path pathFromRoot = Path.of("baremaps-core", "src", "test", "resources", resource); diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/DecompressFileTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/DecompressFileTest.java new file mode 100644 index 00000000..de10032a --- /dev/null +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/DecompressFileTest.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.baremaps.workflow.tasks; + +import static org.junit.jupiter.api.Assertions.*; + +import java.io.IOException; +import java.nio.file.Files; +import org.apache.baremaps.testing.TestFiles; +import org.junit.jupiter.api.Test; + +class DecompressFileTest { + + @Test + void decompressBzip2() throws IOException { + var source = TestFiles.FILE_BZ2; + var target = Files.createTempFile("baremaps", ".txt"); + DecompressFile.decompressBzip2(source, target); + assertTrue(Files.readString(target).contains("test")); + } + + @Test + void decompressGzip() throws IOException { + var source = TestFiles.FILE_GZ; + var target = Files.createTempFile("baremaps", ".txt"); + DecompressFile.decompressGzip(source, target); + assertTrue(Files.readString(target).contains("test")); + } + + @Test + void decompressTarGz() throws IOException { + var source = TestFiles.FILE_TAR_GZ; + var target = Files.createTempDirectory("baremaps"); + DecompressFile.decompressTarGz(source, target); + assertTrue(Files.readString(target.resolve("file.txt")).contains("test")); + } + + @Test + void decompressTarBz2() throws IOException { + var source = TestFiles.FILE_TAR_BZ2; + var target = Files.createTempDirectory("baremaps"); + DecompressFile.decompressTarBz2(source, target); + assertTrue(Files.readString(target.resolve("file.txt")).contains("test")); + } + + @Test + void decompressZip() throws IOException { + var source = TestFiles.FILE_ZIP; + var target = Files.createTempDirectory("baremaps"); + DecompressFile.decompressZip(source, target); + assertTrue(Files.readString(target.resolve("file.txt")).contains("test")); + } +} diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ExecuteCommandTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ExecuteCommandTest.java index f6663da7..c841ecf0 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ExecuteCommandTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ExecuteCommandTest.java @@ -25,7 +25,7 @@ class ExecuteCommandTest { @Test @Disabled void execute() throws Exception { - var path = Paths.get("test.txt").toAbsolutePath(); + var path = Paths.get("file.txt").toAbsolutePath(); new ExecuteCommand(String.format("echo test > %s", path)).execute(new WorkflowContext()); assertTrue(Files.exists(path)); assertTrue(Files.readString(path).contains("test")); diff --git a/baremaps-core/src/test/resources/archives/file.bz2 b/baremaps-core/src/test/resources/archives/file.bz2 new file mode 100644 index 00000000..4250272d Binary files /dev/null and b/baremaps-core/src/test/resources/archives/file.bz2 differ diff --git a/baremaps-core/src/test/resources/archives/file.gz b/baremaps-core/src/test/resources/archives/file.gz new file mode 100644 index 00000000..55a70e35 Binary files /dev/null and b/baremaps-core/src/test/resources/archives/file.gz differ diff --git a/baremaps-core/src/test/resources/archives/file.tar.bz2 b/baremaps-core/src/test/resources/archives/file.tar.bz2 new file mode 100644 index 00000000..f5d017cb Binary files /dev/null and b/baremaps-core/src/test/resources/archives/file.tar.bz2 differ diff --git a/baremaps-core/src/test/resources/archives/file.tar.gz b/baremaps-core/src/test/resources/archives/file.tar.gz new file mode 100644 index 00000000..77812f52 Binary files /dev/null and b/baremaps-core/src/test/resources/archives/file.tar.gz differ diff --git a/baremaps-core/src/test/resources/archives/file.zip b/baremaps-core/src/test/resources/archives/file.zip new file mode 100644 index 00000000..b1011c29 Binary files /dev/null and b/baremaps-core/src/test/resources/archives/file.zip differ
