This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
The following commit(s) were added to refs/heads/main by this push:
new d71e3245 Fix unzip task and add shaded relief workflow (#798)
d71e3245 is described below
commit d71e32454999a7170e22c07f26585eae7608cfbc
Author: Bertil Chapuis <[email protected]>
AuthorDate: Sat Nov 4 01:31:48 2023 +0100
Fix unzip task and add shaded relief workflow (#798)
The unzip task implemented some checks against malicious zip files that
failed with the shaded relief archive. They have now been removed.
---
...reate.run.xml => naturalearth-workflow.run.xml} | 7 ++++-
.../apache/baremaps/workflow/tasks/UnzipFile.java | 36 +++-------------------
examples/shadedrelief/workflow.json | 20 ++++++++++++
3 files changed, 31 insertions(+), 32 deletions(-)
diff --git a/.run/naturalearth-create.run.xml
b/.run/naturalearth-workflow.run.xml
similarity index 54%
rename from .run/naturalearth-create.run.xml
rename to .run/naturalearth-workflow.run.xml
index fb309124..45f5277d 100644
--- a/.run/naturalearth-create.run.xml
+++ b/.run/naturalearth-workflow.run.xml
@@ -1,9 +1,14 @@
<component name="ProjectRunConfigurationManager">
- <configuration default="false" name="naturalearth-create" type="Application"
factoryName="Application">
+ <configuration default="false" name="naturalearth-workflow"
type="Application" factoryName="Application">
<option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" />
<module name="baremaps-cli" />
<option name="PROGRAM_PARAMETERS" value="workflow execute --file
workflow.json" />
<option name="WORKING_DIRECTORY"
value="$PROJECT_DIR$/examples/naturalearth" />
+ <extension
name="software.aws.toolkits.jetbrains.core.execution.JavaAwsConnectionExtension">
+ <option name="credential" />
+ <option name="region" />
+ <option name="useCurrentConnection" value="false" />
+ </extension>
<method v="2">
<option name="Make" enabled="true" />
</method>
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java
b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java
index 5a1181d9..dc89d9c5 100644
---
a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java
+++
b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java
@@ -22,18 +22,9 @@ import java.nio.file.*;
import java.util.zip.ZipFile;
import org.apache.baremaps.workflow.Task;
import org.apache.baremaps.workflow.WorkflowContext;
-import org.apache.baremaps.workflow.WorkflowException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
public record UnzipFile(Path file, Path directory) implements Task {
- private static final long THRESHOLD_ENTRIES = 10000;
- private static final long THRESHOLD_SIZE = 10l << 30;
- private static final double THRESHOLD_RATIO = 100;
-
- private static final Logger logger =
LoggerFactory.getLogger(UnzipFile.class);
-
@Override
public void execute(WorkflowContext context) throws Exception {
var filePath = file.toAbsolutePath();
@@ -41,11 +32,13 @@ public record UnzipFile(Path file, Path directory)
implements Task {
try (var zipFile = new ZipFile(filePath.toFile())) {
var entries = zipFile.entries();
- long totalSizeArchive = 0;
- long totalEntryArchive = 0;
while (entries.hasMoreElements()) {
var ze = entries.nextElement();
+ if (ze.isDirectory()) {
+ continue;
+ }
+
var path = directoryPath.resolve(ze.getName());
var file = path.toFile().getCanonicalFile();
@@ -61,29 +54,10 @@ public record UnzipFile(Path file, Path directory)
implements Task {
try (var input = new BufferedInputStream(zipFile.getInputStream(ze));
var output = new BufferedOutputStream(new
FileOutputStream(path.toFile()))) {
- totalEntryArchive++;
-
- int nBytes = -1;
+ int nBytes;
byte[] buffer = new byte[4096];
- long totalSizeEntry = 0;
-
while ((nBytes = input.read(buffer)) > 0) {
output.write(buffer, 0, nBytes);
- totalSizeEntry += nBytes;
- totalSizeArchive += nBytes;
-
- double compressionRatio = (double) totalSizeEntry / (double)
ze.getCompressedSize();
- if (compressionRatio > THRESHOLD_RATIO) {
- throw new WorkflowException("Archive compression ratio is too
high");
- }
- }
-
- if (totalSizeArchive > THRESHOLD_SIZE) {
- throw new IOException("Archive is too large");
- }
-
- if (totalEntryArchive > THRESHOLD_ENTRIES) {
- throw new IOException("Archive contains too many entries");
}
}
}
diff --git a/examples/shadedrelief/workflow.json
b/examples/shadedrelief/workflow.json
new file mode 100644
index 00000000..48407d52
--- /dev/null
+++ b/examples/shadedrelief/workflow.json
@@ -0,0 +1,20 @@
+{
+ "steps": [
+ {
+ "id": "natural-earth",
+ "needs": [],
+ "tasks": [
+ {
+ "type": "DownloadUrl",
+ "url":
"http://www.shadedrelief.com/ne-draft/World-Base-Map-Shapefiles.zip";,
+ "path": "shadedrelief.zip"
+ },
+ {
+ "type": "UnzipFile",
+ "file": "shadedrelief.zip",
+ "directory": "shadedrelief"
+ }
+ ]
+ }
+ ]
+}
