This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
The following commit(s) were added to refs/heads/main by this push:
new c4445c33 Support FTP downloads and extend IRR whois source files (#698)
c4445c33 is described below
commit c4445c33f0f3a4a41455d7c7925cecbff1f12ba1
Author: Perdjesk <[email protected]>
AuthorDate: Mon Jun 12 18:54:25 2023 +0200
Support FTP downloads and extend IRR whois source files (#698)
* Support FTP downloads. Extend IRR whois source files
* Disable test reliying on third party resource
* Use IllegalArgumentException instead of IOException for unsupported URL
protocols
---
.../baremaps/workflow/tasks/DownloadUrl.java | 56 ++++++++++++++++------
.../baremaps/workflow/tasks/DownloadUrlTest.java | 31 +++++++++++-
examples/ip-to-location/workflow.js | 14 ++++++
3 files changed, 86 insertions(+), 15 deletions(-)
diff --git
a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java
b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java
index 274d0b08..f22bb3a8 100644
---
a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java
+++
b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DownloadUrl.java
@@ -12,8 +12,10 @@
package org.apache.baremaps.workflow.tasks;
+import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
+import java.net.URLConnection;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
@@ -24,6 +26,10 @@ import org.slf4j.LoggerFactory;
public record DownloadUrl(String url, Path path, boolean replaceExisting)
implements Task {
+ private static final String PROTOCOL_FTP = "ftp";
+ private static final String PROTOCOL_HTTP = "http";
+ private static final String PROTOCOL_HTTPS = "https";
+
public DownloadUrl(String url, Path path) {
this(url, path, false);
}
@@ -35,26 +41,48 @@ public record DownloadUrl(String url, Path path, boolean
replaceExisting) implem
var targetUrl = new URL(url);
var targetPath = path.toAbsolutePath();
- if (Files.exists(targetPath) && !replaceExisting) {
- var head = (HttpURLConnection) targetUrl.openConnection();
- head.setFollowRedirects(true);
- head.setRequestMethod("HEAD");
- var contentLength = head.getContentLengthLong();
- head.disconnect();
- if (Files.size(targetPath) == contentLength) {
+ if (isHttp(targetUrl)) {
+ if (Files.exists(targetPath) && !replaceExisting) {
+ var head = (HttpURLConnection) targetUrl.openConnection();
+ head.setInstanceFollowRedirects(true);
+ head.setRequestMethod("HEAD");
+ var contentLength = head.getContentLengthLong();
+ head.disconnect();
+ if (Files.size(targetPath) == contentLength) {
+ logger.info("Skipping download of {} to {}", url, path);
+ return;
+ }
+ }
+
+ var get = (HttpURLConnection) targetUrl.openConnection();
+ get.setInstanceFollowRedirects(true);
+ get.setRequestMethod("GET");
+ urlDownloadToFile(get, targetPath);
+ get.disconnect();
+ } else if (isFtp(targetUrl)) {
+ if (Files.exists(targetPath) && !replaceExisting) {
logger.info("Skipping download of {} to {}", url, path);
return;
}
+ urlDownloadToFile(targetUrl.openConnection(), targetPath);
+ } else {
+ throw new IllegalArgumentException("Unsupported URL protocol (supported:
http(s)/ftp)");
}
+ }
+
+ private static boolean isHttp(URL url) {
+ return url.getProtocol().equalsIgnoreCase(PROTOCOL_HTTP) ||
+ url.getProtocol().equalsIgnoreCase(PROTOCOL_HTTPS);
+ }
+
+ private static boolean isFtp(URL url) {
+ return url.getProtocol().equalsIgnoreCase(PROTOCOL_FTP);
+ }
- var get = (HttpURLConnection) targetUrl.openConnection();
- get.setFollowRedirects(true);
- get.setRequestMethod("GET");
- try (var inputStream = get.getInputStream()) {
- var downloadFile = targetPath.toAbsolutePath();
- Files.createDirectories(downloadFile.getParent());
+ private static void urlDownloadToFile(URLConnection url, Path targetPath)
throws IOException {
+ try (var inputStream = url.getInputStream()) {
+ Files.createDirectories(targetPath.toAbsolutePath().getParent());
Files.copy(inputStream, targetPath, StandardCopyOption.REPLACE_EXISTING);
}
- get.disconnect();
}
}
diff --git
a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/DownloadUrlTest.java
b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/DownloadUrlTest.java
index 05d73881..00b78c35 100644
---
a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/DownloadUrlTest.java
+++
b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/DownloadUrlTest.java
@@ -12,12 +12,14 @@
package org.apache.baremaps.workflow.tasks;
-import static org.junit.jupiter.api.Assertions.*;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.File;
import java.nio.file.Files;
import org.apache.baremaps.utils.FileUtils;
import org.apache.baremaps.workflow.WorkflowContext;
+import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
@@ -34,6 +36,33 @@ class DownloadUrlTest {
assertTrue(Files.readString(file.toPath()).contains("Baremaps"));
}
+ @Disabled("Test relies on third party resource, see TODO comment.")
+ @Test
+ @Tag("integration")
+ void testDownloadFtp() throws Exception {
+ var directory = Files.createTempDirectory("tmp_");
+ var file = directory.resolve("file");
+ // TODO: do not use a 3rd party server, replaces test URL to a baremaps
owned test resource.
+ var task = new DownloadUrl("ftp://whois.in.bell.ca/bell.db.gz";,
+ file);
+ task.execute(new WorkflowContext());
+ assertTrue(file.toFile().length() > 50, "file is less than 50 bytes");
+ FileUtils.deleteRecursively(directory);
+ }
+
+ @Test
+ @Tag("integration")
+ void testDownloadUnsupportedProtocol() throws Exception {
+ var directory = Files.createTempDirectory("tmp_");
+ var file = directory.resolve("file");
+ assertThrows(IllegalArgumentException.class, () -> {
+ var task = new DownloadUrl("file://not-existing-file-243jhks",
+ file);
+ task.execute(new WorkflowContext());
+ }, "Unsupported protocol throws IOException");
+ FileUtils.deleteRecursively(directory);
+ }
+
@Test
@Tag("integration")
void executeFileThatDoesntExist() throws Exception {
diff --git a/examples/ip-to-location/workflow.js
b/examples/ip-to-location/workflow.js
index 92c55fee..5c1f9650 100644
--- a/examples/ip-to-location/workflow.js
+++ b/examples/ip-to-location/workflow.js
@@ -9,6 +9,7 @@
or implied. See the License for the specific language governing permissions
and limitations under
the License.
**/
+// Based on https://www.irr.net/docs/list.html
const nics = [
{url: "https://ftp.afrinic.net/pub/dbase/afrinic.db.gz";, filename:
"afrinic.db"},
{url: "https://ftp.apnic.net/apnic/whois/apnic.db.as-block.gz";, filename:
"apnic.db.as-block.db"},
@@ -32,6 +33,19 @@ const nics = [
{url: "https://ftp.arin.net/pub/rr/arin.db.gz";, filename: "arin.db"},
{url: "https://ftp.lacnic.net/lacnic/dbase/lacnic.db.gz";, filename:
"lacnic.db"},
{url: "https://ftp.ripe.net/ripe/dbase/ripe.db.gz";, filename: "ripe.db"},
+ {url: "ftp://ftp.altdb.net/pub/altdb/altdb.db.gz";, filename: "altdb.db"},
+ {url: "ftp://whois.in.bell.ca/bell.db.gz";, filename: "bell.db"},
+ {url: "ftp://irr.bboi.net/bboi.db.gz";, filename: "bboi.db"},
+ {url: "https://whois.canarie.ca/dbase/canarie.db.gz";, filename:
"canarie.db"},
+ {url: "ftp://irr-mirror.idnic.net/idnic.db.gz";, filename: "idnic.db"},
+ {url: "ftp://ftp.nic.ad.jp/jpirr/jpirr.db.gz";, filename: "jpirr.db"},
+ {url: "ftp://rr.Level3.net/level3.db.gz";, filename: "level3.db"},
+ {url: "ftp://ftp.nestegg.net/irr/nestegg.db.gz";, filename: "nestegg.db"},
+ {url: "ftp://rr1.ntt.net/nttcomRR/nttcom.db.gz";, filename: "nttcom.db"},
+ {url: "ftp://ftp.panix.com/pub/rrdb/panix.db.gz";, filename: "panix.db"},
+ {url: "ftp://ftp.radb.net/radb/dbase/radb.db.gz";, filename: "radb.db"},
+ {url: "ftp://ftp.radb.net/radb/dbase/reach.db.gz";, filename: "reach.db"},
+ {url: "ftp://ftp.bgp.net.br/tc.db.gz";, filename: "tc.db"}
];
export default {"steps": [
