Github user cestella commented on a diff in the pull request:
https://github.com/apache/metron/pull/1011#discussion_r187757704
--- Diff:
metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/geo/GeoEnrichmentLoader.java
---
@@ -167,26 +184,38 @@ protected void loadGeoIpDatabase(CommandLine cli)
throws IOException {
System.out.println("Successfully created and updated new GeoIP
information");
}
- protected File downloadGeoFile(String urlStr, String tmpDir) {
+ protected File downloadGeoFile(String urlStr, String tmpDir, int
numRetries) {
File localFile = null;
- try {
- URL url = new URL(urlStr);
- localFile = new File(tmpDir + new File(url.getPath()).getName());
+ int attempts = 0;
+ boolean valid = false;
+ while (attempts <= numRetries) {
+ try {
+ URL url = new URL(urlStr);
+ localFile = new File(tmpDir + new File(url.getPath()).getName());
- System.out.println("Downloading " + url.toString() + " to " +
localFile.getAbsolutePath());
- if (localFile.exists() && !localFile.delete()) {
- System.err.println("File already exists locally and can't be
deleted. Please delete before continuing");
- System.exit(3);
+ System.out.println("Downloading " + url.toString() + " to " +
localFile.getAbsolutePath());
+ if (localFile.exists() && !localFile.delete()) {
+ System.err.println(
+ "File already exists locally and can't be deleted. Please
delete before continuing");
+ System.exit(3);
+ }
+ FileUtils.copyURLToFile(url, localFile, 5000, 10000);
+ if (!CompressionStrategies.GZIP.test(localFile)) {
+ throw new IOException("Invalid Gzip file");
+ }
+ } catch (MalformedURLException e) {
+ System.err.println("Malformed URL - aborting: " + e);
+ e.printStackTrace();
+ System.exit(4);
+ } catch (IOException e) {
+ System.err.println("Warning: Unable to copy remote GeoIP database
to local file, attempt " + attempts + ": " + e);
+ e.printStackTrace();
--- End diff --
same question, print to stderr?
---
