This is an automated email from the ASF dual-hosted git repository. lewismc pushed a commit to branch NUTCH-2292 in repository https://gitbox.apache.org/repos/asf/nutch.git
commit 10b23cd0b8d53fd250253160162c8d3deb0cb9ea Author: Lewis John McGibbney <[email protected]> AuthorDate: Tue Mar 7 22:52:25 2017 -0800 NUTCH-2292 fix numerous Maven compiler warnings and replace all calls to System.exit --- .gitignore | 7 ++++++- nutch-core/pom.xml | 4 ++-- .../src/main/java/org/apache/nutch/crawl/CrawlDb.java | 3 +-- .../java/org/apache/nutch/crawl/CrawlDbMerger.java | 4 +--- .../java/org/apache/nutch/crawl/CrawlDbReader.java | 4 +--- .../java/org/apache/nutch/crawl/DeduplicationJob.java | 4 +--- .../main/java/org/apache/nutch/crawl/Generator.java | 4 +--- .../src/main/java/org/apache/nutch/crawl/Injector.java | 3 +-- .../src/main/java/org/apache/nutch/crawl/LinkDb.java | 3 +-- .../main/java/org/apache/nutch/crawl/LinkDbMerger.java | 4 +--- .../main/java/org/apache/nutch/crawl/LinkDbReader.java | 4 +--- .../main/java/org/apache/nutch/fetcher/Fetcher.java | 3 +-- .../main/java/org/apache/nutch/hostdb/ReadHostDb.java | 3 +-- .../java/org/apache/nutch/hostdb/UpdateHostDb.java | 3 +-- .../java/org/apache/nutch/indexer/CleaningJob.java | 4 +--- .../apache/nutch/indexer/IndexingFiltersChecker.java | 18 ++++++++---------- .../java/org/apache/nutch/indexer/IndexingJob.java | 4 +--- .../java/org/apache/nutch/net/URLFilterChecker.java | 7 ++----- .../org/apache/nutch/net/URLNormalizerChecker.java | 5 +---- .../org/apache/nutch/parse/ParsePluginsReader.java | 3 +-- .../main/java/org/apache/nutch/parse/ParseSegment.java | 7 ++----- .../java/org/apache/nutch/parse/ParserChecker.java | 6 ++---- .../org/apache/nutch/protocol/RobotRulesParser.java | 3 +-- .../org/apache/nutch/scoring/webgraph/LinkDumper.java | 4 +--- .../org/apache/nutch/scoring/webgraph/LinkRank.java | 3 +-- .../org/apache/nutch/scoring/webgraph/NodeDumper.java | 4 +--- .../apache/nutch/scoring/webgraph/ScoreUpdater.java | 4 +--- .../org/apache/nutch/scoring/webgraph/WebGraph.java | 3 +-- .../java/org/apache/nutch/segment/SegmentMerger.java | 4 +--- .../java/org/apache/nutch/segment/SegmentReader.java | 4 +--- .../java/org/apache/nutch/service/NutchServer.java | 2 +- .../main/java/org/apache/nutch/tools/Benchmark.java | 3 +-- .../org/apache/nutch/tools/CommonCrawlDataDumper.java | 15 +++++++-------- .../main/java/org/apache/nutch/tools/DmozParser.java | 4 ++-- .../java/org/apache/nutch/tools/FreeGenerator.java | 4 +--- .../org/apache/nutch/tools/arc/ArcSegmentCreator.java | 4 +--- .../java/org/apache/nutch/tools/warc/WARCExporter.java | 4 +--- .../main/java/org/apache/nutch/util/CommandRunner.java | 7 ++----- .../java/org/apache/nutch/util/EncodingDetector.java | 2 +- .../org/apache/nutch/tools/proxy/ProxyTestbed.java | 4 ++-- .../java/org/apache/nutch/parse/feed/FeedParser.java | 3 +-- nutch-plugins/index-replace/pom.xml | 4 ++-- .../org/apache/nutch/protocol/http/api/HttpBase.java | 5 ++--- .../java/org/apache/nutch/parse/zip/ZipParser.java | 2 +- nutch-plugins/pom.xml | 3 +-- .../main/java/org/apache/nutch/protocol/file/File.java | 7 ++----- .../main/java/org/apache/nutch/protocol/ftp/Ftp.java | 4 ++-- nutch-plugins/protocol-httpclient/pom.xml | 5 ----- .../net/urlnormalizer/basic/BasicURLNormalizer.java | 1 - .../net/urlnormalizer/regex/RegexURLNormalizer.java | 1 - .../urlnormalizer/regex/TestRegexURLNormalizer.java | 6 +++--- pom.xml | 1 + 52 files changed, 80 insertions(+), 147 deletions(-) diff --git a/.gitignore b/.gitignore index e0cfd33..68acee7 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,9 @@ nutch-plugins/*/target # IntelliJ Idea .idea -**.iml \ No newline at end of file +**.iml + +# Eclipse IDE +*.settings +*.project +*.classpath diff --git a/nutch-core/pom.xml b/nutch-core/pom.xml index 62e2e58..54d927d 100644 --- a/nutch-core/pom.xml +++ b/nutch-core/pom.xml @@ -28,7 +28,7 @@ <artifactId>nutch-core</artifactId> <packaging>jar</packaging> - <name>Apache Nutch</name> + <name>nutch-core</name> <description>Nutch is an open source web-search software. It builds on Hadoop, Tika and Solr, adding web-specifics, such as a crawler, a link-graph database etc. @@ -489,7 +489,7 @@ <resources> <resource> <directory>${project.build.directory}</directory> - <include>${build.finalName}.jar</include> + <include>${project.build.finalName}.jar</include> </resource> <resource> <directory>${project.basedir}</directory> diff --git a/nutch-core/src/main/java/org/apache/nutch/crawl/CrawlDb.java b/nutch-core/src/main/java/org/apache/nutch/crawl/CrawlDb.java index e6b8613..870a38e 100644 --- a/nutch-core/src/main/java/org/apache/nutch/crawl/CrawlDb.java +++ b/nutch-core/src/main/java/org/apache/nutch/crawl/CrawlDb.java @@ -195,8 +195,7 @@ public class CrawlDb extends NutchTool implements Tool { } public static void main(String[] args) throws Exception { - int res = ToolRunner.run(NutchConfiguration.create(), new CrawlDb(), args); - System.exit(res); + ToolRunner.run(NutchConfiguration.create(), new CrawlDb(), args); } public int run(String[] args) throws Exception { diff --git a/nutch-core/src/main/java/org/apache/nutch/crawl/CrawlDbMerger.java b/nutch-core/src/main/java/org/apache/nutch/crawl/CrawlDbMerger.java index e0c15f6..cc1ec03 100644 --- a/nutch-core/src/main/java/org/apache/nutch/crawl/CrawlDbMerger.java +++ b/nutch-core/src/main/java/org/apache/nutch/crawl/CrawlDbMerger.java @@ -172,9 +172,7 @@ public class CrawlDbMerger extends Configured implements Tool { * @param args */ public static void main(String[] args) throws Exception { - int res = ToolRunner.run(NutchConfiguration.create(), new CrawlDbMerger(), - args); - System.exit(res); + ToolRunner.run(NutchConfiguration.create(), new CrawlDbMerger(), args); } public int run(String[] args) throws Exception { diff --git a/nutch-core/src/main/java/org/apache/nutch/crawl/CrawlDbReader.java b/nutch-core/src/main/java/org/apache/nutch/crawl/CrawlDbReader.java index 9e2c145..6b110cc 100644 --- a/nutch-core/src/main/java/org/apache/nutch/crawl/CrawlDbReader.java +++ b/nutch-core/src/main/java/org/apache/nutch/crawl/CrawlDbReader.java @@ -764,9 +764,7 @@ public class CrawlDbReader extends Configured implements Closeable, Tool { } public static void main(String[] args) throws Exception { - int result = ToolRunner.run(NutchConfiguration.create(), - new CrawlDbReader(), args); - System.exit(result); + ToolRunner.run(NutchConfiguration.create(), new CrawlDbReader(), args); } public Object query(Map<String, String> args, Configuration conf, String type, String crawlId) throws Exception { diff --git a/nutch-core/src/main/java/org/apache/nutch/crawl/DeduplicationJob.java b/nutch-core/src/main/java/org/apache/nutch/crawl/DeduplicationJob.java index 70dfce5..adaf16d 100644 --- a/nutch-core/src/main/java/org/apache/nutch/crawl/DeduplicationJob.java +++ b/nutch-core/src/main/java/org/apache/nutch/crawl/DeduplicationJob.java @@ -366,9 +366,7 @@ public class DeduplicationJob extends NutchTool implements Tool { } public static void main(String[] args) throws Exception { - int result = ToolRunner.run(NutchConfiguration.create(), - new DeduplicationJob(), args); - System.exit(result); + ToolRunner.run(NutchConfiguration.create(), new DeduplicationJob(), args); } @Override diff --git a/nutch-core/src/main/java/org/apache/nutch/crawl/Generator.java b/nutch-core/src/main/java/org/apache/nutch/crawl/Generator.java index bce056b..cfcfce7 100644 --- a/nutch-core/src/main/java/org/apache/nutch/crawl/Generator.java +++ b/nutch-core/src/main/java/org/apache/nutch/crawl/Generator.java @@ -715,9 +715,7 @@ public class Generator extends NutchTool implements Tool { * Generate a fetchlist from the crawldb. */ public static void main(String args[]) throws Exception { - int res = ToolRunner - .run(NutchConfiguration.create(), new Generator(), args); - System.exit(res); + ToolRunner.run(NutchConfiguration.create(), new Generator(), args); } public int run(String[] args) throws Exception { diff --git a/nutch-core/src/main/java/org/apache/nutch/crawl/Injector.java b/nutch-core/src/main/java/org/apache/nutch/crawl/Injector.java index 7004146..00bb26e 100644 --- a/nutch-core/src/main/java/org/apache/nutch/crawl/Injector.java +++ b/nutch-core/src/main/java/org/apache/nutch/crawl/Injector.java @@ -441,8 +441,7 @@ public class Injector extends NutchTool implements Tool { } public static void main(String[] args) throws Exception { - int res = ToolRunner.run(NutchConfiguration.create(), new Injector(), args); - System.exit(res); + ToolRunner.run(NutchConfiguration.create(), new Injector(), args); } public int run(String[] args) throws Exception { diff --git a/nutch-core/src/main/java/org/apache/nutch/crawl/LinkDb.java b/nutch-core/src/main/java/org/apache/nutch/crawl/LinkDb.java index 0e56105..dc62f0f 100644 --- a/nutch-core/src/main/java/org/apache/nutch/crawl/LinkDb.java +++ b/nutch-core/src/main/java/org/apache/nutch/crawl/LinkDb.java @@ -294,8 +294,7 @@ public class LinkDb extends NutchTool implements Tool, } public static void main(String[] args) throws Exception { - int res = ToolRunner.run(NutchConfiguration.create(), new LinkDb(), args); - System.exit(res); + ToolRunner.run(NutchConfiguration.create(), new LinkDb(), args); } public int run(String[] args) throws Exception { diff --git a/nutch-core/src/main/java/org/apache/nutch/crawl/LinkDbMerger.java b/nutch-core/src/main/java/org/apache/nutch/crawl/LinkDbMerger.java index bc53059..c7c9907 100644 --- a/nutch-core/src/main/java/org/apache/nutch/crawl/LinkDbMerger.java +++ b/nutch-core/src/main/java/org/apache/nutch/crawl/LinkDbMerger.java @@ -164,9 +164,7 @@ public class LinkDbMerger extends Configured implements Tool, * @param args */ public static void main(String[] args) throws Exception { - int res = ToolRunner.run(NutchConfiguration.create(), new LinkDbMerger(), - args); - System.exit(res); + ToolRunner.run(NutchConfiguration.create(), new LinkDbMerger(), args); } public int run(String[] args) throws Exception { diff --git a/nutch-core/src/main/java/org/apache/nutch/crawl/LinkDbReader.java b/nutch-core/src/main/java/org/apache/nutch/crawl/LinkDbReader.java index 9a4a6c4..b49fe52 100644 --- a/nutch-core/src/main/java/org/apache/nutch/crawl/LinkDbReader.java +++ b/nutch-core/src/main/java/org/apache/nutch/crawl/LinkDbReader.java @@ -154,9 +154,7 @@ public class LinkDbReader extends Configured implements Tool, Closeable { } public static void main(String[] args) throws Exception { - int res = ToolRunner.run(NutchConfiguration.create(), new LinkDbReader(), - args); - System.exit(res); + ToolRunner.run(NutchConfiguration.create(), new LinkDbReader(), args); } public int run(String[] args) throws Exception { diff --git a/nutch-core/src/main/java/org/apache/nutch/fetcher/Fetcher.java b/nutch-core/src/main/java/org/apache/nutch/fetcher/Fetcher.java index 5339511..6e9cdfb 100644 --- a/nutch-core/src/main/java/org/apache/nutch/fetcher/Fetcher.java +++ b/nutch-core/src/main/java/org/apache/nutch/fetcher/Fetcher.java @@ -492,8 +492,7 @@ MapRunnable<Text, CrawlDatum, Text, NutchWritable> { /** Run the fetcher. */ public static void main(String[] args) throws Exception { - int res = ToolRunner.run(NutchConfiguration.create(), new Fetcher(), args); - System.exit(res); + ToolRunner.run(NutchConfiguration.create(), new Fetcher(), args); } public int run(String[] args) throws Exception { diff --git a/nutch-core/src/main/java/org/apache/nutch/hostdb/ReadHostDb.java b/nutch-core/src/main/java/org/apache/nutch/hostdb/ReadHostDb.java index ab3ec0c..778ebc1 100644 --- a/nutch-core/src/main/java/org/apache/nutch/hostdb/ReadHostDb.java +++ b/nutch-core/src/main/java/org/apache/nutch/hostdb/ReadHostDb.java @@ -201,8 +201,7 @@ public class ReadHostDb extends Configured implements Tool { } public static void main(String args[]) throws Exception { - int res = ToolRunner.run(NutchConfiguration.create(), new ReadHostDb(), args); - System.exit(res); + ToolRunner.run(NutchConfiguration.create(), new ReadHostDb(), args); } public int run(String[] args) throws Exception { diff --git a/nutch-core/src/main/java/org/apache/nutch/hostdb/UpdateHostDb.java b/nutch-core/src/main/java/org/apache/nutch/hostdb/UpdateHostDb.java index 519b51b..102ff9a 100644 --- a/nutch-core/src/main/java/org/apache/nutch/hostdb/UpdateHostDb.java +++ b/nutch-core/src/main/java/org/apache/nutch/hostdb/UpdateHostDb.java @@ -170,8 +170,7 @@ public class UpdateHostDb extends Configured implements Tool { } public static void main(String args[]) throws Exception { - int res = ToolRunner.run(NutchConfiguration.create(), new UpdateHostDb(), args); - System.exit(res); + ToolRunner.run(NutchConfiguration.create(), new UpdateHostDb(), args); } public int run(String[] args) throws Exception { diff --git a/nutch-core/src/main/java/org/apache/nutch/indexer/CleaningJob.java b/nutch-core/src/main/java/org/apache/nutch/indexer/CleaningJob.java index 6b3dc80..5a53b81 100644 --- a/nutch-core/src/main/java/org/apache/nutch/indexer/CleaningJob.java +++ b/nutch-core/src/main/java/org/apache/nutch/indexer/CleaningJob.java @@ -205,8 +205,6 @@ public class CleaningJob implements Tool { } public static void main(String[] args) throws Exception { - int result = ToolRunner.run(NutchConfiguration.create(), new CleaningJob(), - args); - System.exit(result); + ToolRunner.run(NutchConfiguration.create(), new CleaningJob(), args); } } diff --git a/nutch-core/src/main/java/org/apache/nutch/indexer/IndexingFiltersChecker.java b/nutch-core/src/main/java/org/apache/nutch/indexer/IndexingFiltersChecker.java index 05caf5a..6c7cfcd 100644 --- a/nutch-core/src/main/java/org/apache/nutch/indexer/IndexingFiltersChecker.java +++ b/nutch-core/src/main/java/org/apache/nutch/indexer/IndexingFiltersChecker.java @@ -23,6 +23,7 @@ import java.io.PrintWriter; import java.lang.invoke.MethodHandles; import java.net.ServerSocket; import java.net.Socket; +import java.net.BindException; import java.net.InetSocketAddress; import java.nio.charset.Charset; import java.util.HashMap; @@ -79,7 +80,7 @@ public class IndexingFiltersChecker extends Configured implements Tool { .getLogger(MethodHandles.lookup().lookupClass()); public IndexingFiltersChecker() { - + //default constructor } public int run(String[] args) throws Exception { @@ -113,8 +114,7 @@ public class IndexingFiltersChecker extends Configured implements Tool { k = nextOne; metadata.put(k, v); } else if (i != args.length - 1) { - System.err.println(usage); - System.exit(-1); + throw new IllegalArgumentException(usage); } else { url =args[i]; } @@ -143,8 +143,8 @@ public class IndexingFiltersChecker extends Configured implements Tool { server.bind(new InetSocketAddress(tcpPort)); LOG.info(server.toString()); } catch (Exception e) { - LOG.error("Could not listen on port " + tcpPort); - System.exit(-1); + LOG.error("Could not listen on port {}: {}" + tcpPort, e); + throw new BindException("Could not listen on port " + tcpPort); } while(true){ @@ -155,7 +155,7 @@ public class IndexingFiltersChecker extends Configured implements Tool { thread.start(); } catch (Exception e) { LOG.error("Accept failed: " + tcpPort); - System.exit(-1); + throw new RuntimeException(e); } } } @@ -193,7 +193,7 @@ public class IndexingFiltersChecker extends Configured implements Tool { in = new BufferedReader(new InputStreamReader(client.getInputStream())); } catch (Exception e) { LOG.error("in or out failed"); - System.exit(-1); + throw new RuntimeException("in or out failed: ", e); } try{ @@ -365,8 +365,6 @@ public class IndexingFiltersChecker extends Configured implements Tool { } public static void main(String[] args) throws Exception { - final int res = ToolRunner.run(NutchConfiguration.create(), - new IndexingFiltersChecker(), args); - System.exit(res); + ToolRunner.run(NutchConfiguration.create(), new IndexingFiltersChecker(), args); } } \ No newline at end of file diff --git a/nutch-core/src/main/java/org/apache/nutch/indexer/IndexingJob.java b/nutch-core/src/main/java/org/apache/nutch/indexer/IndexingJob.java index 0ab1846..7a9f681 100644 --- a/nutch-core/src/main/java/org/apache/nutch/indexer/IndexingJob.java +++ b/nutch-core/src/main/java/org/apache/nutch/indexer/IndexingJob.java @@ -236,9 +236,7 @@ public class IndexingJob extends NutchTool implements Tool { } public static void main(String[] args) throws Exception { - final int res = ToolRunner.run(NutchConfiguration.create(), - new IndexingJob(), args); - System.exit(res); + ToolRunner.run(NutchConfiguration.create(), new IndexingJob(), args); } diff --git a/nutch-core/src/main/java/org/apache/nutch/net/URLFilterChecker.java b/nutch-core/src/main/java/org/apache/nutch/net/URLFilterChecker.java index 89a3d00..472418a 100644 --- a/nutch-core/src/main/java/org/apache/nutch/net/URLFilterChecker.java +++ b/nutch-core/src/main/java/org/apache/nutch/net/URLFilterChecker.java @@ -110,14 +110,13 @@ public class URLFilterChecker { if (args.length == 0) { System.err.println(usage); - System.exit(-1); + throw new IllegalArgumentException(usage); } String filterName = null; if (args[0].equals("-filterName")) { if (args.length != 2) { - System.err.println(usage); - System.exit(-1); + throw new IllegalArgumentException(usage); } filterName = args[1]; } @@ -128,7 +127,5 @@ public class URLFilterChecker { } else { checker.checkAll(); } - - System.exit(0); } } diff --git a/nutch-core/src/main/java/org/apache/nutch/net/URLNormalizerChecker.java b/nutch-core/src/main/java/org/apache/nutch/net/URLNormalizerChecker.java index d8f1c6e..695a0fa 100644 --- a/nutch-core/src/main/java/org/apache/nutch/net/URLNormalizerChecker.java +++ b/nutch-core/src/main/java/org/apache/nutch/net/URLNormalizerChecker.java @@ -99,8 +99,7 @@ public class URLNormalizerChecker { } else if (args[i].equals("-scope")) { scope = args[++i]; } else { - System.err.println(usage); - System.exit(-1); + throw new IllegalArgumentException(usage); } } @@ -111,7 +110,5 @@ public class URLNormalizerChecker { } else { checker.checkAll(scope); } - - System.exit(0); } } diff --git a/nutch-core/src/main/java/org/apache/nutch/parse/ParsePluginsReader.java b/nutch-core/src/main/java/org/apache/nutch/parse/ParsePluginsReader.java index f0fb3a9..39da5c4 100644 --- a/nutch-core/src/main/java/org/apache/nutch/parse/ParsePluginsReader.java +++ b/nutch-core/src/main/java/org/apache/nutch/parse/ParsePluginsReader.java @@ -190,8 +190,7 @@ class ParsePluginsReader { if ((args.length != 0 && args.length != 2) || (args.length == 2 && !"--file".equals(args[0]))) { - System.err.println(usage); - System.exit(1); + throw new IllegalArgumentException(usage); } for (int i = 0; i < args.length; i++) { diff --git a/nutch-core/src/main/java/org/apache/nutch/parse/ParseSegment.java b/nutch-core/src/main/java/org/apache/nutch/parse/ParseSegment.java index 4963920..0c70027 100644 --- a/nutch-core/src/main/java/org/apache/nutch/parse/ParseSegment.java +++ b/nutch-core/src/main/java/org/apache/nutch/parse/ParseSegment.java @@ -237,9 +237,7 @@ public class ParseSegment extends NutchTool implements Tool, } public static void main(String[] args) throws Exception { - int res = ToolRunner.run(NutchConfiguration.create(), new ParseSegment(), - args); - System.exit(res); + ToolRunner.run(NutchConfiguration.create(), new ParseSegment(), args); } public int run(String[] args) throws Exception { @@ -248,8 +246,7 @@ public class ParseSegment extends NutchTool implements Tool, String usage = "Usage: ParseSegment segment [-noFilter] [-noNormalize]"; if (args.length == 0) { - System.err.println(usage); - System.exit(-1); + throw new IllegalArgumentException(usage); } if (args.length > 1) { diff --git a/nutch-core/src/main/java/org/apache/nutch/parse/ParserChecker.java b/nutch-core/src/main/java/org/apache/nutch/parse/ParserChecker.java index dbfa526..f2747de 100644 --- a/nutch-core/src/main/java/org/apache/nutch/parse/ParserChecker.java +++ b/nutch-core/src/main/java/org/apache/nutch/parse/ParserChecker.java @@ -111,7 +111,7 @@ public class ParserChecker implements Tool { metadata.put(k, v); } else if (i != args.length - 1) { LOG.error(usage); - System.exit(-1); + throw new IllegalArgumentException(usage); } else { url = URLUtil.toASCII(args[i]); } @@ -264,9 +264,7 @@ public class ParserChecker implements Tool { } public static void main(String[] args) throws Exception { - int res = ToolRunner.run(NutchConfiguration.create(), new ParserChecker(), - args); - System.exit(res); + ToolRunner.run(NutchConfiguration.create(), new ParserChecker(), args); } } diff --git a/nutch-core/src/main/java/org/apache/nutch/protocol/RobotRulesParser.java b/nutch-core/src/main/java/org/apache/nutch/protocol/RobotRulesParser.java index 2597147..09a856f 100644 --- a/nutch-core/src/main/java/org/apache/nutch/protocol/RobotRulesParser.java +++ b/nutch-core/src/main/java/org/apache/nutch/protocol/RobotRulesParser.java @@ -421,8 +421,7 @@ public abstract class RobotRulesParser implements Tool { public static void main(String[] args) throws Exception { Configuration conf = NutchConfiguration.create(); - int res = ToolRunner.run(conf, new TestRobotRulesParser(conf), args); - System.exit(res); + ToolRunner.run(conf, new TestRobotRulesParser(conf), args); } } diff --git a/nutch-core/src/main/java/org/apache/nutch/scoring/webgraph/LinkDumper.java b/nutch-core/src/main/java/org/apache/nutch/scoring/webgraph/LinkDumper.java index 7c83966..512ecf4 100644 --- a/nutch-core/src/main/java/org/apache/nutch/scoring/webgraph/LinkDumper.java +++ b/nutch-core/src/main/java/org/apache/nutch/scoring/webgraph/LinkDumper.java @@ -391,9 +391,7 @@ public class LinkDumper extends Configured implements Tool { } public static void main(String[] args) throws Exception { - int res = ToolRunner.run(NutchConfiguration.create(), new LinkDumper(), - args); - System.exit(res); + ToolRunner.run(NutchConfiguration.create(), new LinkDumper(), args); } /** diff --git a/nutch-core/src/main/java/org/apache/nutch/scoring/webgraph/LinkRank.java b/nutch-core/src/main/java/org/apache/nutch/scoring/webgraph/LinkRank.java index 6676b7f..956f559 100644 --- a/nutch-core/src/main/java/org/apache/nutch/scoring/webgraph/LinkRank.java +++ b/nutch-core/src/main/java/org/apache/nutch/scoring/webgraph/LinkRank.java @@ -636,8 +636,7 @@ public class LinkRank extends Configured implements Tool { } public static void main(String[] args) throws Exception { - int res = ToolRunner.run(NutchConfiguration.create(), new LinkRank(), args); - System.exit(res); + ToolRunner.run(NutchConfiguration.create(), new LinkRank(), args); } /** diff --git a/nutch-core/src/main/java/org/apache/nutch/scoring/webgraph/NodeDumper.java b/nutch-core/src/main/java/org/apache/nutch/scoring/webgraph/NodeDumper.java index 32ee1e7..7fee74d 100644 --- a/nutch-core/src/main/java/org/apache/nutch/scoring/webgraph/NodeDumper.java +++ b/nutch-core/src/main/java/org/apache/nutch/scoring/webgraph/NodeDumper.java @@ -323,9 +323,7 @@ public class NodeDumper extends Configured implements Tool { } public static void main(String[] args) throws Exception { - int res = ToolRunner.run(NutchConfiguration.create(), new NodeDumper(), - args); - System.exit(res); + ToolRunner.run(NutchConfiguration.create(), new NodeDumper(), args); } /** diff --git a/nutch-core/src/main/java/org/apache/nutch/scoring/webgraph/ScoreUpdater.java b/nutch-core/src/main/java/org/apache/nutch/scoring/webgraph/ScoreUpdater.java index 3c86cf6..797e24d 100644 --- a/nutch-core/src/main/java/org/apache/nutch/scoring/webgraph/ScoreUpdater.java +++ b/nutch-core/src/main/java/org/apache/nutch/scoring/webgraph/ScoreUpdater.java @@ -204,9 +204,7 @@ public class ScoreUpdater extends Configured implements Tool, } public static void main(String[] args) throws Exception { - int res = ToolRunner.run(NutchConfiguration.create(), new ScoreUpdater(), - args); - System.exit(res); + ToolRunner.run(NutchConfiguration.create(), new ScoreUpdater(), args); } /** diff --git a/nutch-core/src/main/java/org/apache/nutch/scoring/webgraph/WebGraph.java b/nutch-core/src/main/java/org/apache/nutch/scoring/webgraph/WebGraph.java index b02c287..e67ebfd 100644 --- a/nutch-core/src/main/java/org/apache/nutch/scoring/webgraph/WebGraph.java +++ b/nutch-core/src/main/java/org/apache/nutch/scoring/webgraph/WebGraph.java @@ -690,8 +690,7 @@ public class WebGraph extends Configured implements Tool { } public static void main(String[] args) throws Exception { - int res = ToolRunner.run(NutchConfiguration.create(), new WebGraph(), args); - System.exit(res); + ToolRunner.run(NutchConfiguration.create(), new WebGraph(), args); } /** diff --git a/nutch-core/src/main/java/org/apache/nutch/segment/SegmentMerger.java b/nutch-core/src/main/java/org/apache/nutch/segment/SegmentMerger.java index 6c53bf2..99c439b 100644 --- a/nutch-core/src/main/java/org/apache/nutch/segment/SegmentMerger.java +++ b/nutch-core/src/main/java/org/apache/nutch/segment/SegmentMerger.java @@ -786,9 +786,7 @@ public class SegmentMerger extends Configured implements Tool, } public static void main(String[] args) throws Exception { - int result = ToolRunner.run(NutchConfiguration.create(), - new SegmentMerger(), args); - System.exit(result); + ToolRunner.run(NutchConfiguration.create(), new SegmentMerger(), args); } } diff --git a/nutch-core/src/main/java/org/apache/nutch/segment/SegmentReader.java b/nutch-core/src/main/java/org/apache/nutch/segment/SegmentReader.java index 7a238d1..534d78a 100644 --- a/nutch-core/src/main/java/org/apache/nutch/segment/SegmentReader.java +++ b/nutch-core/src/main/java/org/apache/nutch/segment/SegmentReader.java @@ -732,8 +732,6 @@ public class SegmentReader extends Configured implements Tool, } public static void main(String[] args) throws Exception { - int result = ToolRunner.run(NutchConfiguration.create(), - new SegmentReader(), args); - System.exit(result); + ToolRunner.run(NutchConfiguration.create(), new SegmentReader(), args); } } diff --git a/nutch-core/src/main/java/org/apache/nutch/service/NutchServer.java b/nutch-core/src/main/java/org/apache/nutch/service/NutchServer.java index 335977e..f5288ee 100644 --- a/nutch-core/src/main/java/org/apache/nutch/service/NutchServer.java +++ b/nutch-core/src/main/java/org/apache/nutch/service/NutchServer.java @@ -228,6 +228,6 @@ public class NutchServer { } public void stop() { - System.exit(0); + Runtime.getRuntime().exit(0); } } diff --git a/nutch-core/src/main/java/org/apache/nutch/tools/Benchmark.java b/nutch-core/src/main/java/org/apache/nutch/tools/Benchmark.java index ff28c1f..b2c40ac 100755 --- a/nutch-core/src/main/java/org/apache/nutch/tools/Benchmark.java +++ b/nutch-core/src/main/java/org/apache/nutch/tools/Benchmark.java @@ -49,8 +49,7 @@ public class Benchmark extends Configured implements Tool { public static void main(String[] args) throws Exception { Configuration conf = NutchConfiguration.create(); - int res = ToolRunner.run(conf, new Benchmark(), args); - System.exit(res); + ToolRunner.run(conf, new Benchmark(), args); } @SuppressWarnings("unused") diff --git a/nutch-core/src/main/java/org/apache/nutch/tools/CommonCrawlDataDumper.java b/nutch-core/src/main/java/org/apache/nutch/tools/CommonCrawlDataDumper.java index 216c503..cdce17d 100644 --- a/nutch-core/src/main/java/org/apache/nutch/tools/CommonCrawlDataDumper.java +++ b/nutch-core/src/main/java/org/apache/nutch/tools/CommonCrawlDataDumper.java @@ -206,8 +206,7 @@ public class CommonCrawlDataDumper extends Configured implements Tool { */ public static void main(String[] args) throws Exception { Configuration conf = NutchConfiguration.create(); - int res = ToolRunner.run(conf, new CommonCrawlDataDumper(), args); - System.exit(res); + ToolRunner.run(conf, new CommonCrawlDataDumper(), args); } /** @@ -270,10 +269,11 @@ public class CommonCrawlDataDumper extends Configured implements Tool { if (linkdb != null) { linkDbReader = new LinkDbReader(fs.getConf(), new Path(linkdb.toString())); } - if (parts == null || parts.size() == 0) { - LOG.error( "No segment directories found in {} ", + if (parts == null || parts.isEmpty()) { + LOG.error("No segment directories found in {} ", segmentRootDir.getAbsolutePath()); - System.exit(1); + throw new IllegalArgumentException("No segment directories found in " + + segmentRootDir.getAbsolutePath()); } LOG.info("Found {} segment parts", parts.size()); if (gzip && !warc) { @@ -283,9 +283,8 @@ public class CommonCrawlDataDumper extends Configured implements Tool { for (Path segmentPart : parts) { LOG.info("Processing segment Part : [ {} ]", segmentPart); - try { - SequenceFile.Reader reader = new SequenceFile.Reader(nutchConfig, - SequenceFile.Reader.file(segmentPart)); + try (SequenceFile.Reader reader = new SequenceFile.Reader(nutchConfig, + SequenceFile.Reader.file(segmentPart))){ Writable key = (Writable) reader.getKeyClass().newInstance(); diff --git a/nutch-core/src/main/java/org/apache/nutch/tools/DmozParser.java b/nutch-core/src/main/java/org/apache/nutch/tools/DmozParser.java index ca95809..fdbb423 100644 --- a/nutch-core/src/main/java/org/apache/nutch/tools/DmozParser.java +++ b/nutch-core/src/main/java/org/apache/nutch/tools/DmozParser.java @@ -296,7 +296,7 @@ public class DmozParser { if (LOG.isErrorEnabled()) { LOG.error(e.toString()); } - System.exit(0); + throw new RuntimeException(e); } } @@ -312,7 +312,7 @@ public class DmozParser { if (LOG.isErrorEnabled()) { LOG.error(e.toString()); } - System.exit(0); + throw new RuntimeException(e); } } diff --git a/nutch-core/src/main/java/org/apache/nutch/tools/FreeGenerator.java b/nutch-core/src/main/java/org/apache/nutch/tools/FreeGenerator.java index 596fe7d..dfab246 100644 --- a/nutch-core/src/main/java/org/apache/nutch/tools/FreeGenerator.java +++ b/nutch-core/src/main/java/org/apache/nutch/tools/FreeGenerator.java @@ -208,8 +208,6 @@ public class FreeGenerator extends Configured implements Tool { } public static void main(String[] args) throws Exception { - int res = ToolRunner.run(NutchConfiguration.create(), new FreeGenerator(), - args); - System.exit(res); + ToolRunner.run(NutchConfiguration.create(), new FreeGenerator(), args); } } diff --git a/nutch-core/src/main/java/org/apache/nutch/tools/arc/ArcSegmentCreator.java b/nutch-core/src/main/java/org/apache/nutch/tools/arc/ArcSegmentCreator.java index 9277991..1546f45 100644 --- a/nutch-core/src/main/java/org/apache/nutch/tools/arc/ArcSegmentCreator.java +++ b/nutch-core/src/main/java/org/apache/nutch/tools/arc/ArcSegmentCreator.java @@ -397,9 +397,7 @@ public class ArcSegmentCreator extends Configured implements Tool, } public static void main(String args[]) throws Exception { - int res = ToolRunner.run(NutchConfiguration.create(), - new ArcSegmentCreator(), args); - System.exit(res); + ToolRunner.run(NutchConfiguration.create(), new ArcSegmentCreator(), args); } public int run(String[] args) throws Exception { diff --git a/nutch-core/src/main/java/org/apache/nutch/tools/warc/WARCExporter.java b/nutch-core/src/main/java/org/apache/nutch/tools/warc/WARCExporter.java index 0d8ec40..479a35a 100644 --- a/nutch-core/src/main/java/org/apache/nutch/tools/warc/WARCExporter.java +++ b/nutch-core/src/main/java/org/apache/nutch/tools/warc/WARCExporter.java @@ -328,8 +328,6 @@ public class WARCExporter extends Configured implements Tool { } public static void main(String[] args) throws Exception { - final int res = ToolRunner.run(NutchConfiguration.create(), - new WARCExporter(), args); - System.exit(res); + ToolRunner.run(NutchConfiguration.create(), new WARCExporter(), args); } } diff --git a/nutch-core/src/main/java/org/apache/nutch/util/CommandRunner.java b/nutch-core/src/main/java/org/apache/nutch/util/CommandRunner.java index 593d590..f390cbf 100644 --- a/nutch-core/src/main/java/org/apache/nutch/util/CommandRunner.java +++ b/nutch-core/src/main/java/org/apache/nutch/util/CommandRunner.java @@ -258,17 +258,14 @@ public class CommandRunner { String usage = "Usage: CommandRunner [-timeout timeoutSecs] commandPath filePath"; if (args.length < 2) { - System.err.println(usage); - System.exit(-1); + throw new IllegalArgumentException(usage); } for (int i = 0; i < args.length; i++) { if (args[i].equals("-timeout")) { timeout = Integer.parseInt(args[++i]); - ; } else if (i != args.length - 2) { - System.err.println(usage); - System.exit(-1); + throw new IllegalArgumentException(usage); } else { commandPath = args[i]; filePath = args[++i]; diff --git a/nutch-core/src/main/java/org/apache/nutch/util/EncodingDetector.java b/nutch-core/src/main/java/org/apache/nutch/util/EncodingDetector.java index 87b26cb..e542276 100644 --- a/nutch-core/src/main/java/org/apache/nutch/util/EncodingDetector.java +++ b/nutch-core/src/main/java/org/apache/nutch/util/EncodingDetector.java @@ -347,7 +347,7 @@ public class EncodingDetector { public static void main(String[] args) throws IOException { if (args.length != 1) { System.err.println("Usage: EncodingDetector <file>"); - System.exit(1); + throw new IllegalArgumentException("Usage: EncodingDetector <file>"); } Configuration conf = NutchConfiguration.create(); diff --git a/nutch-core/src/test/java/org/apache/nutch/tools/proxy/ProxyTestbed.java b/nutch-core/src/test/java/org/apache/nutch/tools/proxy/ProxyTestbed.java index a6b2d8e..892270c 100644 --- a/nutch-core/src/test/java/org/apache/nutch/tools/proxy/ProxyTestbed.java +++ b/nutch-core/src/test/java/org/apache/nutch/tools/proxy/ProxyTestbed.java @@ -63,7 +63,7 @@ public class ProxyTestbed { .println("-delay\tdelay every response by nnn seconds. If delay is negative use a random value up to nnn"); System.err .println("-fake\tif specified, requests to all unknown urls will succeed with fake content"); - System.exit(-1); + throw new IllegalArgumentException(); } Configuration conf = NutchConfiguration.create(); @@ -96,7 +96,7 @@ public class ProxyTestbed { segs.add(new Path(args[++i])); } else { LOG.error("Unknown argument: " + args[i]); - System.exit(-1); + throw new IllegalArgumentException("Unknown argument: " + args[i]); } } diff --git a/nutch-plugins/feed/src/main/java/org/apache/nutch/parse/feed/FeedParser.java b/nutch-plugins/feed/src/main/java/org/apache/nutch/parse/feed/FeedParser.java index 0936187..2e67117 100644 --- a/nutch-plugins/feed/src/main/java/org/apache/nutch/parse/feed/FeedParser.java +++ b/nutch-plugins/feed/src/main/java/org/apache/nutch/parse/feed/FeedParser.java @@ -23,7 +23,6 @@ import java.io.DataInputStream; import java.io.File; import java.io.FileInputStream; import java.util.Date; -import java.util.Iterator; import java.util.List; import java.util.Map.Entry; @@ -198,7 +197,7 @@ public class FeedParser implements Parser { public static void main(String[] args) throws Exception { if (args.length != 1) { System.err.println("Usage: FeedParser <feed>"); - System.exit(1); + throw new IllegalArgumentException("Usage: FeedParser <feed>"); } String name = args[0]; String url = "file:" + name; diff --git a/nutch-plugins/index-replace/pom.xml b/nutch-plugins/index-replace/pom.xml index d39851d..08fdba7 100644 --- a/nutch-plugins/index-replace/pom.xml +++ b/nutch-plugins/index-replace/pom.xml @@ -38,12 +38,12 @@ <dependency> <groupId>org.apache.nutch</groupId> <artifactId>index-basic</artifactId> - <version>${parent.version}</version> + <version>${project.parent.version}</version> </dependency> <dependency> <groupId>org.apache.nutch</groupId> <artifactId>index-metadata</artifactId> - <version>${parent.version}</version> + <version>${project.parent.version}</version> </dependency> </dependencies> diff --git a/nutch-plugins/lib-http/src/main/java/org/apache/nutch/protocol/http/api/HttpBase.java b/nutch-plugins/lib-http/src/main/java/org/apache/nutch/protocol/http/api/HttpBase.java index eb3eb60..f18060c 100644 --- a/nutch-plugins/lib-http/src/main/java/org/apache/nutch/protocol/http/api/HttpBase.java +++ b/nutch-plugins/lib-http/src/main/java/org/apache/nutch/protocol/http/api/HttpBase.java @@ -539,7 +539,7 @@ public abstract class HttpBase implements Protocol { if (args.length == 0) { System.err.println(usage); - System.exit(-1); + throw new IllegalArgumentException(usage); } for (int i = 0; i < args.length; i++) { // parse command line @@ -547,8 +547,7 @@ public abstract class HttpBase implements Protocol { http.timeout = Integer.parseInt(args[++i]) * 1000; } else if (args[i].equals("-verbose")) { // found -verbose option } else if (i != args.length - 1) { - System.err.println(usage); - System.exit(-1); + throw new IllegalArgumentException(usage); } else // root is required parameter url = args[i]; diff --git a/nutch-plugins/parse-zip/src/main/java/org/apache/nutch/parse/zip/ZipParser.java b/nutch-plugins/parse-zip/src/main/java/org/apache/nutch/parse/zip/ZipParser.java index 25a52a1..940c2a7 100644 --- a/nutch-plugins/parse-zip/src/main/java/org/apache/nutch/parse/zip/ZipParser.java +++ b/nutch-plugins/parse-zip/src/main/java/org/apache/nutch/parse/zip/ZipParser.java @@ -121,7 +121,7 @@ public class ZipParser implements Parser { public static void main(String[] args) throws IOException { if (args.length < 1) { System.out.println("ZipParser <zip_file>"); - System.exit(1); + throw new IllegalArgumentException("ZipParser <zip_file>"); } File file = new File(args[0]); String url = "file:"+file.getCanonicalPath(); diff --git a/nutch-plugins/pom.xml b/nutch-plugins/pom.xml index 0fc29e1..864219d 100644 --- a/nutch-plugins/pom.xml +++ b/nutch-plugins/pom.xml @@ -32,7 +32,6 @@ <url>http://nutch.apache.org</url> <modules> - <!--<module>indexer-solr</module>--> <module>creativecommons</module> <module>feed</module> <module>headings</module> @@ -140,7 +139,7 @@ <resources> <resource> <directory>${project.build.directory}</directory> - <include>${build.finalName}.jar</include> + <include>${project.build.finalName}.jar</include> </resource> <resource> <directory>${project.basedir}</directory> diff --git a/nutch-plugins/protocol-file/src/main/java/org/apache/nutch/protocol/file/File.java b/nutch-plugins/protocol-file/src/main/java/org/apache/nutch/protocol/file/File.java index 8a415b6..d1a5208 100644 --- a/nutch-plugins/protocol-file/src/main/java/org/apache/nutch/protocol/file/File.java +++ b/nutch-plugins/protocol-file/src/main/java/org/apache/nutch/protocol/file/File.java @@ -171,7 +171,7 @@ public class File implements Protocol { if (args.length == 0) { System.err.println(usage); - System.exit(-1); + throw new IllegalArgumentException(usage); } for (int i = 0; i < args.length; i++) { @@ -181,7 +181,7 @@ public class File implements Protocol { dumpContent = true; } else if (i != args.length - 1) { System.err.println(usage); - System.exit(-1); + throw new IllegalArgumentException(usage); } else urlString = args[i]; } @@ -192,9 +192,6 @@ public class File implements Protocol { if (maxContentLength != Integer.MIN_VALUE) // set maxContentLength file.setMaxContentLength(maxContentLength); - // set log level - // LOG.setLevel(Level.parse((new String(logLevel)).toUpperCase())); - ProtocolOutput output = file.getProtocolOutput(new Text(urlString), new CrawlDatum()); Content content = output.getContent(); diff --git a/nutch-plugins/protocol-ftp/src/main/java/org/apache/nutch/protocol/ftp/Ftp.java b/nutch-plugins/protocol-ftp/src/main/java/org/apache/nutch/protocol/ftp/Ftp.java index 84aa823..6490621 100644 --- a/nutch-plugins/protocol-ftp/src/main/java/org/apache/nutch/protocol/ftp/Ftp.java +++ b/nutch-plugins/protocol-ftp/src/main/java/org/apache/nutch/protocol/ftp/Ftp.java @@ -181,7 +181,7 @@ public class Ftp implements Protocol { if (args.length == 0) { System.err.println(usage); - System.exit(-1); + throw new IllegalArgumentException(usage); } for (int i = 0; i < args.length; i++) { @@ -199,7 +199,7 @@ public class Ftp implements Protocol { dumpContent = true; } else if (i != args.length - 1) { System.err.println(usage); - System.exit(-1); + throw new IllegalArgumentException(usage); } else { urlString = args[i]; } diff --git a/nutch-plugins/protocol-httpclient/pom.xml b/nutch-plugins/protocol-httpclient/pom.xml index 4fdac6c..fea1da5 100644 --- a/nutch-plugins/protocol-httpclient/pom.xml +++ b/nutch-plugins/protocol-httpclient/pom.xml @@ -55,11 +55,6 @@ <version>${project.parent.version}</version> </dependency> <dependency> - <groupId>org.apache.nutch</groupId> - <artifactId>lib-http</artifactId> - <version>${project.parent.version}</version> - </dependency> - <dependency> <groupId> org.mortbay.jetty</groupId> <artifactId>jetty</artifactId> <version>${jetty.version}</version> diff --git a/nutch-plugins/urlnormalizer-basic/src/main/java/org/apache/nutch/net/urlnormalizer/basic/BasicURLNormalizer.java b/nutch-plugins/urlnormalizer-basic/src/main/java/org/apache/nutch/net/urlnormalizer/basic/BasicURLNormalizer.java index ffd22ce..0d97136 100644 --- a/nutch-plugins/urlnormalizer-basic/src/main/java/org/apache/nutch/net/urlnormalizer/basic/BasicURLNormalizer.java +++ b/nutch-plugins/urlnormalizer-basic/src/main/java/org/apache/nutch/net/urlnormalizer/basic/BasicURLNormalizer.java @@ -293,7 +293,6 @@ public class BasicURLNormalizer extends Configured implements URLNormalizer { System.out.println("failed: " + line); } } - System.exit(0); } } diff --git a/nutch-plugins/urlnormalizer-regex/src/main/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java b/nutch-plugins/urlnormalizer-regex/src/main/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java index 7626f60..4794347 100644 --- a/nutch-plugins/urlnormalizer-regex/src/main/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java +++ b/nutch-plugins/urlnormalizer-regex/src/main/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java @@ -319,7 +319,6 @@ public class RegexURLNormalizer extends Configured implements URLNormalizer { System.out.println("Output url: '" + normalizer.normalize(args[0], scope) + "'"); } - System.exit(0); } } diff --git a/nutch-plugins/urlnormalizer-regex/src/test/java/org/apache/nutch/net/urlnormalizer/regex/TestRegexURLNormalizer.java b/nutch-plugins/urlnormalizer-regex/src/test/java/org/apache/nutch/net/urlnormalizer/regex/TestRegexURLNormalizer.java index 2afbd75..577fb42 100644 --- a/nutch-plugins/urlnormalizer-regex/src/test/java/org/apache/nutch/net/urlnormalizer/regex/TestRegexURLNormalizer.java +++ b/nutch-plugins/urlnormalizer-regex/src/test/java/org/apache/nutch/net/urlnormalizer/regex/TestRegexURLNormalizer.java @@ -149,7 +149,7 @@ public class TestRegexURLNormalizer { public static void main(String[] args) throws Exception { if (args.length == 0) { System.err.println("TestRegexURLNormalizer [-bench <iter>] <scope>"); - System.exit(-1); + throw new IllegalArgumentException("TestRegexURLNormalizer [-bench <iter>] <scope>"); } boolean bench = false; int iter = -1; @@ -163,11 +163,11 @@ public class TestRegexURLNormalizer { } if (scope == null) { System.err.println("Missing required scope name."); - System.exit(-1); + throw new IllegalArgumentException("Missing required scope name."); } if (bench && iter < 0) { System.err.println("Invalid number of iterations: " + iter); - System.exit(-1); + throw new IllegalArgumentException("Invalid number of iterations: " + iter); } TestRegexURLNormalizer test = new TestRegexURLNormalizer(); NormalizedURL[] urls = (NormalizedURL[]) test.testData.get(scope); diff --git a/pom.xml b/pom.xml index ff2147a..949c474 100644 --- a/pom.xml +++ b/pom.xml @@ -41,6 +41,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> + <version>3.6.1</version> <configuration> <source>1.8</source> <target>1.8</target> -- To stop receiving notification emails like this one, please contact "[email protected]" <[email protected]>.
