This is an automated email from the ASF dual-hosted git repository.
lewismc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
The following commit(s) were added to refs/heads/master by this push:
new 5250d62 NUTCH-2851 Random object created and used only once (#571)
5250d62 is described below
commit 5250d62986468b23509a82d2aaa32bdc11cf02a8
Author: Lewis John McGibbney <lewis.mcgibb...@gmail.com>
AuthorDate: Thu Feb 18 07:20:59 2021 -0800
NUTCH-2851 Random object created and used only once (#571)
---
src/java/org/apache/nutch/crawl/Generator.java | 5 +++--
src/java/org/apache/nutch/indexer/IndexingJob.java | 4 +++-
src/java/org/apache/nutch/segment/SegmentReader.java | 5 ++++-
src/java/org/apache/nutch/tools/DmozParser.java | 5 ++++-
4 files changed, 14 insertions(+), 5 deletions(-)
diff --git a/src/java/org/apache/nutch/crawl/Generator.java
b/src/java/org/apache/nutch/crawl/Generator.java
index dcba9bf..00eb18f 100644
--- a/src/java/org/apache/nutch/crawl/Generator.java
+++ b/src/java/org/apache/nutch/crawl/Generator.java
@@ -35,7 +35,6 @@ import org.apache.hadoop.conf.Configurable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.commons.jexl3.JexlExpression;
-import org.antlr.v4.parse.ANTLRParser.throwsSpec_return;
import org.apache.commons.jexl3.JexlContext;
import org.apache.commons.jexl3.MapContext;
import org.apache.hadoop.mapreduce.Counter;
@@ -90,6 +89,8 @@ import org.apache.nutch.util.URLUtil;
**/
public class Generator extends NutchTool implements Tool {
+ private static final Random RANDOM = new Random();
+
protected static final Logger LOG = LoggerFactory
.getLogger(MethodHandles.lookup().lookupClass());
@@ -1013,7 +1014,7 @@ public class Generator extends NutchTool implements Tool {
Job job = NutchJob.getInstance(getConf());
job.setJobName("generate: partition " + segment);
Configuration conf = job.getConfiguration();
- conf.setInt("partition.url.seed", new Random().nextInt());
+ conf.setInt("partition.url.seed", RANDOM.nextInt());
FileInputFormat.addInputPath(job, inputDir);
job.setInputFormatClass(SequenceFileInputFormat.class);
diff --git a/src/java/org/apache/nutch/indexer/IndexingJob.java
b/src/java/org/apache/nutch/indexer/IndexingJob.java
index 0966276..0fe29a7 100644
--- a/src/java/org/apache/nutch/indexer/IndexingJob.java
+++ b/src/java/org/apache/nutch/indexer/IndexingJob.java
@@ -54,6 +54,8 @@ import org.slf4j.LoggerFactory;
public class IndexingJob extends NutchTool implements Tool {
+ private static final Random RANDOM = new Random();
+
private static final Logger LOG = LoggerFactory
.getLogger(MethodHandles.lookup().lookupClass());
@@ -136,7 +138,7 @@ public class IndexingJob extends NutchTool implements Tool {
job.setReduceSpeculativeExecution(false);
final Path tmp = new Path("tmp_" + System.currentTimeMillis() + "-"
- + new Random().nextInt());
+ + RANDOM.nextInt());
FileOutputFormat.setOutputPath(job, tmp);
try {
diff --git a/src/java/org/apache/nutch/segment/SegmentReader.java
b/src/java/org/apache/nutch/segment/SegmentReader.java
index 284daed..2f2fefd 100644
--- a/src/java/org/apache/nutch/segment/SegmentReader.java
+++ b/src/java/org/apache/nutch/segment/SegmentReader.java
@@ -35,6 +35,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Random;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -74,6 +75,8 @@ import org.apache.nutch.util.SegmentReaderUtil;
/** Dump the content of a segment. */
public class SegmentReader extends Configured implements Tool {
+ private static final Random RANDOM = new Random();
+
private static final Logger LOG = LoggerFactory
.getLogger(MethodHandles.lookup().lookupClass());
@@ -220,7 +223,7 @@ public class SegmentReader extends Configured implements
Tool {
job.setJarByClass(SegmentReader.class);
Path tempDir = new Path(conf.get("hadoop.tmp.dir", "/tmp") + "/segread-"
- + new java.util.Random().nextInt());
+ + RANDOM.nextInt());
FileSystem fs = tempDir.getFileSystem(conf);
fs.delete(tempDir, true);
diff --git a/src/java/org/apache/nutch/tools/DmozParser.java
b/src/java/org/apache/nutch/tools/DmozParser.java
index b68facb..8db4817 100644
--- a/src/java/org/apache/nutch/tools/DmozParser.java
+++ b/src/java/org/apache/nutch/tools/DmozParser.java
@@ -54,6 +54,9 @@ import org.apache.nutch.util.NutchConfiguration;
* RDF into a flat file of URLs to be injected.
*/
public class DmozParser {
+
+ private static final Random RANDOM = new Random();
+
private static final Logger LOG = LoggerFactory
.getLogger(MethodHandles.lookup().lookupClass());
@@ -134,7 +137,7 @@ public class DmozParser {
this.includeAdult = includeAdult;
this.topicPattern = topicPattern;
- this.hashSkew = skew != 0 ? skew : new Random().nextInt();
+ this.hashSkew = skew != 0 ? skew : RANDOM.nextInt();
}
//
