[
https://issues.apache.org/jira/browse/NUTCH-2441?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16330855#comment-16330855
]
ASF GitHub Bot commented on NUTCH-2441:
---------------------------------------
lewismc closed pull request #250: fix for NUTCH-2441 ARG_SEGMENT fix for REST
API
URL: https://github.com/apache/nutch/pull/250
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/src/java/org/apache/nutch/crawl/CrawlDb.java
b/src/java/org/apache/nutch/crawl/CrawlDb.java
index 9f37447b3..6235d8496 100644
--- a/src/java/org/apache/nutch/crawl/CrawlDb.java
+++ b/src/java/org/apache/nutch/crawl/CrawlDb.java
@@ -328,13 +328,16 @@ public int run(String[] args) throws Exception {
HadoopFSUtil.getPassDirectoriesFilter(fs));
dirs.addAll(Arrays.asList(HadoopFSUtil.getPaths(paths)));
}
-
- else if(args.containsKey(Nutch.ARG_SEGMENT)) {
- Object segments = args.get(Nutch.ARG_SEGMENT);
- ArrayList<String> segmentList = new ArrayList<>();
+ else if(args.containsKey(Nutch.ARG_SEGMENTS)) {
+ Object segments = args.get(Nutch.ARG_SEGMENTS);
+ ArrayList<String> segmentList = new ArrayList<String>();
if(segments instanceof ArrayList) {
- segmentList = (ArrayList<String>)segments;
+ segmentList = (ArrayList<String>)segments;
+ }
+ else if(segments instanceof Path){
+ segmentList.add(segments.toString());
}
+
for(String segment: segmentList) {
dirs.add(new Path(segment));
}
diff --git a/src/java/org/apache/nutch/crawl/LinkDb.java
b/src/java/org/apache/nutch/crawl/LinkDb.java
index b60398a1d..e9a0b0815 100644
--- a/src/java/org/apache/nutch/crawl/LinkDb.java
+++ b/src/java/org/apache/nutch/crawl/LinkDb.java
@@ -393,16 +393,19 @@ public int run(String[] args) throws Exception {
HadoopFSUtil.getPassDirectoriesFilter(fs));
segs.addAll(Arrays.asList(HadoopFSUtil.getPaths(paths)));
}
- else if(args.containsKey(Nutch.ARG_SEGMENT)) {
- Object segments = args.get(Nutch.ARG_SEGMENT);
- ArrayList<String> segmentList = new ArrayList<>();
+ else if(args.containsKey(Nutch.ARG_SEGMENTS)) {
+ Object segments = args.get(Nutch.ARG_SEGMENTS);
+ ArrayList<String> segmentList = new ArrayList<String>();
if(segments instanceof ArrayList) {
- segmentList = (ArrayList<String>)segments;
+ segmentList = (ArrayList<String>)segments; }
+ else if(segments instanceof Path){
+ segmentList.add(segments.toString());
}
+
for(String segment: segmentList) {
- segs.add(new Path(segment));
+ segs.add(new Path(segment));
}
- }
+ }
else {
String segment_dir = crawlId+"/segments";
File dir = new File(segment_dir);
@@ -425,4 +428,4 @@ else if(args.containsKey(Nutch.ARG_SEGMENT)) {
return results;
}
}
-}
\ No newline at end of file
+}
diff --git a/src/java/org/apache/nutch/fetcher/Fetcher.java
b/src/java/org/apache/nutch/fetcher/Fetcher.java
index 5339511d4..d93486c1d 100644
--- a/src/java/org/apache/nutch/fetcher/Fetcher.java
+++ b/src/java/org/apache/nutch/fetcher/Fetcher.java
@@ -549,15 +549,23 @@ private AtomicInteger getActiveThreads() {
Map<String, Object> results = new HashMap<>();
- Path segment;
- if(args.containsKey(Nutch.ARG_SEGMENT)) {
- Object seg = args.get(Nutch.ARG_SEGMENT);
+ Path segment = null;
+ if(args.containsKey(Nutch.ARG_SEGMENTS)) {
+ Object seg = args.get(Nutch.ARG_SEGMENTS);
if(seg instanceof Path) {
segment = (Path) seg;
}
- else {
+ else if(seg instanceof String){
segment = new Path(seg.toString());
}
+ else if(seg instanceof ArrayList) {
+ String[] segmentsArray = (String[])seg;
+ segment = new Path(segmentsArray[0].toString());
+
+ if(segmentsArray.length > 1){
+ LOG.warn("Only the first segment of segments array is used.");
+ }
+ }
}
else {
String segment_dir = crawlId+"/segments";
diff --git a/src/java/org/apache/nutch/indexer/IndexingJob.java
b/src/java/org/apache/nutch/indexer/IndexingJob.java
index 1e9a08af3..68132332e 100644
--- a/src/java/org/apache/nutch/indexer/IndexingJob.java
+++ b/src/java/org/apache/nutch/indexer/IndexingJob.java
@@ -303,15 +303,17 @@ public static void main(String[] args) throws Exception {
}
}
- if(args.containsKey(Nutch.ARG_SEGMENT)){
- isSegment = true;
- Object seg = args.get(Nutch.ARG_SEGMENT);
- ArrayList<String> segmentList = new ArrayList<>();
- if(seg instanceof ArrayList) {
- segmentList = (ArrayList<String>)seg;
+ if(args.containsKey(Nutch.ARG_SEGMENTS)) {
+ Object segmentsFromArg = args.get(Nutch.ARG_SEGMENTS);
+ ArrayList<String> segmentList = new ArrayList<String>();
+ if(segmentsFromArg instanceof ArrayList) {
+ segmentList = (ArrayList<String>)segmentsFromArg; }
+ else if(segmentsFromArg instanceof Path){
+ segmentList.add(segmentsFromArg.toString());
}
+
for(String segment: segmentList) {
- segments.add(new Path(segment));
+ segments.add(new Path(segment));
}
}
diff --git a/src/java/org/apache/nutch/metadata/Nutch.java
b/src/java/org/apache/nutch/metadata/Nutch.java
index 8d485e5c8..02dfa7268 100644
--- a/src/java/org/apache/nutch/metadata/Nutch.java
+++ b/src/java/org/apache/nutch/metadata/Nutch.java
@@ -77,7 +77,6 @@
public static final Text WRITABLE_FIXED_INTERVAL_KEY = new Text(
FIXED_INTERVAL_KEY);
-
/** For progress of job. Used by the Nutch REST service */
public static final String STAT_PROGRESS = "progress";
/**Used by Nutch REST service */
@@ -95,12 +94,10 @@
/** Argument key to specify the location of a directory of segments for
the REST endpoints.
* Similar to the -dir command in the bin/nutch script **/
public static final String ARG_SEGMENTDIR = "segment_dir";
- /** Argument key to specify the location of individual segment for the
REST endpoints **/
- public static final String ARG_SEGMENT = "segment";
+ /** Argument key to specify the location of individual segment or list
of segments for the REST endpoints. The behavior differs for diffirent
endpoints: CrawlDb, LinkDb and Indexing Jobs take list of segments, Fetcher and
Parse segment take one segment **/
+ public static final String ARG_SEGMENTS = "segment";
/** Argument key to specify the location of hostdb for the REST
endpoints **/
public static final String ARG_HOSTDB = "hostdb";
-
-
/** Title key in the Pub/Sub event metadata for the title of the parsed
page*/
public static final String FETCH_EVENT_TITLE = "title";
/** Content-type key in the Pub/Sub event metadata for the content-type
of the parsed page*/
diff --git a/src/java/org/apache/nutch/parse/ParseSegment.java
b/src/java/org/apache/nutch/parse/ParseSegment.java
index 86453f1f5..5074e0442 100644
--- a/src/java/org/apache/nutch/parse/ParseSegment.java
+++ b/src/java/org/apache/nutch/parse/ParseSegment.java
@@ -275,15 +275,23 @@ public int run(String[] args) throws Exception {
public Map<String, Object> run(Map<String, Object> args, String crawlId)
throws Exception {
Map<String, Object> results = new HashMap<>();
- Path segment;
- if(args.containsKey(Nutch.ARG_SEGMENT)) {
- Object seg = args.get(Nutch.ARG_SEGMENT);
- if(seg instanceof Path) {
- segment = (Path) seg;
- }
- else {
- segment = new Path(seg.toString());
- }
+ Path segment = null;
+ if(args.containsKey(Nutch.ARG_SEGMENTS)) {
+ Object seg = args.get(Nutch.ARG_SEGMENTS);
+ if(seg instanceof Path) {
+ segment = (Path) seg;
+ }
+ else if(seg instanceof String){
+ segment = new Path(seg.toString());
+ }
+ else if(seg instanceof ArrayList) {
+ String[] segmentsArray = (String[])seg;
+ segment = new Path(segmentsArray[0].toString());
+
+ if(segmentsArray.length > 1){
+ LOG.warn("Only the first segment of segments array is used.");
+ }
+ }
}
else {
String segment_dir = crawlId+"/segments";
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> ARG_SEGMENT usage
> -----------------
>
> Key: NUTCH-2441
> URL: https://issues.apache.org/jira/browse/NUTCH-2441
> Project: Nutch
> Issue Type: Improvement
> Components: metadata
> Affects Versions: 1.13
> Reporter: Semyon Semyonov
> Priority: Major
> Fix For: 1.15
>
> Attachments: metadataARG_SEGMENT.patch
>
>
> The class metadata/Nutch.java public static final String ARG_SEGMENT =
> "segment" is not used correctly. In some cases Fetcher and ParseSegment it is
> interpreted as a single segmenet, in others CrawlDb, LinkDb, IndexingJob as
> an array of segments. Such misunderstanding leads to inconsistency of usage
> of the parameter.
> After a discussion with [~wastl-nagel] the proposed solution is to allow the
> usage of both array and a string in all cases. That gives an opportunity to
> not introduce the broken changes.
> A path is proposed.
> *The question left is refactoring, all these five components share the same
> code(two versions of the same code to be precise). Shouldn't we extract a
> method and reduce duplicates? *
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
