Author: ab
Date: Thu Feb 2 02:58:16 2006
New Revision: 374348
URL: http://svn.apache.org/viewcvs?rev=374348&view=rev
Log:
Add functionality to run individual classes in plugins.
Add a user-friendly shortcut to bin/nutch (the "plugin" command).
Fix missing setConf() in Http plugins.
Modified:
lucene/nutch/trunk/bin/nutch
lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginRepository.java
lucene/nutch/trunk/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/Http.java
lucene/nutch/trunk/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
Modified: lucene/nutch/trunk/bin/nutch
URL:
http://svn.apache.org/viewcvs/lucene/nutch/trunk/bin/nutch?rev=374348&r1=374347&r2=374348&view=diff
==============================================================================
--- lucene/nutch/trunk/bin/nutch (original)
+++ lucene/nutch/trunk/bin/nutch Thu Feb 2 02:58:16 2006
@@ -41,6 +41,7 @@
echo " index run the indexer on parsed segments and linkdb"
echo " merge merge several segment indexes"
echo " dedup remove duplicates from a set of segment indexes"
+ echo " plugin load a plugin and run one of its classes main()"
echo " server run a search server"
echo " namenode run the NDFS namenode"
echo " datanode run an NDFS datanode"
@@ -150,6 +151,8 @@
CLASS=org.apache.nutch.indexer.DeleteDuplicates
elif [ "$COMMAND" = "merge" ] ; then
CLASS=org.apache.nutch.indexer.IndexMerger
+elif [ "$COMMAND" = "plugin" ] ; then
+ CLASS=org.apache.nutch.plugin.PluginRepository
elif [ "$COMMAND" = "server" ] ; then
CLASS='org.apache.nutch.searcher.DistributedSearch$Server'
elif [ "$COMMAND" = "namenode" ] ; then
Modified:
lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginRepository.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginRepository.java?rev=374348&r1=374347&r2=374348&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginRepository.java
(original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginRepository.java
Thu Feb 2 02:58:16 2006
@@ -19,6 +19,7 @@
// JDK imports
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
@@ -376,5 +377,47 @@
map.put(plugin.getPluginId(), plugin);
}
return map;
+ }
+
+ /**
+ * Loads all necessary dependencies for a selected plugin, and then
+ * runs one of the classes' main() method.
+ * @param args plugin ID (needs to be activated in the configuration), and
+ * the class name. The rest of arguments is passed to the main method of
the
+ * selected class.
+ * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ if (args.length < 2) {
+ System.err.println("Usage: PluginRepository pluginId className [arg1
arg2 ...]");
+ return;
+ }
+ NutchConf conf = new NutchConf();
+ PluginRepository repo = new PluginRepository(conf);
+ // args[0] - plugin ID
+ PluginDescriptor d = repo.getPluginDescriptor(args[0]);
+ if (d == null) {
+ System.err.println("Plugin '" + args[0] + "' not present or
inactive.");
+ return;
+ }
+ ClassLoader cl = d.getClassLoader();
+ // args[1] - class name
+ Class clazz = null;
+ try {
+ clazz = Class.forName(args[1], true, cl);
+ } catch (Exception e) {
+ System.err.println("Could not load the class '" + args[1] + ": " +
e.getMessage());
+ return;
+ }
+ Method m = null;
+ try {
+ m = clazz.getMethod("main", new Class[]{args.getClass()});
+ } catch (Exception e) {
+ System.err.println("Could not find the 'main(String[])' method in
class " + args[1] + ": " + e.getMessage());
+ return;
+ }
+ String[] subargs = new String[args.length - 2];
+ System.arraycopy(args, 2, subargs, 0, subargs.length);
+ m.invoke(null, new Object[]{subargs});
}
}
Modified:
lucene/nutch/trunk/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/Http.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/Http.java?rev=374348&r1=374347&r2=374348&view=diff
==============================================================================
---
lucene/nutch/trunk/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/Http.java
(original)
+++
lucene/nutch/trunk/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/Http.java
Thu Feb 2 02:58:16 2006
@@ -50,7 +50,9 @@
}
public static void main(String[] args) throws Exception {
- main(new Http(), args);
+ Http http = new Http();
+ http.setConf(new NutchConf());
+ main(http, args);
}
protected Response getResponse(URL url, CrawlDatum datum, boolean redirect)
Modified:
lucene/nutch/trunk/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java?rev=374348&r1=374347&r2=374348&view=diff
==============================================================================
---
lucene/nutch/trunk/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
(original)
+++
lucene/nutch/trunk/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
Thu Feb 2 02:58:16 2006
@@ -86,7 +86,9 @@
}
public static void main(String[] args) throws Exception {
- main(new Http(), args);
+ Http http = new Http();
+ http.setConf(new NutchConf());
+ main(http, args);
}
protected Response getResponse(URL url, CrawlDatum datum, boolean redirect)
