Author: jerome
Date: Sun Sep 11 13:28:07 2005
New Revision: 280176
URL: http://svn.apache.org/viewcvs?rev=280176&view=rev
Log:
Automatically loads active plugins dependencies (add a property, default is on)
Added:
lucene/nutch/trunk/src/java/org/apache/nutch/plugin/CircularDependencyException.java
(with props)
lucene/nutch/trunk/src/java/org/apache/nutch/plugin/MissingDependencyException.java
(with props)
Modified:
lucene/nutch/trunk/conf/nutch-default.xml
lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginManifestParser.java
lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginRepository.java
Modified: lucene/nutch/trunk/conf/nutch-default.xml
URL:
http://svn.apache.org/viewcvs/lucene/nutch/trunk/conf/nutch-default.xml?rev=280176&r1=280175&r2=280176&view=diff
==============================================================================
--- lucene/nutch/trunk/conf/nutch-default.xml (original)
+++ lucene/nutch/trunk/conf/nutch-default.xml Sun Sep 11 13:28:07 2005
@@ -586,8 +586,17 @@
</property>
<property>
+ <name>plugin.auto-activation</name>
+ <value>false</value>
+ <description>Defines if some plugins that are not activated regarding
+ the plugin.includes and plugin.excludes properties must be automaticaly
+ activated if they are needed by some actived plugins.
+ </description>
+</property>
+
+<property>
<name>plugin.includes</name>
-
<value>nutch-extensionpoints|protocol-httpclient|urlfilter-regex|parse-(text|html|js)|index-basic|query-(basic|site|url)</value>
+
<value>protocol-httpclient|urlfilter-regex|parse-(text|html|js)|index-basic|query-(basic|site|url)</value>
<description>Regular expression naming plugin directory names to
include. Any plugin not matching this expression is excluded.
In any case you need at least include the nutch-extensionpoints plugin. By
Added:
lucene/nutch/trunk/src/java/org/apache/nutch/plugin/CircularDependencyException.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/plugin/CircularDependencyException.java?rev=280176&view=auto
==============================================================================
---
lucene/nutch/trunk/src/java/org/apache/nutch/plugin/CircularDependencyException.java
(added)
+++
lucene/nutch/trunk/src/java/org/apache/nutch/plugin/CircularDependencyException.java
Sun Sep 11 13:28:07 2005
@@ -0,0 +1,35 @@
+/*
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.plugin;
+
+
+/**
+ * <code>CircularDependencyException</code> will be thrown if a circular
+ * dependency is detected.
+ *
+ * @author Jérôme Charron
+ */
+public class CircularDependencyException extends Exception {
+
+ public CircularDependencyException(Throwable cause) {
+ super(cause);
+ }
+
+ public CircularDependencyException(String message) {
+ super(message);
+ }
+}
Propchange:
lucene/nutch/trunk/src/java/org/apache/nutch/plugin/CircularDependencyException.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
lucene/nutch/trunk/src/java/org/apache/nutch/plugin/MissingDependencyException.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/plugin/MissingDependencyException.java?rev=280176&view=auto
==============================================================================
---
lucene/nutch/trunk/src/java/org/apache/nutch/plugin/MissingDependencyException.java
(added)
+++
lucene/nutch/trunk/src/java/org/apache/nutch/plugin/MissingDependencyException.java
Sun Sep 11 13:28:07 2005
@@ -0,0 +1,35 @@
+/*
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.plugin;
+
+
+/**
+ * <code>MissingDependencyException</code> will be thrown if a plugin
+ * dependency cannot be found.
+ *
+ * @author Jérôme Charron
+ */
+public class MissingDependencyException extends Exception {
+
+ public MissingDependencyException(Throwable cause) {
+ super(cause);
+ }
+
+ public MissingDependencyException(String message) {
+ super(message);
+ }
+}
Propchange:
lucene/nutch/trunk/src/java/org/apache/nutch/plugin/MissingDependencyException.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginManifestParser.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginManifestParser.java?rev=280176&r1=280175&r2=280176&view=diff
==============================================================================
---
lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginManifestParser.java
(original)
+++
lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginManifestParser.java
Sun Sep 11 13:28:07 2005
@@ -22,7 +22,9 @@
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
-import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
import java.util.logging.Logger;
import java.util.regex.Pattern;
@@ -51,20 +53,52 @@
.startsWith("Windows");
/**
+ * Filters a list of plugins.
+ * The list of plugins is filtered regarding the configuration
+ * properties <code>plugin.excludes</code> and
<code>plugin.includes</code>.
+ */
+ public static Map filter(Map plugins) {
+ Map map = new HashMap();
+ Pattern excludes = Pattern.compile(NutchConf.get().get(
+ "plugin.excludes", ""));
+ Pattern includes = Pattern.compile(NutchConf.get().get(
+ "plugin.includes", ""));
+ if (plugins == null) { return map; }
+
+ Iterator iter = plugins.values().iterator();
+ while (iter.hasNext()) {
+ PluginDescriptor plugin = (PluginDescriptor) iter.next();
+ if (plugin == null) { continue; }
+ String id = plugin.getPluginId();
+ if (id == null) { continue; }
+
+ if (!includes.matcher(id).matches()) {
+ LOG.fine("not including: " + id);
+ continue;
+ }
+ if (excludes.matcher(id).matches()) {
+ LOG.fine("excluding: " + id);
+ continue;
+ }
+ map.put(plugin.getPluginId(), plugin);
+ }
+ return map;
+ }
+
+ /**
* Returns a list with plugin descriptors.
*
* @return ArrayList
*
*/
- public static ArrayList parsePluginFolder() {
- ArrayList list = new ArrayList();
+ public static Map parsePluginFolder() {
+ Map map = new HashMap();
String[] pluginFolders = NutchConf.get().getStrings("plugin.folders");
- Pattern excludes = Pattern.compile(NutchConf.get().get(
- "plugin.excludes", ""));
- Pattern includes = Pattern.compile(NutchConf.get().get(
- "plugin.includes", ""));
- if (pluginFolders == null)
+
+ if (pluginFolders == null) {
throw new IllegalArgumentException("plugin.folders is not
defined");
+ }
+
for (int i = 0; i < pluginFolders.length; i++) {
String name = pluginFolders[i];
File directory = getPluginFolder(name);
@@ -77,35 +111,25 @@
for (int j = 0; j < files.length; j++) {
File oneSubFolder = files[j];
if (oneSubFolder.isDirectory()) {
-
- if (!includes.matcher(oneSubFolder.getName()).matches()) {
- LOG.info("not including: " + oneSubFolder);
- continue;
- }
-
- if (excludes.matcher(oneSubFolder.getName()).matches()) {
- LOG.info("excluding: " + oneSubFolder);
- continue;
- }
-
String manifestPath = oneSubFolder.getAbsolutePath()
+ File.separator + "plugin.xml";
try {
- LOG.info("parsing: " + manifestPath);
- list.add(parseManifestFile(manifestPath));
+ LOG.fine("parsing: " + manifestPath);
+ PluginDescriptor p = parseManifestFile(manifestPath);
+ map.put(p.getPluginId(), p);
} catch (MalformedURLException e) {
- LOG.info(e.toString());
+ LOG.warning(e.toString());
} catch (SAXException e) {
- LOG.info(e.toString());
+ LOG.warning(e.toString());
} catch (IOException e) {
- LOG.info(e.toString());
+ LOG.warning(e.toString());
} catch (ParserConfigurationException e) {
- LOG.info(e.toString());
+ LOG.warning(e.toString());
}
}
}
}
- return list;
+ return map;
}
/**
@@ -118,10 +142,10 @@
URL url = PluginManifestParser.class.getClassLoader().getResource(
name);
if (url == null) {
- LOG.info("Plugins: directory not found: " + name);
+ LOG.warning("Plugins: directory not found: " + name);
return null;
} else if (!"file".equals(url.getProtocol())) {
- LOG.info("Plugins: not a file: url. Can't load plugins from: "
+ LOG.warning("Plugins: not a file: url. Can't load plugins
from: "
+ url);
return null;
}
@@ -184,7 +208,7 @@
}
PluginDescriptor pluginDescriptor = new PluginDescriptor(id, version,
name, providerName, pluginClazz, pPath);
- LOG.fine("plugin: id="+id+" name="+name+" version="+version
+ LOG.fine("plugin: id="+id+" name="+name+" version="+version
+" provider="+providerName+"class="+pluginClazz);
parseExtension(rootElement, pluginDescriptor);
parseExtensionPoints(rootElement, pluginDescriptor);
@@ -289,7 +313,7 @@
String id = oneImplementation.getAttribute("id");
String extensionClass = oneImplementation
.getAttribute("class");
- LOG.info("impl: point=" + pointId + " class="
+ LOG.fine("impl: point=" + pointId + " class="
+ extensionClass);
Extension extension = new Extension(pPluginDescriptor,
pointId, id, extensionClass);
Modified:
lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginRepository.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginRepository.java?rev=280176&r1=280175&r2=280176&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginRepository.java
(original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginRepository.java
Sun Sep 11 13:28:07 2005
@@ -16,14 +16,20 @@
*/
package org.apache.nutch.plugin;
+// JDK imports
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
import java.util.logging.Logger;
+// Nutch imports
import org.apache.nutch.util.LogFormatter;
+import org.apache.nutch.util.NutchConf;
+
/**
* The plugin repositority is a registry of all plugins.
@@ -38,14 +44,19 @@
* @author joa23
*/
public class PluginRepository {
+
+ private final static boolean AUTO =
+ NutchConf.get().getBoolean("plugin.auto-activation", true);
+
private static PluginRepository fInstance;
- private ArrayList fRegisteredPlugins;
+ private List fRegisteredPlugins;
private HashMap fExtensionPoints;
private HashMap fActivatedPlugins;
+
public static final Logger LOG = LogFormatter
.getLogger("org.apache.nutch.plugin.PluginRepository");
@@ -54,66 +65,115 @@
* @see java.lang.Object#Object()
*/
private PluginRepository() throws PluginRuntimeException{
- fActivatedPlugins = new HashMap();
- fExtensionPoints = new HashMap();
- fRegisteredPlugins = getDependencyCheckedPlugins(PluginManifestParser
- .parsePluginFolder());
- installExtensions(fRegisteredPlugins);
+ fActivatedPlugins = new HashMap();
+ fExtensionPoints = new HashMap();
+ Map allPlugins = PluginManifestParser.parsePluginFolder();
+ Map filteredPlugins = PluginManifestParser.filter(allPlugins);
+ fRegisteredPlugins = getDependencyCheckedPlugins(
+ filteredPlugins,
+ AUTO ? allPlugins : filteredPlugins);
+ installExtensionPoints(fRegisteredPlugins);
+ installExtensions(fRegisteredPlugins);
+ displayStatus();
}
+ private void installExtensionPoints(List plugins) {
+ if (plugins == null) { return; }
+
+ for (int i=0; i<plugins.size(); i++) {
+ PluginDescriptor plugin = (PluginDescriptor) plugins.get(i);
+ ExtensionPoint[] points = plugin.getExtenstionPoints();
+ for (int j=0; j<points.length; j++) {
+ ExtensionPoint point = points[j];
+ String xpId = point.getId();
+ LOG.fine("Adding extension point " + xpId);
+ fExtensionPoints.put(xpId, point);
+ }
+ }
+ }
+
/**
* @param pRegisteredPlugins
*/
- private void installExtensions(ArrayList pRegisteredPlugins)
- throws PluginRuntimeException {
- for (int i = 0; i < pRegisteredPlugins.size(); i++) {
- PluginDescriptor descriptor = (PluginDescriptor) pRegisteredPlugins
- .get(i);
- Extension[] extensions = descriptor.getExtensions();
- for (int j = 0; j < extensions.length; j++) {
- Extension extension = extensions[j];
- String xpId = extension.getTargetPoint();
- ExtensionPoint point = getExtensionPoint(xpId);
- if (point == null)
- throw new PluginRuntimeException("extension point: " + xpId
- + " does not exist.");
- point.addExtension(extension);
- }
+ private void installExtensions(List pRegisteredPlugins)
+ throws PluginRuntimeException {
+
+ for (int i = 0; i < pRegisteredPlugins.size(); i++) {
+ PluginDescriptor descriptor = (PluginDescriptor)
pRegisteredPlugins.get(i);
+ Extension[] extensions = descriptor.getExtensions();
+ for (int j = 0; j < extensions.length; j++) {
+ Extension extension = extensions[j];
+ String xpId = extension.getTargetPoint();
+ ExtensionPoint point = getExtensionPoint(xpId);
+ if (point == null) {
+ throw new PluginRuntimeException(
+ "Plugin (" + descriptor.getPluginId() + "), " +
+ "extension point: " + xpId + " does not exist.");
+ }
+ point.addExtension(extension);
}
+ }
}
+ private void getPluginCheckedDependencies(PluginDescriptor plugin,
+ Map plugins, Map dependencies)
+ throws MissingDependencyException,
+ CircularDependencyException {
+
+ if (dependencies == null) { dependencies = new HashMap(); }
+
+ String[] ids = plugin.getDependencies();
+ for (int i=0; i<ids.length; i++) {
+ String id = ids[i];
+ PluginDescriptor dependency = (PluginDescriptor) plugins.get(id);
+ if (dependency == null) {
+ throw new MissingDependencyException(
+ "Missing dependency " + id +
+ " for plugin " + plugin.getPluginId());
+ }
+ if (dependencies.containsKey(id)) {
+ throw new CircularDependencyException(
+ "Circular dependency detected " + id +
+ " for plugin " + plugin.getPluginId());
+ }
+ dependencies.put(id, dependency);
+ getPluginCheckedDependencies((PluginDescriptor) plugins.get(id),
+ plugins, dependencies);
+ }
+ }
+
+ private Map getPluginCheckedDependencies(PluginDescriptor plugin,
+ Map plugins)
+ throws MissingDependencyException,
+ CircularDependencyException {
+ Map dependencies = new HashMap();
+ getPluginCheckedDependencies(plugin, plugins, dependencies);
+ return dependencies;
+ }
+
/**
- * @param pLoadedPlugins
- * @return ArrayList
- */
- private ArrayList getDependencyCheckedPlugins(ArrayList pLoadedPlugins) {
- ArrayList availablePlugins = new ArrayList();
- for (int i = 0; i < pLoadedPlugins.size(); i++) {
- PluginDescriptor descriptor = (PluginDescriptor) pLoadedPlugins
- .get(i);
- String[] dependencyIDs = descriptor.getDependencies();
- boolean available = true;
- for (int j = 0; j < dependencyIDs.length; j++) {
- String id = dependencyIDs[j];
- if (!dependencyIsAvailable(id, pLoadedPlugins)) {
- available = false;
- //LOG.fine("Skipping " + descriptor.getName());
- break;
- }
- }
- if (available) {
- //LOG.fine("Adding " + descriptor.getName());
- availablePlugins.add(descriptor);
- ExtensionPoint[] points = descriptor.getExtenstionPoints();
- for (int j = 0; j < points.length; j++) {
- ExtensionPoint point = points[j];
- String xpId = point.getId();
- //LOG.fine("Adding extension point " + xpId);
- fExtensionPoints.put(xpId, point);
- }
- }
+ * @param filtered is the list of plugin filtred
+ * @param all is the list of all plugins found.
+ * @return List
+ */
+ private List getDependencyCheckedPlugins(Map filtered, Map all) {
+ if (filtered == null) { return null; }
+ Map checked = new HashMap();
+ Iterator iter = filtered.values().iterator();
+ while (iter.hasNext()) {
+ PluginDescriptor plugin = (PluginDescriptor) iter.next();
+ try {
+ checked.putAll(getPluginCheckedDependencies(plugin, all));
+ checked.put(plugin.getPluginId(), plugin);
+ } catch (MissingDependencyException mde) {
+ // Simply ignore this plugin
+ LOG.warning(mde.getMessage());
+ } catch (CircularDependencyException cde) {
+ // Simply ignore this plugin
+ LOG.warning(cde.getMessage());
}
- return availablePlugins;
+ }
+ return new ArrayList(checked.values());
}
/**
@@ -121,7 +181,7 @@
* @param pLoadedPlugins
* @return boolean
*/
- private boolean dependencyIsAvailable(String id, ArrayList pLoadedPlugins)
{
+ private boolean dependencyIsAvailable(String id, List pLoadedPlugins) {
if (pLoadedPlugins != null && id != null) {
for (int i = 0; i < pLoadedPlugins.size(); i++) {
PluginDescriptor descriptor = (PluginDescriptor) pLoadedPlugins
@@ -251,5 +311,31 @@
Plugin object = (Plugin) fActivatedPlugins.get(pluginId);
object.shutDown();
}
+ }
+
+ private void displayStatus() {
+
+ LOG.info("Plugin Auto-activation mode: [" + AUTO + "]");
+
+ LOG.info("Registered Plugins:");
+ if ((fRegisteredPlugins == null) || (fRegisteredPlugins.size() == 0)) {
+ LOG.info("\tNONE");
+ } else {
+ for (int i=0; i<fRegisteredPlugins.size(); i++) {
+ PluginDescriptor plugin = (PluginDescriptor)
fRegisteredPlugins.get(i);
+ LOG.info("\t" + plugin.getName() + " (" + plugin.getPluginId() +
")");
+ }
+ }
+
+ LOG.info("Registered Extension-Points:");
+ if ((fExtensionPoints == null) || (fExtensionPoints.size() == 0)) {
+ LOG.info("\tNONE");
+ } else {
+ Iterator iter = fExtensionPoints.values().iterator();
+ while (iter.hasNext()) {
+ ExtensionPoint ep = (ExtensionPoint) iter.next();
+ LOG.info("\t" + ep.getName() + " (" + ep.getId() + ")");
+ }
+ }
}
}
