Author: jnioche
Date: Tue Jul 20 15:42:23 2010
New Revision: 965887
URL: http://svn.apache.org/viewvc?rev=965887&view=rev
Log:
ported protocol-httpclient to the new API
Modified:
nutch/branches/nutchbase/src/plugin/build.xml
nutch/branches/nutchbase/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
nutch/branches/nutchbase/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
nutch/branches/nutchbase/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java
nutch/branches/nutchbase/src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/TestProtocolHttpClient.java
Modified: nutch/branches/nutchbase/src/plugin/build.xml
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/build.xml?rev=965887&r1=965886&r2=965887&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/plugin/build.xml (original)
+++ nutch/branches/nutchbase/src/plugin/build.xml Tue Jul 20 15:42:23 2010
@@ -40,6 +40,7 @@
<ant dir="protocol-ftp" target="deploy"/>
<ant dir="protocol-file" target="deploy"/>
<ant dir="protocol-http" target="deploy"/>
+ <ant dir="protocol-httpclient" target="deploy"/>
<ant dir="parse-js" target="deploy"/>
<ant dir="parse-rss" target="deploy"/>
<ant dir="parse-tika" target="deploy"/>
@@ -61,7 +62,6 @@
<ant dir="parse-ext" target="deploy"/>
<ant dir="parse-swf" target="deploy"/>
<ant dir="parse-zip" target="deploy"/>
- <ant dir="protocol-httpclient" target="deploy"/>
-->
</target>
@@ -76,6 +76,7 @@
<ant dir="parse-rss" target="test"/>
<ant dir="index-more" target="test"/>
<ant dir="languageidentifier" target="test"/>
+ <ant dir="protocol-httpclient" target="test"/>
<ant dir="urlfilter-automaton" target="test"/>
<ant dir="urlfilter-domain" target="test" />
<ant dir="urlfilter-regex" target="test"/>
@@ -91,7 +92,6 @@
<ant dir="parse-ext" target="test"/>
<ant dir="parse-swf" target="test"/>
<ant dir="parse-zip" target="test"/>
- <ant dir="protocol-httpclient" target="test"/>
-->
</target>
Modified:
nutch/branches/nutchbase/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java?rev=965887&r1=965886&r2=965887&view=diff
==============================================================================
---
nutch/branches/nutchbase/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
(original)
+++
nutch/branches/nutchbase/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
Tue Jul 20 15:42:23 2010
@@ -37,6 +37,7 @@ import org.apache.nutch.protocol.RobotRu
import org.apache.nutch.storage.ProtocolStatusUtils;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.GZIPUtils;
+import org.apache.nutch.util.DeflateUtils;
import org.apache.nutch.util.LogUtil;
import org.apache.nutch.util.MimeUtil;
@@ -89,6 +90,9 @@ public abstract class HttpBase implement
"[email protected]");
+ /** The "Accept-Language" request header value. */
+ protected String acceptLanguage = "en-us,en-gb,en;q=0.7,*;q=0.3";
+
/**
* Maps from host to a Long naming the time it should be unblocked.
* The Long is zero while the host is in use, then set to now+wait when
@@ -161,6 +165,7 @@ public abstract class HttpBase implement
this.maxThreadsPerHost = conf.getInt("fetcher.threads.per.host", 1);
this.userAgent = getAgentString(conf.get("http.agent.name"),
conf.get("http.agent.version"), conf
.get("http.agent.description"), conf.get("http.agent.url"),
conf.get("http.agent.email"));
+ this.acceptLanguage = conf.get("http.accept.language", acceptLanguage);
this.serverDelay = (long) (conf.getFloat("fetcher.server.delay", 1.0f) *
1000);
this.maxCrawlDelay = (conf.getInt("fetcher.max.crawl.delay", -1) * 1000);
// backward-compatible default setting
@@ -329,6 +334,13 @@ public abstract class HttpBase implement
public String getUserAgent() {
return userAgent;
}
+
+ /** Value of "Accept-Language" request header sent by Nutch.
+ * @return The value of the header "Accept-Language" header.
+ */
+ public String getAcceptLanguage() {
+ return acceptLanguage;
+ }
public boolean getUseHttp11() {
return useHttp11;
@@ -474,6 +486,7 @@ public abstract class HttpBase implement
logger.info("http.timeout = " + timeout);
logger.info("http.content.limit = " + maxContent);
logger.info("http.agent = " + userAgent);
+ logger.info("http.accept.language = " + acceptLanguage);
logger.info(Protocol.CHECK_BLOCKING + " = " + checkBlocking);
logger.info(Protocol.CHECK_ROBOTS + " = " + checkRobots);
if (checkBlocking) {
@@ -505,6 +518,23 @@ public abstract class HttpBase implement
return content;
}
+ public byte[] processDeflateEncoded(byte[] compressed, URL url) throws
IOException {
+
+ if (LOGGER.isTraceEnabled()) { LOGGER.trace("inflating...."); }
+
+ byte[] content = DeflateUtils.inflateBestEffort(compressed,
getMaxContent());
+
+ if (content == null)
+ throw new IOException("inflateBestEffort returned null");
+
+ if (LOGGER.isTraceEnabled()) {
+ LOGGER.trace("fetched " + compressed.length
+ + " bytes of compressed content (expanded to "
+ + content.length + " bytes) from " + url);
+ }
+ return content;
+ }
+
protected static void main(HttpBase http, String[] args) throws Exception {
@SuppressWarnings("unused")
boolean verbose = false;
Modified:
nutch/branches/nutchbase/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java?rev=965887&r1=965886&r2=965887&view=diff
==============================================================================
---
nutch/branches/nutchbase/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
(original)
+++
nutch/branches/nutchbase/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
Tue Jul 20 15:42:23 2010
@@ -21,6 +21,9 @@ import java.io.InputStream;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import org.xml.sax.SAXException;
@@ -44,8 +47,9 @@ import org.apache.commons.httpclient.par
import org.apache.commons.httpclient.protocol.Protocol;
// Nutch imports
+import org.apache.nutch.storage.WebPage;
+import org.apache.nutch.storage.WebPage.Field;
import org.apache.nutch.util.LogUtil;
-import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.net.protocols.Response;
import org.apache.nutch.protocol.ProtocolException;
import org.apache.nutch.protocol.http.api.HttpBase;
@@ -53,377 +57,406 @@ import org.apache.hadoop.conf.Configurat
import org.apache.nutch.util.NutchConfiguration;
/**
- * This class is a protocol plugin that configures an HTTP client for
- * Basic, Digest and NTLM authentication schemes for web server as well
- * as proxy server. It takes care of HTTPS protocol as well as cookies
- * in a single fetch session.
- *
+ * This class is a protocol plugin that configures an HTTP client for Basic,
+ * Digest and NTLM authentication schemes for web server as well as proxy
+ * server. It takes care of HTTPS protocol as well as cookies in a single fetch
+ * session.
+ *
* @author Susam Pal
*/
public class Http extends HttpBase {
- public static final Log LOG = LogFactory.getLog(Http.class);
+ public static final Log LOG = LogFactory.getLog(Http.class);
- private static MultiThreadedHttpConnectionManager connectionManager =
- new MultiThreadedHttpConnectionManager();
+ private static MultiThreadedHttpConnectionManager connectionManager =
new MultiThreadedHttpConnectionManager();
- // Since the Configuration has not yet been set,
- // then an unconfigured client is returned.
- private static HttpClient client = new HttpClient(connectionManager);
- private static String defaultUsername;
- private static String defaultPassword;
- private static String defaultRealm;
- private static String defaultScheme;
- private static String authFile;
- private static String agentHost;
- private static boolean authRulesRead = false;
- private static Configuration conf;
-
- int maxThreadsTotal = 10;
-
- private String proxyUsername;
- private String proxyPassword;
- private String proxyRealm;
-
-
- /**
- * Returns the configured HTTP client.
- *
- * @return HTTP client
- */
- static synchronized HttpClient getClient() {
- return client;
- }
-
- /**
- * Constructs this plugin.
- */
- public Http() {
- super(LOG);
- }
-
- /**
- * Reads the configuration from the Nutch configuration files and sets
- * the configuration.
- *
- * @param conf Configuration
- */
- public void setConf(Configuration conf) {
- super.setConf(conf);
- this.conf = conf;
- this.maxThreadsTotal = conf.getInt("fetcher.threads.fetch", 10);
- this.proxyUsername = conf.get("http.proxy.username", "");
- this.proxyPassword = conf.get("http.proxy.password", "");
- this.proxyRealm = conf.get("http.proxy.realm", "");
- agentHost = conf.get("http.agent.host", "");
- authFile = conf.get("http.auth.file", "");
- configureClient();
- try {
- setCredentials();
- } catch (Exception ex) {
- if (LOG.isFatalEnabled()) {
- LOG.fatal("Could not read " + authFile + " : " + ex.getMessage());
- ex.printStackTrace(LogUtil.getErrorStream(LOG));
- }
- }
- }
-
- /**
- * Main method.
- *
- * @param args Command line arguments
- */
- public static void main(String[] args) throws Exception {
- Http http = new Http();
- http.setConf(NutchConfiguration.create());
- main(http, args);
- }
-
- /**
- * Fetches the <code>url</code> with a configured HTTP client and
- * gets the response.
- *
- * @param url URL to be fetched
- * @param datum Crawl data
- * @param redirect Follow redirects if and only if true
- * @return HTTP response
- */
- protected Response getResponse(URL url, CrawlDatum datum, boolean redirect)
- throws ProtocolException, IOException {
- resolveCredentials(url);
- return new HttpResponse(this, url, datum, redirect);
- }
-
- /**
- * Configures the HTTP client
- */
- private void configureClient() {
-
- // Set up an HTTPS socket factory that accepts self-signed certs.
- Protocol https = new Protocol("https",
- new DummySSLProtocolSocketFactory(), 443);
- Protocol.registerProtocol("https", https);
-
- HttpConnectionManagerParams params = connectionManager.getParams();
- params.setConnectionTimeout(timeout);
- params.setSoTimeout(timeout);
- params.setSendBufferSize(BUFFER_SIZE);
- params.setReceiveBufferSize(BUFFER_SIZE);
- params.setMaxTotalConnections(maxThreadsTotal);
- if (maxThreadsTotal > maxThreadsPerHost) {
- params.setDefaultMaxConnectionsPerHost(maxThreadsPerHost);
- } else {
- params.setDefaultMaxConnectionsPerHost(maxThreadsTotal);
- }
-
- // executeMethod(HttpMethod) seems to ignore the connection timeout on the
connection manager.
- // set it explicitly on the HttpClient.
- client.getParams().setConnectionManagerTimeout(timeout);
-
- HostConfiguration hostConf = client.getHostConfiguration();
- ArrayList headers = new ArrayList();
- // Set the User Agent in the header
- headers.add(new Header("User-Agent", userAgent));
- // prefer English
- headers.add(new Header("Accept-Language", "en-us,en-gb,en;q=0.7,*;q=0.3"));
- // prefer UTF-8
- headers.add(new Header("Accept-Charset",
"utf-8,ISO-8859-1;q=0.7,*;q=0.7"));
- // prefer understandable formats
- headers.add(new Header("Accept",
-
"text/html,application/xml;q=0.9,application/xhtml+xml,text/xml;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5"));
- // accept gzipped content
- headers.add(new Header("Accept-Encoding", "x-gzip, gzip, deflate"));
- hostConf.getParams().setParameter("http.default-headers", headers);
-
- // HTTP proxy server details
- if (useProxy) {
- hostConf.setProxy(proxyHost, proxyPort);
-
- if (proxyUsername.length() > 0) {
-
- AuthScope proxyAuthScope = getAuthScope(
- this.proxyHost, this.proxyPort, this.proxyRealm);
-
- NTCredentials proxyCredentials = new NTCredentials(
- this.proxyUsername, this.proxyPassword,
- this.agentHost, this.proxyRealm);
-
- client.getState().setProxyCredentials(
- proxyAuthScope, proxyCredentials);
- }
- }
-
- }
-
- /**
- * Reads authentication configuration file (defined as
- * 'http.auth.file' in Nutch configuration file) and sets the
- * credentials for the configured authentication scopes in the HTTP
- * client object.
- *
- * @throws ParserConfigurationException If a document builder can not
- * be created.
- * @throws SAXException If any parsing error occurs.
- * @throws IOException If any I/O error occurs.
- */
- private static synchronized void setCredentials() throws
- ParserConfigurationException, SAXException, IOException {
-
- if (authRulesRead)
- return;
-
- authRulesRead = true; // Avoid re-attempting to read
-
- InputStream is = conf.getConfResourceAsInputStream(authFile);
- if (is != null) {
- Document doc = DocumentBuilderFactory.newInstance()
- .newDocumentBuilder().parse(is);
-
- Element rootElement = doc.getDocumentElement();
- if (!"auth-configuration".equals(rootElement.getTagName())) {
- if (LOG.isWarnEnabled())
- LOG.warn("Bad auth conf file: root element <"
- + rootElement.getTagName() + "> found in " + authFile
- + " - must be <auth-configuration>");
- }
-
- // For each set of credentials
- NodeList credList = rootElement.getChildNodes();
- for (int i = 0; i < credList.getLength(); i++) {
- Node credNode = credList.item(i);
- if (!(credNode instanceof Element))
- continue;
-
- Element credElement = (Element) credNode;
- if (!"credentials".equals(credElement.getTagName())) {
- if (LOG.isWarnEnabled())
- LOG.warn("Bad auth conf file: Element <"
- + credElement.getTagName() + "> not recognized in "
- + authFile + " - expected <credentials>");
- continue;
- }
-
- String username = credElement.getAttribute("username");
- String password = credElement.getAttribute("password");
-
- // For each authentication scope
- NodeList scopeList = credElement.getChildNodes();
- for (int j = 0; j < scopeList.getLength(); j++) {
- Node scopeNode = scopeList.item(j);
- if (!(scopeNode instanceof Element))
- continue;
-
- Element scopeElement = (Element) scopeNode;
-
- if ("default".equals(scopeElement.getTagName())) {
-
- // Determine realm and scheme, if any
- String realm = scopeElement.getAttribute("realm");
- String scheme = scopeElement.getAttribute("scheme");
-
- // Set default credentials
- defaultUsername = username;
- defaultPassword = password;
- defaultRealm = realm;
- defaultScheme = scheme;
-
- if (LOG.isTraceEnabled()) {
- LOG.trace("Credentials - username: " + username
- + "; set as default"
- + " for realm: " + realm + "; scheme: " + scheme);
- }
-
- } else if ("authscope".equals(scopeElement.getTagName())) {
-
- // Determine authentication scope details
- String host = scopeElement.getAttribute("host");
- int port = -1; // For setting port to AuthScope.ANY_PORT
- try {
- port = Integer.parseInt(
- scopeElement.getAttribute("port"));
- } catch (Exception ex) {
- // do nothing, port is already set to any port
- }
- String realm = scopeElement.getAttribute("realm");
- String scheme = scopeElement.getAttribute("scheme");
-
- // Set credentials for the determined scope
- AuthScope authScope = getAuthScope(host, port, realm, scheme);
- NTCredentials credentials = new NTCredentials(
- username, password, agentHost, realm);
-
- client.getState().setCredentials(authScope, credentials);
-
- if (LOG.isTraceEnabled()) {
- LOG.trace("Credentials - username: " + username
- + "; set for AuthScope - " + "host: " + host
- + "; port: " + port + "; realm: " + realm
- + "; scheme: " + scheme);
- }
-
- } else {
- if (LOG.isWarnEnabled())
- LOG.warn("Bad auth conf file: Element <"
- + scopeElement.getTagName() + "> not recognized in "
- + authFile + " - expected <authscope>");
- }
- }
- is.close();
- }
- }
- }
-
- /**
- * If credentials for the authentication scope determined from the
- * specified <code>url</code> is not already set in the HTTP client,
- * then this method sets the default credentials to fetch the
- * specified <code>url</code>. If credentials are found for the
- * authentication scope, the method returns without altering the
- * client.
- *
- * @param url URL to be fetched
- */
- private void resolveCredentials(URL url) {
-
- if (defaultUsername != null && defaultUsername.length() > 0) {
-
- int port = url.getPort();
- if (port == -1) {
- if ("https".equals(url.getProtocol()))
- port = 443;
- else
- port = 80;
- }
-
- AuthScope scope = new AuthScope(url.getHost(), port);
-
- if (client.getState().getCredentials(scope) != null) {
- if (LOG.isTraceEnabled())
- LOG.trace("Pre-configured credentials with scope - host: "
- + url.getHost() + "; port: " + port
- + "; found for url: " + url);
-
- // Credentials are already configured, so do nothing and return
- return;
- }
-
- if (LOG.isTraceEnabled())
- LOG.trace("Pre-configured credentials with scope - host: "
- + url.getHost() + "; port: " + port
- + "; not found for url: " + url);
-
- AuthScope serverAuthScope = getAuthScope(
- url.getHost(), port, defaultRealm, defaultScheme);
-
- NTCredentials serverCredentials = new NTCredentials(
- defaultUsername, defaultPassword,
- agentHost, defaultRealm);
-
- client.getState().setCredentials(
- serverAuthScope, serverCredentials);
- }
- }
-
- /**
- * Returns an authentication scope for the specified
- * <code>host</code>, <code>port</code>, <code>realm</code> and
- * <code>scheme</code>.
- *
- * @param host Host name or address.
- * @param port Port number.
- * @param realm Authentication realm.
- * @param scheme Authentication scheme.
- */
- private static AuthScope getAuthScope(String host, int port,
- String realm, String scheme) {
-
- if (host.length() == 0)
- host = null;
-
- if (port < 0)
- port = -1;
-
- if (realm.length() == 0)
- realm = null;
-
- if (scheme.length() == 0)
- scheme = null;
-
- return new AuthScope(host, port, realm, scheme);
- }
-
- /**
- * Returns an authentication scope for the specified
- * <code>host</code>, <code>port</code> and <code>realm</code>.
- *
- * @param host Host name or address.
- * @param port Port number.
- * @param realm Authentication realm.
- */
- private static AuthScope getAuthScope(String host, int port,
- String realm) {
+ // Since the Configuration has not yet been set,
+ // then an unconfigured client is returned.
+ private static HttpClient client = new HttpClient(connectionManager);
+ private static String defaultUsername;
+ private static String defaultPassword;
+ private static String defaultRealm;
+ private static String defaultScheme;
+ private static String authFile;
+ private static String agentHost;
+ private static boolean authRulesRead = false;
+ private static Configuration conf;
+
+ int maxThreadsTotal = 10;
+
+ private String proxyUsername;
+ private String proxyPassword;
+ private String proxyRealm;
+
+ private static final Collection<WebPage.Field> FIELDS = new
HashSet<WebPage.Field>();
+
+ static {
+ FIELDS.add(WebPage.Field.MODIFIED_TIME);
+ FIELDS.add(WebPage.Field.HEADERS);
+ }
+
+ @Override
+ public Collection<Field> getFields() {
+ return FIELDS;
+ }
+
+ /**
+ * Returns the configured HTTP client.
+ *
+ * @return HTTP client
+ */
+ static synchronized HttpClient getClient() {
+ return client;
+ }
+
+ /**
+ * Constructs this plugin.
+ */
+ public Http() {
+ super(LOG);
+ }
+
+ /**
+ * Reads the configuration from the Nutch configuration files and sets
the
+ * configuration.
+ *
+ * @param conf
+ * Configuration
+ */
+ public void setConf(Configuration conf) {
+ super.setConf(conf);
+ this.conf = conf;
+ this.maxThreadsTotal = conf.getInt("fetcher.threads.fetch", 10);
+ this.proxyUsername = conf.get("http.proxy.username", "");
+ this.proxyPassword = conf.get("http.proxy.password", "");
+ this.proxyRealm = conf.get("http.proxy.realm", "");
+ agentHost = conf.get("http.agent.host", "");
+ authFile = conf.get("http.auth.file", "");
+ configureClient();
+ try {
+ setCredentials();
+ } catch (Exception ex) {
+ if (LOG.isFatalEnabled()) {
+ LOG.fatal("Could not read " + authFile + " : "
+ + ex.getMessage());
+ ex.printStackTrace(LogUtil.getErrorStream(LOG));
+ }
+ }
+ }
+
+ /**
+ * Main method.
+ *
+ * @param args
+ * Command line arguments
+ */
+ public static void main(String[] args) throws Exception {
+ Http http = new Http();
+ http.setConf(NutchConfiguration.create());
+ main(http, args);
+ }
+
+ /**
+ * Fetches the <code>url</code> with a configured HTTP client and gets
the
+ * response.
+ *
+ * @param url
+ * URL to be fetched
+ * @param datum
+ * Crawl data
+ * @param redirect
+ * Follow redirects if and only if true
+ * @return HTTP response
+ */
+ protected Response getResponse(URL url, WebPage page, boolean redirect)
+ throws ProtocolException, IOException {
+ resolveCredentials(url);
+ return new HttpResponse(this, url, page, redirect);
+ }
+
+ /**
+ * Configures the HTTP client
+ */
+ private void configureClient() {
+
+ // Set up an HTTPS socket factory that accepts self-signed
certs.
+ Protocol https = new Protocol("https",
+ new DummySSLProtocolSocketFactory(), 443);
+ Protocol.registerProtocol("https", https);
+
+ HttpConnectionManagerParams params =
connectionManager.getParams();
+ params.setConnectionTimeout(timeout);
+ params.setSoTimeout(timeout);
+ params.setSendBufferSize(BUFFER_SIZE);
+ params.setReceiveBufferSize(BUFFER_SIZE);
+ params.setMaxTotalConnections(maxThreadsTotal);
+ if (maxThreadsTotal > maxThreadsPerHost) {
+
params.setDefaultMaxConnectionsPerHost(maxThreadsPerHost);
+ } else {
+ params.setDefaultMaxConnectionsPerHost(maxThreadsTotal);
+ }
+
+ // executeMethod(HttpMethod) seems to ignore the connection
timeout on
+ // the connection manager.
+ // set it explicitly on the HttpClient.
+ client.getParams().setConnectionManagerTimeout(timeout);
+
+ HostConfiguration hostConf = client.getHostConfiguration();
+ ArrayList headers = new ArrayList();
+ // Set the User Agent in the header
+ headers.add(new Header("User-Agent", userAgent));
+ // prefer English
+ headers.add(new Header("Accept-Language",
+ "en-us,en-gb,en;q=0.7,*;q=0.3"));
+ // prefer UTF-8
+ headers.add(new Header("Accept-Charset",
+ "utf-8,ISO-8859-1;q=0.7,*;q=0.7"));
+ // prefer understandable formats
+ headers.add(new Header(
+ "Accept",
+
"text/html,application/xml;q=0.9,application/xhtml+xml,text/xml;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5"));
+ // accept gzipped content
+ headers.add(new Header("Accept-Encoding", "x-gzip, gzip,
deflate"));
+ hostConf.getParams().setParameter("http.default-headers",
headers);
+
+ // HTTP proxy server details
+ if (useProxy) {
+ hostConf.setProxy(proxyHost, proxyPort);
+
+ if (proxyUsername.length() > 0) {
+
+ AuthScope proxyAuthScope =
getAuthScope(this.proxyHost,
+ this.proxyPort,
this.proxyRealm);
+
+ NTCredentials proxyCredentials = new
NTCredentials(
+ this.proxyUsername,
this.proxyPassword, this.agentHost,
+ this.proxyRealm);
+
+
client.getState().setProxyCredentials(proxyAuthScope,
+ proxyCredentials);
+ }
+ }
+
+ }
+
+ /**
+ * Reads authentication configuration file (defined as 'http.auth.file'
in
+ * Nutch configuration file) and sets the credentials for the configured
+ * authentication scopes in the HTTP client object.
+ *
+ * @throws ParserConfigurationException
+ * If a document builder can not be created.
+ * @throws SAXException
+ * If any parsing error occurs.
+ * @throws IOException
+ * If any I/O error occurs.
+ */
+ private static synchronized void setCredentials()
+ throws ParserConfigurationException, SAXException,
IOException {
+
+ if (authRulesRead)
+ return;
+
+ authRulesRead = true; // Avoid re-attempting to read
+
+ InputStream is = conf.getConfResourceAsInputStream(authFile);
+ if (is != null) {
+ Document doc = DocumentBuilderFactory.newInstance()
+ .newDocumentBuilder().parse(is);
+
+ Element rootElement = doc.getDocumentElement();
+ if
(!"auth-configuration".equals(rootElement.getTagName())) {
+ if (LOG.isWarnEnabled())
+ LOG.warn("Bad auth conf file: root
element <"
+ +
rootElement.getTagName() + "> found in "
+ + authFile + " - must
be <auth-configuration>");
+ }
+
+ // For each set of credentials
+ NodeList credList = rootElement.getChildNodes();
+ for (int i = 0; i < credList.getLength(); i++) {
+ Node credNode = credList.item(i);
+ if (!(credNode instanceof Element))
+ continue;
+
+ Element credElement = (Element) credNode;
+ if
(!"credentials".equals(credElement.getTagName())) {
+ if (LOG.isWarnEnabled())
+ LOG.warn("Bad auth conf file:
Element <"
+ +
credElement.getTagName()
+ + "> not
recognized in " + authFile
+ + " - expected
<credentials>");
+ continue;
+ }
+
+ String username =
credElement.getAttribute("username");
+ String password =
credElement.getAttribute("password");
+
+ // For each authentication scope
+ NodeList scopeList =
credElement.getChildNodes();
+ for (int j = 0; j < scopeList.getLength(); j++)
{
+ Node scopeNode = scopeList.item(j);
+ if (!(scopeNode instanceof Element))
+ continue;
+
+ Element scopeElement = (Element)
scopeNode;
+
+ if
("default".equals(scopeElement.getTagName())) {
+
+ // Determine realm and scheme,
if any
+ String realm =
scopeElement.getAttribute("realm");
+ String scheme =
scopeElement.getAttribute("scheme");
+
+ // Set default credentials
+ defaultUsername = username;
+ defaultPassword = password;
+ defaultRealm = realm;
+ defaultScheme = scheme;
+
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Credentials
- username: " + username
+ + ";
set as default" + " for realm: "
+ + realm
+ "; scheme: " + scheme);
+ }
+
+ } else if
("authscope".equals(scopeElement.getTagName())) {
+
+ // Determine authentication
scope details
+ String host =
scopeElement.getAttribute("host");
+ int port = -1; // For setting
port to AuthScope.ANY_PORT
+ try {
+ port =
Integer.parseInt(scopeElement
+
.getAttribute("port"));
+ } catch (Exception ex) {
+ // do nothing, port is
already set to any port
+ }
+ String realm =
scopeElement.getAttribute("realm");
+ String scheme =
scopeElement.getAttribute("scheme");
+
+ // Set credentials for the
determined scope
+ AuthScope authScope =
getAuthScope(host, port, realm,
+ scheme);
+ NTCredentials credentials = new
NTCredentials(username,
+ password,
agentHost, realm);
+
+ client.getState()
+
.setCredentials(authScope, credentials);
+
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Credentials
- username: " + username
+ + ";
set for AuthScope - " + "host: "
+ + host
+ "; port: " + port + "; realm: "
+ + realm
+ "; scheme: " + scheme);
+ }
+
+ } else {
+ if (LOG.isWarnEnabled())
+ LOG.warn("Bad auth conf
file: Element <"
+ +
scopeElement.getTagName()
+ + ">
not recognized in " + authFile
+ + " -
expected <authscope>");
+ }
+ }
+ is.close();
+ }
+ }
+ }
+
+ /**
+ * If credentials for the authentication scope determined from the
specified
+ * <code>url</code> is not already set in the HTTP client, then this
method
+ * sets the default credentials to fetch the specified
<code>url</code>. If
+ * credentials are found for the authentication scope, the method
returns
+ * without altering the client.
+ *
+ * @param url
+ * URL to be fetched
+ */
+ private void resolveCredentials(URL url) {
+
+ if (defaultUsername != null && defaultUsername.length() > 0) {
+
+ int port = url.getPort();
+ if (port == -1) {
+ if ("https".equals(url.getProtocol()))
+ port = 443;
+ else
+ port = 80;
+ }
+
+ AuthScope scope = new AuthScope(url.getHost(), port);
+
+ if (client.getState().getCredentials(scope) != null) {
+ if (LOG.isTraceEnabled())
+ LOG.trace("Pre-configured credentials
with scope - host: "
+ + url.getHost() + ";
port: " + port
+ + "; found for url: " +
url);
+
+ // Credentials are already configured, so do
nothing and return
+ return;
+ }
+
+ if (LOG.isTraceEnabled())
+ LOG.trace("Pre-configured credentials with
scope - host: "
+ + url.getHost() + "; port: " +
port
+ + "; not found for url: " +
url);
+
+ AuthScope serverAuthScope = getAuthScope(url.getHost(),
port,
+ defaultRealm, defaultScheme);
+
+ NTCredentials serverCredentials = new NTCredentials(
+ defaultUsername, defaultPassword,
agentHost, defaultRealm);
+
+ client.getState()
+ .setCredentials(serverAuthScope,
serverCredentials);
+ }
+ }
+
+ /**
+ * Returns an authentication scope for the specified <code>host</code>,
+ * <code>port</code>, <code>realm</code> and <code>scheme</code>.
+ *
+ * @param host
+ * Host name or address.
+ * @param port
+ * Port number.
+ * @param realm
+ * Authentication realm.
+ * @param scheme
+ * Authentication scheme.
+ */
+ private static AuthScope getAuthScope(String host, int port, String
realm,
+ String scheme) {
+
+ if (host.length() == 0)
+ host = null;
+
+ if (port < 0)
+ port = -1;
+
+ if (realm.length() == 0)
+ realm = null;
+
+ if (scheme.length() == 0)
+ scheme = null;
+
+ return new AuthScope(host, port, realm, scheme);
+ }
+
+ /**
+ * Returns an authentication scope for the specified <code>host</code>,
+ * <code>port</code> and <code>realm</code>.
+ *
+ * @param host
+ * Host name or address.
+ * @param port
+ * Port number.
+ * @param realm
+ * Authentication realm.
+ */
+ private static AuthScope getAuthScope(String host, int port, String
realm) {
- return getAuthScope(host, port, realm, "");
- }
-}
+ return getAuthScope(host, port, realm, "");
+ }
+}
Modified:
nutch/branches/nutchbase/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java?rev=965887&r1=965886&r2=965887&view=diff
==============================================================================
---
nutch/branches/nutchbase/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java
(original)
+++
nutch/branches/nutchbase/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java
Tue Jul 20 15:42:23 2010
@@ -31,12 +31,12 @@ import org.apache.commons.httpclient.par
import org.apache.commons.httpclient.HttpException;
// Nutch imports
-import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.metadata.Metadata;
import org.apache.nutch.metadata.SpellCheckedMetadata;
import org.apache.nutch.net.protocols.HttpDateFormat;
import org.apache.nutch.net.protocols.Response;
import org.apache.nutch.protocol.http.api.HttpBase;
+import org.apache.nutch.storage.WebPage;
/**
* An HTTP response.
@@ -56,13 +56,13 @@ public class HttpResponse implements Res
* @param http An instance of the implementation class
* of this plugin
* @param url URL to be fetched
- * @param datum Crawl data
+ * @param page WebPage
* @param followRedirects Whether to follow redirects; follows
* redirect if and only if this is true
* @return HTTP response
* @throws IOException When an error occurs
*/
- HttpResponse(Http http, URL url, CrawlDatum datum,
+ HttpResponse(Http http, URL url, WebPage page,
boolean followRedirects) throws IOException {
// Prepare GET method for HTTP request
@@ -70,9 +70,9 @@ public class HttpResponse implements Res
GetMethod get = new GetMethod(url.toString());
get.setFollowRedirects(followRedirects);
get.setDoAuthentication(true);
- if (datum.getModifiedTime() > 0) {
+ if (page.getModifiedTime() > 0) {
get.setRequestHeader("If-Modified-Since",
- HttpDateFormat.toString(datum.getModifiedTime()));
+ HttpDateFormat.toString(page.getModifiedTime()));
}
// Set HTTP parameters
Modified:
nutch/branches/nutchbase/src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/TestProtocolHttpClient.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/TestProtocolHttpClient.java?rev=965887&r1=965886&r2=965887&view=diff
==============================================================================
---
nutch/branches/nutchbase/src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/TestProtocolHttpClient.java
(original)
+++
nutch/branches/nutchbase/src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/TestProtocolHttpClient.java
Tue Jul 20 15:42:23 2010
@@ -17,176 +17,196 @@
package org.apache.nutch.protocol.httpclient;
-import java.net.URL;
import java.net.MalformedURLException;
+import java.net.URL;
+
import junit.framework.TestCase;
-import org.mortbay.jetty.Server;
-import org.mortbay.jetty.servlet.ServletHttpContext;
-import org.mortbay.jetty.servlet.ServletHandler;
-import org.mortbay.http.SocketListener;
-import org.mortbay.http.handler.ResourceHandler;
+
import org.apache.hadoop.conf.Configuration;
-import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.net.protocols.Response;
+import org.apache.nutch.storage.WebPage;
+import org.mortbay.jetty.Server;
+import org.mortbay.jetty.nio.SelectChannelConnector;
+import org.mortbay.jetty.servlet.Context;
+import org.mortbay.jetty.servlet.ServletHolder;
/**
* Test cases for protocol-httpclient.
- *
+ *
* @author Susam Pal
*/
public class TestProtocolHttpClient extends TestCase {
- private Server server;
- private Configuration conf;
- private static final String RES_DIR = System.getProperty("test.data", ".");
- private int port;
- private Http http = new Http();
-
- protected void setUp() throws Exception {
-
- ServletHttpContext context = new ServletHttpContext();
- context.setContextPath("/");
- context.setResourceBase(RES_DIR);
- context.addServlet("JSP", "*.jsp", "org.apache.jasper.servlet.JspServlet");
- context.addHandler(new ResourceHandler());
-
- server = new Server();
- server.addContext(context);
-
- conf = new Configuration();
- conf.addResource("nutch-default.xml");
- conf.addResource("nutch-site-test.xml");
-
- http = new Http();
- http.setConf(conf);
- }
-
- protected void tearDown() throws Exception {
- server.stop();
- }
-
- /**
- * Tests whether the client can remember cookies.
- *
- * @throws Exception If an error occurs or the test case fails.
- */
- public void testCookies() throws Exception {
- startServer(47500);
- fetchPage("/cookies.jsp", 200);
- fetchPage("/cookies.jsp?cookie=yes", 200);
- }
-
- /**
- * Tests that no pre-emptive authorization headers are sent by the
- * client.
- *
- * @throws Exception If an error occurs or the test case fails.
- */
- public void testNoPreemptiveAuth() throws Exception {
- startServer(47500);
- fetchPage("/noauth.jsp", 200);
- }
-
- /**
- * Tests default credentials.
- *
- * @throws Exception If an error occurs or the test case fails.
- */
- public void testDefaultCredentials() throws Exception {
- startServer(47502);
- fetchPage("/basic.jsp", 200);
- }
-
- /**
- * Tests basic authentication scheme for various realms.
- *
- * @throws Exception If an error occurs or the test case fails.
- */
- public void testBasicAuth() throws Exception {
- startServer(47500);
- fetchPage("/basic.jsp", 200);
- fetchPage("/basic.jsp?case=1", 200);
- fetchPage("/basic.jsp?case=2", 200);
- server.start();
- }
-
- /**
- * Tests that authentication happens for a defined realm and not for
- * other realms for a host:port when an extra <code>authscope</code>
- * tag is not defined to match all other realms.
- *
- * @throws Exception If an error occurs or the test case fails.
- */
- public void testOtherRealmsNoAuth() throws Exception {
- startServer(47501);
- fetchPage("/basic.jsp", 200);
- fetchPage("/basic.jsp?case=1", 401);
- fetchPage("/basic.jsp?case=2", 401);
- }
-
- /**
- * Tests Digest authentication scheme.
- *
- * @throws Exception If an error occurs or the test case fails.
- */
- public void testDigestAuth() throws Exception {
- startServer(47500);
- fetchPage("/digest.jsp", 200);
- }
-
- /**
- * Tests NTLM authentication scheme.
- *
- * @throws Exception If an error occurs or the test case fails.
- */
- public void testNtlmAuth() throws Exception {
- startServer(47501);
- fetchPage("/ntlm.jsp", 200);
- }
-
- /**
- * Starts the Jetty server at a specified port.
- *
- * @param portno Port number.
- * @throws Exception When an error occurs.
- */
- private void startServer(int portno) throws Exception {
- port = portno;
- SocketListener listener = new SocketListener();
- listener.setHost("127.0.0.1");
- listener.setPort(port);
- server.addListener(listener);
- server.start();
- }
-
- /**
- * Fetches the specified <code>page</code> from the local Jetty server
- * and checks whether the HTTP response status code matches with the
- * expected code.
- *
- * @param page Page to be fetched.
- * @param expectedCode HTTP response status code expected while
- * fetching the page.
- * @throws Exception When an error occurs or test case fails.
- */
- private void fetchPage(String page, int expectedCode)
- throws Exception {
- URL url = new URL("http", "127.0.0.1", port, page);
- Response response = null;
- response = http.getResponse(url, new CrawlDatum(), true);
-
- int code = response.getCode();
- assertEquals("HTTP Status Code for " + url, expectedCode, code);
- }
-
- /**
- * Returns an URL to the specified page.
- *
- * @param page Page available in the local Jetty
- * server.
- * @throws MalformedURLException If an URL can not be formed.
- */
- private URL getURL(String page) throws MalformedURLException {
- return new URL("http", "127.0.0.1", port, page);
- }
+ private Server server;
+ private Configuration conf;
+ private static final String RES_DIR = System.getProperty("test.data",
".");
+ private int port;
+ private Http http = new Http();
+
+ protected void setUp() throws Exception {
+
+ server = new Server();
+
+// Context scontext = new Context();
+// scontext.setContextPath("/");
+// scontext.setResourceBase(RES_DIR);
+// // servlet handler?
+// scontext.addServlet("JSP", "*.jsp",
+// "org.apache.jasper.servlet.JspServlet");
+// scontext.addHandler(new ResourceHandler());
+
+ Context root = new Context(server,"/",Context.SESSIONS);
+ root.setContextPath("/");
+ root.setResourceBase(RES_DIR);
+ ServletHolder sh = new
ServletHolder(org.apache.jasper.servlet.JspServlet.class);
+ root.addServlet(sh, "*.jsp");
+
+ conf = new Configuration();
+ conf.addResource("nutch-default.xml");
+ conf.addResource("nutch-site-test.xml");
+
+ http = new Http();
+ http.setConf(conf);
+ }
+
+ protected void tearDown() throws Exception {
+ server.stop();
+ }
+
+ /**
+ * Tests whether the client can remember cookies.
+ *
+ * @throws Exception
+ * If an error occurs or the test case fails.
+ */
+ public void testCookies() throws Exception {
+ startServer(47500);
+ fetchPage("/cookies.jsp", 200);
+ fetchPage("/cookies.jsp?cookie=yes", 200);
+ }
+
+ /**
+ * Tests that no pre-emptive authorization headers are sent by the
client.
+ *
+ * @throws Exception
+ * If an error occurs or the test case fails.
+ */
+ public void testNoPreemptiveAuth() throws Exception {
+ startServer(47500);
+ fetchPage("/noauth.jsp", 200);
+ }
+
+ /**
+ * Tests default credentials.
+ *
+ * @throws Exception
+ * If an error occurs or the test case fails.
+ */
+ public void testDefaultCredentials() throws Exception {
+ startServer(47502);
+ fetchPage("/basic.jsp", 200);
+ }
+
+ /**
+ * Tests basic authentication scheme for various realms.
+ *
+ * @throws Exception
+ * If an error occurs or the test case fails.
+ */
+ public void testBasicAuth() throws Exception {
+ startServer(47500);
+ fetchPage("/basic.jsp", 200);
+ fetchPage("/basic.jsp?case=1", 200);
+ fetchPage("/basic.jsp?case=2", 200);
+ server.start();
+ }
+
+ /**
+ * Tests that authentication happens for a defined realm and not for
other
+ * realms for a host:port when an extra <code>authscope</code> tag is
not
+ * defined to match all other realms.
+ *
+ * @throws Exception
+ * If an error occurs or the test case fails.
+ */
+ public void testOtherRealmsNoAuth() throws Exception {
+ startServer(47501);
+ fetchPage("/basic.jsp", 200);
+ fetchPage("/basic.jsp?case=1", 401);
+ fetchPage("/basic.jsp?case=2", 401);
+ }
+
+ /**
+ * Tests Digest authentication scheme.
+ *
+ * @throws Exception
+ * If an error occurs or the test case fails.
+ */
+ public void testDigestAuth() throws Exception {
+ startServer(47500);
+ fetchPage("/digest.jsp", 200);
+ }
+
+ /**
+ * Tests NTLM authentication scheme.
+ *
+ * @throws Exception
+ * If an error occurs or the test case fails.
+ */
+ public void testNtlmAuth() throws Exception {
+ startServer(47501);
+ fetchPage("/ntlm.jsp", 200);
+ }
+
+ /**
+ * Starts the Jetty server at a specified port.
+ *
+ * @param portno
+ * Port number.
+ * @throws Exception
+ * When an error occurs.
+ */
+ private void startServer(int portno) throws Exception {
+ port = portno;
+
+ SelectChannelConnector connector1 = new
SelectChannelConnector();
+ connector1.setHost("127.0.0.1");
+ connector1.setPort(port);
+
+ server.addConnector(connector1);
+ server.start();
+ }
+
+ /**
+ * Fetches the specified <code>page</code> from the local Jetty server
and
+ * checks whether the HTTP response status code matches with the
expected
+ * code.
+ *
+ * @param page
+ * Page to be fetched.
+ * @param expectedCode
+ * HTTP response status code expected while fetching the
page.
+ * @throws Exception
+ * When an error occurs or test case fails.
+ */
+ private void fetchPage(String page, int expectedCode) throws Exception {
+ URL url = new URL("http", "127.0.0.1", port, page);
+ Response response = null;
+ response = http.getResponse(url, new WebPage(), true);
+
+ int code = response.getCode();
+ assertEquals("HTTP Status Code for " + url, expectedCode, code);
+ }
+
+ /**
+ * Returns an URL to the specified page.
+ *
+ * @param page
+ * Page available in the local Jetty server.
+ * @throws MalformedURLException
+ * If an URL can not be formed.
+ */
+ private URL getURL(String page) throws MalformedURLException {
+ return new URL("http", "127.0.0.1", port, page);
+ }
}
