svn commit: r1411041 - in /manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main: java/org/apache/manifoldcf/crawler/connectors/wiki/ native2ascii/org/apache/manifoldcf/crawler/connectors/wiki/

kwright Sun, 18 Nov 2012 17:47:17 -0800

Author: kwright
Date: Mon Nov 19 01:46:48 2012
New Revision: 1411041

URL: http://svn.apache.org/viewvc?rev=1411041&view=rev
Log:
Add support for wiki User-Agent header, so I can connect to wikipedia


Modified:
    
manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConfig.java
    
manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
    
manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/wiki/common_en_US.properties
    
manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/wiki/common_ja_JP.properties

Modified: 
manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConfig.java
URL: 
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConfig.java?rev=1411041&r1=1411040&r2=1411041&view=diff
==============================================================================
--- 
manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConfig.java
 (original)
+++ 
manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConfig.java
 Mon Nov 19 01:46:48 2012
@@ -35,7 +35,11 @@ public class WikiConfig
   public static final String PARAM_PORT = "Server port";
   /** Path */
   public static final String PARAM_PATH = "Server path";
-       
+  
+  /** Email */
+  public static final String PARAM_EMAIL = "Email";
+  
+  // Login info
   public static final String PARAM_LOGIN = "serverlogin";
   public static final String PARAM_PASSWORD = "serverpass";
   public static final String PARAM_DOMAIN = "serverdomain";

Modified: 
manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
URL: 
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java?rev=1411041&r1=1411040&r2=1411041&view=diff
==============================================================================
--- 
manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
 (original)
+++ 
manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
 Mon Nov 19 01:46:48 2012
@@ -48,6 +48,7 @@ import org.apache.http.NameValuePair;
 import org.apache.http.message.BasicNameValuePair;
 import org.apache.http.protocol.HTTP;
 import org.apache.http.util.EntityUtils;
+import org.apache.http.message.BasicHeader;
 
 import org.apache.http.conn.ConnectTimeoutException;
 import org.apache.http.client.CircularRedirectException;
@@ -58,6 +59,8 @@ import java.util.*;
 import java.io.*;
 import java.net.*;
 
+import java.util.concurrent.TimeUnit;
+
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -89,6 +92,9 @@ public class WikiConnector extends org.a
   /** Base URL */
   protected String baseURL = null;
   
+  /** The user-agent for this connector instance */
+  protected String userAgent = null;
+
   protected String serverLogin = null;
   protected String serverPass = null;
   protected String serverDomain = null;
@@ -140,6 +146,12 @@ public class WikiConnector extends org.a
   {
     if (hasBeenSetup == false)
     {
+      String emailAddress = params.getParameter(WikiConfig.PARAM_EMAIL);
+      if (emailAddress != null)
+        userAgent = "Mozilla/5.0 (ApacheManifoldCFWikiReader; 
"+((emailAddress==null)?"":emailAddress)+")";
+      else
+        userAgent = null;
+
       String protocol = params.getParameter(WikiConfig.PARAM_PROTOCOL);
       if (protocol == null || protocol.length() == 0)
         protocol = "http";
@@ -627,9 +639,8 @@ public class WikiConnector extends org.a
   public void poll()
     throws ManifoldCFException
   {
-    // MHL
-    //if (connectionManager != null)
-    //  connectionManager.closeIdleConnections(60000L);
+    if (connectionManager != null)
+      connectionManager.closeIdleConnections(60000L,TimeUnit.MILLISECONDS);
   }
 
   /** Close the connection.  Call this before discarding the connection.
@@ -644,7 +655,8 @@ public class WikiConnector extends org.a
     serverPass = null;
     serverDomain = null;
     baseURL = null;
-    
+    userAgent = null;
+
     if (httpClient != null) {
       httpClient = null;
     }
@@ -828,12 +840,19 @@ public class WikiConnector extends org.a
     throws ManifoldCFException, IOException
   {
     tabsArray.add(Messages.getString(locale,"WikiConnector.Server"));
+    tabsArray.add(Messages.getString(locale,"WikiConnector.Email"));
 
     out.print(
 "<script type=\"text/javascript\">\n"+
 "<!--\n"+
 "function checkConfig()\n"+
 "{\n"+
+"  if (editconnection.email.value != \"\" && 
editconnection.email.value.indexOf(\"@\") == -1)\n"+
+"  {\n"+
+"    
alert(\""+Messages.getBodyJavascriptString(locale,"WikiConnector.NeedAValidEmailAddress")+"\");\n"+
+"    editconnection.email.focus();\n"+
+"    return false;\n"+
+"  }\n"+
 "  if (editconnection.serverport.value != \"\" && 
!isInteger(editconnection.serverport.value))\n"+
 "  {\n"+
 "    
alert(\""+Messages.getBodyJavascriptString(locale,"WikiConnector.WikiServerPortMustBeAValidInteger")+"\");\n"+
@@ -851,6 +870,13 @@ public class WikiConnector extends org.a
 "\n"+
 "function checkConfigForSave()\n"+
 "{\n"+
+"  if (editconnection.email.value == \"\")\n"+
+"  {\n"+
+"    
alert(\""+Messages.getBodyJavascriptString(locale,"WikiConnector.EmailAddressRequiredToBeIncludedInAllRequestHeaders")+"\");\n"+
+"    
SelectTab(\""+Messages.getBodyJavascriptString(locale,"WikiConnector.Email")+"\");\n"+
+"    editconnection.email.focus();\n"+
+"    return false;\n"+
+"  }\n"+
 "  if (editconnection.servername.value == \"\")\n"+
 "  {\n"+
 "    
alert(\""+Messages.getBodyJavascriptString(locale,"WikiConnector.PleaseSupplyAValidWikiServerName")+"\");\n"+
@@ -893,6 +919,10 @@ public class WikiConnector extends org.a
     Locale locale, ConfigParams parameters, String tabName)
     throws ManifoldCFException, IOException
   {
+    String email = parameters.getParameter(WikiConfig.PARAM_EMAIL);
+    if (email == null)
+      email = "";
+
     String protocol = parameters.getParameter(WikiConfig.PARAM_PROTOCOL);
     if (protocol == null)
       protocol = "http";
@@ -922,6 +952,25 @@ public class WikiConnector extends org.a
       domain = "";
     }
     
+    // Email tab
+    if (tabName.equals(Messages.getString(locale,"RSSConnector.Email")))
+    {
+      out.print(
+"<table class=\"displaytable\">\n"+
+"  <tr><td class=\"separator\" colspan=\"2\"><hr/></td></tr>\n"+
+"  <tr>\n"+
+"    <td class=\"description\"><nobr>" + 
Messages.getBodyString(locale,"WikiConnector.EmailAddressToContactColon") + 
"</nobr></td><td class=\"value\"><input type=\"text\" size=\"32\" 
name=\"email\" 
value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(email)+"\"/></td>\n"+
+"  </tr>\n"+
+"</table>\n"
+      );
+    }
+    else
+    {
+      out.print(
+"<input type=\"hidden\" name=\"email\" 
value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(email)+"\"/>\n"
+      );
+    }
+
     if (tabName.equals(Messages.getString(locale,"WikiConnector.Server")))
     {
       out.print(
@@ -1004,6 +1053,10 @@ public class WikiConnector extends org.a
     Locale locale, ConfigParams parameters)
     throws ManifoldCFException
   {
+    String email = variableContext.getParameter("email");
+    if (email != null)
+      parameters.setParameter(WikiConfig.PARAM_EMAIL,email);
+
     String protocol = variableContext.getParameter("serverprotocol");
     if (protocol != null)
       parameters.setParameter(WikiConfig.PARAM_PROTOCOL,protocol);
@@ -1613,7 +1666,10 @@ public class WikiConnector extends org.a
   protected HttpRequestBase getInitializedGetMethod(String URL)
     throws IOException
   {
-    return new HttpGet(URL);
+    HttpGet method = new HttpGet(URL);
+    if (userAgent != null)
+      method.setHeader(new BasicHeader("User-Agent",userAgent));
+    return method;
   }
 
   /** Create an initialize a post method */
@@ -1621,6 +1677,9 @@ public class WikiConnector extends org.a
     throws IOException
   {
     HttpPost method = new HttpPost(URL);
+    if (userAgent != null)
+      method.setHeader(new BasicHeader("User-Agent",userAgent));
+
     List<NameValuePair> pairs = new ArrayList<NameValuePair>();
     
     for (String key : params.keySet()) {

Modified: 
manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/wiki/common_en_US.properties
URL: 
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/wiki/common_en_US.properties?rev=1411041&r1=1411040&r2=1411041&view=diff
==============================================================================
--- 
manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/wiki/common_en_US.properties
 (original)
+++ 
manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/wiki/common_en_US.properties
 Mon Nov 19 01:46:48 2012
@@ -23,6 +23,7 @@ WikiConnector.ServerPassword=API passwor
 WikiConnector.ServerDomain=API domain:
 WikiConnector.NamespaceAndTitles=Namespace and Titles
 WikiConnector.NamespaceAndTitles2=Namespaces and titles:
+WikiConnector.Email=Email
 WikiConnector.Namespace=Namespace
 WikiConnector.TitlePrefix=Title prefix
 WikiConnector.Security=Security
@@ -40,6 +41,9 @@ WikiConnector.AllDefaultNamespaceDocumen
 WikiConnector.WikiServerPortMustBeAValidInteger=Wiki server port must be a 
valid integer
 WikiConnector.PathMustStartWithACharacter=Path must start with a '/' character
 WikiConnector.PleaseSupplyAValidWikiServerName=Please supply a valid Wiki 
server name
+WikiConnector.NeedAValidEmailAddress=Need a valid email address
+WikiConnector.EmailAddressRequiredToBeIncludedInAllRequestHeaders=Email 
address required to be included in all request headers
+WikiConnector.EmailAddressToContactColon=Email address to contact:
 WikiConnector.Parameters=Parameters:
 WikiConnector.certificates= certificate(s)
 WikiConnector.DeleteNamespaceTitle=Delete namespace/title #

Modified: 
manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/wiki/common_ja_JP.properties
URL: 
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/wiki/common_ja_JP.properties?rev=1411041&r1=1411040&r2=1411041&view=diff
==============================================================================
--- 
manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/wiki/common_ja_JP.properties
 (original)
+++ 
manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/wiki/common_ja_JP.properties
 Mon Nov 19 01:46:48 2012
@@ -23,6 +23,7 @@ WikiConnector.ServerPassword=API passwor
 WikiConnector.ServerDomain=API domain:
 WikiConnector.NamespaceAndTitles=ååç©ºéã¨é¡å
 WikiConnector.NamespaceAndTitles2=ååç©ºéã¨é¡åï¼
+WikiConnector.Email=Email
 WikiConnector.Namespace=ååç©ºéã
 WikiConnector.TitlePrefix=é¡åæ¥åé è¾
 WikiConnector.Security=Security
@@ -36,10 +37,13 @@ WikiConnector.Add=è¿½å 
 WikiConnector.AddNamespacePrefix=ååç©ºé/æ¥åé è¾ã®è¿½å 
 WikiConnector.UseDefault=ããã©ã«ããä½¿ã
 WikiConnector.TransientError=ä¸æçã¨ã©ã¼ï¼ 
-WikiConnector.=AllDefaultNamespaceDocumentsIncluded=ãã¹ã¦ã®ããã©ã«ãååç©ºéãå«ã
+WikiConnector.AllDefaultNamespaceDocumentsIncluded=ãã¹ã¦ã®ããã©ã«ãååç©ºéãå«ã
 
WikiConnector.WikiServerPortMustBeAValidInteger=Wikiãµã¼ããã¼ãã«ã¯æ´æ°ãå¥åãã¦ãã ãã
 
WikiConnector.PathMustStartWithACharacter=ãã¹åã®åé ã¯ã/ãã«ãã¦ãã ãã
 
WikiConnector.PleaseSupplyAValidWikiServerName=æ£ããWikiãµã¼ãåãå¥åãã¦ãã ãã
+WikiConnector.NeedAValidEmailAddress=Need a valid email address
+WikiConnector.EmailAddressRequiredToBeIncludedInAllRequestHeaders=Email 
address required to be included in all request headers
+WikiConnector.EmailAddressToContactColon=Email address to contact:
 WikiConnector.Parameters=å¼æ°ï¼
 WikiConnector.certificates= è¨¼æè¨¼
 WikiConnector.DeleteNamespaceTitle=ååç©ºéï¼é¡åãåé¤ï¼ #

svn commit: r1411041 - in /manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main: java/org/apache/manifoldcf/crawler/connectors/wiki/ native2ascii/org/apache/manifoldcf/crawler/connectors/wiki/

Reply via email to