Modified: manifoldcf/branches/CONNECTORS-1196/connectors/kafka/connector/src/main/resources/org/apache/manifoldcf/agents/output/kafka/viewConfiguration.html URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/connectors/kafka/connector/src/main/resources/org/apache/manifoldcf/agents/output/kafka/viewConfiguration.html?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/connectors/kafka/connector/src/main/resources/org/apache/manifoldcf/agents/output/kafka/viewConfiguration.html (original) +++ manifoldcf/branches/CONNECTORS-1196/connectors/kafka/connector/src/main/resources/org/apache/manifoldcf/agents/output/kafka/viewConfiguration.html Tue Mar 14 21:17:50 2017 @@ -16,17 +16,16 @@ --> <table class="displaytable"> - <tr> - <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('KafkaConnector.IPColon'))</nobr> - + <tr> + <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('KafkaConnector.IPColon'))</nobr></td> <td class="value">$Encoder.bodyEscape($IP)</td> -</tr> -<tr> + </tr> + <tr> <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('KafkaConnector.PortColon'))</nobr></td> -<td class="value">$Encoder.bodyEscape($PORT)</td> -</tr> -<tr> + <td class="value">$Encoder.bodyEscape($PORT)</td> + </tr> + <tr> <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('KafkaConnector.TopicColon'))</nobr></td> -<td class="value">$Encoder.bodyEscape($TOPIC)</td> -</tr> + <td class="value">$Encoder.bodyEscape($TOPIC)</td> + </tr> </table> \ No newline at end of file
Modified: manifoldcf/branches/CONNECTORS-1196/connectors/kafka/connector/src/test/java/org/apache/manifoldcf/agents/output/kafka/APISanityHSQLDBIT.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/connectors/kafka/connector/src/test/java/org/apache/manifoldcf/agents/output/kafka/APISanityHSQLDBIT.java?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/connectors/kafka/connector/src/test/java/org/apache/manifoldcf/agents/output/kafka/APISanityHSQLDBIT.java (original) +++ manifoldcf/branches/CONNECTORS-1196/connectors/kafka/connector/src/test/java/org/apache/manifoldcf/agents/output/kafka/APISanityHSQLDBIT.java Tue Mar 14 21:17:50 2017 @@ -31,7 +31,6 @@ public class APISanityHSQLDBIT extends B @Test public void sanityCheck() throws Exception { - try { int i; // Create a basic file system connection, and save it. @@ -59,14 +58,6 @@ public class APISanityHSQLDBIT extends B child = new ConfigurationNode("configuration"); - //Testing Repository Connector parameters - // MHL - /* - ConfigurationNode cmisBindingNode = new ConfigurationNode("_PARAMETER_"); - cmisBindingNode.setAttribute("name", CmisConfig.BINDING_PARAM); - cmisBindingNode.setValue(CmisConfig.BINDING_DEFAULT_VALUE); - child.addChild(child.getChildCount(), cmisBindingNode); - */ connectionObject.addChild(connectionObject.getChildCount(), child); requestObject = new Configuration(); @@ -199,10 +190,6 @@ public class APISanityHSQLDBIT extends B waitJobDeleted(jobIDString, 120000L); // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around. - } catch (Exception e) { - e.printStackTrace(); - throw e; - } } protected void startJob(String jobIDString) Modified: manifoldcf/branches/CONNECTORS-1196/connectors/kafka/connector/src/test/java/org/apache/manifoldcf/agents/output/kafka/BaseITHSQLDB.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/connectors/kafka/connector/src/test/java/org/apache/manifoldcf/agents/output/kafka/BaseITHSQLDB.java?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/connectors/kafka/connector/src/test/java/org/apache/manifoldcf/agents/output/kafka/BaseITHSQLDB.java (original) +++ manifoldcf/branches/CONNECTORS-1196/connectors/kafka/connector/src/test/java/org/apache/manifoldcf/agents/output/kafka/BaseITHSQLDB.java Tue Mar 14 21:17:50 2017 @@ -17,6 +17,8 @@ package org.apache.manifoldcf.agents.output.kafka; +import java.io.File; + import java.util.Properties; import org.junit.After; import static org.junit.Assert.fail; @@ -24,7 +26,7 @@ import org.junit.Before; public class BaseITHSQLDB extends org.apache.manifoldcf.crawler.tests.BaseITHSQLDB { - static KafkaLocal kafka; + KafkaLocal kafka; protected String[] getConnectorNames() { return new String[]{"CMIS"}; @@ -48,41 +50,41 @@ public class BaseITHSQLDB extends org.ap Properties kafkaProperties = new Properties(); Properties zkProperties = new Properties(); - try { - //load properties - kafkaProperties.put("broker.id", "0"); - kafkaProperties.put("port", "9092"); - kafkaProperties.put("num.network.threads", "3"); - kafkaProperties.put("num.io.threads", "8"); - kafkaProperties.put("socket.send.buffer.bytes", "102400"); - kafkaProperties.put("socket.receive.buffer.bytes", "102400"); - kafkaProperties.put("socket.request.max.bytes", "104857600"); - kafkaProperties.put("log.dirs", "/tmp/kafka-logs"); - kafkaProperties.put("num.partitions", "1"); - kafkaProperties.put("num.recovery.threads.per.data.dir", "1"); - kafkaProperties.put("log.retention.hours", "168"); - kafkaProperties.put("log.segment.bytes", "1073741824"); - kafkaProperties.put("log.retention.check.interval.ms", "300000"); - kafkaProperties.put("log.cleaner.enable", "false"); - kafkaProperties.put("zookeeper.connect", "localhost:2181"); - kafkaProperties.put("zookeeper.connection.timeout.ms", "6000"); - - zkProperties.put("dataDir", "/tmp/zookeeper"); - zkProperties.put("clientPort", "2181"); - zkProperties.put("maxClientCnxns", "0"); - - //kafkaProperties.load(Class.class.getResourceAsStream("/kafkalocal.properties")); - //zkProperties.load(Class.class.getResourceAsStream("/zklocal.properties")); - System.out.println("Kafka is starting..."); - - //start kafka - kafka = new KafkaLocal(kafkaProperties, zkProperties); - Thread.sleep(5000); - } catch (Exception e) { - e.printStackTrace(System.out); - fail("Error running local Kafka broker"); - e.printStackTrace(System.out); - } + String tmpDir = System.getProperty("java.io.tmpdir"); + File logDir = new File(tmpDir, "kafka-logs"); + logDir.mkdir(); + File zookeeperDir = new File(tmpDir, "zookeeper"); + zookeeperDir.mkdir(); + + //load properties + kafkaProperties.put("broker.id", "0"); + kafkaProperties.put("port", "9092"); + kafkaProperties.put("num.network.threads", "3"); + kafkaProperties.put("num.io.threads", "8"); + kafkaProperties.put("socket.send.buffer.bytes", "102400"); + kafkaProperties.put("socket.receive.buffer.bytes", "102400"); + kafkaProperties.put("socket.request.max.bytes", "104857600"); + kafkaProperties.put("log.dirs", logDir.getAbsolutePath()); + kafkaProperties.put("num.partitions", "1"); + kafkaProperties.put("num.recovery.threads.per.data.dir", "1"); + kafkaProperties.put("log.retention.hours", "168"); + kafkaProperties.put("log.segment.bytes", "1073741824"); + kafkaProperties.put("log.retention.check.interval.ms", "300000"); + kafkaProperties.put("log.cleaner.enable", "false"); + kafkaProperties.put("zookeeper.connect", "localhost:2181"); + kafkaProperties.put("zookeeper.connection.timeout.ms", "6000"); + + zkProperties.put("dataDir", zookeeperDir.getAbsolutePath()); + zkProperties.put("clientPort", "2181"); + zkProperties.put("maxClientCnxns", "0"); + + //kafkaProperties.load(Class.class.getResourceAsStream("/kafkalocal.properties")); + //zkProperties.load(Class.class.getResourceAsStream("/zklocal.properties")); + System.out.println("Kafka is starting..."); + + //start kafka + kafka = new KafkaLocal(kafkaProperties, zkProperties); + Thread.sleep(5000); } @After Modified: manifoldcf/branches/CONNECTORS-1196/connectors/kafka/connector/src/test/java/org/apache/manifoldcf/agents/output/kafka/BaseUIHSQLDB.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/connectors/kafka/connector/src/test/java/org/apache/manifoldcf/agents/output/kafka/BaseUIHSQLDB.java?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/connectors/kafka/connector/src/test/java/org/apache/manifoldcf/agents/output/kafka/BaseUIHSQLDB.java (original) +++ manifoldcf/branches/CONNECTORS-1196/connectors/kafka/connector/src/test/java/org/apache/manifoldcf/agents/output/kafka/BaseUIHSQLDB.java Tue Mar 14 21:17:50 2017 @@ -17,8 +17,17 @@ package org.apache.manifoldcf.agents.output.kafka; +import java.io.File; + +import java.util.Properties; +import org.junit.After; +import static org.junit.Assert.fail; +import org.junit.Before; + public class BaseUIHSQLDB extends org.apache.manifoldcf.crawler.tests.ConnectorBaseUIHSQLDB { + KafkaLocal kafka; + protected String[] getConnectorNames() { return new String[]{"Test Connector"}; } @@ -35,4 +44,52 @@ public class BaseUIHSQLDB extends org.ap return new String[]{"org.apache.manifoldcf.agents.output.kafka.KafkaOutputConnector"}; } + @Before + public void setupKafka() + throws Exception { + Properties kafkaProperties = new Properties(); + Properties zkProperties = new Properties(); + + String tmpDir = System.getProperty("java.io.tmpdir"); + File logDir = new File(tmpDir, "kafka-logs"); + logDir.mkdir(); + File zookeeperDir = new File(tmpDir, "zookeeper"); + zookeeperDir.mkdir(); + + //load properties + kafkaProperties.put("broker.id", "0"); + kafkaProperties.put("port", "9092"); + kafkaProperties.put("num.network.threads", "3"); + kafkaProperties.put("num.io.threads", "8"); + kafkaProperties.put("socket.send.buffer.bytes", "102400"); + kafkaProperties.put("socket.receive.buffer.bytes", "102400"); + kafkaProperties.put("socket.request.max.bytes", "104857600"); + kafkaProperties.put("log.dirs", logDir.getAbsolutePath()); + kafkaProperties.put("num.partitions", "1"); + kafkaProperties.put("num.recovery.threads.per.data.dir", "1"); + kafkaProperties.put("log.retention.hours", "168"); + kafkaProperties.put("log.segment.bytes", "1073741824"); + kafkaProperties.put("log.retention.check.interval.ms", "300000"); + kafkaProperties.put("log.cleaner.enable", "false"); + kafkaProperties.put("zookeeper.connect", "localhost:2181"); + kafkaProperties.put("zookeeper.connection.timeout.ms", "6000"); + + zkProperties.put("dataDir", zookeeperDir.getAbsolutePath()); + zkProperties.put("clientPort", "2181"); + zkProperties.put("maxClientCnxns", "0"); + + //kafkaProperties.load(Class.class.getResourceAsStream("/kafkalocal.properties")); + //zkProperties.load(Class.class.getResourceAsStream("/zklocal.properties")); + System.out.println("Kafka is starting..."); + + //start kafka + kafka = new KafkaLocal(kafkaProperties, zkProperties); + Thread.sleep(5000); + } + + @After + public void cleanUpKafka() { + kafka.stop(); + } + } Modified: manifoldcf/branches/CONNECTORS-1196/connectors/kafka/connector/src/test/java/org/apache/manifoldcf/agents/output/kafka/NavigationHSQLDBUI.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/connectors/kafka/connector/src/test/java/org/apache/manifoldcf/agents/output/kafka/NavigationHSQLDBUI.java?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/connectors/kafka/connector/src/test/java/org/apache/manifoldcf/agents/output/kafka/NavigationHSQLDBUI.java (original) +++ manifoldcf/branches/CONNECTORS-1196/connectors/kafka/connector/src/test/java/org/apache/manifoldcf/agents/output/kafka/NavigationHSQLDBUI.java Tue Mar 14 21:17:50 2017 @@ -21,6 +21,7 @@ import java.util.Locale; import org.apache.manifoldcf.core.tests.HTMLTester; import org.junit.Test; +import org.junit.Ignore; public class NavigationHSQLDBUI extends BaseUIHSQLDB { Modified: manifoldcf/branches/CONNECTORS-1196/connectors/nulloutput/connector/src/main/java/org/apache/manifoldcf/agents/output/nullconnector/NullConnector.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/connectors/nulloutput/connector/src/main/java/org/apache/manifoldcf/agents/output/nullconnector/NullConnector.java?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/connectors/nulloutput/connector/src/main/java/org/apache/manifoldcf/agents/output/nullconnector/NullConnector.java (original) +++ manifoldcf/branches/CONNECTORS-1196/connectors/nulloutput/connector/src/main/java/org/apache/manifoldcf/agents/output/nullconnector/NullConnector.java Tue Mar 14 21:17:50 2017 @@ -136,7 +136,20 @@ public class NullConnector extends org.a { // Establish a session getSession(); - activities.recordActivity(null,INGEST_ACTIVITY,new Long(document.getBinaryLength()),documentURI,"OK",null); + final StringBuffer sb = new StringBuffer(); + final Iterator<String> metadataKeys = document.getFields(); + boolean needComma = false; + while (metadataKeys.hasNext()) { + final String key = metadataKeys.next(); + final String[] values = document.getFieldAsStrings(key); + if (needComma) { + sb.append(","); + } else { + needComma = true; + } + sb.append("\"").append(key).append("\":").append(Integer.toString(values.length)); + } + activities.recordActivity(null,INGEST_ACTIVITY,new Long(document.getBinaryLength()),documentURI,"OK",sb.toString()); return DOCUMENTSTATUS_ACCEPTED; } Propchange: manifoldcf/branches/CONNECTORS-1196/connectors/sharepoint/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Tue Mar 14 21:17:50 2017 @@ -61,3 +61,4 @@ /manifoldcf/branches/CONNECTORS-891/connectors/sharepoint:1568778-1568807 /manifoldcf/branches/CONNECTORS-912/connectors/sharepoint:1579605-1582723 /manifoldcf/branches/CONNECTORS-990/connectors/sharepoint:1610284-1610707 +/manifoldcf/trunk/connectors/sharepoint:1783604-1786950 Modified: manifoldcf/branches/CONNECTORS-1196/connectors/sharepoint/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/sharepoint/editConfiguration_Server.html URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/connectors/sharepoint/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/sharepoint/editConfiguration_Server.html?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/connectors/sharepoint/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/sharepoint/editConfiguration_Server.html (original) +++ manifoldcf/branches/CONNECTORS-1196/connectors/sharepoint/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/sharepoint/editConfiguration_Server.html Tue Mar 14 21:17:50 2017 @@ -47,6 +47,11 @@ #else <option value="5.0">SharePoint Services 5.0 (2013)</option> #end + #if($SERVERVERSION == '6.0') + <option value="6.0" selected="true">SharePoint Services 6.0 (2016)</option> + #else + <option value="6.0">SharePoint Services 6.0 (2016)</option> + #end </select> </td> </tr> Modified: manifoldcf/branches/CONNECTORS-1196/connectors/sharepoint/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/sharepoint/viewConfiguration.html URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/connectors/sharepoint/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/sharepoint/viewConfiguration.html?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/connectors/sharepoint/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/sharepoint/viewConfiguration.html (original) +++ manifoldcf/branches/CONNECTORS-1196/connectors/sharepoint/connector/src/main/resources/org/apache/manifoldcf/authorities/authorities/sharepoint/viewConfiguration.html Tue Mar 14 21:17:50 2017 @@ -27,6 +27,8 @@ SharePoint Services 4.0 (2010) #elseif($SERVERVERSION == '5.0') SharePoint Services 5.0 (2013) + #elseif($SERVERVERSION == '6.0') + SharePoint Services 6.0 (2016) #else Unknown #end Modified: manifoldcf/branches/CONNECTORS-1196/connectors/sharepoint/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/sharepoint/editConfiguration_Server.html URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/connectors/sharepoint/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/sharepoint/editConfiguration_Server.html?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/connectors/sharepoint/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/sharepoint/editConfiguration_Server.html (original) +++ manifoldcf/branches/CONNECTORS-1196/connectors/sharepoint/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/sharepoint/editConfiguration_Server.html Tue Mar 14 21:17:50 2017 @@ -47,6 +47,11 @@ #else <option value="5.0">SharePoint Services 5.0 (2013)</option> #end + #if($SERVERVERSION == '6.0') + <option value="6.0" selected="true">SharePoint Services 6.0 (2016)</option> + #else + <option value="6.0">SharePoint Services 6.0 (2016)</option> + #end </select> </td> </tr> Modified: manifoldcf/branches/CONNECTORS-1196/connectors/sharepoint/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/sharepoint/viewConfiguration.html URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/connectors/sharepoint/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/sharepoint/viewConfiguration.html?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/connectors/sharepoint/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/sharepoint/viewConfiguration.html (original) +++ manifoldcf/branches/CONNECTORS-1196/connectors/sharepoint/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/sharepoint/viewConfiguration.html Tue Mar 14 21:17:50 2017 @@ -27,6 +27,8 @@ SharePoint Services 4.0 (2010) #elseif($SERVERVERSION == '5.0') SharePoint Services 5.0 (2013) + #elseif($SERVERVERSION == '6.0') + SharePoint Services 6.0 (2016) #else Unknown #end Modified: manifoldcf/branches/CONNECTORS-1196/connectors/slack/connector/src/main/java/org/apache/manifoldcf/crawler/notifications/slack/SlackConnector.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/connectors/slack/connector/src/main/java/org/apache/manifoldcf/crawler/notifications/slack/SlackConnector.java?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/connectors/slack/connector/src/main/java/org/apache/manifoldcf/crawler/notifications/slack/SlackConnector.java (original) +++ manifoldcf/branches/CONNECTORS-1196/connectors/slack/connector/src/main/java/org/apache/manifoldcf/crawler/notifications/slack/SlackConnector.java Tue Mar 14 21:17:50 2017 @@ -66,7 +66,7 @@ public class SlackConnector extends org. @Override public void connect(ConfigParams configParameters) { super.connect(configParameters); - this.webHookUrl = configParameters.getParameter(SlackConfig.WEBHOOK_URL_PARAM); + this.webHookUrl = configParameters.getObfuscatedParameter(SlackConfig.WEBHOOK_URL_PARAM); String proxyHost = configParameters.getParameter(SlackConfig.PROXY_HOST_PARAM); String proxyPortString = configParameters.getParameter(SlackConfig.PROXY_PORT_PARAM); @@ -330,7 +330,13 @@ public class SlackConnector extends org. } private static void fillInServerConfigurationMap(Map<String, Object> paramMap, IPasswordMapperActivity mapper, ConfigParams parameters) { - String webHookUrl = getEmptyOnNull(parameters, SlackConfig.WEBHOOK_URL_PARAM); + String webHookUrl = parameters.getObfuscatedParameter(SlackConfig.WEBHOOK_URL_PARAM); + if (webHookUrl == null) { + webHookUrl = StringUtils.EMPTY; + } else { + webHookUrl = mapper.mapPasswordToKey(webHookUrl); + } + String proxyHost = getEmptyOnNull(parameters, SlackConfig.PROXY_HOST_PARAM); String proxyPort = getEmptyOnNull(parameters, SlackConfig.PROXY_PORT_PARAM); String proxyUsername = getEmptyOnNull(parameters, SlackConfig.PROXY_USERNAME_PARAM); @@ -339,7 +345,7 @@ public class SlackConnector extends org. if(proxyPassword == null) { proxyPassword = StringUtils.EMPTY; } else { - mapper.mapPasswordToKey(proxyPassword); + proxyPassword = mapper.mapPasswordToKey(proxyPassword); } String proxyDomain = getEmptyOnNull(parameters, SlackConfig.PROXY_DOMAIN_PARAM); @@ -380,7 +386,7 @@ public class SlackConnector extends org. final String webHookUrl = variableContext.getParameter("webHookUrl"); if (webHookUrl != null) { - parameters.setParameter(SlackConfig.WEBHOOK_URL_PARAM, webHookUrl); + parameters.setObfuscatedParameter(SlackConfig.WEBHOOK_URL_PARAM, variableContext.mapKeyToPassword(webHookUrl)); } final String proxyHost = variableContext.getParameter("proxyHost"); Modified: manifoldcf/branches/CONNECTORS-1196/connectors/slack/connector/src/main/resources/org/apache/manifoldcf/crawler/notifications/slack/ConfigurationView.html URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/connectors/slack/connector/src/main/resources/org/apache/manifoldcf/crawler/notifications/slack/ConfigurationView.html?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/connectors/slack/connector/src/main/resources/org/apache/manifoldcf/crawler/notifications/slack/ConfigurationView.html (original) +++ manifoldcf/branches/CONNECTORS-1196/connectors/slack/connector/src/main/resources/org/apache/manifoldcf/crawler/notifications/slack/ConfigurationView.html Tue Mar 14 21:17:50 2017 @@ -22,7 +22,9 @@ limitations under the License. <nobr>$Encoder.bodyEscape($ResourceBundle.getString('SlackConnector.WebHookUrlColon'))</nobr> </td> <td class="value"> - <nobr>$Encoder.bodyEscape($WEBHOOK_URL)</nobr> + #if($WEBHOOK_URL && !$WEBHOOK_URL.empty) + <nobr>https://hooks.slack.com/services/********/********/************************</nobr> + #end </td> </tr> <tr> Modified: manifoldcf/branches/CONNECTORS-1196/connectors/slack/connector/src/main/resources/org/apache/manifoldcf/crawler/notifications/slack/Configuration_Server.html URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/connectors/slack/connector/src/main/resources/org/apache/manifoldcf/crawler/notifications/slack/Configuration_Server.html?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/connectors/slack/connector/src/main/resources/org/apache/manifoldcf/crawler/notifications/slack/Configuration_Server.html (original) +++ manifoldcf/branches/CONNECTORS-1196/connectors/slack/connector/src/main/resources/org/apache/manifoldcf/crawler/notifications/slack/Configuration_Server.html Tue Mar 14 21:17:50 2017 @@ -24,7 +24,7 @@ limitations under the License. <nobr>$Encoder.bodyEscape($ResourceBundle.getString('SlackConnector.WebHookUrlColon'))</nobr> </td> <td class="value"> - <input id="webHookUrl" name="webHookUrl" type="text" size="80" value="$Encoder.attributeEscape($WEBHOOK_URL)"/> + <input id="webHookUrl" name="webHookUrl" type="password" size="80" value="$Encoder.attributeEscape($WEBHOOK_URL)"/> </td> </tr> <tr> Modified: manifoldcf/branches/CONNECTORS-1196/connectors/tika/pom.xml URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/connectors/tika/pom.xml?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/connectors/tika/pom.xml (original) +++ manifoldcf/branches/CONNECTORS-1196/connectors/tika/pom.xml Tue Mar 14 21:17:50 2017 @@ -204,46 +204,14 @@ <version>${httpcomponent.httpclient.version}</version> </dependency> <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - <version>2.1.3</version> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - <version>2.3.2</version> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-annotations</artifactId> - <version>2.3.0</version> - </dependency> - <dependency> <groupId>org.apache.tika</groupId> <artifactId>tika-core</artifactId> <version>${tika.version}</version> - <exclusions> - <exclusion> - <artifactId>jhighlight</artifactId> - <groupId>com.uwyn</groupId> - </exclusion> - </exclusions> </dependency> <dependency> <groupId>org.apache.tika</groupId> <artifactId>tika-parsers</artifactId> <version>${tika.version}</version> - <exclusions> - <exclusion> - <artifactId>jhighlight</artifactId> - <groupId>com.uwyn</groupId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.codelibs</groupId> - <artifactId>jhighlight</artifactId> - <version>${jhighlight.version}</version> </dependency> <dependency> <groupId>de.l3s.boilerpipe</groupId> Modified: manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConfig.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConfig.java?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConfig.java (original) +++ manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConfig.java Tue Mar 14 21:17:50 2017 @@ -44,6 +44,8 @@ public class WebcrawlerConfig /** Robots usage (a parameter) */ public static final String PARAMETER_ROBOTSUSAGE = "Robots usage"; + /** Meta robots tags usage (a parameter) */ + public static final String PARAMETER_META_ROBOTS_TAGS_USAGE = "Meta robots tags usage"; /** Email (a parameter) */ public static final String PARAMETER_EMAIL = "Email address"; /** Proxy host name (parameter) */ Modified: manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java (original) +++ manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java Tue Mar 14 21:17:50 2017 @@ -123,6 +123,9 @@ public class WebcrawlerConnector extends protected static final int ROBOTS_DATA = 1; protected static final int ROBOTS_ALL = 2; + protected static final int META_ROBOTS_NONE = 0; + protected static final int META_ROBOTS_ALL = 1; + // Relationship types public final static String REL_LINK = "link"; public final static String REL_REDIRECT = "redirect"; @@ -162,6 +165,8 @@ public class WebcrawlerConnector extends /** Robots usage flag */ protected int robotsUsage = ROBOTS_ALL; + /** Meta robots tag usage flag */ + protected int metaRobotsTagsUsage = META_ROBOTS_ALL; /** The user-agent for this connector instance */ protected String userAgent = null; /** The email address for this connector instance */ @@ -364,8 +369,6 @@ public class WebcrawlerConnector extends // Handle everything else if (!isInitialized) { - String x; - // Either set this from the connection name, or just have one. Right now, we have one. String throttleGroupName = ""; @@ -375,15 +378,21 @@ public class WebcrawlerConnector extends userAgent = "Mozilla/5.0 (ApacheManifoldCFWebCrawler; "+emailAddress+")"; from = emailAddress; - x = params.getParameter(WebcrawlerConfig.PARAMETER_ROBOTSUSAGE); + String robotsTxt = params.getParameter(WebcrawlerConfig.PARAMETER_ROBOTSUSAGE); robotsUsage = ROBOTS_ALL; - if (x == null || x.length() == 0 || x.equals("all")) + if (robotsTxt == null || robotsTxt.length() == 0 || robotsTxt.equals("all")) robotsUsage = ROBOTS_ALL; - else if (x.equals("none")) + else if (robotsTxt.equals("none")) robotsUsage = ROBOTS_NONE; - else if (x.equals("data")) + else if (robotsTxt.equals("data")) robotsUsage = ROBOTS_DATA; + String metaRobots = params.getParameter(WebcrawlerConfig.PARAMETER_META_ROBOTS_TAGS_USAGE); + if (metaRobots == null || metaRobots.length() == 0 || metaRobots.equals("all")) + metaRobotsTagsUsage = META_ROBOTS_ALL; + else if (metaRobots.equals("none")) + metaRobotsTagsUsage = META_ROBOTS_NONE; + throttleDescription = new ThrottleDescription(params); credentialsDescription = new CredentialsDescription(params); trustsDescription = new TrustsDescription(params); @@ -1889,6 +1898,9 @@ public class WebcrawlerConnector extends String robotsUsage = parameters.getParameter(WebcrawlerConfig.PARAMETER_ROBOTSUSAGE); if (robotsUsage == null) robotsUsage = "all"; + String metaRobotsTagsUsage = parameters.getParameter(WebcrawlerConfig.PARAMETER_META_ROBOTS_TAGS_USAGE); + if (metaRobotsTagsUsage == null) + metaRobotsTagsUsage = "all"; String proxyHost = parameters.getParameter(WebcrawlerConfig.PARAMETER_PROXYHOST); if (proxyHost == null) proxyHost = ""; @@ -1985,13 +1997,23 @@ public class WebcrawlerConnector extends " </select>\n"+ " </td>\n"+ " </tr>\n"+ +" <tr>\n"+ +" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.MetaRobotsTagsUsage") + "</nobr></td>\n"+ +" <td class=\"value\">\n"+ +" <select name=\"metarobotstagsusage\" size=\"3\">\n"+ +" <option value=\"none\" "+(metaRobotsTagsUsage.equals("none")?"selected=\"selected\"":"")+">" + Messages.getBodyString(locale,"WebcrawlerConnector.DontLookAtMetaRobotsTags") + "</option>\n"+ +" <option value=\"all\" "+(metaRobotsTagsUsage.equals("all")?"selected=\"selected\"":"")+">" + Messages.getBodyString(locale,"WebcrawlerConnector.ObeyMetaRobotsTags") + "</option>\n"+ +" </select>\n"+ +" </td>\n"+ +" </tr>\n"+ "</table>\n" ); } else { out.print( -"<input type=\"hidden\" name=\"robotsusage\" value=\""+robotsUsage+"\"/>\n" +"<input type=\"hidden\" name=\"robotsusage\" value=\""+robotsUsage+"\"/>\n"+ +"<input type=\"hidden\" name=\"metarobotstagsusage\" value=\""+metaRobotsTagsUsage+"\"/>\n" ); } @@ -2883,6 +2905,9 @@ public class WebcrawlerConnector extends String robotsUsage = variableContext.getParameter("robotsusage"); if (robotsUsage != null) parameters.setParameter(WebcrawlerConfig.PARAMETER_ROBOTSUSAGE,robotsUsage); + String obeyMetaRobotsTags = variableContext.getParameter("metarobotstagsusage"); + if (obeyMetaRobotsTags != null) + parameters.setParameter(WebcrawlerConfig.PARAMETER_META_ROBOTS_TAGS_USAGE, obeyMetaRobotsTags); String proxyHost = variableContext.getParameter("proxyhost"); if (proxyHost != null) parameters.setParameter(WebcrawlerConfig.PARAMETER_PROXYHOST,proxyHost); @@ -3277,11 +3302,16 @@ public class WebcrawlerConnector extends String email = parameters.getParameter(WebcrawlerConfig.PARAMETER_EMAIL); String robots = parameters.getParameter(WebcrawlerConfig.PARAMETER_ROBOTSUSAGE); if (robots.equals("none")) - robots = "Ignore robots.txt"; + robots = Messages.getBodyString(locale,"WebcrawlerConnector.DontLookAtRobotsTxt"); else if (robots.equals("data")) - robots = "Obey robots.txt for data fetches only"; + robots = Messages.getBodyString(locale,"WebcrawlerConnector.ObeyRobotsTxtForDataFetchesOnly"); else if (robots.equals("all")) - robots = "Obey robots.txt for all fetches"; + robots = Messages.getBodyString(locale,"WebcrawlerConnector.ObeyRobotsTxtForAllFetches"); + String metaRobotsTagsUsage = parameters.getParameter(WebcrawlerConfig.PARAMETER_META_ROBOTS_TAGS_USAGE); + if (metaRobotsTagsUsage == null || metaRobotsTagsUsage.equals("all")) + metaRobotsTagsUsage = Messages.getBodyString(locale,"WebcrawlerConnector.ObeyMetaRobotsTags"); + else if (metaRobotsTagsUsage.equals("none")) + metaRobotsTagsUsage = Messages.getBodyString(locale,"WebcrawlerConnector.DontLookAtMetaRobotsTags"); String proxyHost = parameters.getParameter(WebcrawlerConfig.PARAMETER_PROXYHOST); if (proxyHost == null) proxyHost = ""; @@ -3299,9 +3329,13 @@ public class WebcrawlerConnector extends "<table class=\"displaytable\">\n"+ " <tr>\n"+ " <td class=\"description\" colspan=\"1\"><nobr>"+Messages.getBodyString(locale,"WebcrawlerConnector.EmailAddress")+"</nobr></td>\n"+ -" <td class=\"value\" colspan=\"1\">"+Encoder.bodyEscape(email)+"</td>\n"+ +" <td class=\"value\" colspan=\"3\">"+Encoder.bodyEscape(email)+"</td>\n"+ +" </tr>\n"+ +" <tr>\n"+ " <td class=\"description\" colspan=\"1\"><nobr>"+Messages.getBodyString(locale,"WebcrawlerConnector.RobotsUsage")+"</nobr></td>\n"+ " <td class=\"value\" colspan=\"1\"><nobr>"+Encoder.bodyEscape(robots)+"</nobr></td>\n"+ +" <td class=\"description\" colspan=\"1\"><nobr>"+Messages.getBodyString(locale,"WebcrawlerConnector.MetaRobotsTagsUsage")+"</nobr></td>\n"+ +" <td class=\"value\" colspan=\"1\">"+Encoder.bodyEscape(metaRobotsTagsUsage)+"</td>\n"+ " </tr>\n"+ " <tr>\n"+ " <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.ProxyHostColon") + "</nobr></td>\n"+ @@ -6048,7 +6082,7 @@ public class WebcrawlerConnector extends if (Logging.connectors.isDebugEnabled() && redirectHandler.shouldIndex() == false) Logging.connectors.debug("Web: Not indexing document '"+documentIdentifier+"' because of redirection"); // For html, we don't want any actions, because we don't do form submission. - ProcessActivityHTMLHandler htmlHandler = new ProcessActivityHTMLHandler(documentIdentifier,activities,filter); + ProcessActivityHTMLHandler htmlHandler = new ProcessActivityHTMLHandler(documentIdentifier,activities,filter,metaRobotsTagsUsage); handleHTML(documentIdentifier,htmlHandler); if (Logging.connectors.isDebugEnabled() && htmlHandler.shouldIndex() == false) Logging.connectors.debug("Web: Not indexing document '"+documentIdentifier+"' because of HTML robots or content tags prohibiting indexing"); @@ -6124,11 +6158,13 @@ public class WebcrawlerConnector extends { boolean allowIndex = true; boolean allowFollow = true; + boolean obeyMetaRobotsTags = true; /** Constructor. */ - public ProcessActivityHTMLHandler(String documentIdentifier, IProcessActivity activities, DocumentURLFilter filter) + public ProcessActivityHTMLHandler(String documentIdentifier, IProcessActivity activities, DocumentURLFilter filter, int metaRobotTagsUsage) { super(documentIdentifier,activities,filter,"html",REL_LINK); + this.obeyMetaRobotsTags = metaRobotTagsUsage == META_ROBOTS_ALL; } /** Decide whether we should index. */ @@ -6152,7 +6188,7 @@ public class WebcrawlerConnector extends throws ManifoldCFException { String name = (String)metaAttributes.get("name"); - if (name != null && name.toLowerCase(Locale.ROOT).equals("robots")) + if (obeyMetaRobotsTags && name != null && name.toLowerCase(Locale.ROOT).equals("robots")) { String contentValue = (String)metaAttributes.get("content"); if (contentValue != null) Modified: manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/webcrawler/common_en_US.properties URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/webcrawler/common_en_US.properties?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/webcrawler/common_en_US.properties (original) +++ manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/webcrawler/common_en_US.properties Tue Mar 14 21:17:50 2017 @@ -29,6 +29,9 @@ WebcrawlerConnector.RobotsTxtUsage=Robot WebcrawlerConnector.DontLookAtRobotsTxt=Don't look at robots.txt WebcrawlerConnector.ObeyRobotsTxtForDataFetchesOnly=Obey robots.txt for data fetches only WebcrawlerConnector.ObeyRobotsTxtForAllFetches=Obey robots.txt for all fetches +WebcrawlerConnector.MetaRobotsTagsUsage=Meta robots tags usage: +WebcrawlerConnector.DontLookAtMetaRobotsTags=Don't look at meta robots tags +WebcrawlerConnector.ObeyMetaRobotsTags=Obey meta robots tags WebcrawlerConnector.Throttles=Throttles: WebcrawlerConnector.BinRegularExpression=Bin regular expression WebcrawlerConnector.CaseInsensitive=Case insensitive? Modified: manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/webcrawler/common_es_ES.properties URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/webcrawler/common_es_ES.properties?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/webcrawler/common_es_ES.properties (original) +++ manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/webcrawler/common_es_ES.properties Tue Mar 14 21:17:50 2017 @@ -21,182 +21,185 @@ WebcrawlerConnector.Certificates=Certifi WebcrawlerConnector.Proxy=Proxy WebcrawlerConnector.ProxyHostColon=host Proxy: WebcrawlerConnector.ProxyPortColon=Puerto de proxy: -WebcrawlerConnector.ProxyAuthenticationDomainColon=Dominio de autenticación proxy: -WebcrawlerConnector.ProxyAuthenticationUserNameColon=Nombre de usuario de autenticación proxy: -WebcrawlerConnector.ProxyAuthenticationPasswordColon=Contraseña de autenticación proxy: -WebcrawlerConnector.EmailAddressToContact=Dirección de correo electrónico para ponerse en contacto: +WebcrawlerConnector.ProxyAuthenticationDomainColon=Dominio de autenticación proxy: +WebcrawlerConnector.ProxyAuthenticationUserNameColon=Nombre de usuario de autenticación proxy: +WebcrawlerConnector.ProxyAuthenticationPasswordColon=Contraseña de autenticación proxy: +WebcrawlerConnector.EmailAddressToContact=Dirección de correo electrónico para ponerse en contacto: WebcrawlerConnector.RobotsTxtUsage=uso robots.txtV: WebcrawlerConnector.DontLookAtRobotsTxt=No mire robots.txt WebcrawlerConnector.ObeyRobotsTxtForDataFetchesOnly=Obedecer robots.txt para datos obtiene solamente WebcrawlerConnector.ObeyRobotsTxtForAllFetches=Obedecer robots.txt para todas las recuperaciones +WebcrawlerConnector.MetaRobotsTagsUsage=Meta robots tags usage: +WebcrawlerConnector.DontLookAtMetaRobotsTags=Don't look at meta robots tags +WebcrawlerConnector.ObeyMetaRobotsTags=Obey meta robots tags WebcrawlerConnector.Throttles=aceleradores: -WebcrawlerConnector.BinRegularExpression=Expresión regular Bin -WebcrawlerConnector.CaseInsensitive=mayúsculas y minúsculas? +WebcrawlerConnector.BinRegularExpression=Expresión regular Bin +WebcrawlerConnector.CaseInsensitive=mayúsculas y minúsculas? WebcrawlerConnector.MaxConnections=conexiones Max WebcrawlerConnector.MaxKbytesSec=Max Kbytes / seg WebcrawlerConnector.MaxFetchesMin=Max obtiene / min WebcrawlerConnector.Delete=Delete -WebcrawlerConnector.DeleteBinRegularExpression=Eliminar expresión regular bin # -WebcrawlerConnector.Add=Añadir -WebcrawlerConnector.AddBinRegularExpression=Añadir expresión regular bin -WebcrawlerConnector.PageAccessCredentials=Credenciales de acceso a la página: -WebcrawlerConnector.URLRegularExpression=Expresión regular de URL -WebcrawlerConnector.AddPageAuthenticationUrlRegularExpression=Añadir página de autenticación url expresión regular -WebcrawlerConnector.PageAccessCredentials=Credenciales de acceso a la página: -WebcrawlerConnector.URLRegularExpression=Expresión regular de URL +WebcrawlerConnector.DeleteBinRegularExpression=Eliminar expresión regular bin # +WebcrawlerConnector.Add=Añadir +WebcrawlerConnector.AddBinRegularExpression=Añadir expresión regular bin +WebcrawlerConnector.PageAccessCredentials=Credenciales de acceso a la página: +WebcrawlerConnector.URLRegularExpression=Expresión regular de URL +WebcrawlerConnector.AddPageAuthenticationUrlRegularExpression=Añadir página de autenticación url expresión regular +WebcrawlerConnector.PageAccessCredentials=Credenciales de acceso a la página: +WebcrawlerConnector.URLRegularExpression=Expresión regular de URL WebcrawlerConnector.CredentialType=tipo de Credencial WebcrawlerConnector.CredentialDomain=dominio de credenciales WebcrawlerConnector.UserName=Nombre de usuario -WebcrawlerConnector.NoPageAccessCredentials=Sin credenciales de acceso a la página -WebcrawlerConnector.SessionBasedAccessCredentials=Sesión-credenciales de acceso basados: -WebcrawlerConnector.LoginPages=páginas Iniciar sesión -WebcrawlerConnector.LoginURLRegularExpression=Expresión regular Ingresar URL -WebcrawlerConnector.PageType=tipo de Página -WebcrawlerConnector.FormNamelinkTargetRegularExpression=Expresión regular identificación -WebcrawlerConnector.OverrideFormParameters=Parámetros de forma de anulación -WebcrawlerConnector.OverrideTargetURL=URL de destino de anulación -WebcrawlerConnector.NoLoginPagesSpecified=Ninguna página de inicio de sesión especificado -WebcrawlerConnector.NoSessionBasedAccessCredentials=Ninguna sesión-credenciales de acceso basados +WebcrawlerConnector.NoPageAccessCredentials=Sin credenciales de acceso a la página +WebcrawlerConnector.SessionBasedAccessCredentials=Sesión-credenciales de acceso basados: +WebcrawlerConnector.LoginPages=páginas Iniciar sesión +WebcrawlerConnector.LoginURLRegularExpression=Expresión regular Ingresar URL +WebcrawlerConnector.PageType=tipo de Página +WebcrawlerConnector.FormNamelinkTargetRegularExpression=Expresión regular identificación +WebcrawlerConnector.OverrideFormParameters=Parámetros de forma de anulación +WebcrawlerConnector.OverrideTargetURL=URL de destino de anulación +WebcrawlerConnector.NoLoginPagesSpecified=Ninguna página de inicio de sesión especificado +WebcrawlerConnector.NoSessionBasedAccessCredentials=Ninguna sesión-credenciales de acceso basados WebcrawlerConnector.TrustCertificates=certificados fiduciarios: -WebcrawlerConnector.URLRegularExpression=Expresión regular de URL +WebcrawlerConnector.URLRegularExpression=Expresión regular de URL WebcrawlerConnector.Certificate=Certificado WebcrawlerConnector.TrustEverything=La confianza de todo WebcrawlerConnector.NoTrustCertificates=No hay certificados de confianza -WebcrawlerConnector.Description=Descripción +WebcrawlerConnector.Description=Descripción WebcrawlerConnector.Reorder=reordenar? WebcrawlerConnector.RemoveJSPSessions=Retire sesiones JSP? WebcrawlerConnector.RemoveASPSessions=Retire sesiones ASP? WebcrawlerConnector.RemovePHPSessions=Retire las sesiones de PHP? WebcrawlerConnector.RemoveBVSessions=Retire sesiones BV? WebcrawlerConnector.DeleteUrlRegexp2=Eliminar regexp url -WebcrawlerConnector.NoCanonicalizationSpecified=No canonicalización especificada - todas las direcciones URL se reordenan y se han eliminado todas las sesiones -WebcrawlerConnector.AddUrlRegexp=Añadir regexp url +WebcrawlerConnector.NoCanonicalizationSpecified=No canonicalización especificada - todas las direcciones URL se reordenan y se han eliminado todas las sesiones +WebcrawlerConnector.AddUrlRegexp=Añadir regexp url WebcrawlerConnector.IncludeInCrawl=Incluir en rastreo: -WebcrawlerConnector.IncludeInIndex=Incluir en el índice: -WebcrawlerConnector.IncludeOnlyHostsMatchingSeeds=Incluya sólo los hosts que emparejan semillas? +WebcrawlerConnector.IncludeInIndex=Incluir en el Ãndice: +WebcrawlerConnector.IncludeOnlyHostsMatchingSeeds=Incluya sólo los hosts que emparejan semillas? WebcrawlerConnector.ExcludeFromCrawl=Excluir de rastreo: -WebcrawlerConnector.ExcludeFromIndex=Excluir del índice: -WebcrawlerConnector.ExcludeContentFromIndex=Excluir contenido del índice: -WebcrawlerConnector.DeleteToken=eliminar símbolo # +WebcrawlerConnector.ExcludeFromIndex=Excluir del Ãndice: +WebcrawlerConnector.ExcludeContentFromIndex=Excluir contenido del Ãndice: +WebcrawlerConnector.DeleteToken=eliminar sÃmbolo # WebcrawlerConnector.NoAccessTokensPresent=No hay tokens de acceso actuales -WebcrawlerConnector.AddAccessToken=Añadir token de acceso +WebcrawlerConnector.AddAccessToken=Añadir token de acceso WebcrawlerConnector.DeleteMetadata=eliminar metadatos # WebcrawlerConnector.NoMetadataPresent=Sin metadatos presente -WebcrawlerConnector.AddMetadata=Añadir metadatos +WebcrawlerConnector.AddMetadata=Añadir metadatos WebcrawlerConnector.Seeds=Semillas: -WebcrawlerConnector.URLCanonicalization=canonicalización URL: +WebcrawlerConnector.URLCanonicalization=canonicalización URL: WebcrawlerConnector.URLRegexp=URL regexp WebcrawlerConnector.AccessTokens=tokens de acceso: WebcrawlerConnector.NoAccessTokensSpecified=No hay tokens de acceso especificados WebcrawlerConnector.Metadata=metadatos: WebcrawlerConnector.NoMetadataSpecified=Sin metadatos especificada -WebcrawlerConnector.BasicAuthentication=autenticación básica -WebcrawlerConnector.NTLMAuthentication=Autenticación NTLM +WebcrawlerConnector.BasicAuthentication=autenticación básica +WebcrawlerConnector.NTLMAuthentication=Autenticación NTLM WebcrawlerConnector.UploadCertificate=Cargue certificado: -WebcrawlerConnector.DeletePageAuthenticationUrlRegularExpression=Eliminar página de autenticación url expresión regular # -WebcrawlerConnector.SessionBasedAccessCredentials=Sesión-credenciales de acceso basados: -WebcrawlerConnector.URLRegularExpression=Expresión regular de URL -WebcrawlerConnector.LoginPages=páginas Iniciar sesión +WebcrawlerConnector.DeletePageAuthenticationUrlRegularExpression=Eliminar página de autenticación url expresión regular # +WebcrawlerConnector.SessionBasedAccessCredentials=Sesión-credenciales de acceso basados: +WebcrawlerConnector.URLRegularExpression=Expresión regular de URL +WebcrawlerConnector.LoginPages=páginas Iniciar sesión WebcrawlerConnector.TrustCertificates=certificados fiduciarios: -WebcrawlerConnector.URLRegularExpression=Expresión regular de URL +WebcrawlerConnector.URLRegularExpression=Expresión regular de URL WebcrawlerConnector.Certificate=Certificado -WebcrawlerConnector.DeleteSessionAuthenticationUrlRegularExpression=Eliminar sesión de autenticación url expresión regular # -WebcrawlerConnector.LoginURLRegularExpression=Expresión regular Ingresar URL -WebcrawlerConnector.PageType=tipo de Página +WebcrawlerConnector.DeleteSessionAuthenticationUrlRegularExpression=Eliminar sesión de autenticación url expresión regular # +WebcrawlerConnector.LoginURLRegularExpression=Expresión regular Ingresar URL +WebcrawlerConnector.PageType=tipo de Página WebcrawlerConnector.NoTrustCertificatesSpecified=No hay certificados de confianza especificados -WebcrawlerConnector.AddUrlRegularExpressionForTruststore=Añadir expresión regular url para almacén de confianza +WebcrawlerConnector.AddUrlRegularExpressionForTruststore=Añadir expresión regular url para almacén de confianza WebcrawlerConnector.TrustEverything=La confianza de todo -WebcrawlerConnector.NoSessionBasedAccessCredentialsSpecified=Ninguna sesión-credenciales de acceso basados especifican -WebcrawlerConnector.NoPageAccessCredentialsSpecified=Sin credenciales de acceso a la página especifican -WebcrawlerConnector.AuthenticationType=Tipo de autenticación +WebcrawlerConnector.NoSessionBasedAccessCredentialsSpecified=Ninguna sesión-credenciales de acceso basados especifican +WebcrawlerConnector.NoPageAccessCredentialsSpecified=Sin credenciales de acceso a la página especifican +WebcrawlerConnector.AuthenticationType=Tipo de autenticación WebcrawlerConnector.Domain=Dominio WebcrawlerConnector.UserName=Nombre de usuario -WebcrawlerConnector.Password=Contraseña -WebcrawlerConnector.EmailAaddressRequired=Dirección de correo electrónico requerida, que se incluirá en todos los encabezados de solicitud -WebcrawlerConnector.MaximumConnectionsMustBeAnInteger=Conexiones máximas deben ser un número entero -WebcrawlerConnector.NeedAValidEmailAddress=Necesitas una dirección de correo electrónico válida -WebcrawlerConnector.MaximumKbytesPerSecondMustBeAnInteger=Kbytes máximos por segundo debe ser un número entero -WebcrawlerConnector.MaximumFetchesPerMinuteMustBeAnInteger=Recuperaciones máximas por minuto debe ser un número entero +WebcrawlerConnector.Password=Contraseña +WebcrawlerConnector.EmailAaddressRequired=Dirección de correo electrónico requerida, que se incluirá en todos los encabezados de solicitud +WebcrawlerConnector.MaximumConnectionsMustBeAnInteger=Conexiones máximas deben ser un número entero +WebcrawlerConnector.NeedAValidEmailAddress=Necesitas una dirección de correo electrónico válida +WebcrawlerConnector.MaximumKbytesPerSecondMustBeAnInteger=Kbytes máximos por segundo debe ser un número entero +WebcrawlerConnector.MaximumFetchesPerMinuteMustBeAnInteger=Recuperaciones máximas por minuto debe ser un número entero WebcrawlerConnector.CredentialMustHaveNonNullUserName=Credencial debe tener el nombre de usuario no nulo -WebcrawlerConnector.MatchExpressionMustBeAValidRegularExpression=Expresión Partido debe ser una expresión regular válida -WebcrawlerConnector.ParameterMustHaveNonEmptyName=Parámetro debe tener no-nombre vacío -WebcrawlerConnector.ParameterCanEitherBeHidden=Parámetro tampoco se puede ocultar o no, pero no pueden ser a la vez -WebcrawlerConnector.MaximumKbytesPerSecondMustBeAnInteger=Kbytes máximos por segundo debe ser un número entero -WebcrawlerConnector.MaximumFetchesPerMinuteMustBeAnInteger=Recuperaciones máximas por minuto debe ser un número entero -WebcrawlerConnector.AValidRegularExpressionIsRequired=Se requiere una expresión regular válida +WebcrawlerConnector.MatchExpressionMustBeAValidRegularExpression=Expresión Partido debe ser una expresión regular válida +WebcrawlerConnector.ParameterMustHaveNonEmptyName=Parámetro debe tener no-nombre vacÃo +WebcrawlerConnector.ParameterCanEitherBeHidden=Parámetro tampoco se puede ocultar o no, pero no pueden ser a la vez +WebcrawlerConnector.MaximumKbytesPerSecondMustBeAnInteger=Kbytes máximos por segundo debe ser un número entero +WebcrawlerConnector.MaximumFetchesPerMinuteMustBeAnInteger=Recuperaciones máximas por minuto debe ser un número entero +WebcrawlerConnector.AValidRegularExpressionIsRequired=Se requiere una expresión regular válida WebcrawlerConnector.CredentialMustIncludeANonNullUserName=Credencial debe incluir un no-Nombre de usuario nula -WebcrawlerConnector.ParameterNameMustBeARegularExpression=Nombre del parámetro debe ser una expresión regular +WebcrawlerConnector.ParameterNameMustBeARegularExpression=Nombre del parámetro debe ser una expresión regular WebcrawlerConnector.SpecifyATrustCertificateFileToUploadFirst=Especifique un archivo de certificado de confianza para subir primero, o compruebe 'La confianza de todo' -WebcrawlerConnector.NoBandwidthOrConnectionThrottlingSpecified=No ancho de banda o la conexión de estrangulamiento especificado -WebcrawlerConnector.DeleteLoginPage=Eliminar la página de inicio de sesión # -WebcrawlerConnector.ParameterRegularExpression=Expresión regular Parámetro +WebcrawlerConnector.NoBandwidthOrConnectionThrottlingSpecified=No ancho de banda o la conexión de estrangulamiento especificado +WebcrawlerConnector.DeleteLoginPage=Eliminar la página de inicio de sesión # +WebcrawlerConnector.ParameterRegularExpression=Expresión regular Parámetro WebcrawlerConnector.Value=Valor -WebcrawlerConnector.Password=Contraseña -WebcrawlerConnector.DeleteParameter=eliminar parámetro # -WebcrawlerConnector.ForLoginPage= para la página de inicio de sesión # +WebcrawlerConnector.Password=Contraseña +WebcrawlerConnector.DeleteParameter=eliminar parámetro # +WebcrawlerConnector.ForLoginPage= para la página de inicio de sesión # WebcrawlerConnector.ForCredential= para credenciales # -WebcrawlerConnector.AddParameterToLoginPage=Añadir parámetro a la página iniciar sesión # -WebcrawlerConnector.AddLoginPageToCredential=Añadir página de acceso a la credencial # -WebcrawlerConnector.FormName=Nombre del formulario/id/acción +WebcrawlerConnector.AddParameterToLoginPage=Añadir parámetro a la página iniciar sesión # +WebcrawlerConnector.AddLoginPageToCredential=Añadir página de acceso a la credencial # +WebcrawlerConnector.FormName=Nombre del formulario/id/acción WebcrawlerConnector.LinkTarget=objetivo Enlace -WebcrawlerConnector.RedirectionTo=La redirección de -WebcrawlerConnector.PageContent=contenido de la página -WebcrawlerConnector.AddSessionAuthenticationUrlRegularExpression=Agregar sesión de autenticación url expresión regular -WebcrawlerConnector.DeleteTrustUrlRegularExpression=Eliminar la confianza url expresión regular # +WebcrawlerConnector.RedirectionTo=La redirección de +WebcrawlerConnector.PageContent=contenido de la página +WebcrawlerConnector.AddSessionAuthenticationUrlRegularExpression=Agregar sesión de autenticación url expresión regular +WebcrawlerConnector.DeleteTrustUrlRegularExpression=Eliminar la confianza url expresión regular # WebcrawlerConnector.TrustEverything=La confianza de todo -WebcrawlerConnector.EmailAddress=Dirección de correo electrónico: +WebcrawlerConnector.EmailAddress=Dirección de correo electrónico: WebcrawlerConnector.RobotsUsage=uso de Robots: WebcrawlerConnector.BandwidthThrottling=del ancho de banda: -WebcrawlerConnector.BinRegularExpression=Expresión regular Bin -WebcrawlerConnector.CaseInsensitive=mayúsculas y minúsculas +WebcrawlerConnector.BinRegularExpression=Expresión regular Bin +WebcrawlerConnector.CaseInsensitive=mayúsculas y minúsculas WebcrawlerConnector.MaxConnections=conexiones Max WebcrawlerConnector.MaxKbytesSec=Max kbytes/segundo WebcrawlerConnector.MaxFetchesMin=Max obtiene/min WebcrawlerConnector.NoBandwidthThrottling=No del ancho de banda WebcrawlerConnector.Seeds=Semillas WebcrawlerConnector.SeedsColon=Semillas: -WebcrawlerConnector.Canonicalization=canonicalización +WebcrawlerConnector.Canonicalization=canonicalización WebcrawlerConnector.Inclusions=inclusiones WebcrawlerConnector.Exclusions=Exclusiones WebcrawlerConnector.Security=Seguridad WebcrawlerConnector.Metadata=metadatos -WebcrawlerConnector.FoundAnIllegalRegularExpressionIn=Encontró un expresión regular ilegal en +WebcrawlerConnector.FoundAnIllegalRegularExpressionIn=Encontró un expresión regular ilegal en WebcrawlerConnector.ErrorWas=Error era: WebcrawlerConnector.TypeInAnAccessToken=Escriba un token de acceso WebcrawlerConnector.TypeInMetadataName=Escriba el nombre de metadatos WebcrawlerConnector.TypeInMetadataValue=Tipo de valor metadatos -WebcrawlerConnector.NeedAValidEmailAddress=Necesitas una dirección de correo electrónico válida -WebcrawlerConnector.MaximumKbytesPerSecondMustBeAnInteger=Kbytes máximos por segundo debe ser un número entero -WebcrawlerConnector.MaximumFetchesPerMinuteMustBeAnInteger=Recuperaciones máximas por minuto debe ser un número entero +WebcrawlerConnector.NeedAValidEmailAddress=Necesitas una dirección de correo electrónico válida +WebcrawlerConnector.MaximumKbytesPerSecondMustBeAnInteger=Kbytes máximos por segundo debe ser un número entero +WebcrawlerConnector.MaximumFetchesPerMinuteMustBeAnInteger=Recuperaciones máximas por minuto debe ser un número entero WebcrawlerConnector.CredentialMustHaveNonNullUserName=Credencial debe tener el nombre de usuario no nulo -WebcrawlerConnector.MatchExpressionMustBeAValidRegularExpression=Expresión Partido debe ser una expresión regular válida -WebcrawlerConnector.ParameterMustHaveNonEmptyName=Parámetro debe tener no-nombre vacío -WebcrawlerConnector.ParameterCanEitherBeHidden=Parámetro tampoco se puede ocultar o no, pero no pueden ser a la vez -WebcrawlerConnector.MaximumKbytesPerSecondMustBeAnInteger=Kbytes máximos por segundo debe ser un número entero -WebcrawlerConnector.MaximumFetchesPerMinuteMustBeAnInteger=Recuperaciones máximas por minuto debe ser un número entero -WebcrawlerConnector.AValidRegularExpressionIsRequired=Se requiere una expresión regular válida +WebcrawlerConnector.MatchExpressionMustBeAValidRegularExpression=Expresión Partido debe ser una expresión regular válida +WebcrawlerConnector.ParameterMustHaveNonEmptyName=Parámetro debe tener no-nombre vacÃo +WebcrawlerConnector.ParameterCanEitherBeHidden=Parámetro tampoco se puede ocultar o no, pero no pueden ser a la vez +WebcrawlerConnector.MaximumKbytesPerSecondMustBeAnInteger=Kbytes máximos por segundo debe ser un número entero +WebcrawlerConnector.MaximumFetchesPerMinuteMustBeAnInteger=Recuperaciones máximas por minuto debe ser un número entero +WebcrawlerConnector.AValidRegularExpressionIsRequired=Se requiere una expresión regular válida WebcrawlerConnector.CredentialMustIncludeANonNullUserName=Credencial debe incluir un no-Nombre de usuario nula -WebcrawlerConnector.AValidRegularExpressionIsRequired=Se requiere una expresión regular válida -WebcrawlerConnector.ParameterNameMustBeARegularExpression=Nombre del parámetro debe ser una expresión regular -WebcrawlerConnector.SpecifyATrustCertificateFileToUploadFirst=Especifique un archivo de certificado de confianza para subir primero, o comprobación 'La confianza de todo' -WebcrawlerConnector.NoBandwidthOrConnectionThrottlingSpecified=No ancho de banda o la conexión de estrangulamiento especificado -WebcrawlerConnector.DeleteLoginPage=Eliminar la página de inicio de sesión # -WebcrawlerConnector.ParameterRegularExpression=Expresión regular Parámetro +WebcrawlerConnector.AValidRegularExpressionIsRequired=Se requiere una expresión regular válida +WebcrawlerConnector.ParameterNameMustBeARegularExpression=Nombre del parámetro debe ser una expresión regular +WebcrawlerConnector.SpecifyATrustCertificateFileToUploadFirst=Especifique un archivo de certificado de confianza para subir primero, o comprobación 'La confianza de todo' +WebcrawlerConnector.NoBandwidthOrConnectionThrottlingSpecified=No ancho de banda o la conexión de estrangulamiento especificado +WebcrawlerConnector.DeleteLoginPage=Eliminar la página de inicio de sesión # +WebcrawlerConnector.ParameterRegularExpression=Expresión regular Parámetro WebcrawlerConnector.Value=Valor -WebcrawlerConnector.Password=Contraseña -WebcrawlerConnector.DeleteParameter=eliminar parámetro # -WebcrawlerConnector.ForLoginPage= para la página de inicio de sesión # +WebcrawlerConnector.Password=Contraseña +WebcrawlerConnector.DeleteParameter=eliminar parámetro # +WebcrawlerConnector.ForLoginPage= para la página de inicio de sesión # WebcrawlerConnector.ForCredential= para credenciales # -WebcrawlerConnector.AddParameterToLoginPage=Añadir parámetro a la página iniciar sesión # -WebcrawlerConnector.AddLoginPageToCredential=Añadir página de acceso a la credencial # -WebcrawlerConnector.InvalidUrlsInSeedsList=URL no válidos en la lista de semillas: +WebcrawlerConnector.AddParameterToLoginPage=Añadir parámetro a la página iniciar sesión # +WebcrawlerConnector.AddLoginPageToCredential=Añadir página de acceso a la credencial # +WebcrawlerConnector.InvalidUrlsInSeedsList=URL no válidos en la lista de semillas: WebcrawlerConnector.yes=si WebcrawlerConnector.no=no WebcrawlerConnector.ExcludedHeadersColon=cabeceras excluidos: WebcrawlerConnector.URLMappings=URL Asignaciones -WebcrawlerConnector.RemoveRegexp=Retire expresión regular # -WebcrawlerConnector.AddRegexp=Añadir expresión regular +WebcrawlerConnector.RemoveRegexp=Retire expresión regular # +WebcrawlerConnector.AddRegexp=Añadir expresión regular WebcrawlerConnector.Remove=Eliminar -WebcrawlerConnector.NoMappingsSpecifiedWillAcceptAllUrls=Ningún trazar un mapa de especificado; aceptará todas las URL +WebcrawlerConnector.NoMappingsSpecifiedWillAcceptAllUrls=Ningún trazar un mapa de especificado; aceptará todas las URL WebcrawlerConnector.URLMappingsColon=URL trazar un mapa: WebcrawlerConnector.MatchMustHaveARegexpValue=Partido debe tener un valor de expresiones regulares Modified: manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/webcrawler/common_ja_JP.properties URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/webcrawler/common_ja_JP.properties?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/webcrawler/common_ja_JP.properties (original) +++ manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/webcrawler/common_ja_JP.properties Tue Mar 14 21:17:50 2017 @@ -29,6 +29,9 @@ WebcrawlerConnector.RobotsTxtUsage=Robot WebcrawlerConnector.DontLookAtRobotsTxt=robots.txtãå©ç¨ããªã WebcrawlerConnector.ObeyRobotsTxtForDataFetchesOnly=ãã¼ã¿åå¾ã®å ´åã®ã¿ã«robots.txtã«å¾ã WebcrawlerConnector.ObeyRobotsTxtForAllFetches=ãã¹ã¦robots.txtã«å¾ã +WebcrawlerConnector.MetaRobotsTagsUsage=Meta robots tags usage: +WebcrawlerConnector.DontLookAtMetaRobotsTags=Don't look at meta robots tags +WebcrawlerConnector.ObeyMetaRobotsTags=Obey meta robots tags WebcrawlerConnector.Throttles=ã¹ãããã«ï¼ WebcrawlerConnector.BinRegularExpression=Binæ£è¦è¡¨ç¾ WebcrawlerConnector.CaseInsensitive=大ï¼å°æåãåºå¥ããªã Modified: manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/webcrawler/common_zh_CN.properties URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/webcrawler/common_zh_CN.properties?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/webcrawler/common_zh_CN.properties (original) +++ manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/webcrawler/common_zh_CN.properties Tue Mar 14 21:17:50 2017 @@ -29,6 +29,9 @@ WebcrawlerConnector.RobotsTxtUsage=ä½� WebcrawlerConnector.DontLookAtRobotsTxt=ä¸ä½¿ç¨robots.txt WebcrawlerConnector.ObeyRobotsTxtForDataFetchesOnly=åªå¨æåæ°æ®æ¶æä»robots.txt WebcrawlerConnector.ObeyRobotsTxtForAllFetches=æ¯æ¬¡æååæä»robots.txt +WebcrawlerConnector.MetaRobotsTagsUsage=Meta robots tags usage: +WebcrawlerConnector.DontLookAtMetaRobotsTags=Don't look at meta robots tags +WebcrawlerConnector.ObeyMetaRobotsTags=Obey meta robots tags WebcrawlerConnector.Throttles=éæµå¨: WebcrawlerConnector.BinRegularExpression=Binæ£åè¡¨è¾¾å¼ WebcrawlerConnector.CaseInsensitive=ä¸åºå大å°å Modified: manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/webcrawler/tests/NavigationHSQLDBUI.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/webcrawler/tests/NavigationHSQLDBUI.java?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/webcrawler/tests/NavigationHSQLDBUI.java (original) +++ manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/webcrawler/tests/NavigationHSQLDBUI.java Tue Mar 14 21:17:50 2017 @@ -134,6 +134,8 @@ public class NavigationHSQLDBUI extends form = window.findForm(testerInstance.createStringDescription("editconnection")); selectbox = form.findSelectbox(testerInstance.createStringDescription("robotsusage")); selectbox.selectValue(testerInstance.createStringDescription("none")); + selectbox = form.findSelectbox(testerInstance.createStringDescription("metarobotstagsusage")); + selectbox.selectValue(testerInstance.createStringDescription("none")); // Bandwidth link = window.findLink(testerInstance.createStringDescription("Bandwidth tab")); link.click(); Modified: manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/pom.xml URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/pom.xml?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/pom.xml (original) +++ manifoldcf/branches/CONNECTORS-1196/connectors/webcrawler/pom.xml Tue Mar 14 21:17:50 2017 @@ -233,6 +233,12 @@ <scope>test</scope> </dependency> <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <version>${mockito.version}</version> + <scope>test</scope> + </dependency> + <dependency> <groupId>${project.groupId}</groupId> <artifactId>mcf-core</artifactId> <version>${project.version}</version> Modified: manifoldcf/branches/CONNECTORS-1196/dist-license/DEPENDENCIES.txt URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/dist-license/DEPENDENCIES.txt?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/dist-license/DEPENDENCIES.txt (original) +++ manifoldcf/branches/CONNECTORS-1196/dist-license/DEPENDENCIES.txt Tue Mar 14 21:17:50 2017 @@ -1,6 +1,6 @@ ManifoldCF requires ------------------ -* JRE 1.7 or above +* JRE 1.8 or above * Many other libraries, available from the ManifoldCF XXX-lib distribution For running ManifoldCF: Modified: manifoldcf/branches/CONNECTORS-1196/dist-license/NOTICE.txt URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/dist-license/NOTICE.txt?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/dist-license/NOTICE.txt (original) +++ manifoldcf/branches/CONNECTORS-1196/dist-license/NOTICE.txt Tue Mar 14 21:17:50 2017 @@ -1,5 +1,5 @@ Apache ManifoldCF -Copyright 2010-2016 The Apache Software Foundation +Copyright 2010-2017 The Apache Software Foundation This product includes software developed by The Apache Software Foundation (http://www.apache.org/). Modified: manifoldcf/branches/CONNECTORS-1196/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java (original) +++ manifoldcf/branches/CONNECTORS-1196/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java Tue Mar 14 21:17:50 2017 @@ -40,13 +40,13 @@ public class IdleCleanupThread extends T { super(); this.processID = processID; - setName("Idle cleanup thread"); + setName("Agents idle cleanup thread"); setDaemon(true); } public void run() { - Logging.agents.debug("Start up idle cleanup thread"); + Logging.agents.debug("Start up agents idle cleanup thread"); try { // Create a thread context object. Modified: manifoldcf/branches/CONNECTORS-1196/framework/build.xml URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/framework/build.xml?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/framework/build.xml (original) +++ manifoldcf/branches/CONNECTORS-1196/framework/build.xml Tue Mar 14 21:17:50 2017 @@ -34,7 +34,7 @@ <include name="zookeeper*.jar"/> <include name="json*.jar"/> <include name="commons-codec*.jar"/> - <include name="commons-collections*.jar"/> + <include name="commons-collections-*.jar"/> <include name="commons-el*.jar"/> <include name="commons-fileupload*.jar"/> <include name="httpcore*.jar"/> @@ -86,6 +86,7 @@ <include name="jackson-annotations*.jar"/> <include name="tika-parsers*.jar"/> <include name="geoapi*.jar"/> + <include name="jsr-275*.jar"/> <include name="sis-metadata*.jar"/> <include name="sis-utility*.jar"/> <include name="sis-netcdf*.jar"/> @@ -94,11 +95,15 @@ <include name="jackcess*.jar"/> <include name="junrar*.jar"/> <include name="commons-vfs2*.jar"/> + <include name="maven-scm*.jar"/> + <include name="plexus-utils*.jar"/> + <include name="regexp*.jar"/> <include name="commons-csv*.jar"/> <include name="commons-exec*.jar"/> <include name="netcdf4*.jar"/> <include name="httpservices*.jar"/> <include name="grib*.jar"/> + <include name="jna*.jar"/> <include name="cdm*.jar"/> <include name="udunits*.jar"/> <include name="jcommander*.jar"/> @@ -1393,6 +1398,7 @@ <include name="jackson-annotations*.jar"/> <include name="tika-parsers*.jar"/> <include name="geoapi*.jar"/> + <include name="jsr-275*.jar"/> <include name="sis-metadata*.jar"/> <include name="sis-utility*.jar"/> <include name="sis-netcdf*.jar"/> @@ -1401,11 +1407,16 @@ <include name="jackcess*.jar"/> <include name="junrar*.jar"/> <include name="commons-vfs2*.jar"/> + <include name="maven-scm*.jar"/> + <include name="plexus-utils*.jar"/> + <include name="regexp*.jar"/> <include name="commons-csv*.jar"/> <include name="commons-exec*.jar"/> <include name="netcdf4*.jar"/> <include name="httpservices*.jar"/> <include name="grib*.jar"/> + <include name="jna*.jar"/> + <include name="jj2000*.jar"/> <include name="cdm*.jar"/> <include name="udunits*.jar"/> <include name="jcommander*.jar"/> Modified: manifoldcf/branches/CONNECTORS-1196/framework/connector-common/pom.xml URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/framework/connector-common/pom.xml?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/framework/connector-common/pom.xml (original) +++ manifoldcf/branches/CONNECTORS-1196/framework/connector-common/pom.xml Tue Mar 14 21:17:50 2017 @@ -82,17 +82,6 @@ <groupId>org.apache.tika</groupId> <artifactId>tika-core</artifactId> <version>${tika.version}</version> - <exclusions> - <exclusion> - <artifactId>jhighlight</artifactId> - <groupId>com.uwyn</groupId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.codelibs</groupId> - <artifactId>jhighlight</artifactId> - <version>${jhighlight.version}</version> </dependency> <dependency> <groupId>javax.activation</groupId> Modified: manifoldcf/branches/CONNECTORS-1196/framework/core/src/test/resources/org/apache/manifoldcf/core/tests/VirtualBrowser.py URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/framework/core/src/test/resources/org/apache/manifoldcf/core/tests/VirtualBrowser.py?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/framework/core/src/test/resources/org/apache/manifoldcf/core/tests/VirtualBrowser.py (original) +++ manifoldcf/branches/CONNECTORS-1196/framework/core/src/test/resources/org/apache/manifoldcf/core/tests/VirtualBrowser.py Tue Mar 14 21:17:50 2017 @@ -1493,7 +1493,8 @@ class JSDocObject( Javascript.JSObject ) return Javascript.JSObject.get_value( self, member_name ) def set_value( self, member_name, value ): - raise Exception("Cannot set properties of document object") + if member_name != "onkeypress": + raise Exception("Cannot set properties of document object") # Class representing a form in Javascript class JSFormObject( Javascript.JSObject ): Propchange: manifoldcf/branches/CONNECTORS-1196/framework/core/src/test/resources/org/apache/manifoldcf/core/tests/VirtualBrowser.py ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Tue Mar 14 21:17:50 2017 @@ -11,3 +11,4 @@ /incubator/lcf/branches/CONNECTORS-32/legacy-tests/VirtualBrowser.py:1092556-1094216 /incubator/lcf/branches/CONNECTORS-335/framework/core/src/test/resource/org/apache/manifoldcf/core/tests/VirtualBrowser.py:1224860-1225060 /manifoldcf/branches/CONNECTORS-737/framework/core/src/test/resources/org/apache/manifoldcf/core/tests/VirtualBrowser.py:1501867-1503249 +/manifoldcf/trunk/framework/core/src/test/resources/org/apache/manifoldcf/core/tests/VirtualBrowser.py:1783604-1786950 Modified: manifoldcf/branches/CONNECTORS-1196/framework/example-multiprocess-zk-proprietary/properties.xml URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/framework/example-multiprocess-zk-proprietary/properties.xml?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/framework/example-multiprocess-zk-proprietary/properties.xml (original) +++ manifoldcf/branches/CONNECTORS-1196/framework/example-multiprocess-zk-proprietary/properties.xml Tue Mar 14 21:17:50 2017 @@ -28,7 +28,7 @@ <!-- ZooKeeper lock manager configuration --> <property name="org.apache.manifoldcf.lockmanagerclass" value="org.apache.manifoldcf.core.lockmanager.ZooKeeperLockManager"/> <property name="org.apache.manifoldcf.zookeeper.connectstring" value="localhost:8349"/> - <property name="org.apache.manifoldcf.zookeeper.sessiontimeout" value="2000"/> + <property name="org.apache.manifoldcf.zookeeper.sessiontimeout" value="300000"/> <!-- Point to a specific (common) logging file --> <property name="org.apache.manifoldcf.logconfigfile" value="./logging.ini"/> <!-- Specify the connectors to be loaded --> Modified: manifoldcf/branches/CONNECTORS-1196/framework/example-multiprocess-zk/properties.xml URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/framework/example-multiprocess-zk/properties.xml?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/framework/example-multiprocess-zk/properties.xml (original) +++ manifoldcf/branches/CONNECTORS-1196/framework/example-multiprocess-zk/properties.xml Tue Mar 14 21:17:50 2017 @@ -28,7 +28,7 @@ <!-- ZooKeeper lock manager configuration --> <property name="org.apache.manifoldcf.lockmanagerclass" value="org.apache.manifoldcf.core.lockmanager.ZooKeeperLockManager"/> <property name="org.apache.manifoldcf.zookeeper.connectstring" value="localhost:8349"/> - <property name="org.apache.manifoldcf.zookeeper.sessiontimeout" value="2000"/> + <property name="org.apache.manifoldcf.zookeeper.sessiontimeout" value="300000"/> <!-- Point to a specific (common) logging file --> <property name="org.apache.manifoldcf.logconfigfile" value="./logging.ini"/> <!-- Specify the connectors to be loaded --> Modified: manifoldcf/branches/CONNECTORS-1196/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/system/IdleCleanupThread.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/system/IdleCleanupThread.java?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/system/IdleCleanupThread.java (original) +++ manifoldcf/branches/CONNECTORS-1196/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/system/IdleCleanupThread.java Tue Mar 14 21:17:50 2017 @@ -41,7 +41,7 @@ public class IdleCleanupThread extends T throws ManifoldCFException { super(); - setName("Idle cleanup thread"); + setName("Authorities idle cleanup thread"); setDaemon(true); } Modified: manifoldcf/branches/CONNECTORS-1196/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java (original) +++ manifoldcf/branches/CONNECTORS-1196/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java Tue Mar 14 21:17:50 2017 @@ -43,13 +43,13 @@ public class IdleCleanupThread extends T { super(); this.processID = processID; - setName("Idle cleanup thread"); + setName("Crawler idle cleanup thread"); setDaemon(true); } public void run() { - Logging.threads.debug("Start up idle cleanup thread"); + Logging.threads.debug("Start up crawler idle cleanup thread"); try { // Create a thread context object. Modified: manifoldcf/branches/CONNECTORS-1196/lib-license/NOTICE.txt URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/lib-license/NOTICE.txt?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/lib-license/NOTICE.txt (original) +++ manifoldcf/branches/CONNECTORS-1196/lib-license/NOTICE.txt Tue Mar 14 21:17:50 2017 @@ -1,5 +1,5 @@ Apache ManifoldCF -Copyright 2010-2016 The Apache Software Foundation +Copyright 2010-2017 The Apache Software Foundation This product includes software developed by The Apache Software Foundation (http://www.apache.org/). Modified: manifoldcf/branches/CONNECTORS-1196/pom.xml URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/pom.xml?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/pom.xml (original) +++ manifoldcf/branches/CONNECTORS-1196/pom.xml Tue Mar 14 21:17:50 2017 @@ -43,7 +43,7 @@ <!-- dependency versions --> <junit.version>4.12</junit.version> <mockito.version>1.9.5</mockito.version> - <wiremock.version>1.34</wiremock.version> + <wiremock.version>2.5.1</wiremock.version> <postgresql.version>9.1-901.jdbc4</postgresql.version> <mysql.version>5.1.33</mysql.version> <hsqldb.version>2.3.2</hsqldb.version> @@ -84,8 +84,7 @@ <zookeeper.version>3.4.8</zookeeper.version> <xmlbeans.version>2.6.0</xmlbeans.version> <poi.version>3.15-beta1</poi.version> - <tika.version>1.13</tika.version> - <jhighlight.version>1.0.3</jhighlight.version> + <tika.version>1.14</tika.version> <boilerpipe.version>1.1.0</boilerpipe.version> <hadoop.version>2.6.0</hadoop.version> <tomcat.version>6.0.35</tomcat.version> Modified: manifoldcf/branches/CONNECTORS-1196/site/build.xml URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/site/build.xml?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/site/build.xml (original) +++ manifoldcf/branches/CONNECTORS-1196/site/build.xml Tue Mar 14 21:17:50 2017 @@ -263,7 +263,7 @@ <target name="download-dejavu-fonts" depends="calculate-dejavu-condition-before" unless="dejavuAvailableBefore"> <mkdir dir="fonts"/> - <get src="http://superb-sea2.dl.sourceforge.net/project/dejavu/dejavu/2.33/dejavu-fonts-ttf-2.33.zip"; dest="fonts"/> + <get src="http://superb-dca2.dl.sourceforge.net/project/dejavu/dejavu/2.33/dejavu-fonts-ttf-2.33.zip"; dest="fonts"/> </target> <target name="calculate-ipa-condition-before"> Modified: manifoldcf/branches/CONNECTORS-1196/site/src/documentation/content/xdocs/en_US/end-user-documentation.xml URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/site/src/documentation/content/xdocs/en_US/end-user-documentation.xml?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/site/src/documentation/content/xdocs/en_US/end-user-documentation.xml (original) +++ manifoldcf/branches/CONNECTORS-1196/site/src/documentation/content/xdocs/en_US/end-user-documentation.xml Tue Mar 14 21:17:50 2017 @@ -1,4 +1,4 @@ -<?xml version="1.0"?> +<?xml version="1.0"?> <!-- Licensed to the Apache Software Foundation (ASF) under one or more @@ -2729,7 +2729,7 @@ curl -XGET http://localhost:9200/index/_ <br/><br/> <figure src="images/en_US/web-configure-robots.PNG" alt="Web Connection, Robots tab" width="80%"/> <br/><br/> - <p>Select how the connection will interpret robots.txt. Remember that you have an interest in crawling people's sites as politely as is possible.</p> + <p>Select how the connection will interpret robots.txt and <meta name="robots ...> tags on HTML pages. Remember that you have an interest in crawling people's sites as politely as is possible.</p> <p>The "Bandwidth" tab allows you to specify a list of bandwidth rules. Each rule has a regular expression matched against a URL's throttle bin. Throttle bins, in connections of the Web type, are simply the server name part of the URL. Each rule allows you to select a maximum bandwidth, number of connections, and fetch rate. You can have as many rules as you like; if a URL matches more than one rule, then the most conservative value will be used.</p> Modified: manifoldcf/branches/CONNECTORS-1196/site/src/documentation/resources/images/en_US/web-configure-robots.PNG URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/site/src/documentation/resources/images/en_US/web-configure-robots.PNG?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== Binary files - no diff available. Modified: manifoldcf/branches/CONNECTORS-1196/site/src/documentation/skinconf.xml URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1196/site/src/documentation/skinconf.xml?rev=1786952&r1=1786951&r2=1786952&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-1196/site/src/documentation/skinconf.xml (original) +++ manifoldcf/branches/CONNECTORS-1196/site/src/documentation/skinconf.xml Tue Mar 14 21:17:50 2017 @@ -90,7 +90,7 @@ which will be used to configure the chos <favicon-url>images/favicon.ico</favicon-url> <!-- The following are used to construct a copyright statement --> - <year>2009-2016</year> + <year>2009-2017</year> <vendor>The Apache Software Foundation.</vendor> <copyright-link>http://www.apache.org/licenses/</copyright-link> <!-- Optional text trademark statement below the copyright statement -->
