Modified: manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java?rev=1793547&r1=1793546&r2=1793547&view=diff ============================================================================== --- manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java (original) +++ manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java Tue May 2 16:00:56 2017 @@ -1563,344 +1563,90 @@ public class WebcrawlerConnector extends tabsArray.add(Messages.getString(locale,"WebcrawlerConnector.Certificates")); tabsArray.add(Messages.getString(locale,"WebcrawlerConnector.Proxy")); - out.print( -"<script type=\"text/javascript\">\n"+ -"<!--\n"+ -"function checkConfig()\n"+ -"{\n"+ -" if (editconnection.email.value != \"\" && editconnection.email.value.indexOf(\"@\") == -1)\n"+ -" {\n"+ -" alert(\""+Messages.getBodyJavascriptString(locale,"WebcrawlerConnector.NeedAValidEmailAddress")+"\");\n"+ -" editconnection.email.focus();\n"+ -" return false;\n"+ -" }\n"+ -"\n"+ -" // If the Bandwidth tab is up, check to be sure we have valid numbers and regexps everywhere.\n"+ -" var i = 0;\n"+ -" var count = editconnection.bandwidth_count.value;\n"+ -" while (i < count)\n"+ -" {\n"+ -" var connections = eval(\"editconnection.connections_bandwidth_\"+i+\".value\");\n"+ -" if (connections != \"\" && !isInteger(connections))\n"+ -" {\n"+ -" alert(\""+Messages.getBodyJavascriptString(locale,"WebcrawlerConnector.MaximumConnectionsMustBeAnInteger")+"\");\n"+ -" eval(\"editconnection.connections_bandwidth_\"+i+\".focus()\");\n"+ -" return false;\n"+ -" }\n"+ -" var rate = eval(\"editconnection.rate_bandwidth_\"+i+\".value\");\n"+ -" if (rate != \"\" && !isInteger(rate))\n"+ -" {\n"+ -" alert(\""+Messages.getBodyJavascriptString(locale,"WebcrawlerConnector.MaximumKbytesPerSecondMustBeAnInteger")+"\");\n"+ -" eval(\"editconnection.rate_bandwidth_\"+i+\".focus()\");\n"+ -" return false;\n"+ -" }\n"+ -" var fetches = eval(\"editconnection.fetches_bandwidth_\"+i+\".value\");\n"+ -" if (fetches != \"\" && !isInteger(fetches))\n"+ -" {\n"+ -" alert(\""+Messages.getBodyJavascriptString(locale,"WebcrawlerConnector.MaximumFetchesPerMinuteMustBeAnInteger")+"\");\n"+ -" eval(\"editconnection.fetches_bandwidth_\"+i+\".focus()\");\n"+ -" return false;\n"+ -" }\n"+ -"\n"+ -" i = i + 1;\n"+ -" }\n"+ -" \n"+ -" // Make sure access credentials are all legal\n"+ -" i = 0;\n"+ -" count = editconnection.acredential_count.value;\n"+ -" while (i < count)\n"+ -" {\n"+ -" var username = eval(\"editconnection.username_acredential_\"+i+\".value\");\n"+ -" if (username == \"\")\n"+ -" {\n"+ -" alert(\""+Messages.getBodyJavascriptString(locale,"WebcrawlerConnector.CredentialMustHaveNonNullUserName")+"\");\n"+ -" eval(\"editconnection.username_acredential_\"+i+\".focus()\");\n"+ -" return false;\n"+ -" }\n"+ -" i = i + 1;\n"+ -" }\n"+ -"\n"+ -" // Make sure session credentials are all legal\n"+ -" i = 0;\n"+ -" count = editconnection.scredential_count.value;\n"+ -" while (i < count)\n"+ -" {\n"+ -" var loginpagecount = eval(\"editconnection.scredential_\"+i+\"_loginpagecount.value\");\n"+ -" var j = 0;\n"+ -" while (j < loginpagecount)\n"+ -" {\n"+ -" var matchregexp = eval(\"editconnection.scredential_\"+i+\"_\"+j+\"_matchregexp.value\");\n"+ -" if (!isRegularExpression(matchregexp))\n"+ -" {\n"+ -" alert(\""+Messages.getBodyJavascriptString(locale,"WebcrawlerConnector.MatchExpressionMustBeAValidRegularExpression")+"\");\n"+ -" eval(\"editconnection.scredential_\"+i+\"_\"+j+\"_matchregexp.focus()\");\n"+ -" return false;\n"+ -" }\n"+ -" if (eval(\"editconnection.scredential_\"+i+\"_\"+j+\"_type.value\") == \"form\")\n"+ -" {\n"+ -" var paramcount = eval(\"editconnection.scredential_\"+i+\"_\"+j+\"_loginparamcount.value\");\n"+ -" var k = 0;\n"+ -" while (k < paramcount)\n"+ -" {\n"+ -" var paramname = eval(\"editconnection.scredential_\"+i+\"_\"+j+\"_\"+k+\"_param.value\");\n"+ -" if (paramname == \"\")\n"+ -" {\n"+ -" alert(\""+Messages.getBodyJavascriptString(locale,"WebcrawlerConnector.ParameterMustHaveNonEmptyName")+"\");\n"+ -" eval(\"editconnection.scredential_\"+i+\"_\"+j+\"_\"+k+\"_param.focus()\");\n"+ -" return false;\n"+ -" }\n"+ -" var paramvalue = eval(\"editconnection.scredential_\"+i+\"_\"+j+\"_\"+k+\"_value.value\");\n"+ -" var parampassword = eval(\"editconnection.scredential_\"+i+\"_\"+j+\"_\"+k+\"_password.value\");\n"+ -" if (paramvalue != \"\" && parampassword != \"\")\n"+ -" {\n"+ -" alert(\""+Messages.getBodyJavascriptString(locale,"WebcrawlerConnector.ParameterCanEitherBeHidden")+"\");\n"+ -" eval(\"editconnection.scredential_\"+i+\"_\"+j+\"_\"+k+\"_value.focus()\");\n"+ -" return false;\n"+ -" }\n"+ -" k = k + 1;\n"+ -" }\n"+ -" }\n"+ -" j = j + 1;\n"+ -" }\n"+ -" i = i + 1;\n"+ -" }\n"+ -" return true;\n"+ -"}\n"+ -"\n"+ -"function checkConfigForSave()\n"+ -"{\n"+ -" if (editconnection.email.value == \"\")\n"+ -" {\n"+ -" alert(\"" + Messages.getBodyJavascriptString(locale,"WebcrawlerConnector.EmailAaddressRequired") + "\");\n"+ -" SelectTab(\"" + Messages.getBodyJavascriptString(locale,"WebcrawlerConnector.Email") + "\");\n"+ -" editconnection.email.focus();\n"+ -" return false;\n"+ -" }\n"+ -" return true;\n"+ -"}\n"+ -"\n"+ -"function deleteRegexp(i)\n"+ -"{\n"+ -" // Set the operation\n"+ -" eval(\"editconnection.op_bandwidth_\"+i+\".value=\\\"Delete\\\"\");\n"+ -" // Submit\n"+ -" if (editconnection.bandwidth_count.value==i)\n"+ -" postFormSetAnchor(\"bandwidth\");\n"+ -" else\n"+ -" postFormSetAnchor(\"bandwidth_\"+i)\n"+ -" // Undo, so we won't get two deletes next time\n"+ -" eval(\"editconnection.op_bandwidth_\"+i+\".value=\\\"Continue\\\"\");\n"+ -"}\n"+ -"\n"+ -"function addRegexp()\n"+ -"{\n"+ -" if (editconnection.connections_bandwidth.value != \"\" && !isInteger(editconnection.connections_bandwidth.value))\n"+ -" {\n"+ -" alert(\"" + Messages.getBodyJavascriptString(locale,"WebcrawlerConnector.MaximumConnectionsMustBeAnInteger")+"\");\n"+ -" editconnection.connections_bandwidth.focus();\n"+ -" return;\n"+ -" }\n"+ -" if (editconnection.rate_bandwidth.value != \"\" && !isInteger(editconnection.rate_bandwidth.value))\n"+ -" {\n"+ -" alert(\""+Messages.getBodyJavascriptString(locale,"WebcrawlerConnector.MaximumKbytesPerSecondMustBeAnInteger")+"\");\n"+ -" editconnection.rate_bandwidth.focus();\n"+ -" return;\n"+ -" }\n"+ -" if (editconnection.fetches_bandwidth.value != \"\" && !isInteger(editconnection.fetches_bandwidth.value))\n"+ -" {\n"+ -" alert(\""+Messages.getBodyJavascriptString(locale,"WebcrawlerConnector.MaximumFetchesPerMinuteMustBeAnInteger")+"\");\n"+ -" editconnection.fetches_bandwidth.focus();\n"+ -" return;\n"+ -" }\n"+ -" if (!isRegularExpression(editconnection.regexp_bandwidth.value))\n"+ -" {\n"+ -" alert(\""+Messages.getBodyJavascriptString(locale,"WebcrawlerConnector.AValidRegularExpressionIsRequired")+"\");\n"+ -" editconnection.regexp_bandwidth.focus();\n"+ -" return;\n"+ -" }\n"+ -" editconnection.bandwidth_op.value=\"Add\";\n"+ -" postFormSetAnchor(\"bandwidth\");\n"+ -"}\n"+ -"\n"+ -"function deleteARegexp(i)\n"+ -"{\n"+ -" // Set the operation\n"+ -" eval(\"editconnection.op_acredential_\"+i+\".value=\\\"Delete\\\"\");\n"+ -" // Submit\n"+ -" if (editconnection.acredential_count.value==i)\n"+ -" postFormSetAnchor(\"acredential\");\n"+ -" else\n"+ -" postFormSetAnchor(\"acredential_\"+i)\n"+ -" // Undo, so we won't get two deletes next time\n"+ -" eval(\"editconnection.op_acredential_\"+i+\".value=\\\"Continue\\\"\");\n"+ -"}\n"+ -"\n"+ -"function addARegexp()\n"+ -"{\n"+ -" if (editconnection.username_acredential.value == \"\")\n"+ -" {\n"+ -" alert(\""+Messages.getBodyJavascriptString(locale,"WebcrawlerConnector.CredentialMustIncludeANonNullUserName")+"\");\n"+ -" editconnection.username_acredential.focus();\n"+ -" return;\n"+ -" }\n"+ -" if (!isRegularExpression(editconnection.regexp_acredential.value))\n"+ -" {\n"+ -" alert(\""+Messages.getBodyJavascriptString(locale,"WebcrawlerConnector.AValidRegularExpressionIsRequired")+"\");\n"+ -" editconnection.regexp_acredential.focus();\n"+ -" return;\n"+ -" }\n"+ -" editconnection.acredential_op.value=\"Add\";\n"+ -" postFormSetAnchor(\"acredential\");\n"+ -"}\n"+ -"\n"+ -"function deleteSRegexp(i)\n"+ -"{\n"+ -" // Set the operation\n"+ -" eval(\"editconnection.scredential_\"+i+\"_op.value=\\\"Delete\\\"\");\n"+ -" // Submit\n"+ -" if (editconnection.scredential_count.value==i)\n"+ -" postFormSetAnchor(\"scredential\");\n"+ -" else\n"+ -" postFormSetAnchor(\"scredential_\"+i)\n"+ -" // Undo, so we won't get two deletes next time\n"+ -" eval(\"editconnection.scredential_\"+i+\"_op.value=\\\"Continue\\\"\");\n"+ -"}\n"+ -"\n"+ -"function addSRegexp()\n"+ -"{\n"+ -" if (!isRegularExpression(editconnection.scredential_regexp.value))\n"+ -" {\n"+ -" alert(\""+Messages.getBodyJavascriptString(locale,"WebcrawlerConnector.AValidRegularExpressionIsRequired")+"\");\n"+ -" editconnection.scredential_regexp.focus();\n"+ -" return;\n"+ -" }\n"+ -" editconnection.scredential_op.value=\"Add\";\n"+ -" postFormSetAnchor(\"scredential\");\n"+ -"}\n"+ -"\n"+ -"function deleteLoginPage(credential,loginpage)\n"+ -"{\n"+ -" // Set the operation\n"+ -" eval(\"editconnection.scredential_\"+credential+\"_\"+loginpage+\"_op.value=\\\"Delete\\\"\");\n"+ -" // Submit\n"+ -" if (eval(\"editconnection.scredential_\"+credential+\"_loginpagecount.value\")==credential)\n"+ -" postFormSetAnchor(\"scredential_loginpage\");\n"+ -" else\n"+ -" postFormSetAnchor(\"scredential_\"+credential+\"_\"+loginpage)\n"+ -" // Undo, so we won't get two deletes next time\n"+ -" eval(\"editconnection.scredential_\"+credential+\"_\"+loginpage+\"_op.value=\\\"Continue\\\"\");\n"+ -"\n"+ -"}\n"+ -" \n"+ -"function addLoginPage(credential)\n"+ -"{\n"+ -" if (!isRegularExpression(eval(\"editconnection.scredential_\"+credential+\"_loginpageregexp.value\")))\n"+ -" {\n"+ -" alert(\""+Messages.getBodyJavascriptString(locale,"WebcrawlerConnector.AValidRegularExpressionIsRequired")+"\");\n"+ -" eval(\"editconnection.scredential_\"+credential+\"_loginpageregexp.focus()\");\n"+ -" return;\n"+ -" }\n"+ -" if (!isRegularExpression(eval(\"editconnection.scredential_\"+credential+\"_loginpagematchregexp.value\")))\n"+ -" {\n"+ -" alert(\""+Messages.getBodyJavascriptString(locale,"WebcrawlerConnector.AValidRegularExpressionIsRequired")+"\");\n"+ -" eval(\"editconnection.scredential_\"+credential+\"_loginpagematchregexp.focus()\");\n"+ -" return;\n"+ -" }\n"+ -" eval(\"editconnection.scredential_\"+credential+\"_loginpageop.value=\\\"Add\\\"\");\n"+ -" postFormSetAnchor(\"scredential_\"+credential);\n"+ -"}\n"+ -" \n"+ -"function deleteLoginPageParameter(credential,loginpage,parameter)\n"+ -"{\n"+ -" // Set the operation\n"+ -" eval(\"editconnection.scredential_\"+credential+\"_\"+loginpage+\"_\"+parameter+\"_op.value=\\\"Delete\\\"\");\n"+ -" // Submit\n"+ -" if (eval(\"editconnection.scredential_\"+credential+\"_\"+loginpage+\"_loginparamcount.value\")==credential)\n"+ -" postFormSetAnchor(\"scredential_\"+credential+\"_loginparam\");\n"+ -" else\n"+ -" postFormSetAnchor(\"scredential_\"+credential+\"_\"+loginpage+\"_\"+parameter)\n"+ -" // Undo, so we won't get two deletes next time\n"+ -" eval(\"editconnection.scredential_\"+credential+\"_\"+loginpage+\"_\"+parameter+\"_op.value=\\\"Continue\\\"\");\n"+ -"}\n"+ -" \n"+ -"function addLoginPageParameter(credential,loginpage)\n"+ -"{\n"+ -" if (!isRegularExpression(eval(\"editconnection.scredential_\"+credential+\"_\"+loginpage+\"_loginparamname.value\")))\n"+ -" {\n"+ -" alert(\""+Messages.getBodyJavascriptString(locale,"WebcrawlerConnector.ParameterNameMustBeARegularExpression")+"\");\n"+ -" eval(\"editconnection.scredential_\"+credential+\"_\"+loginpage+\"_loginparamname.focus()\");\n"+ -" return;\n"+ -" }\n"+ -" if (eval(\"editconnection.scredential_\"+credential+\"_\"+loginpage+\"_loginparamvalue.value\") != \"\" && eval(\"editconnection.scredential_\"+credential+\"_\"+loginpage+\"_loginparampassword.value\") != \"\")\n"+ -" {\n"+ -" alert(\""+Messages.getBodyJavascriptString(locale,"WebcrawlerConnector.ParameterCanEitherBeHidden")+"\");\n"+ -" eval(\"editconnection.scredential_\"+credential+\"_\"+loginpage+\"_loginparamvalue.focus()\");\n"+ -" return;\n"+ -" }\n"+ -" eval(\"editconnection.scredential_\"+credential+\"_\"+loginpage+\"_loginparamop.value=\\\"Add\\\"\");\n"+ -" postFormSetAnchor(\"scredential_\"+credential+\"_\"+loginpage);\n"+ -"}\n"+ -" \n"+ -"function deleteTRegexp(i)\n"+ -"{\n"+ -" // Set the operation\n"+ -" eval(\"editconnection.op_trust_\"+i+\".value=\\\"Delete\\\"\");\n"+ -" // Submit\n"+ -" if (editconnection.trust_count.value==i)\n"+ -" postFormSetAnchor(\"trust\");\n"+ -" else\n"+ -" postFormSetAnchor(\"trust_\"+i);\n"+ -" // Undo, so we won't get two deletes next time\n"+ -" eval(\"editconnection.op_trust_\"+i+\".value=\\\"Continue\\\"\");\n"+ -"}\n"+ -"\n"+ -"function addTRegexp()\n"+ -"{\n"+ -" if (editconnection.certificate_trust.value == \"\" && editconnection.all_trust.checked == false)\n"+ -" {\n"+ -" alert(\""+Messages.getBodyJavascriptString(locale,"WebcrawlerConnector.SpecifyATrustCertificateFileToUploadFirst")+"\");\n"+ -" editconnection.certificate_trust.focus();\n"+ -" return;\n"+ -" }\n"+ -" if (!isRegularExpression(editconnection.regexp_trust.value))\n"+ -" {\n"+ -" alert(\""+Messages.getBodyJavascriptString(locale,"WebcrawlerConnector.AValidRegularExpressionIsRequired")+"\");\n"+ -" editconnection.regexp_trust.focus();\n"+ -" return;\n"+ -" }\n"+ -" editconnection.trust_op.value=\"Add\";\n"+ -" postFormSetAnchor(\"trust\");\n"+ -"}\n"+ -" \n"+ -"//-->\n"+ -"</script>\n" - ); + final Map<String,Object> velocityContext = new HashMap<String,Object>(); + Messages.outputResourceWithVelocity(out, locale, "editConfiguration.js.vm", velocityContext); } - - /** Output the configuration body section. - * This method is called in the body section of the connector's configuration page. Its purpose is to present the required form elements for editing. - * The coder can presume that the HTML that is output from this configuration will be within appropriate <html>, <body>, and <form> tags. The name of the - * form is "editconnection". - *@param threadContext is the local thread context. - *@param out is the output to which any HTML should be sent. - *@param parameters are the configuration parameters, as they currently exist, for this connection being configured. - *@param tabName is the current tab name. - */ - @Override - public void outputConfigurationBody(IThreadContext threadContext, IHTTPOutput out, - Locale locale, ConfigParams parameters, String tabName) - throws ManifoldCFException, IOException + + private void fillInEmailTab(Map<String,Object> velocityContext, IHTTPOutput out, ConfigParams parameters) { - String email = parameters.getParameter(WebcrawlerConfig.PARAMETER_EMAIL); if (email == null) email = ""; + + velocityContext.put("EMAIL",email); + } + + private void fillInRobotsTab(Map<String,Object> velocityContext, IHTTPOutput out, ConfigParams parameters) + { String robotsUsage = parameters.getParameter(WebcrawlerConfig.PARAMETER_ROBOTSUSAGE); if (robotsUsage == null) robotsUsage = "all"; String metaRobotsTagsUsage = parameters.getParameter(WebcrawlerConfig.PARAMETER_META_ROBOTS_TAGS_USAGE); if (metaRobotsTagsUsage == null) metaRobotsTagsUsage = "all"; + + velocityContext.put("ROBOTSUSAGE",robotsUsage); + velocityContext.put("METAROBOTSTAGSUSAGE",metaRobotsTagsUsage); + } + + private void fillInBandwidthTab(Map<String,Object> velocityContext, IHTTPOutput out, ConfigParams parameters) + { + int i = 0; + int binCounter = 0; + List<Map<String,String>> throttlesMapList = new ArrayList<>(); + while (i < parameters.getChildCount()) + { + ConfigNode cn = parameters.getChild(i++); + if (cn.getType().equals(WebcrawlerConfig.NODE_BINDESC)) + { + Map<String,String> throttleMap = new HashMap<>(); + // A bin description node! Look for all its parameters. + String regexp = cn.getAttributeValue(WebcrawlerConfig.ATTR_BINREGEXP); + String isCaseInsensitive = cn.getAttributeValue(WebcrawlerConfig.ATTR_INSENSITIVE); + String maxConnections = null; + String maxKBPerSecond = null; + String maxFetchesPerMinute = null; + int j = 0; + while (j < cn.getChildCount()) + { + ConfigNode childNode = cn.getChild(j++); + if (childNode.getType().equals(WebcrawlerConfig.NODE_MAXCONNECTIONS)) + maxConnections = childNode.getAttributeValue(WebcrawlerConfig.ATTR_VALUE); + else if (childNode.getType().equals(WebcrawlerConfig.NODE_MAXKBPERSECOND)) + maxKBPerSecond = childNode.getAttributeValue(WebcrawlerConfig.ATTR_VALUE); + else if (childNode.getType().equals(WebcrawlerConfig.NODE_MAXFETCHESPERMINUTE)) + maxFetchesPerMinute = childNode.getAttributeValue(WebcrawlerConfig.ATTR_VALUE); + } + if (maxConnections == null) + maxConnections = ""; + if (maxKBPerSecond == null) + maxKBPerSecond = ""; + if (maxFetchesPerMinute == null) + maxFetchesPerMinute = ""; + if(regexp == null) + regexp = ""; + + if (isCaseInsensitive == null || isCaseInsensitive.length() == 0) + isCaseInsensitive = "false"; + + throttleMap.put("regexp",regexp); + throttleMap.put("isCaseInsensitive",isCaseInsensitive); + throttleMap.put("maxConnections",maxConnections); + throttleMap.put("maxKBPerSecond",maxKBPerSecond); + throttleMap.put("maxFetchesPerMinute",maxFetchesPerMinute); + throttlesMapList.add(throttleMap); + binCounter++; + } + } + if (parameters.getChildCount() == 0) + { + velocityContext.put("BRANDNEW",true); + } + velocityContext.put("THROTTLESMAPLIST",throttlesMapList); + } + + private void fillInProxyTab(Map<String,Object> velocityContext, IHTTPOutput out, ConfigParams parameters) + { String proxyHost = parameters.getParameter(WebcrawlerConfig.PARAMETER_PROXYHOST); if (proxyHost == null) proxyHost = ""; @@ -1919,969 +1665,217 @@ public class WebcrawlerConnector extends else proxyAuthPassword = out.mapPasswordToKey(proxyAuthPassword); - // Proxy tab - if (tabName.equals(Messages.getString(locale,"WebcrawlerConnector.Proxy"))) - { - out.print( -"<table class=\"displaytable\">\n"+ -" <tr><td class=\"separator\" colspan=\"2\"><hr/></td></tr>\n"+ -" <tr>\n"+ -" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.ProxyHostColon") + "</nobr></td>\n"+ -" <td class=\"value\"><input type=\"text\" size=\"40\" name=\"proxyhost\" value=\""+Encoder.attributeEscape(proxyHost)+"\"/></td>\n"+ -" </tr>\n"+ -" <tr>\n"+ -" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.ProxyPortColon") + "</nobr></td>\n"+ -" <td class=\"value\"><input type=\"text\" size=\"5\" name=\"proxyport\" value=\""+Encoder.attributeEscape(proxyPort)+"\"/></td>\n"+ -" </tr>\n"+ -" <tr>\n"+ -" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.ProxyAuthenticationDomainColon") + "</nobr></td>\n"+ -" <td class=\"value\"><input type=\"text\" size=\"32\" name=\"proxyauthdomain\" value=\""+Encoder.attributeEscape(proxyAuthDomain)+"\"/></td>\n"+ -" </tr>\n"+ -" <tr>\n"+ -" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.ProxyAuthenticationUserNameColon") + "</nobr></td>\n"+ -" <td class=\"value\"><input type=\"text\" size=\"32\" name=\"proxyauthusername\" value=\""+Encoder.attributeEscape(proxyAuthUsername)+"\"/></td>\n"+ -" </tr>\n"+ -" <tr>\n"+ -" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.ProxyAuthenticationPasswordColon") + "</nobr></td>\n"+ -" <td class=\"value\"><input type=\"password\" size=\"16\" name=\"proxyauthpassword\" value=\""+Encoder.attributeEscape(proxyAuthPassword)+"\"/></td>\n"+ -" </tr>\n"+ -"</table>\n" - ); - } - else - { - out.print( -"<input type=\"hidden\" name=\"proxyhost\" value=\""+Encoder.attributeEscape(proxyHost)+"\"/>\n"+ -"<input type=\"hidden\" name=\"proxyport\" value=\""+Encoder.attributeEscape(proxyPort)+"\"/>\n"+ -"<input type=\"hidden\" name=\"proxyauthusername\" value=\""+Encoder.attributeEscape(proxyAuthUsername)+"\"/>\n"+ -"<input type=\"hidden\" name=\"proxyauthdomain\" value=\""+Encoder.attributeEscape(proxyAuthDomain)+"\"/>\n"+ -"<input type=\"hidden\" name=\"proxyauthpassword\" value=\""+Encoder.attributeEscape(proxyAuthPassword)+"\"/>\n" - ); - } - - // Email tab - if (tabName.equals(Messages.getString(locale,"WebcrawlerConnector.Email"))) - { - out.print( -"<table class=\"displaytable\">\n"+ -" <tr><td class=\"separator\" colspan=\"2\"><hr/></td></tr>\n"+ -" <tr>\n"+ -" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.EmailAddressToContact") + "</nobr></td>\n"+ -" <td class=\"value\">\n"+ -" <input type=\"text\" size=\"32\" name=\"email\" value=\""+Encoder.attributeEscape(email)+"\"/>\n"+ -" </td>\n"+ -" </tr>\n"+ -"</table>\n" - ); - } - else - { - out.print( -"<input type=\"hidden\" name=\"email\" value=\""+Encoder.attributeEscape(email)+"\"/>\n" - ); - } - - // Robots tab - if (tabName.equals(Messages.getString(locale,"WebcrawlerConnector.Robots"))) - { - out.print( -"<table class=\"displaytable\">\n"+ -" <tr><td class=\"separator\" colspan=\"2\"><hr/></td></tr>\n"+ -" <tr>\n"+ -" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.RobotsTxtUsage") + "</nobr></td>\n"+ -" <td class=\"value\">\n"+ -" <select name=\"robotsusage\" size=\"3\">\n"+ -" <option value=\"none\" "+(robotsUsage.equals("none")?"selected=\"selected\"":"")+">" + Messages.getBodyString(locale,"WebcrawlerConnector.DontLookAtRobotsTxt") + "</option>\n"+ -" <option value=\"data\" "+(robotsUsage.equals("data")?"selected=\"selected\"":"")+">" + Messages.getBodyString(locale,"WebcrawlerConnector.ObeyRobotsTxtForDataFetchesOnly") + "</option>\n"+ -" <option value=\"all\" "+(robotsUsage.equals("all")?"selected=\"selected\"":"")+">" + Messages.getBodyString(locale,"WebcrawlerConnector.ObeyRobotsTxtForAllFetches") + "</option>\n"+ -" </select>\n"+ -" </td>\n"+ -" </tr>\n"+ -" <tr>\n"+ -" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.MetaRobotsTagsUsage") + "</nobr></td>\n"+ -" <td class=\"value\">\n"+ -" <select name=\"metarobotstagsusage\" size=\"3\">\n"+ -" <option value=\"none\" "+(metaRobotsTagsUsage.equals("none")?"selected=\"selected\"":"")+">" + Messages.getBodyString(locale,"WebcrawlerConnector.DontLookAtMetaRobotsTags") + "</option>\n"+ -" <option value=\"all\" "+(metaRobotsTagsUsage.equals("all")?"selected=\"selected\"":"")+">" + Messages.getBodyString(locale,"WebcrawlerConnector.ObeyMetaRobotsTags") + "</option>\n"+ -" </select>\n"+ -" </td>\n"+ -" </tr>\n"+ -"</table>\n" - ); - } - else - { - out.print( -"<input type=\"hidden\" name=\"robotsusage\" value=\""+robotsUsage+"\"/>\n"+ -"<input type=\"hidden\" name=\"metarobotstagsusage\" value=\""+metaRobotsTagsUsage+"\"/>\n" - ); - } - - // Bandwidth tab - if (tabName.equals(Messages.getString(locale,"WebcrawlerConnector.Bandwidth"))) - { - out.print( -"<table class=\"displaytable\">\n"+ -" <tr><td class=\"separator\" colspan=\"2\"><hr/></td></tr>\n"+ -" <tr>\n"+ -" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.Throttles") + "</nobr></td>\n"+ -" <td class=\"boxcell\">\n"+ -" <table class=\"formtable\">\n"+ -" <tr class=\"formheaderrow\">\n"+ -" <td class=\"formcolumnheader\"></td>\n"+ -" <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.BinRegularExpression") + "</nobr></td>\n"+ -" <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.CaseInsensitive") + "</nobr></td>\n"+ -" <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.MaxConnections") + "</nobr></td>\n"+ -" <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.MaxKbytesSec") + "</nobr></td>\n"+ -" <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.MaxFetchesMin") + "</nobr></td>\n"+ -" </tr>\n" - ); - int i = 0; - int binCounter = 0; - while (i < parameters.getChildCount()) - { - ConfigNode cn = parameters.getChild(i++); - if (cn.getType().equals(WebcrawlerConfig.NODE_BINDESC)) - { - // A bin description node! Look for all its parameters. - String regexp = cn.getAttributeValue(WebcrawlerConfig.ATTR_BINREGEXP); - String isCaseInsensitive = cn.getAttributeValue(WebcrawlerConfig.ATTR_INSENSITIVE); - String maxConnections = null; - String maxKBPerSecond = null; - String maxFetchesPerMinute = null; - int j = 0; - while (j < cn.getChildCount()) - { - ConfigNode childNode = cn.getChild(j++); - if (childNode.getType().equals(WebcrawlerConfig.NODE_MAXCONNECTIONS)) - maxConnections = childNode.getAttributeValue(WebcrawlerConfig.ATTR_VALUE); - else if (childNode.getType().equals(WebcrawlerConfig.NODE_MAXKBPERSECOND)) - maxKBPerSecond = childNode.getAttributeValue(WebcrawlerConfig.ATTR_VALUE); - else if (childNode.getType().equals(WebcrawlerConfig.NODE_MAXFETCHESPERMINUTE)) - maxFetchesPerMinute = childNode.getAttributeValue(WebcrawlerConfig.ATTR_VALUE); - } - if (maxConnections == null) - maxConnections = ""; - if (maxKBPerSecond == null) - maxKBPerSecond = ""; - if (maxFetchesPerMinute == null) - maxFetchesPerMinute = ""; - - if (isCaseInsensitive == null || isCaseInsensitive.length() == 0) - isCaseInsensitive = "false"; - - // It's prefix will be... - String prefix = "bandwidth_" + Integer.toString(binCounter); - out.print( -" <tr class=\""+(((binCounter % 2)==0)?"evenformrow":"oddformrow")+"\">\n"+ -" <td class=\"formcolumncell\">\n"+ -" <a name=\""+prefix+"\">\n"+ -" <input type=\"button\" value=\"" + Messages.getAttributeString(locale,"WebcrawlerConnector.Delete") + "\" alt=\""+Messages.getAttributeString(locale,"WebcrawlerConnector.DeleteBinRegularExpression")+Integer.toString(binCounter+1)+"\" onclick='javascript:deleteRegexp("+Integer.toString(binCounter)+");'/>\n"+ -" <input type=\"hidden\" name=\""+"op_"+prefix+"\" value=\"Continue\"/>\n"+ -" <input type=\"hidden\" name=\""+"regexp_"+prefix+"\" value=\""+Encoder.attributeEscape(regexp)+"\"/>\n"+ -" </a>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr>"+Encoder.bodyEscape(regexp)+"</nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"checkbox\" name=\"insensitive_"+prefix+"\" value=\"true\" "+(isCaseInsensitive.equals("true")?"checked=\"\"":"")+" /></nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"text\" size=\"5\" name=\"connections_"+prefix+"\" value=\""+maxConnections+"\"/></nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"text\" size=\"5\" name=\"rate_"+prefix+"\" value=\""+maxKBPerSecond+"\"/></nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"text\" size=\"5\" name=\"fetches_"+prefix+"\" value=\""+maxFetchesPerMinute+"\"/></nobr>\n"+ -" </td>\n"+ -" </tr>\n" - ); - binCounter++; - } - } - - // If it looks like this is a brand-new configuration, add in a default throttle. - // This only works because other nodes must get created on the first post, and cannot then be deleted. - if (parameters.getChildCount() == 0) - { - // It's prefix will be... - String prefix = "bandwidth_" + Integer.toString(binCounter); - out.print( -" <tr class=\""+(((binCounter % 2)==0)?"evenformrow":"oddformrow")+"\">\n"+ -" <td class=\"formcolumncell\">\n"+ -" <a name=\""+prefix+"\">\n"+ -" <input type=\"button\" value=\"" + Messages.getAttributeString(locale,"WebcrawlerConnector.Delete") + "\" alt=\""+ Messages.getAttributeString(locale,"WebcrawlerConnector.DeleteBinRegularExpression") +Integer.toString(binCounter+1)+"\" onclick='javascript:deleteRegexp("+Integer.toString(binCounter)+");'/>\n"+ -" <input type=\"hidden\" name=\""+"op_"+prefix+"\" value=\"Continue\"/>\n"+ -" <input type=\"hidden\" name=\""+"regexp_"+prefix+"\" value=\"\"/>\n"+ -" </a>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr></nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"checkbox\" name=\"insensitive_"+prefix+"\" value=\"false\"/></nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"text\" size=\"5\" name=\"connections_"+prefix+"\" value=\"2\"/></nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"text\" size=\"5\" name=\"rate_"+prefix+"\" value=\"64\"/></nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"text\" size=\"5\" name=\"fetches_"+prefix+"\" value=\"12\"/></nobr>\n"+ -" </td>\n"+ -" </tr>\n" - ); - binCounter++; - } + velocityContext.put("PROXYHOST",proxyHost); + velocityContext.put("PROXYPORT",proxyPort); + velocityContext.put("PROXYAUTHDOMAIN",proxyAuthDomain); + velocityContext.put("PROXYAUTHUSERNAME",proxyAuthUsername); + velocityContext.put("PROXYAUTHPASSWORD",proxyAuthPassword); + } - if (binCounter == 0) - { - out.print( -" <tr class=\"formrow\"><td class=\"formmessage\" colspan=\"6\">"+Messages.getBodyString(locale,"WebcrawlerConnector.NoBandwidthOrConnectionThrottlingSpecified")+"</td></tr>\n" - ); - } - out.print( -" <tr class=\"formrow\"><td class=\"formseparator\" colspan=\"6\"><hr/></td></tr>\n"+ -" <tr class=\"formrow\">\n"+ -" <td class=\"formcolumncell\">\n"+ -" <a name=\"bandwidth\">\n"+ -" <input type=\"button\" value=\"" + Messages.getAttributeString(locale,"WebcrawlerConnector.Add") + "\" alt=\"" + Messages.getAttributeString(locale,"WebcrawlerConnector.AddBinRegularExpression") + "\" onclick=\"javascript:addRegexp();\"/>\n"+ -" </a>\n"+ -" <input type=\"hidden\" name=\"bandwidth_count\" value=\""+binCounter+"\"/>\n"+ -" <input type=\"hidden\" name=\"bandwidth_op\" value=\"Continue\"/>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"text\" size=\"30\" name=\"regexp_bandwidth\" value=\"\"/></nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"checkbox\" name=\"insensitive_bandwidth\" value=\"true\"/></nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"text\" size=\"5\" name=\"connections_bandwidth\" value=\"\"/></nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"text\" size=\"5\" name=\"rate_bandwidth\" value=\"\"/></nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"text\" size=\"5\" name=\"fetches_bandwidth\" value=\"\"/></nobr>\n"+ -" </td>\n"+ -" </tr>\n"+ -" </table>\n"+ -" </td>\n"+ -" </tr>\n"+ -"</table>\n" - ); - } - else + private void fillInCertificatesTab(Map<String,Object> velocityContext, IHTTPOutput out, ConfigParams parameters) throws ManifoldCFException + { + int i = 0; + List<Map<String,String>> trustMapList = new ArrayList<>(); + while (i < parameters.getChildCount()) { - // Hiddens for bandwidth tab. - int i = 0; - int binCounter = 0; - while (i < parameters.getChildCount()) + ConfigNode cn = parameters.getChild(i++); + if (cn.getType().equals(WebcrawlerConfig.NODE_TRUST)) { - ConfigNode cn = parameters.getChild(i++); - if (cn.getType().equals(WebcrawlerConfig.NODE_BINDESC)) - { - // A bin description node! Look for all its parameters. - String regexp = cn.getAttributeValue(WebcrawlerConfig.ATTR_BINREGEXP); - String isCaseInsensitive = cn.getAttributeValue(WebcrawlerConfig.ATTR_INSENSITIVE); - String maxConnections = null; - String maxKBPerSecond = null; - String maxFetchesPerMinute = null; - int j = 0; - while (j < cn.getChildCount()) - { - ConfigNode childNode = cn.getChild(j++); - if (childNode.getType().equals(WebcrawlerConfig.NODE_MAXCONNECTIONS)) - maxConnections = childNode.getAttributeValue(WebcrawlerConfig.ATTR_VALUE); - else if (childNode.getType().equals(WebcrawlerConfig.NODE_MAXKBPERSECOND)) - maxKBPerSecond = childNode.getAttributeValue(WebcrawlerConfig.ATTR_VALUE); - else if (childNode.getType().equals(WebcrawlerConfig.NODE_MAXFETCHESPERMINUTE)) - maxFetchesPerMinute = childNode.getAttributeValue(WebcrawlerConfig.ATTR_VALUE); - } - if (maxConnections == null) - maxConnections = ""; - if (maxKBPerSecond == null) - maxKBPerSecond = ""; - if (maxFetchesPerMinute == null) - maxFetchesPerMinute = ""; - if (isCaseInsensitive == null || isCaseInsensitive.length() == 0) - isCaseInsensitive = "false"; - - // It's prefix will be... - String prefix = "bandwidth_" + Integer.toString(binCounter); - out.print( -"<input type=\"hidden\" name=\""+"regexp_"+prefix+"\" value=\""+Encoder.attributeEscape(regexp)+"\"/>\n"+ -"<input type=\"hidden\" name=\""+"insensitive_"+prefix+"\" value=\""+isCaseInsensitive+"\"/>\n"+ -"<input type=\"hidden\" name=\""+"connections_"+prefix+"\" value=\""+maxConnections+"\"/>\n"+ -"<input type=\"hidden\" name=\""+"rate_"+prefix+"\" value=\""+maxKBPerSecond+"\"/>\n"+ -"<input type=\"hidden\" name=\""+"fetches_"+prefix+"\" value=\""+maxFetchesPerMinute+"\"/>\n" - ); - binCounter++; - } - } + Map<String,String> trustMap = new HashMap<>(); - // If it looks like this is a brand-new configuration, add in a default throttle. - // This only works because other nodes must get created on the first post, and cannot then be deleted. - if (parameters.getChildCount() == 0) - { - // It's prefix will be... - String prefix = "bandwidth_" + Integer.toString(binCounter); - out.print( -"<input type=\"hidden\" name=\""+"regexp_"+prefix+"\" value=\"\"/>\n"+ -"<input type=\"hidden\" name=\""+"insensitive_"+prefix+"\" value=\"false\"/>\n"+ -"<input type=\"hidden\" name=\""+"connections_"+prefix+"\" value=\"2\"/>\n"+ -"<input type=\"hidden\" name=\""+"rate_"+prefix+"\" value=\"64\"/>\n"+ -"<input type=\"hidden\" name=\""+"fetches_"+prefix+"\" value=\"12\"/>\n" - ); - binCounter++; - } + // A bin description node! Look for all its parameters. + String regexp = cn.getAttributeValue(WebcrawlerConfig.ATTR_URLREGEXP); + String trustEverything = cn.getAttributeValue(WebcrawlerConfig.ATTR_TRUSTEVERYTHING); - out.print( -"<input type=\"hidden\" name=\"bandwidth_count\" value=\""+binCounter+"\"/>\n" - ); - } + trustMap.put("trustEverything",trustEverything); + trustMap.put("regexp",regexp); - // Access Credentials tab - if (tabName.equals(Messages.getString(locale,"WebcrawlerConnector.AccessCredentials"))) - { - out.print( -"<table class=\"displaytable\">\n"+ -" <tr><td class=\"separator\" colspan=\"2\"><hr/></td></tr>\n"+ -" <tr>\n"+ -" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.PageAccessCredentials") + "</nobr></td>\n"+ -" <td class=\"boxcell\">\n"+ -" <table class=\"formtable\">\n"+ -" <tr class=\"formheaderrow\">\n"+ -" <td class=\"formcolumnheader\"></td>\n"+ -" <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.URLRegularExpression") + "</nobr></td>\n"+ -" <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.AuthenticationType") + "</nobr></td>\n"+ -" <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.Domain") + "</nobr></td>\n"+ -" <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.UserName") + "</nobr></td>\n"+ -" <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.Password") + "</nobr></td>\n"+ -" </tr>\n" - ); - int i = 0; - int accessCounter = 0; - while (i < parameters.getChildCount()) - { - ConfigNode cn = parameters.getChild(i++); - if (cn.getType().equals(WebcrawlerConfig.NODE_ACCESSCREDENTIAL)) + if (trustEverything != null && trustEverything.equals("true")) { - // A bin description node! Look for all its parameters. - String type = cn.getAttributeValue(WebcrawlerConfig.ATTR_TYPE); - if (!type.equals(WebcrawlerConfig.ATTRVALUE_SESSION)) - { - String regexp = cn.getAttributeValue(WebcrawlerConfig.ATTR_URLREGEXP); - String domain = cn.getAttributeValue(WebcrawlerConfig.ATTR_DOMAIN); - if (domain == null) - domain = ""; - String userName = cn.getAttributeValue(WebcrawlerConfig.ATTR_USERNAME); - String password = out.mapPasswordToKey(ManifoldCF.deobfuscate(cn.getAttributeValue(WebcrawlerConfig.ATTR_PASSWORD))); - - // It's prefix will be... - String prefix = "acredential_" + Integer.toString(accessCounter); - out.print( -" <tr class=\""+(((accessCounter % 2)==0)?"evenformrow":"oddformrow")+"\">\n"+ -" <td class=\"formcolumncell\">\n"+ -" <a name=\""+prefix+"\">\n"+ -" <input type=\"button\" value=\"" + Messages.getAttributeString(locale,"WebcrawlerConnector.Delete") + "\" alt=\""+Messages.getAttributeString(locale,"WebcrawlerConnector.DeletePageAuthenticationUrlRegularExpression")+Integer.toString(accessCounter+1)+"\" onclick='javascript:deleteARegexp("+Integer.toString(accessCounter)+");'/>\n"+ -" <input type=\"hidden\" name=\"op_"+prefix+"\" value=\"Continue\"/>\n"+ -" <input type=\"hidden\" name=\"regexp_"+prefix+"\" value=\""+Encoder.attributeEscape(regexp)+"\"/>\n"+ -" </a>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr>"+Encoder.bodyEscape(regexp)+"</nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"radio\" name=\"type_"+prefix+"\" value=\"basic\" "+(type.equals("basic")?"checked=\"\"":"")+" /> " + Messages.getBodyString(locale,"WebcrawlerConnector.BasicAuthentication") + "</nobr><br/>\n"+ -" <nobr><input type=\"radio\" name=\"type_"+prefix+"\" value=\"ntlm\" "+(type.equals("ntlm")?"checked=\"\"":"")+" /> " + Messages.getBodyString(locale,"WebcrawlerConnector.NTLMAuthentication") + "</nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"text\" size=\"16\" name=\""+"domain_"+prefix+"\" value=\""+Encoder.attributeEscape(domain)+"\"/></nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"text\" size=\"16\" name=\""+"username_"+prefix+"\" value=\""+Encoder.attributeEscape(userName)+"\"/></nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"password\" size=\"16\" name=\""+"password_"+prefix+"\" value=\""+Encoder.attributeEscape(password)+"\"/></nobr>\n"+ -" </td>\n"+ -" </tr>\n" - ); - accessCounter++; - } } - } - - if (accessCounter == 0) - { - out.print( -" <tr class=\"formrow\"><td class=\"formmessage\" colspan=\"6\">" + Messages.getBodyString(locale,"WebcrawlerConnector.NoPageAccessCredentialsSpecified") + "</td></tr>\n" - ); - } - out.print( -" <tr class=\"formrow\"><td class=\"formseparator\" colspan=\"6\"><hr/></td></tr>\n"+ -" <tr class=\"formrow\">\n"+ -" <td class=\"formcolumncell\">\n"+ -" <a name=\"acredential\">\n"+ -" <input type=\"button\" value=\"" + Messages.getAttributeString(locale,"WebcrawlerConnector.Add") + "\" alt=\"" + Messages.getAttributeString(locale,"WebcrawlerConnector.AddPageAuthenticationUrlRegularExpression") + "\" onclick=\"javascript:addARegexp();\"/>\n"+ -" </a>\n"+ -" <input type=\"hidden\" name=\"acredential_count\" value=\""+accessCounter+"\"/>\n"+ -" <input type=\"hidden\" name=\"acredential_op\" value=\"Continue\"/>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"text\" size=\"30\" name=\"regexp_acredential\" value=\"\"/></nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"radio\" name=\"type_acredential\" value=\"basic\" checked=\"\" /> " + Messages.getBodyString(locale,"WebcrawlerConnector.BasicAuthentication") + "</nobr><br/>\n"+ -" <nobr><input type=\"radio\" name=\"type_acredential\" value=\"ntlm\" /> " + Messages.getBodyString(locale,"WebcrawlerConnector.NTLMAuthentication") + "</nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"text\" size=\"16\" name=\"domain_acredential\" value=\"\"/></nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"text\" size=\"16\" name=\"username_acredential\" value=\"\"/></nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"password\" size=\"16\" name=\"password_acredential\" value=\"\"/></nobr>\n"+ -" </td>\n"+ -" </tr>\n"+ -" </table>\n"+ -" </td>\n"+ -" </tr>\n"+ -" \n"+ -" <tr><td class=\"separator\" colspan=\"2\"><hr/></td></tr>\n"+ -"\n"+ -" <tr>\n"+ -" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.SessionBasedAccessCredentials") + "</nobr></td>\n"+ -" <td class=\"boxcell\">\n"+ -" <table class=\"formtable\">\n"+ -" <tr class=\"formheaderrow\">\n"+ -" <td class=\"formcolumnheader\"></td>\n"+ -" <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.URLRegularExpression") + "</nobr></td>\n"+ -" <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.LoginPages") + "</nobr></td>\n"+ -" </tr>\n" - ); - i = 0; - accessCounter = 0; - while (i < parameters.getChildCount()) - { - ConfigNode cn = parameters.getChild(i++); - if (cn.getType().equals(WebcrawlerConfig.NODE_ACCESSCREDENTIAL)) + else { - // A bin description node! Look for all its parameters. - String type = cn.getAttributeValue(WebcrawlerConfig.ATTR_TYPE); - if (type.equals(WebcrawlerConfig.ATTRVALUE_SESSION)) + String trustStore = cn.getAttributeValue(WebcrawlerConfig.ATTR_TRUSTSTORE); + IKeystoreManager localTruststore = KeystoreManagerFactory.make("",trustStore); + String[] truststoreContents = localTruststore.getContents(); + + // Each trust store will have only at most one cert in it at this level. These individual certs are assembled into the proper trust store + // for each individual url at fetch time. + + if (truststoreContents.length == 1) { - String regexp = cn.getAttributeValue(WebcrawlerConfig.ATTR_URLREGEXP); - - // It's prefix will be... - String prefix = "scredential_" + Integer.toString(accessCounter); - out.print( -" <tr class=\""+(((accessCounter % 2)==0)?"evenformrow":"oddformrow")+"\">\n"+ -" <td class=\"formcolumncell\">\n"+ -" <a name=\""+prefix+"\">\n"+ -" <input type=\"button\" value=\"" + Messages.getAttributeString(locale,"WebcrawlerConnector.Delete") + "\" alt=\""+Messages.getAttributeString(locale,"WebcrawlerConnector.DeleteSessionAuthenticationUrlRegularExpression")+Integer.toString(accessCounter+1)+"\" onclick='javascript:deleteSRegexp("+Integer.toString(accessCounter)+");'/>\n"+ -" <input type=\"hidden\" name=\""+prefix+"_op"+"\" value=\"Continue\"/>\n"+ -" <input type=\"hidden\" name=\""+prefix+"_regexp"+"\" value=\""+Encoder.attributeEscape(regexp)+"\"/>\n"+ -" </a>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr>"+Encoder.bodyEscape(regexp)+"</nobr>\n"+ -" </td>\n"+ -" <td class=\"boxcell\">\n"+ -" <table class=\"formtable\">\n"+ -" <tr class=\"formheaderrow\">\n"+ -" <td class=\"formcolumnheader\"></td>\n"+ -" <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.LoginURLRegularExpression") + "</nobr></td>\n"+ -" <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.PageType") + "</nobr></td>\n"+ -" <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.FormNamelinkTargetRegularExpression") + "</nobr></td>\n"+ -" <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.OverrideTargetURL") + "</nobr></td>\n"+ -" <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.OverrideFormParameters") + "</nobr></td>\n"+ -" </tr>\n" - ); - int q = 0; - int authPageCounter = 0; - while (q < cn.getChildCount()) - { - ConfigNode authPageNode = cn.getChild(q++); - if (authPageNode.getType().equals(WebcrawlerConfig.NODE_AUTHPAGE)) - { - String pageRegexp = authPageNode.getAttributeValue(WebcrawlerConfig.ATTR_URLREGEXP); - String pageType = authPageNode.getAttributeValue(WebcrawlerConfig.ATTR_TYPE); - String matchRegexp = authPageNode.getAttributeValue(WebcrawlerConfig.ATTR_MATCHREGEXP); - if (matchRegexp == null) - matchRegexp = ""; - String overrideTargetURL = authPageNode.getAttributeValue(WebcrawlerConfig.ATTR_OVERRIDETARGETURL); - if (overrideTargetURL == null) - overrideTargetURL = ""; - String authpagePrefix = prefix + "_" + authPageCounter; - out.print( -" <tr class=\""+(((authPageCounter % 2)==0)?"evenformrow":"oddformrow")+"\">\n"+ -" <td class=\"formcolumncell\">\n"+ -" <a name=\""+authpagePrefix+"\">\n"+ -" <input type=\"button\" value=\"Delete\" alt=\""+Messages.getAttributeString(locale,"WebcrawlerConnector.DeleteLoginPage")+(authPageCounter+1)+" for url regular expression #"+Integer.toString(accessCounter+1)+"\" onclick='javascript:deleteLoginPage("+Integer.toString(accessCounter)+","+Integer.toString(authPageCounter)+");'/>\n"+ -" <input type=\"hidden\" name=\""+authpagePrefix+"_op"+"\" value=\"Continue\"/>\n"+ -" <input type=\"hidden\" name=\""+authpagePrefix+"_regexp"+"\" value=\""+Encoder.attributeEscape(pageRegexp)+"\"/>\n"+ -" <input type=\"hidden\" name=\""+authpagePrefix+"_type"+"\" value=\""+pageType+"\"/>\n"+ -" </a>\n"+ -" </td>\n"+ -"\n"+ -" <td class=\"formcolumncell\"><nobr>"+Encoder.bodyEscape(pageRegexp)+"</nobr></td>\n"+ -" <td class=\"formcolumncell\"><nobr>"+pageType+"</nobr></td>\n"+ -" <td class=\"formcolumncell\"><nobr><input type=\"text\" size=\"30\" name=\""+authpagePrefix+"_matchregexp"+"\" value=\""+Encoder.attributeEscape(matchRegexp)+"\"/></nobr></td>\n"+ -" <td class=\"formcolumncell\"><nobr><input type=\"text\" size=\"30\" name=\""+authpagePrefix+"_overridetargeturl"+"\" value=\""+Encoder.attributeEscape(overrideTargetURL)+"\"/></nobr></td>\n" - ); - if (pageType.equals(WebcrawlerConfig.ATTRVALUE_FORM)) - { - out.print( -" <td class=\"boxcell\">\n"+ -" <table class=\"formtable\">\n"+ -" <tr class=\"formheaderrow\">\n"+ -" <td class=\"formcolumnheader\"></td>\n"+ -" <td class=\"formcolumnheader\"><nobr>"+Messages.getBodyString(locale,"WebcrawlerConnector.ParameterRegularExpression")+"</nobr></td>\n"+ -" <td class=\"formcolumnheader\"><nobr>"+Messages.getBodyString(locale,"WebcrawlerConnector.Value")+"</nobr></td>\n"+ -" <td class=\"formcolumnheader\"><nobr>"+Messages.getBodyString(locale,"WebcrawlerConnector.Password")+"</nobr></td>\n"+ -" </tr>\n" - ); - int z = 0; - int paramCounter = 0; - while (z < authPageNode.getChildCount()) - { - ConfigNode paramNode = authPageNode.getChild(z++); - if (paramNode.getType().equals(WebcrawlerConfig.NODE_AUTHPARAMETER)) - { - String param = paramNode.getAttributeValue(WebcrawlerConfig.ATTR_NAMEREGEXP); - if (param == null) - param = ""; - String value = paramNode.getAttributeValue(WebcrawlerConfig.ATTR_VALUE); - if (value == null) - value = ""; - String password = paramNode.getAttributeValue(WebcrawlerConfig.ATTR_PASSWORD); - if (password == null) - password = ""; - else - password = out.mapPasswordToKey(ManifoldCF.deobfuscate(password)); - String authParamPrefix = authpagePrefix + "_" + paramCounter; - out.print( -" <tr class=\""+(((paramCounter % 2)==0)?"evenformrow":"oddformrow")+"\">\n"+ -" <td class=\"formcolumncell\">\n"+ -" <a name=\""+authParamPrefix+"\">\n"+ -" <input type=\"button\" value=\"Delete\" alt=\""+Messages.getAttributeString(locale,"WebcrawlerConnector.DeleteParameter")+(paramCounter+1)+Messages.getAttributeString(locale,"WebcrawlerConnector.ForLoginPage")+(authPageCounter+1)+Messages.getAttributeString(locale,"WebcrawlerConnector.ForCredential")+(accessCounter+1)+"\" onclick='javascript:deleteLoginPageParameter("+accessCounter+","+authPageCounter+","+paramCounter+");'/>\n"+ -" <input type=\"hidden\" name=\""+authParamPrefix+"_op"+"\" value=\"Continue\"/>\n"+ -" </a>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"text\" size=\"30\" name=\""+authParamPrefix+"_param"+"\" value=\""+Encoder.attributeEscape(param)+"\"/></nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"text\" size=\"15\" name=\""+authParamPrefix+"_value"+"\" value=\""+Encoder.attributeEscape(value)+"\"/></nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"password\" size=\"15\" name=\""+authParamPrefix+"_password"+"\" value=\""+Encoder.attributeEscape(password)+"\"/></nobr>\n"+ -" </td>\n"+ -" </tr>\n" - ); - paramCounter++; - } - } - out.print( -" <tr class=\"formrow\"><td class=\"formseparator\" colspan=\"4\"><hr/></td></tr>\n"+ -" <tr class=\"formrow\">\n"+ -" <td class=\"formcolumncell\">\n"+ -" <a name=\""+authpagePrefix+"_loginparam"+"\">\n"+ -" <input type=\"button\" value=\"Add\" alt=\""+Messages.getAttributeString(locale,"WebcrawlerConnector.AddParameterToLoginPage")+(authPageCounter+1)+Messages.getAttributeString(locale,"WebcrawlerConnector.ForCredential")+(accessCounter+1)+"\" onclick='javascript:addLoginPageParameter("+accessCounter+","+authPageCounter+");'/>\n"+ -" </a>\n"+ -" <input type=\"hidden\" name=\""+authpagePrefix+"_loginparamcount"+"\" value=\""+paramCounter+"\"/>\n"+ -" <input type=\"hidden\" name=\""+authpagePrefix+"_loginparamop"+"\" value=\"Continue\"/>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"text\" size=\"30\" name=\""+authpagePrefix+"_loginparamname"+"\" value=\"\"/></nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"text\" size=\"15\" name=\""+authpagePrefix+"_loginparamvalue"+"\" value=\"\"/></nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"password\" size=\"15\" name=\""+authpagePrefix+"_loginparampassword"+"\" value=\"\"/></nobr>\n"+ -" </td>\n"+ -" </tr>\n"+ -" </table>\n"+ -" </td>\n" - ); - } - else - { - out.print( -" <td class=\"formcolumncell\"></td>\n" - ); - } - out.print( -" </tr>\n" - ); - authPageCounter++; - } - } - out.print( -" <tr class=\"formrow\"><td class=\"formseparator\" colspan=\"6\"><hr/></td></tr>\n"+ -" <tr class=\"formrow\">\n"+ -" <td class=\"formcolumncell\">\n"+ -" <a name=\""+prefix+"_loginpage"+"\">\n"+ -" <input type=\"button\" value=\"Add\" alt=\""+Messages.getAttributeString(locale,"WebcrawlerConnector.AddLoginPageToCredential")+(accessCounter+1)+"\" onclick='javascript:addLoginPage("+accessCounter+");'/>\n"+ -" </a>\n"+ -" <input type=\"hidden\" name=\""+prefix+"_loginpagecount"+"\" value=\""+authPageCounter+"\"/>\n"+ -" <input type=\"hidden\" name=\""+prefix+"_loginpageop"+"\" value=\"Continue\"/>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"text\" size=\"30\" name=\""+prefix+"_loginpageregexp"+"\" value=\"\"/></nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"radio\" name=\""+prefix+"_loginpagetype"+"\" value=\""+WebcrawlerConfig.ATTRVALUE_FORM+"\" checked=\"\"/>"+Messages.getBodyString(locale,"WebcrawlerConnector.FormName")+"</nobr><br/>\n"+ -" <nobr><input type=\"radio\" name=\""+prefix+"_loginpagetype"+"\" value=\""+WebcrawlerConfig.ATTRVALUE_LINK+"\"/>"+Messages.getBodyString(locale,"WebcrawlerConnector.LinkTarget")+"</nobr>\n"+ -" <nobr><input type=\"radio\" name=\""+prefix+"_loginpagetype"+"\" value=\""+WebcrawlerConfig.ATTRVALUE_REDIRECTION+"\"/>"+Messages.getBodyString(locale,"WebcrawlerConnector.RedirectionTo")+"</nobr>\n"+ -" <nobr><input type=\"radio\" name=\""+prefix+"_loginpagetype"+"\" value=\""+WebcrawlerConfig.ATTRVALUE_CONTENT+"\"/>"+Messages.getBodyString(locale,"WebcrawlerConnector.PageContent")+"</nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"text\" size=\"30\" name=\""+prefix+"_loginpagematchregexp"+"\" value=\"\"/></nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"text\" size=\"30\" name=\""+prefix+"_loginpageoverridetargeturl"+"\" value=\"\"/></nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" </td>\n"+ -" </tr>\n"+ -"\n"+ -" </table>\n"+ -" </td>\n"+ -" </tr>\n" - ); - accessCounter++; + String alias = truststoreContents[0]; + String description = localTruststore.getDescription(alias); + String shortenedDescription = description; + if (shortenedDescription.length() > 100) + shortenedDescription = shortenedDescription.substring(0,100) + "..."; + + trustMap.put("trustStore",trustStore); + trustMap.put("shortenedDescription",shortenedDescription); } } + trustMapList.add(trustMap); } - - if (accessCounter == 0) - { - out.print( -" <tr class=\"formrow\"><td class=\"formmessage\" colspan=\"3\">" + Messages.getBodyString(locale,"WebcrawlerConnector.NoSessionBasedAccessCredentialsSpecified") + "</td></tr>\n" - ); - } - out.print( -" <tr class=\"formrow\"><td class=\"formseparator\" colspan=\"3\"><hr/></td></tr>\n"+ -" <tr class=\"formrow\">\n"+ -" <td class=\"formcolumncell\">\n"+ -" <a name=\"scredential\">\n"+ -" <input type=\"button\" value=\"" + Messages.getAttributeString(locale,"WebcrawlerConnector.Add") + "\" alt=\""+Messages.getAttributeString(locale,"WebcrawlerConnector.AddSessionAuthenticationUrlRegularExpression")+"\" onclick=\"javascript:addSRegexp();\"/>\n"+ -" </a>\n"+ -" <input type=\"hidden\" name=\"scredential_count\" value=\""+accessCounter+"\"/>\n"+ -" <input type=\"hidden\" name=\"scredential_op\" value=\"Continue\"/>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"text\" size=\"30\" name=\"scredential_regexp\" value=\"\"/></nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" </td>\n"+ -" </tr>\n"+ -" </table>\n"+ -" </td>\n"+ -" </tr>\n"+ -"\n"+ -"</table>\n" - ); } - else + velocityContext.put("TRUSTMAPLIST",trustMapList); + } + + private void fillInAccessTab(Map<String,Object> velocityContext, IHTTPOutput out, ConfigParams parameters) throws ManifoldCFException + { + int i = 0; + List<Map<String,String>> pageAccessMapList = new ArrayList<>(); + while (i < parameters.getChildCount()) { - // Hiddens for Access Credentials tab. - - // Page credentials first. - int i = 0; - int accessCounter = 0; - while (i < parameters.getChildCount()) + ConfigNode cn = parameters.getChild(i++); + if (cn.getType().equals(WebcrawlerConfig.NODE_ACCESSCREDENTIAL)) { - ConfigNode cn = parameters.getChild(i++); - if (cn.getType().equals(WebcrawlerConfig.NODE_ACCESSCREDENTIAL)) + Map<String,String> pageAccessMap = new HashMap<>(); + + // A bin description node! Look for all its parameters. + String type = cn.getAttributeValue(WebcrawlerConfig.ATTR_TYPE); + if (!type.equals(WebcrawlerConfig.ATTRVALUE_SESSION)) { - // A bin description node! Look for all its parameters. - String type = cn.getAttributeValue(WebcrawlerConfig.ATTR_TYPE); - if (!type.equals(WebcrawlerConfig.ATTRVALUE_SESSION)) - { - String regexp = cn.getAttributeValue(WebcrawlerConfig.ATTR_URLREGEXP); - String domain = cn.getAttributeValue(WebcrawlerConfig.ATTR_DOMAIN); - if (domain == null) - domain = ""; - String userName = cn.getAttributeValue(WebcrawlerConfig.ATTR_USERNAME); - String password = out.mapPasswordToKey(ManifoldCF.deobfuscate(cn.getAttributeValue(WebcrawlerConfig.ATTR_PASSWORD))); - - // It's prefix will be... - String prefix = "acredential_" + Integer.toString(accessCounter); - out.print( -"<input type=\"hidden\" name=\""+"regexp_"+prefix+"\" value=\""+Encoder.attributeEscape(regexp)+"\"/>\n"+ -"<input type=\"hidden\" name=\""+"type_"+prefix+"\" value=\""+type+"\"/>\n"+ -"<input type=\"hidden\" name=\""+"domain_"+prefix+"\" value=\""+Encoder.attributeEscape(domain)+"\"/>\n"+ -"<input type=\"hidden\" name=\""+"username_"+prefix+"\" value=\""+Encoder.attributeEscape(userName)+"\"/>\n"+ -"<input type=\"hidden\" name=\""+"password_"+prefix+"\" value=\""+Encoder.attributeEscape(password)+"\"/>\n" - ); - accessCounter++; - } + String regexp = cn.getAttributeValue(WebcrawlerConfig.ATTR_URLREGEXP); + if(regexp == null) + regexp = ""; + String domain = cn.getAttributeValue(WebcrawlerConfig.ATTR_DOMAIN); + if (domain == null) + domain = ""; + String userName = cn.getAttributeValue(WebcrawlerConfig.ATTR_USERNAME); + String password = out.mapPasswordToKey(ManifoldCF.deobfuscate(cn.getAttributeValue(WebcrawlerConfig.ATTR_PASSWORD))); + + pageAccessMap.put("regexp",regexp); + pageAccessMap.put("domain",domain); + pageAccessMap.put("userName",userName); + pageAccessMap.put("password",password); + pageAccessMap.put("type",type); + + pageAccessMapList.add(pageAccessMap); } } - out.print( -"<input type=\"hidden\" name=\"acredential_count\" value=\""+accessCounter+"\"/>\n" - ); + } + velocityContext.put("PAGEACCESSMAPLIST",pageAccessMapList); - // Now, session credentials - i = 0; - accessCounter = 0; - while (i < parameters.getChildCount()) + i = 0; + List<Map<String,Object>> sessionAccessMapList = new ArrayList<>(); + while (i < parameters.getChildCount()) + { + ConfigNode cn = parameters.getChild(i++); + if (cn.getType().equals(WebcrawlerConfig.NODE_ACCESSCREDENTIAL)) { - ConfigNode cn = parameters.getChild(i++); - if (cn.getType().equals(WebcrawlerConfig.NODE_ACCESSCREDENTIAL)) + // A bin description node! Look for all its parameters. + String type = cn.getAttributeValue(WebcrawlerConfig.ATTR_TYPE); + if (type.equals(WebcrawlerConfig.ATTRVALUE_SESSION)) { - // A bin description node! Look for all its parameters. - String type = cn.getAttributeValue(WebcrawlerConfig.ATTR_TYPE); - if (type.equals(WebcrawlerConfig.ATTRVALUE_SESSION)) + Map<String,Object> sessionAccessMap = new HashMap<>(); + String regexp = cn.getAttributeValue(WebcrawlerConfig.ATTR_URLREGEXP); + if(regexp == null) + regexp = ""; + sessionAccessMap.put("regexp",regexp); + + int q = 0; + List<Map<String,Object>> authPageMapList = new ArrayList<>(); + while (q < cn.getChildCount()) { - String regexp = cn.getAttributeValue(WebcrawlerConfig.ATTR_URLREGEXP); - // It's identifier will be... - String prefix = "scredential_" + Integer.toString(accessCounter); - out.print( -"<input type=\"hidden\" name=\""+prefix+"_regexp"+"\" value=\""+Encoder.attributeEscape(regexp)+"\"/>\n" - ); - // Loop through login pages... - int q = 0; - int authPageCounter = 0; - while (q < cn.getChildCount()) + ConfigNode authPageNode = cn.getChild(q++); + if (authPageNode.getType().equals(WebcrawlerConfig.NODE_AUTHPAGE)) { - ConfigNode authPageNode = cn.getChild(q++); - if (authPageNode.getType().equals(WebcrawlerConfig.NODE_AUTHPAGE)) + Map<String,Object> authPageMap = new HashMap<>(); + + String pageRegexp = authPageNode.getAttributeValue(WebcrawlerConfig.ATTR_URLREGEXP); + String pageType = authPageNode.getAttributeValue(WebcrawlerConfig.ATTR_TYPE); + String matchRegexp = authPageNode.getAttributeValue(WebcrawlerConfig.ATTR_MATCHREGEXP); + if (matchRegexp == null) + matchRegexp = ""; + String overrideTargetURL = authPageNode.getAttributeValue(WebcrawlerConfig.ATTR_OVERRIDETARGETURL); + if (overrideTargetURL == null) + overrideTargetURL = ""; + + authPageMap.put("pageRegexp",pageRegexp); + authPageMap.put("pageType",pageType); + authPageMap.put("matchRegexp",matchRegexp); + authPageMap.put("overrideTargetURL",overrideTargetURL); + + if (pageType.equals(WebcrawlerConfig.ATTRVALUE_FORM)) { - String pageRegexp = authPageNode.getAttributeValue(WebcrawlerConfig.ATTR_URLREGEXP); - String pageType = authPageNode.getAttributeValue(WebcrawlerConfig.ATTR_TYPE); - String matchRegexp = authPageNode.getAttributeValue(WebcrawlerConfig.ATTR_MATCHREGEXP); - if (matchRegexp == null) - matchRegexp = ""; - String overrideTargetURL = authPageNode.getAttributeValue(WebcrawlerConfig.ATTR_OVERRIDETARGETURL); - if (overrideTargetURL == null) - overrideTargetURL = ""; - String authpagePrefix = prefix + "_" + authPageCounter; - out.print( -"<input type=\"hidden\" name=\""+authpagePrefix+"_regexp"+"\" value=\""+Encoder.attributeEscape(pageRegexp)+"\"/>\n"+ -"<input type=\"hidden\" name=\""+authpagePrefix+"_type"+"\" value=\""+pageType+"\"/>\n"+ -"<input type=\"hidden\" name=\""+authpagePrefix+"_matchregexp"+"\" value=\""+Encoder.attributeEscape(matchRegexp)+"\"/>\n"+ -"<input type=\"hidden\" name=\""+authpagePrefix+"_overridetargeturl"+"\" value=\""+Encoder.attributeEscape(overrideTargetURL)+"\"/>\n" - ); - if (pageType.equals(WebcrawlerConfig.ATTRVALUE_FORM)) + int z = 0; + List<Map<String,String>> authPageParamMapList = new ArrayList<>(); + while (z < authPageNode.getChildCount()) { - int z = 0; - int paramCounter = 0; - while (z < authPageNode.getChildCount()) + ConfigNode paramNode = authPageNode.getChild(z++); + if (paramNode.getType().equals(WebcrawlerConfig.NODE_AUTHPARAMETER)) { - ConfigNode paramNode = authPageNode.getChild(z++); - if (paramNode.getType().equals(WebcrawlerConfig.NODE_AUTHPARAMETER)) - { - String param = paramNode.getAttributeValue(WebcrawlerConfig.ATTR_NAMEREGEXP); - if (param == null) - param = ""; - String value = paramNode.getAttributeValue(WebcrawlerConfig.ATTR_VALUE); - if (value == null) - value = ""; - String password = paramNode.getAttributeValue(WebcrawlerConfig.ATTR_PASSWORD); - if (password == null) - password = ""; - else - password = out.mapPasswordToKey(ManifoldCF.deobfuscate(password)); - String authParamPrefix = authpagePrefix + "_" + paramCounter; - out.print( -"<input type=\"hidden\" name=\""+authParamPrefix+"_param"+"\" value=\""+Encoder.attributeEscape(param)+"\"/>\n"+ -"<input type=\"hidden\" name=\""+authParamPrefix+"_value"+"\" value=\""+Encoder.attributeEscape(value)+"\"/>\n"+ -"<input type=\"hidden\" name=\""+authParamPrefix+"_password"+"\" value=\""+Encoder.attributeEscape(password)+"\"/>\n" - ); - paramCounter++; - } + Map<String,String> authPageParamMap = new HashMap<>(); + + String param = paramNode.getAttributeValue(WebcrawlerConfig.ATTR_NAMEREGEXP); + if (param == null) + param = ""; + String value = paramNode.getAttributeValue(WebcrawlerConfig.ATTR_VALUE); + if (value == null) + value = ""; + String password = paramNode.getAttributeValue(WebcrawlerConfig.ATTR_PASSWORD); + if (password == null) + password = ""; + else + password = out.mapPasswordToKey(ManifoldCF.deobfuscate(password)); + + authPageParamMap.put("param",param); + authPageParamMap.put("value",value); + authPageParamMap.put("password",password); + + authPageParamMapList.add(authPageParamMap); } - out.print( -"<input type=\"hidden\" name=\""+authpagePrefix+"_loginparamcount"+"\" value=\""+paramCounter+"\"/>\n" - ); } - authPageCounter++; + authPageMap.put("authPageParamMapList",authPageParamMapList); } + authPageMapList.add(authPageMap); } - out.print( -"<input type=\"hidden\" name=\""+prefix+"_loginpagecount"+"\" value=\""+authPageCounter+"\"/>\n" - ); - accessCounter++; } + sessionAccessMap.put("authPageMapList",authPageMapList); + sessionAccessMapList.add(sessionAccessMap); } } - out.print( -"<input type=\"hidden\" name=\"scredential_count\" value=\""+accessCounter+"\"/>\n" - ); - } - - // "Certificates" tab - if (tabName.equals(Messages.getString(locale,"WebcrawlerConnector.Certificates"))) - { - out.print( -"<table class=\"displaytable\">\n"+ -" <tr><td class=\"separator\" colspan=\"2\"><hr/></td></tr>\n"+ -" <tr>\n"+ -" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.TrustCertificates") + "</nobr></td>\n"+ -" <td class=\"boxcell\">\n"+ -" <table class=\"formtable\">\n"+ -" <tr class=\"formheaderrow\">\n"+ -" <td class=\"formcolumnheader\"></td>\n"+ -" <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.URLRegularExpression") + "</nobr></td>\n"+ -" <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.Certificate") + "</nobr></td>\n"+ -" </tr>\n" - ); - int i = 0; - int trustsCounter = 0; - while (i < parameters.getChildCount()) - { - ConfigNode cn = parameters.getChild(i++); - if (cn.getType().equals(WebcrawlerConfig.NODE_TRUST)) - { - // It's prefix will be... - String prefix = "trust_" + Integer.toString(trustsCounter); - // A bin description node! Look for all its parameters. - String regexp = cn.getAttributeValue(WebcrawlerConfig.ATTR_URLREGEXP); - String trustEverything = cn.getAttributeValue(WebcrawlerConfig.ATTR_TRUSTEVERYTHING); - if (trustEverything != null && trustEverything.equals("true")) - { - // We trust everything that matches this regexp - out.print( -" <tr class=\""+(((trustsCounter % 2)==0)?"evenformrow":"oddformrow")+"\">\n"+ -" <td class=\"formcolumncell\">\n"+ -" <a name=\""+prefix+"\"><input type=\"button\" value=\"Delete\" alt=\""+Messages.getAttributeString(locale,"WebcrawlerConnector.DeleteTrustUrlRegularExpression")+Integer.toString(trustsCounter+1)+"\" onclick='javascript:deleteTRegexp("+Integer.toString(trustsCounter)+");'/>\n"+ -" <input type=\"hidden\" name=\""+"op_"+prefix+"\" value=\"Continue\"/>\n"+ -" <input type=\"hidden\" name=\""+"regexp_"+prefix+"\" value=\""+Encoder.attributeEscape(regexp)+"\"/>\n"+ -" <input type=\"hidden\" name=\""+"trustall_"+prefix+"\" value=\"true\"/>\n"+ -" <input type=\"hidden\" name=\""+"truststore_"+prefix+"\" value=\"\"/>\n"+ -" </a>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr>"+Encoder.bodyEscape(regexp)+"</nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><i>"+Messages.getBodyString(locale,"WebcrawlerConnector.TrustEverything")+"</i></nobr>\n"+ -" </td>\n"+ -" </tr>\n" - ); - trustsCounter++; - } - else - { - String trustStore = cn.getAttributeValue(WebcrawlerConfig.ATTR_TRUSTSTORE); - IKeystoreManager localTruststore = KeystoreManagerFactory.make("",trustStore); - String[] truststoreContents = localTruststore.getContents(); - - // Each trust store will have only at most one cert in it at this level. These individual certs are assembled into the proper trust store - // for each individual url at fetch time. - - if (truststoreContents.length == 1) - { - String alias = truststoreContents[0]; - String description = localTruststore.getDescription(alias); - String shortenedDescription = description; - if (shortenedDescription.length() > 100) - shortenedDescription = shortenedDescription.substring(0,100) + "..."; - out.print( -" <tr class=\""+(((trustsCounter % 2)==0)?"evenformrow":"oddformrow")+"\">\n"+ -" <td class=\"formcolumncell\">\n"+ -" <a name=\""+prefix+"\">\n"+ -" <input type=\"button\" value=\"Delete\" alt=\""+Messages.getAttributeString(locale,"WebcrawlerConnector.DeleteTrustUrlRegularExpression")+Integer.toString(trustsCounter+1)+"\" onclick='javascript:deleteTRegexp("+Integer.toString(trustsCounter)+");'/>\n"+ -" <input type=\"hidden\" name=\""+"op_"+prefix+"\" value=\"Continue\"/>\n"+ -" <input type=\"hidden\" name=\""+"regexp_"+prefix+"\" value=\""+Encoder.attributeEscape(regexp)+"\"/>\n"+ -" <input type=\"hidden\" name=\""+"trustall_"+prefix+"\" value=\"false\"/>\n"+ -" <input type=\"hidden\" name=\""+"truststore_"+prefix+"\" value=\""+Encoder.attributeEscape(trustStore)+"\"/>\n"+ -" </a>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr>"+Encoder.bodyEscape(regexp)+"</nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr>"+Encoder.bodyEscape(shortenedDescription)+"</nobr>\n"+ -" </td>\n"+ -" </tr>\n" - ); - trustsCounter++; - } - } + } + velocityContext.put("SESSIONACCESSMAPLIST",sessionAccessMapList); + } + + /** Output the configuration body section. + * This method is called in the body section of the connector's configuration page. Its purpose is to present the required form elements for editing. + * The coder can presume that the HTML that is output from this configuration will be within appropriate <html>, <body>, and <form> tags. The name of the + * form is "editconnection". + *@param threadContext is the local thread context. + *@param out is the output to which any HTML should be sent. + *@param parameters are the configuration parameters, as they currently exist, for this connection being configured. + *@param tabName is the current tab name. + */ + @Override + public void outputConfigurationBody(IThreadContext threadContext, IHTTPOutput out, + Locale locale, ConfigParams parameters, String tabName) + throws ManifoldCFException, IOException + { - } - } + final Map<String,Object> velocityContext = new HashMap<String,Object>(); + velocityContext.put("TABNAME",tabName); - if (trustsCounter == 0) - { - out.print( -" <tr class=\"formrow\"><td class=\"formmessage\" colspan=\"3\">" + Messages.getBodyString(locale,"WebcrawlerConnector.NoTrustCertificatesSpecified") + "</td></tr>\n" - ); - } - out.print( -" <tr class=\"formrow\"><td class=\"formseparator\" colspan=\"3\"><hr/></td></tr>\n"+ -" <tr class=\"formrow\">\n"+ -" <td class=\"formcolumncell\">\n"+ -" <a name=\"trust\"><input type=\"button\" value=\"" + Messages.getAttributeString(locale,"WebcrawlerConnector.Add") + "\" alt=\"" + Messages.getAttributeString(locale,"WebcrawlerConnector.AddUrlRegularExpressionForTruststore") + "\" onclick=\"javascript:addTRegexp();\"/></a>\n"+ -" <input type=\"hidden\" name=\"trust_count\" value=\""+trustsCounter+"\"/>\n"+ -" <input type=\"hidden\" name=\"trust_op\" value=\"Continue\"/>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr><input type=\"text\" size=\"30\" name=\"regexp_trust\" value=\"\"/></nobr>\n"+ -" </td>\n"+ -" <td class=\"formcolumncell\">\n"+ -" <nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.UploadCertificate") + " <input name=\"certificate_trust\" size=\"50\" type=\"file\"/> <input name=\"all_trust\" type=\"checkbox\" value=\"true\">" + Messages.getBodyString(locale,"WebcrawlerConnector.TrustEverything") + "</input></nobr>\n"+ -" </td>\n"+ -" </tr>\n"+ -" </table>\n"+ -" </td>\n"+ -" </tr>\n"+ -"</table>\n" - ); - } - else - { - // Hiddens for Certificates tab. - int i = 0; - int trustsCounter = 0; - while (i < parameters.getChildCount()) - { - ConfigNode cn = parameters.getChild(i++); - if (cn.getType().equals(WebcrawlerConfig.NODE_TRUST)) - { - // It's prefix will be... - String prefix = "trust_" + Integer.toString(trustsCounter); + fillInEmailTab(velocityContext,out,parameters); + fillInRobotsTab(velocityContext,out,parameters); + fillInBandwidthTab(velocityContext,out,parameters); + fillInAccessTab(velocityContext,out,parameters); + fillInCertificatesTab(velocityContext,out,parameters); + fillInProxyTab(velocityContext,out,parameters); - // A bin description node! Look for all its parameters. - String regexp = cn.getAttributeValue(WebcrawlerConfig.ATTR_URLREGEXP); - String trustEverything = cn.getAttributeValue(WebcrawlerConfig.ATTR_TRUSTEVERYTHING); - if (trustEverything != null && trustEverything.equals("true")) - { - // We trust everything that matches this regexp - out.print( -"<input type=\"hidden\" name=\""+"regexp_"+prefix+"\" value=\""+Encoder.attributeEscape(regexp)+"\"/>\n"+ -"<input type=\"hidden\" name=\""+"truststore_"+prefix+"\" value=\"\"/>\n"+ -"<input type=\"hidden\" name=\""+"trustall_"+prefix+"\" value=\"true\"/>\n" - ); - trustsCounter++; - } - else - { - String trustStore = cn.getAttributeValue(WebcrawlerConfig.ATTR_TRUSTSTORE); - out.print( -"<input type=\"hidden\" name=\""+"regexp_"+prefix+"\" value=\""+Encoder.attributeEscape(regexp)+"\"/>\n"+ -"<input type=\"hidden\" name=\""+"truststore_"+prefix+"\" value=\""+Encoder.attributeEscape(trustStore)+"\"/>\n"+ -"<input type=\"hidden\" name=\""+"trustall_"+prefix+"\" value=\"false\"/>\n" - ); - trustsCounter++; - } - } - } - out.print( -"<input type=\"hidden\" name=\"trust_count\" value=\""+trustsCounter+"\"/>\n" - ); - } + // Email tab + Messages.outputResourceWithVelocity(out,locale,"editConfiguration_Email.html.vm",velocityContext); + // Robots tab + Messages.outputResourceWithVelocity(out,locale,"editConfiguration_Robots.html.vm",velocityContext); + //Bandwidth tab + Messages.outputResourceWithVelocity(out,locale,"editConfiguration_Bandwidth.html.vm",velocityContext); + // Access Credentials tab + Messages.outputResourceWithVelocity(out,locale,"editConfiguration_Access.html.vm",velocityContext); + //Certificates tab + Messages.outputResourceWithVelocity(out,locale,"editConfiguration_Certificates.html.vm",velocityContext); + // Proxy tab + Messages.outputResourceWithVelocity(out,locale,"editConfiguration_Proxy.html.vm",velocityContext); } @@ -3299,367 +2293,18 @@ public class WebcrawlerConnector extends Locale locale, ConfigParams parameters) throws ManifoldCFException, IOException { - String email = parameters.getParameter(WebcrawlerConfig.PARAMETER_EMAIL); - String robots = parameters.getParameter(WebcrawlerConfig.PARAMETER_ROBOTSUSAGE); - if (robots.equals("none")) - robots = Messages.getBodyString(locale,"WebcrawlerConnector.DontLookAtRobotsTxt"); - else if (robots.equals("data")) - robots = Messages.getBodyString(locale,"WebcrawlerConnector.ObeyRobotsTxtForDataFetchesOnly"); - else if (robots.equals("all")) - robots = Messages.getBodyString(locale,"WebcrawlerConnector.ObeyRobotsTxtForAllFetches"); - String metaRobotsTagsUsage = parameters.getParameter(WebcrawlerConfig.PARAMETER_META_ROBOTS_TAGS_USAGE); - if (metaRobotsTagsUsage == null || metaRobotsTagsUsage.equals("all")) - metaRobotsTagsUsage = Messages.getBodyString(locale,"WebcrawlerConnector.ObeyMetaRobotsTags"); - else if (metaRobotsTagsUsage.equals("none")) - metaRobotsTagsUsage = Messages.getBodyString(locale,"WebcrawlerConnector.DontLookAtMetaRobotsTags"); - String proxyHost = parameters.getParameter(WebcrawlerConfig.PARAMETER_PROXYHOST); - if (proxyHost == null) - proxyHost = ""; - String proxyPort = parameters.getParameter(WebcrawlerConfig.PARAMETER_PROXYPORT); - if (proxyPort == null) - proxyPort = ""; - String proxyAuthDomain = parameters.getParameter(WebcrawlerConfig.PARAMETER_PROXYAUTHDOMAIN); - if (proxyAuthDomain == null) - proxyAuthDomain = ""; - String proxyAuthUsername = parameters.getParameter(WebcrawlerConfig.PARAMETER_PROXYAUTHUSERNAME); - if (proxyAuthUsername == null) - proxyAuthUsername = ""; - out.print( -"<table class=\"displaytable\">\n"+ -" <tr>\n"+ -" <td class=\"description\" colspan=\"1\"><nobr>"+Messages.getBodyString(locale,"WebcrawlerConnector.EmailAddress")+"</nobr></td>\n"+ -" <td class=\"value\" colspan=\"3\">"+Encoder.bodyEscape(email)+"</td>\n"+ -" </tr>\n"+ -" <tr>\n"+ -" <td class=\"description\" colspan=\"1\"><nobr>"+Messages.getBodyString(locale,"WebcrawlerConnector.RobotsUsage")+"</nobr></td>\n"+ -" <td class=\"value\" colspan=\"1\"><nobr>"+Encoder.bodyEscape(robots)+"</nobr></td>\n"+ -" <td class=\"description\" colspan=\"1\"><nobr>"+Messages.getBodyString(locale,"WebcrawlerConnector.MetaRobotsTagsUsage")+"</nobr></td>\n"+ -" <td class=\"value\" colspan=\"1\">"+Encoder.bodyEscape(metaRobotsTagsUsage)+"</td>\n"+ -" </tr>\n"+ -" <tr>\n"+ -" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.ProxyHostColon") + "</nobr></td>\n"+ -" <td class=\"value\">"+Encoder.bodyEscape(proxyHost)+"</td>\n"+ -" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.ProxyPortColon") + "</nobr></td>\n"+ -" <td class=\"value\">"+Encoder.bodyEscape(proxyPort)+"</td>\n"+ -" </tr>\n"+ -" <tr>\n"+ -" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.ProxyAuthenticationDomainColon") + "</nobr></td>\n"+ -" <td class=\"value\">"+Encoder.bodyEscape(proxyAuthDomain)+"</td>\n"+ -" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"WebcrawlerConnector.ProxyAuthenticationUserNameColon") + "</nobr></td>\n"+ -" <td class=\"value\">"+Encoder.bodyEscape(proxyAuthUsername)+"</td>\n"+ -" </tr>\n"+ -" <tr>\n"+ -" <td class=\"description\" colspan=\"1\"><nobr>"+Messages.getBodyString(locale,"WebcrawlerConnector.BandwidthThrottling")+"</nobr></td>\n"+ -" <td class=\"boxcell\" colspan=\"3\">\n"+ -" <table class=\"formtable\">\n"+ -" <tr class=\"formheaderrow\">\n"+ -" <td class=\"formcolumnheader\"><nobr>"+Messages.getBodyString(locale,"WebcrawlerConnector.BinRegularExpression")+"</nobr></td>\n"+ -" <td class=\"formcolumnheader\"><nobr>"+Messages.getBodyString(locale,"WebcrawlerConnector.CaseInsensitive")+"</nobr></td>\n"+ -" <td class=\"formcolumnheader\"><nobr>"+Messages.getBodyString(locale,"WebcrawlerConnector.MaxConnections")+"</nobr></td>\n"+ -" <td class=\"formcolumnheader\"><nobr>"+Messages.getBodyString(locale,"WebcrawlerConnector.MaxKbytesSec")+"</nobr></td>\n"+ -" <td class=\"formcolumnheader\"><nobr>"+Messages.getBodyString(locale,"WebcrawlerConnector.MaxFetchesMin")+"</nobr></td>\n"+ -" </tr>\n" - ); - int i = 0; - int instanceNumber = 0; - while (i < parameters.getChildCount()) - { - ConfigNode cn = parameters.getChild(i++); - if (cn.getType().equals(WebcrawlerConfig.NODE_BINDESC)) - { - // A bin description node! Look for all its parameters. - String regexp = cn.getAttributeValue(WebcrawlerConfig.ATTR_BINREGEXP); - String isCaseInsensitive = cn.getAttributeValue(WebcrawlerConfig.ATTR_INSENSITIVE); - String maxConnections = null; - String maxKBPerSecond = null; - String maxFetchesPerMinute = null; - int j = 0; - while (j < cn.getChildCount()) - { - ConfigNode childNode = cn.getChild(j++); - if (childNode.getType().equals(WebcrawlerConfig.NODE_MAXCONNECTIONS)) - maxConnections = childNode.getAttributeValue(WebcrawlerConfig.ATTR_VALUE); - else if (childNode.getType().equals(WebcrawlerConfig.NODE_MAXKBPERSECOND)) - maxKBPerSecond = childNode.getAttributeValue(WebcrawlerConfig.ATTR_VALUE); - else if (childNode.getType().equals(WebcrawlerConfig.NODE_MAXFETCHESPERMINUTE)) - maxFetchesPerMinute = childNode.getAttributeValue(WebcrawlerConfig.ATTR_VALUE); - } - if (maxConnections == null) - maxConnections = "Not limited"; - if (maxKBPerSecond == null) - maxKBPerSecond = "Not limited"; - if (maxFetchesPerMinute == null) - maxFetchesPerMinute = "Not limited"; - if (isCaseInsensitive == null || isCaseInsensitive.length() == 0) - isCaseInsensitive = "false"; - out.print( -" <tr class=\""+(((instanceNumber % 2)==0)?"evenformrow":"oddformrow")+"\">\n"+ -" <td class=\"formcolumncell\"><nobr>"+Encoder.bodyEscape(regexp)+"</nobr></td>\n"+ -" <td class=\"formcolumncell\">"+isCaseInsensitive+"</td>\n"+ -" <td class=\"formcolumncell\"><nobr>"+maxConnections+"</nobr></td>\n"+
[... 1790 lines stripped ...]
