Author: kwright
Date: Sat Nov 21 09:00:02 2020
New Revision: 1883688
URL: http://svn.apache.org/viewvc?rev=1883688&view=rev
Log:
Move solr ingester code to solr connector area
Added:
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/java/org/apache/manifoldcf/crawler/
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/solr/
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/solr/Messages.java
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/solr/SolrIngesterConfig.java
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/solr/SolrIngesterConnector.java
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/solr/
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/solr/common_en_US.properties
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/solr/common_es_ES.properties
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/solr/common_fr_FR.properties
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/resources/
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/resources/org/
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/resources/org/apache/
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/resources/org/apache/manifoldcf/
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/resources/org/apache/manifoldcf/crawler/
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/solr/
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/solr/editConfiguration_connector.html
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/solr/editConfiguration_connector.js
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/solr/editSpecification_job_parameters.html
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/solr/editSpecification_job_parameters.js
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/solr/editSpecification_job_security.html
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/solr/editSpecification_job_security.js
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/solr/viewConfiguration_connector.html
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/solr/viewSpecification_job_parameters.html
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/solr/viewSpecification_job_security.html
Removed:
manifoldcf/branches/CONNECTORS-1653/connectors/solringester/
Modified:
manifoldcf/branches/CONNECTORS-1653/connectors/pom.xml
manifoldcf/branches/CONNECTORS-1653/connectors/solr/build.xml
Modified: manifoldcf/branches/CONNECTORS-1653/connectors/pom.xml
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1653/connectors/pom.xml?rev=1883688&r1=1883687&r2=1883688&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1653/connectors/pom.xml (original)
+++ manifoldcf/branches/CONNECTORS-1653/connectors/pom.xml Sat Nov 21 09:00:02
2020
@@ -78,7 +78,6 @@
<module>html-extractor</module>
<module>mongodb</module>
<module>csws</module>
- <module>solringester</module>
</modules>
</project>
Modified: manifoldcf/branches/CONNECTORS-1653/connectors/solr/build.xml
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1653/connectors/solr/build.xml?rev=1883688&r1=1883687&r2=1883688&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1653/connectors/solr/build.xml (original)
+++ manifoldcf/branches/CONNECTORS-1653/connectors/solr/build.xml Sat Nov 21
09:00:02 2020
@@ -90,6 +90,10 @@
<param name="connector-label" value="Solr"/>
<param name="connector-class"
value="org.apache.manifoldcf.agents.output.solr.SolrConnector"/>
</antcall>
+ <antcall target="general-add-repository-connector">
+ <param name="connector-label" value="Solr"/>
+ <param name="connector-class"
value="org.apache.manifoldcf.crawler.connectors.solr.SolrIngesterConnector"/>
+ </antcall>
</target>
</project>
Added:
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/solr/Messages.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/solr/Messages.java?rev=1883688&view=auto
==============================================================================
---
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/solr/Messages.java
(added)
+++
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/solr/Messages.java
Sat Nov 21 09:00:02 2020
@@ -0,0 +1,121 @@
+/* $Id: Messages.java 1295926 2012-03-01 21:56:27Z kwright $ */
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+* http://www.apache.org/licenses/LICENSE-2.0
+ *
+* Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.manifoldcf.crawler.connectors.solr;
+
+import java.util.Locale;
+import java.util.Map;
+import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
+import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+
+public class Messages extends org.apache.manifoldcf.ui.i18n.Messages {
+
+ public static final String DEFAULT_BUNDLE_NAME =
"org.apache.manifoldcf.crawler.connectors.solr.common";
+
+ public static final String DEFAULT_PATH_NAME =
"org.apache.manifoldcf.crawler.connectors.solr";
+
+ /**
+ * Constructor - do no instantiate
+ */
+ protected Messages() {
+ }
+
+ public static String getString(Locale locale, String messageKey) {
+ return getString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+ }
+
+ public static String getAttributeString(Locale locale, String messageKey) {
+ return getAttributeString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+ }
+
+ public static String getBodyString(Locale locale, String messageKey) {
+ return getBodyString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+ }
+
+ public static String getAttributeJavascriptString(Locale locale, String
messageKey) {
+ return getAttributeJavascriptString(DEFAULT_BUNDLE_NAME, locale,
messageKey, null);
+ }
+
+ public static String getBodyJavascriptString(Locale locale, String
messageKey) {
+ return getBodyJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey,
null);
+ }
+
+ public static String getString(Locale locale, String messageKey, Object[]
args) {
+ return getString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+ }
+
+ public static String getAttributeString(Locale locale, String messageKey,
Object[] args) {
+ return getAttributeString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+ }
+
+ public static String getBodyString(Locale locale, String messageKey,
Object[] args) {
+ return getBodyString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+ }
+
+ public static String getAttributeJavascriptString(Locale locale, String
messageKey, Object[] args) {
+ return getAttributeJavascriptString(DEFAULT_BUNDLE_NAME, locale,
messageKey, args);
+ }
+
+ public static String getBodyJavascriptString(Locale locale, String
messageKey, Object[] args) {
+ return getBodyJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey,
args);
+ }
+
+ // More general methods which allow bundlenames and class loaders to be
specified.
+ public static String getString(String bundleName, Locale locale, String
messageKey, Object[] args) {
+ return getString(Messages.class, bundleName, locale, messageKey, args);
+ }
+
+ public static String getAttributeString(String bundleName, Locale locale,
String messageKey, Object[] args) {
+ return getAttributeString(Messages.class, bundleName, locale, messageKey,
args);
+ }
+
+ public static String getBodyString(String bundleName, Locale locale, String
messageKey, Object[] args) {
+ return getBodyString(Messages.class, bundleName, locale, messageKey, args);
+ }
+
+ public static String getAttributeJavascriptString(String bundleName, Locale
locale, String messageKey, Object[] args) {
+ return getAttributeJavascriptString(Messages.class, bundleName, locale,
messageKey, args);
+ }
+
+ public static String getBodyJavascriptString(String bundleName, Locale
locale, String messageKey, Object[] args) {
+ return getBodyJavascriptString(Messages.class, bundleName, locale,
messageKey, args);
+ }
+
+ // Resource output
+ public static void outputResource(IHTTPOutput output, Locale locale, String
resourceKey,
+ Map<String, String> substitutionParameters, boolean mapToUpperCase)
+ throws ManifoldCFException {
+ outputResource(output, Messages.class, DEFAULT_PATH_NAME, locale,
resourceKey,
+ substitutionParameters, mapToUpperCase);
+ }
+
+ public static void outputResourceWithVelocity(IHTTPOutput output, Locale
locale, String resourceKey,
+ Map<String, String> substitutionParameters, boolean mapToUpperCase)
+ throws ManifoldCFException {
+ outputResourceWithVelocity(output, Messages.class, DEFAULT_BUNDLE_NAME,
DEFAULT_PATH_NAME, locale, resourceKey,
+ substitutionParameters, mapToUpperCase);
+ }
+
+ public static void outputResourceWithVelocity(IHTTPOutput output, Locale
locale, String resourceKey,
+ Map<String, Object> contextObjects)
+ throws ManifoldCFException {
+ outputResourceWithVelocity(output, Messages.class, DEFAULT_BUNDLE_NAME,
DEFAULT_PATH_NAME, locale, resourceKey,
+ contextObjects);
+ }
+
+
+}
Added:
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/solr/SolrIngesterConfig.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/solr/SolrIngesterConfig.java?rev=1883688&view=auto
==============================================================================
---
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/solr/SolrIngesterConfig.java
(added)
+++
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/solr/SolrIngesterConfig.java
Sat Nov 21 09:00:02 2020
@@ -0,0 +1,51 @@
+/* $Id: Messages.java 1295926 2012-03-01 21:56:27Z kwright $ */
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+* http://www.apache.org/licenses/LICENSE-2.0
+ *
+* Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.manifoldcf.crawler.connectors.solr;
+
+public class SolrIngesterConfig {
+
+ // Repository connection Configuration parameters
+ public static final String PARAM_SOLRADDRESS = "solrAddress";
+ public static final String PARAM_SOLRUSERNAME = "solrUsername";
+ public static final String PARAM_SOLRPASSWORD = "solrPassword";
+// public static final String PARAM_SOLRPORT = "solrPort";
+ //public static final String SOLRPORTPORT_DEFAULT = "8983";
+ public static final String PARAM_CONNECTIONTIMEOUT = "connectionTimeout";
+ public static final String SOLRUSERNAME_DEFAULT = "";
+ public static final String SOLRPASSWORD_DEFAULT = "";
+ public static final String PARAM_SOCKETTIMEOUT = "socketTimeout";
+ public static final String SOLRADDRESS_DEFAULT =
"http://localhost:8983/solr";;
+ public static final String CONNECTIONTIMEOUT_DEFAULT = "60000";
+ public static final String SOCKETTIMEOUT_DEFAULT = "180000";
+
+ // Specification nodes and values
+ public static final String NODE_FIELDMAP = "fieldmap";
+ public static final String ATTRIBUTE_SOURCE = "source";
+ public static final String ATTRIBUTE_TARGET = "target";
+ public static final String ATTRIBUTE_VALUE = "value";
+ public static final String SECURITY_ACTIVATED = "securityactivated";
+ public static final String SECURITY_FIELD = "securityfield";
+ public static final String SECURITY_FIELD2 = "securityfield2";
+ public static final String ID_FIELD = "fieldid";
+ public static final String ROWS_NUMBER = "rowsnumber";
+ public static final String DATE_FIELD = "fielddate";
+ public static final String CONTENT_FIELD = "fieldcontent";
+ public static final String FILTER_CONDITION = "filter";
+ public static final String COLLECTION_NAME = "collection";
+
+}
Added:
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/solr/SolrIngesterConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/solr/SolrIngesterConnector.java?rev=1883688&view=auto
==============================================================================
---
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/solr/SolrIngesterConnector.java
(added)
+++
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/solr/SolrIngesterConnector.java
Sat Nov 21 09:00:02 2020
@@ -0,0 +1,1279 @@
+/* $Id: SvnConnector.java 994959 2010-09-08 10:04:42Z krycek $ */
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.manifoldcf.crawler.connectors.solr;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.security.KeyManagementException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import javax.net.ssl.SSLContext;
+
+import org.apache.http.Header;
+import org.apache.http.HttpException;
+import org.apache.http.HttpHeaders;
+import org.apache.http.HttpRequest;
+import org.apache.http.HttpRequestInterceptor;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.auth.Credentials;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.config.SocketConfig;
+import org.apache.http.conn.HttpClientConnectionManager;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.impl.auth.BasicScheme;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.DefaultRedirectStrategy;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.protocol.HttpContext;
+import org.apache.http.protocol.HttpRequestExecutor;
+import org.apache.http.ssl.SSLContexts;
+import org.apache.http.ssl.TrustStrategy;
+import org.apache.http.util.EntityUtils;
+import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
+import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
+import org.apache.manifoldcf.connectorcommon.common.XThreadStringBuffer;
+import org.apache.manifoldcf.core.interfaces.ConfigParams;
+import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
+import org.apache.manifoldcf.core.interfaces.IPostParameters;
+import org.apache.manifoldcf.core.interfaces.IThreadContext;
+import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.core.interfaces.Specification;
+import org.apache.manifoldcf.core.interfaces.SpecificationNode;
+import org.apache.manifoldcf.core.system.ManifoldCF;
+import org.apache.manifoldcf.core.util.URLEncoder;
+import org.apache.manifoldcf.crawler.connectors.BaseRepositoryConnector;
+import org.apache.manifoldcf.crawler.interfaces.IExistingVersions;
+import org.apache.manifoldcf.crawler.interfaces.IProcessActivity;
+import org.apache.manifoldcf.crawler.interfaces.ISeedingActivity;
+import org.apache.manifoldcf.crawler.system.Logging;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpClientUtil;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.params.CursorMarkParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+
+public class SolrIngesterConnector extends BaseRepositoryConnector {
+
+ public static final String _rcsid = "@(#)$Id: solringesterConnector.java
994959 2010-09-08 10:04:42Z redguy $";
+
+ /**
+ * Deny access token for default authority
+ */
+ private final static String defaultAuthorityDenyToken = "__nosecurity__";
+
+ private final static String ACTION_PARAM_NAME = "action";
+
+ private final static String ACTION_CHECK = "check";
+
+ private final static String ACTION_SEED = "seed";
+
+ private final static String ACTION_ITEMS = "items";
+
+ private final static String ACTION_ITEM = "item";
+
+ /** Connection timeout */
+ private int connectionTimeout = -1;
+
+ /** Socket timeout */
+ private int socketTimeout = -1;
+
+ /** Session timeout */
+ private long sessionTimeout = -1L;
+
+ protected final static long sessionExpirationInterval = 300000L;
+
+ private String solringesterLogin = null;
+
+ private String solringesterPassword = null;
+
+ /** Connection manager */
+ private HttpClientConnectionManager connectionManager = null;
+
+ /** HttpClientBuilder */
+ private HttpClientBuilder builder = null;
+
+ /** Httpclient instance */
+ private CloseableHttpClient httpClient = null;
+
+ private HttpSolrClient httpSolrClient = null;
+
+ /** Connection timeout */
+ private String connectionTimeoutString = null;
+
+ /** Socket timeout */
+ private String socketTimeoutString = null;
+
+ private String solringesterEntryPoint = null;
+
+ private static final String date_target_field = "last_modified";
+
+ private final static String versionField = "_version_";
+
+ protected static final String RELATIONSHIP_RELATED = "related";
+
+ private static final String EDIT_CONFIGURATION_CONNECTOR_JS =
"editConfiguration_connector.js";
+ private static final String EDIT_CONFIGURATION_CONNECTOR_HTML =
"editConfiguration_connector.html";
+ private static final String VIEW_CONFIGURATION_CONNECTOR_HTML =
"viewConfiguration_connector.html";
+
+ private static final String EDIT_SPECIFICATION_JOB_PARAMETERS_JS =
"editSpecification_job_parameters.js";
+ private static final String EDIT_SPECIFICATION_JOB_PARAMETERS_HTML =
"editSpecification_job_parameters.html";
+ private static final String VIEW_SPECIFICATION_JOB_PARAMETERS_HTML =
"viewSpecification_job_parameters.html";
+
+ private static final String EDIT_SPECIFICATION_JOB_SECURITY_JS =
"editSpecification_job_security.js";
+ private static final String EDIT_SPECIFICATION_JOB_SECURITY_HTML =
"editSpecification_job_security.html";
+ private static final String VIEW_SPECIFICATION_JOB_SECURITY_HTML =
"viewSpecification_job_security.html";
+
+ protected final static String ACTIVITY_GET = "get document";
+
+ /**
+ * Constructor.
+ */
+ public SolrIngesterConnector() {
+ }
+
+ @Override
+ public int getMaxDocumentRequest() {
+ return 100;
+ }
+
+ @Override
+ public String[] getRelationshipTypes() {
+ return new String[] { RELATIONSHIP_RELATED };
+ }
+
+ @Override
+ public int getConnectorModel() {
+ return SolrIngesterConnector.MODEL_ADD_CHANGE;
+ }
+
+ @Override
+ public String[] getActivitiesList() {
+ return new String[] { ACTIVITY_GET };
+ }
+
+ /**
+ * For any given document, list the bins that it is a member of.
+ */
+ @Override
+ public String[] getBinNames(final String documentIdentifier) {
+ // Return the host name
+ return new String[] { solringesterEntryPoint };
+ }
+
+ // All methods below this line will ONLY be called if a connect() call
succeeded
+ // on this instance!
+ /**
+ * Connect. The configuration parameters are included.
+ *
+ * @param configParams
+ * are the configuration parameters for this connection. Note well:
There are no exceptions allowed from this call, since it is
+ * expected to mainly establish connection parameters.
+ */
+ @Override
+ public void connect(final ConfigParams configParams) {
+ super.connect(configParams);
+ solringesterEntryPoint =
configParams.getParameter(SolrIngesterConfig.PARAM_SOLRADDRESS);
+ solringesterLogin =
configParams.getParameter(SolrIngesterConfig.PARAM_SOLRUSERNAME);
+ solringesterPassword = "";
+ try {
+ solringesterPassword =
ManifoldCF.deobfuscate(configParams.getParameter(SolrIngesterConfig.PARAM_SOLRPASSWORD));
+ } catch (final ManifoldCFException ignore) {
+ }
+ connectionTimeoutString =
configParams.getParameter(SolrIngesterConfig.PARAM_CONNECTIONTIMEOUT);
+ socketTimeoutString =
configParams.getParameter(SolrIngesterConfig.PARAM_SOCKETTIMEOUT);
+
+ if (Logging.connectors.isDebugEnabled()) {
+ Logging.connectors.debug("SolrIngester: Connection to '" +
solringesterEntryPoint + "'");
+ }
+
+ }
+
+ @Override
+ public void disconnect() throws ManifoldCFException {
+ socketTimeoutString = null;
+ connectionTimeoutString = null;
+ sessionTimeout = -1L;
+ httpClient = null;
+ httpSolrClient = null;
+ if (connectionManager != null) {
+ connectionManager.shutdown();
+ }
+ connectionManager = null;
+ solringesterLogin = null;
+ solringesterPassword = null;
+ solringesterEntryPoint = null;
+ super.disconnect();
+ }
+
+ protected void expireSession() throws ManifoldCFException {
+ httpClient = null;
+ httpSolrClient = null;
+ if (connectionManager != null) {
+ connectionManager.shutdown();
+ }
+ connectionManager = null;
+ sessionTimeout = -1L;
+ }
+
+ @Override
+ public void poll() throws ManifoldCFException {
+ if (System.currentTimeMillis() >= sessionTimeout) {
+ expireSession();
+ }
+ if (connectionManager != null) {
+ connectionManager.closeIdleConnections(60000L, TimeUnit.MILLISECONDS);
+ }
+ }
+
+ protected void getSession() throws ManifoldCFException {
+ if (sessionTimeout == -1L) {
+
+ try {
+ this.connectionTimeout = Integer.parseInt(connectionTimeoutString);
+ } catch (final NumberFormatException e) {
+ throw new ManifoldCFException("Bad connection timeout number: " +
connectionTimeoutString);
+ }
+ try {
+ this.socketTimeout = Integer.parseInt(socketTimeoutString);
+ } catch (final NumberFormatException e) {
+ throw new ManifoldCFException("Bad socket timeout number: " +
socketTimeoutString);
+ }
+
+ final TrustStrategy acceptingTrustStrategy = (cert, authType) -> true;
+ SSLConnectionSocketFactory sslsf =
SSLConnectionSocketFactory.getSocketFactory();
+ try {
+ final SSLContext sslContext =
SSLContexts.custom().loadTrustMaterial(null, acceptingTrustStrategy).build();
+ sslsf = new SSLConnectionSocketFactory(sslContext,
NoopHostnameVerifier.INSTANCE);
+ } catch (KeyManagementException | NoSuchAlgorithmException |
KeyStoreException e) {
+ throw new ManifoldCFException("SSL context initialization failure", e);
+ }
+
+ final PoolingHttpClientConnectionManager poolingConnectionManager = new
PoolingHttpClientConnectionManager(
+ RegistryBuilder.<ConnectionSocketFactory>create().register("http",
PlainConnectionSocketFactory.getSocketFactory()).register("https",
sslsf).build());
+ poolingConnectionManager.setDefaultMaxPerRoute(1);
+ poolingConnectionManager.setValidateAfterInactivity(2000);
+
poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom().setTcpNoDelay(true).setSoTimeout(socketTimeout).build());
+
+ this.connectionManager = poolingConnectionManager;
+
+ final RequestConfig.Builder requestBuilder =
RequestConfig.custom().setCircularRedirectsAllowed(true).setSocketTimeout(socketTimeout).setExpectContinueEnabled(false)
+
.setConnectTimeout(connectionTimeout).setConnectionRequestTimeout(socketTimeout);
+
+ final List<Header> headers = new ArrayList<>();
+ if (solringesterLogin != null && !solringesterLogin.isEmpty() &&
solringesterPassword != null) {
+ final String auth = solringesterLogin + ":" + solringesterPassword;
+ final String encodedAuth =
Base64.getEncoder().encodeToString(auth.getBytes(StandardCharsets.ISO_8859_1));
+ final String authHeader = "Basic " + new String(encodedAuth);
+ final Header basicAuthheader = new
BasicHeader(HttpHeaders.AUTHORIZATION, authHeader);
+ headers.add(basicAuthheader);
+ }
+
+ this.builder =
HttpClients.custom().setDefaultHeaders(headers).setConnectionManager(connectionManager).disableAutomaticRetries().setDefaultRequestConfig(requestBuilder.build());
+ builder.setRequestExecutor(new
HttpRequestExecutor(socketTimeout)).setRedirectStrategy(new
DefaultRedirectStrategy());
+ this.httpClient = builder.build();
+ this.httpSolrClient = new
HttpSolrClient.Builder(solringesterEntryPoint).withHttpClient(httpClient).withConnectionTimeout(connectionTimeout).withSocketTimeout(socketTimeout).build();
+ }
+ sessionTimeout = System.currentTimeMillis() + sessionExpirationInterval;
+
+ }
+
+ @Override
+ public String check() throws ManifoldCFException {
+ getSession();
+
+ final HttpGet httpGet = new HttpGet(solringesterEntryPoint);
+ CloseableHttpResponse response = null;
+ try {
+ try {
+ response = this.httpClient.execute(httpGet);
+ } catch (final IOException e) {
+ return "Connection error: " + e.getMessage();
+ }
+ final int responseCode = response.getStatusLine().getStatusCode();
+ if (responseCode != 200) {
+ return "Bad response: " + response.getStatusLine();
+ }
+ return super.check();
+ } finally {
+ if (response != null) {
+ try {
+ response.close();
+ } catch (final IOException e) {
+ return "Connection error: " + e.getMessage();
+ }
+ }
+ }
+ }
+
+ @Override
+ public String addSeedDocuments(final ISeedingActivity activities, final
Specification spec, final String lastSeedVersion, final long seedTime, final
int jobMode)
+ throws ManifoldCFException, ServiceInterruption {
+
+ long startTime;
+
+ String idFieldName = null;
+ String collection = null;
+ final String dateField = null;
+ final String contentField = null;
+ String rowsNumberString = null;
+ String filter = null;
+
+ if (Logging.connectors.isDebugEnabled()) {
+ Logging.connectors.debug("SolrIngester: Connection to '" +
solringesterEntryPoint + "'");
+ }
+
+ // Retrieve configuration parameters
+ for (int l = 0; l < spec.getChildCount(); l++) {
+ final SpecificationNode sn = spec.getChild(l);
+ if (sn.getType() == SolrIngesterConfig.COLLECTION_NAME) {
+ collection = sn.getAttributeValue(SolrIngesterConfig.ATTRIBUTE_VALUE);
+ }
+ if (sn.getType() == SolrIngesterConfig.ID_FIELD) {
+ idFieldName = sn.getAttributeValue(SolrIngesterConfig.ATTRIBUTE_VALUE);
+ }
+
+ if (sn.getType() == SolrIngesterConfig.ROWS_NUMBER) {
+ rowsNumberString =
sn.getAttributeValue(SolrIngesterConfig.ATTRIBUTE_VALUE);
+ }
+
+ if (sn.getType() == SolrIngesterConfig.FILTER_CONDITION) {
+ filter = sn.getAttributeValue(SolrIngesterConfig.ATTRIBUTE_VALUE);
+ }
+ }
+
+ if (lastSeedVersion == null) {
+ startTime = 0L;
+ } else {
+ // Unpack seed time from seed version string
+ startTime = new Long(lastSeedVersion).longValue();
+ }
+
+ getSession();
+
+ final SimpleDateFormat sdf = new
SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT);
+
+ final StringBuilder url = new StringBuilder(solringesterEntryPoint);
+ url.append("?").append(ACTION_PARAM_NAME).append("=").append(ACTION_SEED);
+ if (startTime > 0) {
+ url.append("&startTime=").append(sdf.format(new Date(startTime)));
+ }
+ url.append("&endTime=").append(sdf.format(new Date(seedTime)));
+
+ // ExecuteSeedingThread t = new ExecuteSeedingThread(client,
url.toString());
+
+ long dateSolr;
+ if (lastSeedVersion != null) {
+ dateSolr = new Long(lastSeedVersion).longValue();
+ } else {
+ dateSolr = 0L;
+ }
+
+ // String dateSolr = sdf.format(dateSolr);
+
+ final int rowsNumber = Integer.valueOf(rowsNumberString);
+ try {
+ SolrQuery query;
+ if (filter == null || filter == "*:*") {
+ query = new SolrQuery("*:*").setRows(rowsNumber).setSort(idFieldName,
SolrQuery.ORDER.asc);
+ } else {
+ query = new
SolrQuery("*:*").addFilterQuery(filter).setRows(rowsNumber).setSort(idFieldName,
SolrQuery.ORDER.asc);
+ }
+ query.setFields(idFieldName);
+ String cursorMark = CursorMarkParams.CURSOR_MARK_START;
+ boolean done = false;
+ while (!done) {
+ query.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark);
+ QueryResponse response;
+
+ response = httpSolrClient.query(collection, query);
+ final String nextCursorMark = response.getNextCursorMark();
+ final SolrDocumentList documents = response.getResults();
+ for (final SolrDocument document : documents) {
+ activities.addSeedDocument((String)
document.getFieldValue(idFieldName));
+ }
+ if (cursorMark.equals(nextCursorMark)) {
+ done = true;
+ }
+ cursorMark = nextCursorMark;
+ }
+ } catch (final SolrServerException | IOException e) {
+ Logging.connectors.error("Unable to perform Solr requests", e);
+ throw new ManifoldCFException("Unable to perform Solr requests", e);
+ }
+
+ return new Long(seedTime).toString();
+
+ }
+
+ @Override
+ public void processDocuments(final String[] documentIdentifiers, final
IExistingVersions statuses, final Specification spec, final IProcessActivity
activities, final int jobMode,
+ final boolean usesDefaultAuthority) throws ManifoldCFException,
ServiceInterruption {
+
+ getSession();
+
+ if (Logging.connectors.isDebugEnabled()) {
+ Logging.connectors.debug("SolrIngester: ProcessDocuments method");
+ }
+ // Get parameters configuration
+ String collection = null;
+ String idFieldName = null;
+ boolean securityActivated = false;
+ String dateField = null;
+ String contentField = null;
+ String securityField = null;
+ String securityField2 = null;
+ String rowsNumberString = null;
+
+ String errorCode = null;
+ String description = "";
+
+ final long startFetchTime = System.currentTimeMillis();
+
+
+ // Hashmap
+ final Map<String, String> mapFields = new HashMap<String, String>();
+
+ for (int l = 0; l < spec.getChildCount(); l++) {
+ final SpecificationNode sn = spec.getChild(l);
+ if (sn.getType() == SolrIngesterConfig.COLLECTION_NAME) {
+ collection = sn.getAttributeValue(SolrIngesterConfig.ATTRIBUTE_VALUE);
+ }
+ if (sn.getType() == SolrIngesterConfig.SECURITY_ACTIVATED) {
+ securityActivated =
Boolean.parseBoolean(sn.getAttributeValue(SolrIngesterConfig.ATTRIBUTE_VALUE));
+ }
+ if (sn.getType() == SolrIngesterConfig.ID_FIELD) {
+ idFieldName = sn.getAttributeValue(SolrIngesterConfig.ATTRIBUTE_VALUE);
+ }
+ if (sn.getType() == SolrIngesterConfig.DATE_FIELD) {
+ dateField = sn.getAttributeValue(SolrIngesterConfig.ATTRIBUTE_VALUE);
+ }
+ if (sn.getType() == SolrIngesterConfig.CONTENT_FIELD) {
+ contentField =
sn.getAttributeValue(SolrIngesterConfig.ATTRIBUTE_VALUE);
+ }
+ if (sn.getType() == SolrIngesterConfig.SECURITY_FIELD) {
+ securityField =
sn.getAttributeValue(SolrIngesterConfig.ATTRIBUTE_VALUE);
+ }
+ if (sn.getType() == SolrIngesterConfig.SECURITY_FIELD2) {
+ securityField2 =
sn.getAttributeValue(SolrIngesterConfig.ATTRIBUTE_VALUE);
+ }
+ if (sn.getType().equals(SolrIngesterConfig.NODE_FIELDMAP)) {
+ final String source =
sn.getAttributeValue(SolrIngesterConfig.ATTRIBUTE_SOURCE);
+ final String target =
sn.getAttributeValue(SolrIngesterConfig.ATTRIBUTE_TARGET);
+ mapFields.put(source, target);
+ }
+ if (sn.getType() == SolrIngesterConfig.ROWS_NUMBER) {
+ rowsNumberString =
sn.getAttributeValue(SolrIngesterConfig.ATTRIBUTE_VALUE);
+ }
+ }
+
+ final String versionString = null;
+
+ final StringBuilder url = new StringBuilder(solringesterEntryPoint);
+
+ url.append("?").append(ACTION_PARAM_NAME).append("=").append(ACTION_ITEMS);
+ for (int i = 0; i < documentIdentifiers.length; i++) {
+ url.append("&id[]=").append(URLEncoder.encode(documentIdentifiers[i]));
+ }
+
+ /*
+ * Step 1 query that gets idFieldName and versionFieldName
+ *
+ */
+ final String documentIdentifiersString = "\"" + String.join("\" OR \"",
documentIdentifiers) + "\"";
+ final int rowsNumber = Integer.valueOf(rowsNumberString);
+ final HashMap<String, String> existingIds = new HashMap<String, String>();
+ if (Logging.connectors.isDebugEnabled()) {
+ Logging.connectors.debug("SolrIngester: docidentifiers size '" +
documentIdentifiers.length);
+
+ }
+ if (documentIdentifiers.length > 0) {
+ try {
+ SolrQuery query;
+ query = new SolrQuery("*:*").setRows(rowsNumber).setSort(idFieldName,
SolrQuery.ORDER.asc);
+ query.setFields(idFieldName, versionField);
+ query.addFilterQuery(idFieldName + ":(" + documentIdentifiersString +
")");
+
+ String cursorMark = CursorMarkParams.CURSOR_MARK_START;
+ boolean done = false;
+ while (!done) {
+ query.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark);
+ QueryResponse response;
+ response = httpSolrClient.query(collection, query);
+
+ final String nextCursorMark = response.getNextCursorMark();
+ final SolrDocumentList documents = response.getResults();
+ for (final SolrDocument document : documents) {
+ existingIds.put((String) document.getFieldValue(idFieldName),
String.valueOf(document.getFieldValue(versionField)));
+ }
+ if (cursorMark.equals(nextCursorMark)) {
+ done = true;
+ }
+ cursorMark = nextCursorMark;
+ }
+
+ /*
+ * End step 1
+ */
+
+ /*
+ * Step 2 Compare the version stored in MCF and store id without
version or with different version
+ *
+ */
+ final Set<String> toProcess = new HashSet<String>();
+ final Iterator it = existingIds.entrySet().iterator();
+
+ while (it.hasNext()) {
+
+ final Map.Entry pair = (Map.Entry) it.next();
+
+ final String idStringBis = (String) pair.getKey();
+ final String versionStringBis = (String) pair.getValue();
+
+ if (versionStringBis.length() == 0 ||
activities.checkDocumentNeedsReindexing(idStringBis, versionStringBis)) {
+ toProcess.add("\"" + (String) pair.getKey() + "\"");
+
+ }
+ // it.remove(); // avoids a ConcurrentModificationException
+ }
+
+ /*
+ * Step 3 : Build query from toProcess list and index all the content
+ */
+
+ if (toProcess.size() > 0) {
+ final List<String> listToProcess = new ArrayList<String>(toProcess);
+ final String listToProcessString = String.join(" OR ",
listToProcess);
+
+ // Process the document
+ final RepositoryDocument doc = new RepositoryDocument();
+
+ query = new
SolrQuery("*:*").setRows(rowsNumber).setSort(idFieldName, SolrQuery.ORDER.asc);
+ query.setFields("*");
+ query.addFilterQuery(idFieldName + ":(" + listToProcessString + ")");
+
+ cursorMark = CursorMarkParams.CURSOR_MARK_START;
+ done = false;
+ while (!done) {
+ query.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark);
+ QueryResponse response;
+
+ SolrDocumentList documents = null;
+
+ response = httpSolrClient.query(collection, query);
+
+ final String nextCursorMark = response.getNextCursorMark();
+ documents = response.getResults();
+ InputStream is = null;
+ for (final SolrDocument document : documents) {
+ for (final Map.Entry<String, String> entry :
mapFields.entrySet()) {
+ ArrayList<Object> listFieldValues = null;
+ if (document.getFieldValues(entry.getKey()) != null) {
+ listFieldValues = (ArrayList<Object>)
document.getFieldValues(entry.getKey());
+ if (listFieldValues != null) {
+
+ // TODO
+ // For now only supports String fields (not int, long,
date etc...)
+ final String[] tablistFieldValues =
listFieldValues.toArray(new String[0]);
+ doc.addField(entry.getValue(), tablistFieldValues);
+
+ }
+ }
+ }
+
+ // TODO
+ // For now you can indicate the date field of the source. But
the date field of the target is hardcoded and its value is last_modified
+ doc.addField(date_target_field, (Date)
document.getFieldValue(dateField));
+ doc.setFileName((String) document.getFieldValue(idFieldName));
+
+ if (Logging.connectors.isDebugEnabled()) {
+ Logging.connectors.debug("doc '" + (String)
document.getFieldValue(idFieldName));
+ }
+
+ // Content part
+ ArrayList<Object> contentFieldValues = null;
+ String contentFieldValuesString = "";
+
+ if (document.getFieldValues(contentField) != null) {
+ contentFieldValues = (ArrayList<Object>)
document.getFieldValues(contentField);
+ if (contentFieldValues != null) {
+ final String[] tabContentFieldValues =
contentFieldValues.toArray(new String[0]);
+ for (final String s : tabContentFieldValues) {
+ contentFieldValuesString = contentFieldValuesString + " "
+ s;
+ }
+ }
+ } else if (document.getFieldValue(contentField) != null) {
+
+ contentFieldValuesString = (String)
document.getFieldValue(contentField);
+ } else {
+ contentFieldValuesString = "";
+ }
+
+ is = new
ByteArrayInputStream(contentFieldValuesString.getBytes());
+
+ // security part
+
+ if (securityActivated = true) {
+
+
+ if (Logging.connectors.isDebugEnabled()) {
+ Logging.connectors.debug("Security part");
+ }
+
+ if (document.getFieldValues(securityField) != null &&
document.getFieldValues(securityField2) != null) {
+ ArrayList<Object> securityFieldValues = null;
+ ArrayList<Object> securityFieldValues2 = null;
+
+ securityFieldValues = (ArrayList<Object>)
document.getFieldValues(securityField);
+ securityFieldValues2 = (ArrayList<Object>)
document.getFieldValues(securityField2);
+ if (Logging.connectors.isDebugEnabled()) {
+ Logging.connectors.debug("Security field 1 : " +
securityFieldValues.toString());
+ Logging.connectors.debug("Security field 2 : " +
securityFieldValues2.toString());
+ }
+
+ String[] securityValues = null;
+ final ArrayList<Object> all = new ArrayList<Object>();
+ all.addAll(securityFieldValues);
+ all.addAll(securityFieldValues2);
+
+ final HashSet hs = new HashSet();
+ hs.addAll(all);
+ all.clear();
+ all.addAll(hs);
+ securityValues = all.toArray(new String[0]);
+ doc.setSecurity(RepositoryDocument.SECURITY_TYPE_DOCUMENT,
securityValues, new String[] { defaultAuthorityDenyToken });
+ securityFieldValues = null;
+ securityFieldValues2 = null;
+ } else if (document.getFieldValues(securityField) != null) {
+ ArrayList<Object> securityFieldValues = null;
+ securityFieldValues = (ArrayList<Object>)
document.getFieldValues(securityField);
+ String[] tabsecurityFieldValues =
securityFieldValues.toArray(new String[0]);
+ doc.setSecurity(RepositoryDocument.SECURITY_TYPE_DOCUMENT,
tabsecurityFieldValues, new String[] { defaultAuthorityDenyToken });
+ securityFieldValues = null;
+ tabsecurityFieldValues = null;
+ } else if (document.getFieldValues(securityField2) != null) {
+ ArrayList<Object> securityFieldValues2 = null;
+ securityFieldValues2 = (ArrayList<Object>)
document.getFieldValues(securityField2);
+ String[] tabsecurityFieldValues2 =
securityFieldValues2.toArray(new String[0]);
+ doc.setSecurity(RepositoryDocument.SECURITY_TYPE_DOCUMENT,
tabsecurityFieldValues2, new String[] { defaultAuthorityDenyToken });
+ securityFieldValues2 = null;
+ tabsecurityFieldValues2 = null;
+ } else {
+ doc.setSecurity(RepositoryDocument.SECURITY_TYPE_DOCUMENT,
new String[] { "__nosecurity__" }, new String[] { "__nosecurity__" });
+ }
+
+ }
+
+ // Can only index while background thread is running!
+ try {
+ doc.setBinary(is, is.available());
+ // doc.setFileName(documentIdentifier);
+
activities.ingestDocumentWithException(String.valueOf(document.getFieldValue(idFieldName)),
String.valueOf(document.getFieldValue(versionField)),
+ String.valueOf(document.getFieldValue(idFieldName)), doc);
+ activities.recordActivity(new Long(startFetchTime),
ACTIVITY_GET, doc.getBinaryLength(),
document.getFieldValue(idFieldName).toString(), "OK", "", null);
+ is.close();
+ } catch (final IOException e) {
+ errorCode = "ERROR";
+ description = "Unable to perform Solr requests";
+ Logging.connectors.error("Unable to perform Solr requests", e);
+ }
+
+ }
+ if (cursorMark.equals(nextCursorMark)) {
+ done = true;
+ }
+ cursorMark = nextCursorMark;
+ }
+
+ }
+ } catch (final SolrServerException | IOException e) {
+ errorCode = "ERROR";
+ description = "Unable to perform Solr requests";
+ Logging.connectors.error("Unable to perform Solr requests", e);
+ }
+
+ /*
+ * Etape 4 Iterate into documentIdentifiers and check if each id is
present into existingIds. If not, delete the doc */
+
+ for (int i = 0; i < documentIdentifiers.length; i++) {
+ if (!existingIds.containsKey(documentIdentifiers[i])) {
+ activities.deleteDocument(documentIdentifiers[i]);
+ activities.recordActivity(new Long(startFetchTime), ACTIVITY_GET,
0L, documentIdentifiers[i], "NOTFOUND", "Document not found in Solr", null);
+ } else if (errorCode != null) {
+ activities.recordActivity(new Long(startFetchTime), ACTIVITY_GET,
0L, documentIdentifiers[i], errorCode, description, null);
+ }
+
+ }
+ }
+
+
+
+ }
+
+ @Override
+ public String processSpecificationPost(final IPostParameters
variableContext, final Locale locale, final Specification os, final int
connectionSequenceNumber) throws ManifoldCFException {
+ final String seqPrefix = "s" + connectionSequenceNumber + "_";
+
+ String x;
+
+ x = variableContext.getParameter(seqPrefix + "fieldmapping_count");
+ if (x != null && x.length() > 0) {
+ // About to gather the fieldmapping nodes, so get rid of the old ones.
+ int i = 0;
+ while (i < os.getChildCount()) {
+ final SpecificationNode node = os.getChild(i);
+ if (node.getType().equals(SolrIngesterConfig.NODE_FIELDMAP)) {
+ os.removeChild(i);
+ } else {
+ i++;
+ }
+ }
+ final int count = Integer.parseInt(x);
+ i = 0;
+ while (i < count) {
+ final String prefix = seqPrefix + "fieldmapping_";
+ final String suffix = "_" + Integer.toString(i);
+ final String op = variableContext.getParameter(prefix + "op" + suffix);
+ if (op == null || !op.equals("Delete")) {
+ // Gather the fieldmap etc.
+ final String source = variableContext.getParameter(prefix + "source"
+ suffix);
+ String target = variableContext.getParameter(prefix + "target" +
suffix);
+ if (target == null) {
+ target = "";
+ }
+ final SpecificationNode node = new
SpecificationNode(SolrIngesterConfig.NODE_FIELDMAP);
+ node.setAttribute(SolrIngesterConfig.ATTRIBUTE_SOURCE, source);
+ node.setAttribute(SolrIngesterConfig.ATTRIBUTE_TARGET, target);
+ os.addChild(os.getChildCount(), node);
+ }
+ i++;
+ }
+
+ final String addop = variableContext.getParameter(seqPrefix +
"fieldmapping_op");
+ if (addop != null && addop.equals("Add")) {
+ final String source = variableContext.getParameter(seqPrefix +
"fieldmapping_source");
+ String target = variableContext.getParameter(seqPrefix +
"fieldmapping_target");
+ if (target == null) {
+ target = "";
+ }
+ final SpecificationNode node = new
SpecificationNode(SolrIngesterConfig.NODE_FIELDMAP);
+ node.setAttribute(SolrIngesterConfig.ATTRIBUTE_SOURCE, source);
+ node.setAttribute(SolrIngesterConfig.ATTRIBUTE_TARGET, target);
+ os.addChild(os.getChildCount(), node);
+ }
+
+ final SpecificationNode node2 = new
SpecificationNode(SolrIngesterConfig.SECURITY_ACTIVATED);
+ final String securityActivated = variableContext.getParameter(seqPrefix
+ "securityactivated");
+
+ if (securityActivated != null) {
+ node2.setAttribute(SolrIngesterConfig.ATTRIBUTE_VALUE,
securityActivated);
+ } else {
+ node2.setAttribute(SolrIngesterConfig.ATTRIBUTE_VALUE, "false");
+ }
+ os.addChild(os.getChildCount(), node2);
+
+ final SpecificationNode node3 = new
SpecificationNode(SolrIngesterConfig.SECURITY_FIELD);
+ final String securityField = variableContext.getParameter(seqPrefix +
"securityfield");
+
+ node3.setAttribute(SolrIngesterConfig.ATTRIBUTE_VALUE, securityField);
+
+ os.addChild(os.getChildCount(), node3);
+
+ final SpecificationNode node4 = new
SpecificationNode(SolrIngesterConfig.ID_FIELD);
+ final String fieldId = variableContext.getParameter(seqPrefix +
"fieldid");
+
+ node4.setAttribute(SolrIngesterConfig.ATTRIBUTE_VALUE, fieldId);
+
+ os.addChild(os.getChildCount(), node4);
+
+ final SpecificationNode node5 = new
SpecificationNode(SolrIngesterConfig.DATE_FIELD);
+ final String fieldDate = variableContext.getParameter(seqPrefix +
"fielddate");
+
+ node5.setAttribute(SolrIngesterConfig.ATTRIBUTE_VALUE, fieldDate);
+
+ os.addChild(os.getChildCount(), node5);
+
+ final SpecificationNode node6 = new
SpecificationNode(SolrIngesterConfig.CONTENT_FIELD);
+ final String fieldContent = variableContext.getParameter(seqPrefix +
"fieldcontent");
+
+ node6.setAttribute(SolrIngesterConfig.ATTRIBUTE_VALUE, fieldContent);
+
+ os.addChild(os.getChildCount(), node6);
+
+ final SpecificationNode node7 = new
SpecificationNode(SolrIngesterConfig.COLLECTION_NAME);
+ final String collectionName = variableContext.getParameter(seqPrefix +
"collection");
+
+ node7.setAttribute(SolrIngesterConfig.ATTRIBUTE_VALUE, collectionName);
+
+ os.addChild(os.getChildCount(), node7);
+
+ final SpecificationNode node8 = new
SpecificationNode(SolrIngesterConfig.FILTER_CONDITION);
+ final String filter = variableContext.getParameter(seqPrefix + "filter");
+
+ node8.setAttribute(SolrIngesterConfig.ATTRIBUTE_VALUE, filter);
+
+ os.addChild(os.getChildCount(), node8);
+
+ final SpecificationNode node9 = new
SpecificationNode(SolrIngesterConfig.ROWS_NUMBER);
+ final String rowsNumber = variableContext.getParameter(seqPrefix +
"rowsnumber");
+
+ node9.setAttribute(SolrIngesterConfig.ATTRIBUTE_VALUE, rowsNumber);
+
+ os.addChild(os.getChildCount(), node9);
+
+ final SpecificationNode node10 = new
SpecificationNode(SolrIngesterConfig.SECURITY_FIELD2);
+ final String securityField2 = variableContext.getParameter(seqPrefix +
"securityfield2");
+
+ node10.setAttribute(SolrIngesterConfig.ATTRIBUTE_VALUE, securityField2);
+
+ os.addChild(os.getChildCount(), node10);
+ }
+
+ return null;
+ }
+
+ /**
+ * Output the configuration header section. This method is called in the
head section of the connector's configuration page. Its purpose is
+ * to add the required tabs to the list, and to output any javascript
methods that might be needed by the configuration editing HTML.
+ *
+ * @param threadContext
+ * is the local thread context.
+ * @param out
+ * is the output to which any HTML should be sent.
+ * @param parameters
+ * are the configuration parameters, as they currently exist, for
this connection being configured.
+ * @param tabsArray
+ * is an array of tab names. Add to this array any tab names that
are specific to the connector.
+ */
+ @Override
+ public void outputConfigurationHeader(final IThreadContext threadContext,
final IHTTPOutput out, final Locale locale, final ConfigParams parameters,
final List<String> tabsArray)
+ throws ManifoldCFException, IOException {
+ tabsArray.add(Messages.getString(locale,
"SolrIngester.SolrIngesterTabName"));
+ Messages.outputResourceWithVelocity(out, locale,
EDIT_CONFIGURATION_CONNECTOR_JS, null);
+ }
+
+ /**
+ * Output the configuration body section. This method is called in the body
section of the connector's configuration page. Its purpose is to
+ * present the required form elements for editing. The coder can presume
that the HTML that is output from this configuration will be within
+ * appropriate <html>, <body>, and <form> tags. The name of the form is
"editconnection".
+ *
+ * @param threadContext
+ * is the local thread context.
+ * @param out
+ * is the output to which any HTML should be sent.
+ * @param parameters
+ * are the configuration parameters, as they currently exist, for
this connection being configured.
+ * @param tabName
+ * is the current tab name.
+ */
+ @Override
+ public void outputConfigurationBody(final IThreadContext threadContext,
final IHTTPOutput out, final Locale locale, final ConfigParams parameters,
final String tabName)
+ throws ManifoldCFException, IOException {
+ final Map<String, Object> velocityContext = new HashMap<>();
+ velocityContext.put("TabName", tabName);
+ fillInServerTab(velocityContext, out, parameters);
+ Messages.outputResourceWithVelocity(out, locale,
EDIT_CONFIGURATION_CONNECTOR_HTML, velocityContext);
+ }
+
+ /**
+ * Process a configuration post. This method is called at the start of the
connector's configuration page, whenever there is a possibility
+ * that form data for a connection has been posted. Its purpose is to gather
form information and modify the configuration parameters
+ * accordingly. The name of the posted form is "editconnection".
+ *
+ * @param threadContext
+ * is the local thread context.
+ * @param variableContext
+ * is the set of variables available from the post, including
binary file post information.
+ * @param parameters
+ * are the configuration parameters, as they currently exist, for
this connection being configured.
+ * @return null if all is well, or a string error message if there is an
error that should prevent saving of the connection (and cause a
+ * redirection to an error page).
+ */
+ @Override
+ public String processConfigurationPost(final IThreadContext threadContext,
final IPostParameters variableContext, final Locale locale, final ConfigParams
parameters) throws ManifoldCFException {
+ final String solrAddress = variableContext.getParameter("solrAddress");
+ if (solrAddress != null) {
+ parameters.setParameter(SolrIngesterConfig.PARAM_SOLRADDRESS,
solrAddress);
+ }
+
+ final String solrUsername = variableContext.getParameter("solrUsername");
+ if (solrUsername != null) {
+ parameters.setParameter(SolrIngesterConfig.PARAM_SOLRUSERNAME,
solrUsername);
+ }
+
+ final String solrPassword = variableContext.getParameter("solrPassword");
+ if (solrPassword != null) {
+ parameters.setParameter(SolrIngesterConfig.PARAM_SOLRPASSWORD,
ManifoldCF.obfuscate(variableContext.mapKeyToPassword(solrPassword)));
+ }
+
+ final String connectionTimeout =
variableContext.getParameter(SolrIngesterConfig.PARAM_CONNECTIONTIMEOUT);
+ if (connectionTimeout != null) {
+ parameters.setParameter(SolrIngesterConfig.PARAM_CONNECTIONTIMEOUT,
connectionTimeout);
+ }
+
+ final String socketTimeout =
variableContext.getParameter(SolrIngesterConfig.PARAM_SOCKETTIMEOUT);
+ if (socketTimeout != null) {
+ parameters.setParameter(SolrIngesterConfig.PARAM_SOCKETTIMEOUT,
socketTimeout);
+ }
+
+ return null;
+ }
+
+ /**
+ * View configuration. This method is called in the body section of the
connector's view configuration page. Its purpose is to present the
+ * connection information to the user. The coder can presume that the HTML
that is output from this configuration will be within appropriate
+ * <html> and <body> tags.
+ *
+ * @param threadContext
+ * is the local thread context.
+ * @param out
+ * is the output to which any HTML should be sent.
+ * @param parameters
+ * are the configuration parameters, as they currently exist, for
this connection being configured.
+ */
+ @Override
+ public void viewConfiguration(final IThreadContext threadContext, final
IHTTPOutput out, final Locale locale, final ConfigParams parameters) throws
ManifoldCFException, IOException {
+ final Map<String, Object> velocityContext = new HashMap<>();
+ fillInServerTab(velocityContext, out, parameters);
+ Messages.outputResourceWithVelocity(out, locale,
VIEW_CONFIGURATION_CONNECTOR_HTML, velocityContext);
+ }
+
+ protected static void fillInServerTab(final Map<String, Object>
velocityContext, final IHTTPOutput out, final ConfigParams parameters) throws
ManifoldCFException {
+
+ String solrAddress =
parameters.getParameter(SolrIngesterConfig.PARAM_SOLRADDRESS);
+
+ if (solrAddress == null) {
+ solrAddress = SolrIngesterConfig.SOLRADDRESS_DEFAULT;
+ }
+
+ String solrUsername =
parameters.getParameter(SolrIngesterConfig.PARAM_SOLRUSERNAME);
+
+ if (solrUsername == null) {
+ solrUsername = SolrIngesterConfig.SOLRUSERNAME_DEFAULT;
+ }
+
+ String solrPassword =
parameters.getParameter(SolrIngesterConfig.PARAM_SOLRPASSWORD);
+
+ if (solrPassword == null) {
+ solrPassword = SolrIngesterConfig.SOLRPASSWORD_DEFAULT;
+ }
+
+ String connectionTimeout =
parameters.getParameter(SolrIngesterConfig.PARAM_CONNECTIONTIMEOUT);
+ if (connectionTimeout == null) {
+ connectionTimeout = SolrIngesterConfig.CONNECTIONTIMEOUT_DEFAULT;
+ }
+
+ String socketTimeout =
parameters.getParameter(SolrIngesterConfig.PARAM_SOCKETTIMEOUT);
+ if (socketTimeout == null) {
+ socketTimeout = SolrIngesterConfig.SOCKETTIMEOUT_DEFAULT;
+ }
+ // Fill in context
+
+ velocityContext.put("SOLRADDRESS", solrAddress);
+ velocityContext.put("SOLRUSERNAME", solrUsername);
+ velocityContext.put("SOLRPASSWORD", solrPassword);
+ velocityContext.put("CONNECTIONTIMEOUT", connectionTimeout);
+ velocityContext.put("SOCKETTIMEOUT", socketTimeout);
+
+ }
+
+ /**
+ * Output the specification header section. This method is called in the
head section of a job page which has selected a pipeline connection
+ * of the current type. Its purpose is to add the required tabs to the list,
and to output any javascript methods that might be needed by
+ * the job editing HTML.
+ *
+ * @param out
+ * is the output to which any HTML should be sent.
+ * @param locale
+ * is the preferred local of the output.
+ * @param os
+ * is the current pipeline specification for this connection.
+ * @param connectionSequenceNumber
+ * is the unique number of this connection within the job.
+ * @param tabsArray
+ * is an array of tab names. Add to this array any tab names that
are specific to the connector.
+ */
+ @Override
+ public void outputSpecificationHeader(final IHTTPOutput out, final Locale
locale, final Specification os, final int connectionSequenceNumber, final
List<String> tabsArray)
+ throws ManifoldCFException, IOException {
+ final Map<String, Object> paramMap = new HashMap<>();
+ paramMap.put("SEQNUM", Integer.toString(connectionSequenceNumber));
+
+ tabsArray.add(Messages.getString(locale, "SolrIngester.SecurityTabName"));
+ tabsArray.add(Messages.getString(locale,
"SolrIngester.ParametersTabName"));
+
+ // Fill in the specification header map, using data from all tabs.
+
+ Messages.outputResourceWithVelocity(out, locale,
EDIT_SPECIFICATION_JOB_PARAMETERS_JS, paramMap);
+ Messages.outputResourceWithVelocity(out, locale,
EDIT_SPECIFICATION_JOB_SECURITY_JS, paramMap);
+ }
+
+ /**
+ * Output the specification body section. This method is called in the body
section of a job page which has selected a pipeline connection
+ * of the current type. Its purpose is to present the required form elements
for editing. The coder can presume that the HTML that is output
+ * from this configuration will be within appropriate <html>, <body>, and
<form> tags. The name of the form is "editjob".
+ *
+ * @param out
+ * is the output to which any HTML should be sent.
+ * @param locale
+ * is the preferred local of the output.
+ * @param os
+ * is the current pipeline specification for this job.
+ * @param connectionSequenceNumber
+ * is the unique number of this connection within the job.
+ * @param actualSequenceNumber
+ * is the connection within the job that has currently been
selected.
+ * @param tabName
+ * is the current tab name.
+ */
+ @Override
+ public void outputSpecificationBody(final IHTTPOutput out, final Locale
locale, final Specification os, final int connectionSequenceNumber, final int
actualSequenceNumber, final String tabName)
+ throws ManifoldCFException, IOException {
+ final Map<String, Object> paramMap = new HashMap<>();
+
+ // Set the tab name
+ paramMap.put("TABNAME", tabName);
+ paramMap.put("SEQNUM", Integer.toString(connectionSequenceNumber));
+ paramMap.put("SELECTEDNUM", Integer.toString(actualSequenceNumber));
+
+ // Fill in the field mapping tab data
+ fillInFieldMappingSpecificationMap(paramMap, os);
+ // fillInSecuritySpecificationMap(paramMap, os);
+
+ Messages.outputResourceWithVelocity(out, locale,
EDIT_SPECIFICATION_JOB_PARAMETERS_HTML, paramMap);
+ Messages.outputResourceWithVelocity(out, locale,
EDIT_SPECIFICATION_JOB_SECURITY_HTML, paramMap);
+ }
+
+ /**
+ * View specification. This method is called in the body section of a job's
view page. Its purpose is to present the pipeline specification
+ * information to the user. The coder can presume that the HTML that is
output from this configuration will be within appropriate <html> and
+ * <body> tags.
+ *
+ * @param out
+ * is the output to which any HTML should be sent.
+ * @param locale
+ * is the preferred local of the output.
+ * @param connectionSequenceNumber
+ * is the unique number of this connection within the job.
+ * @param os
+ * is the current pipeline specification for this job.
+ */
+
+ @Override
+ public void viewSpecification(final IHTTPOutput out, final Locale locale,
final Specification os, final int connectionSequenceNumber) throws
ManifoldCFException, IOException {
+ final Map<String, Object> paramMap = new HashMap<>();
+ paramMap.put("SEQNUM", Integer.toString(connectionSequenceNumber));
+
+ // Fill in the map with data from all tabs
+ fillInFieldMappingSpecificationMap(paramMap, os);
+
+ Messages.outputResourceWithVelocity(out, locale,
VIEW_SPECIFICATION_JOB_PARAMETERS_HTML, paramMap);
+ Messages.outputResourceWithVelocity(out, locale,
VIEW_SPECIFICATION_JOB_SECURITY_HTML, paramMap);
+
+ }
+
+ protected static void fillInFieldMappingSpecificationMap(final Map<String,
Object> paramMap, final Specification os) {
+
+ String securityActivated = "false";
+ String securityField = "";
+ String securityField2 = "";
+ String idField = "id";
+ String dateField = "last_modified";
+ String contentField = "content";
+ String filterCondition = "*:*";
+ String collection = "techproducts";
+ String rowsNumber = "50";
+
+ final List<Map<String, String>> fieldMappings = new ArrayList<>();
+
+ for (int i = 0; i < os.getChildCount(); i++) {
+ final SpecificationNode sn = os.getChild(i);
+
+ if (sn.getType().equals(SolrIngesterConfig.NODE_FIELDMAP)) {
+ final String source =
sn.getAttributeValue(SolrIngesterConfig.ATTRIBUTE_SOURCE);
+ String target =
sn.getAttributeValue(SolrIngesterConfig.ATTRIBUTE_TARGET);
+ String targetDisplay;
+ if (target == null) {
+ target = "";
+ targetDisplay = "(remove)";
+ } else {
+ targetDisplay = target;
+ }
+ final Map<String, String> fieldMapping = new HashMap<>();
+ fieldMapping.put("SOURCE", source);
+ fieldMapping.put("TARGET", target);
+ fieldMapping.put("TARGETDISPLAY", targetDisplay);
+
+ fieldMappings.add(fieldMapping);
+ } else if (sn.getType().equals(SolrIngesterConfig.SECURITY_ACTIVATED)) {
+ securityActivated =
sn.getAttributeValue(SolrIngesterConfig.ATTRIBUTE_VALUE);
+
+ } else if (sn.getType().equals(SolrIngesterConfig.SECURITY_FIELD)) {
+ securityField =
sn.getAttributeValue(SolrIngesterConfig.ATTRIBUTE_VALUE);
+
+ } else if (sn.getType().equals(SolrIngesterConfig.SECURITY_FIELD2)) {
+ securityField2 =
sn.getAttributeValue(SolrIngesterConfig.ATTRIBUTE_VALUE);
+
+ } else if (sn.getType().equals(SolrIngesterConfig.ID_FIELD)) {
+ idField = sn.getAttributeValue(SolrIngesterConfig.ATTRIBUTE_VALUE);
+
+ } else if (sn.getType().equals(SolrIngesterConfig.DATE_FIELD)) {
+ dateField = sn.getAttributeValue(SolrIngesterConfig.ATTRIBUTE_VALUE);
+
+ } else if (sn.getType().equals(SolrIngesterConfig.CONTENT_FIELD)) {
+ contentField =
sn.getAttributeValue(SolrIngesterConfig.ATTRIBUTE_VALUE);
+
+ } else if (sn.getType().equals(SolrIngesterConfig.FILTER_CONDITION)) {
+ filterCondition =
sn.getAttributeValue(SolrIngesterConfig.ATTRIBUTE_VALUE);
+
+ } else if (sn.getType().equals(SolrIngesterConfig.COLLECTION_NAME)) {
+ collection = sn.getAttributeValue(SolrIngesterConfig.ATTRIBUTE_VALUE);
+
+ } else if (sn.getType().equals(SolrIngesterConfig.ROWS_NUMBER)) {
+ rowsNumber = sn.getAttributeValue(SolrIngesterConfig.ATTRIBUTE_VALUE);
+
+ }
+ }
+ // Prep for field mappings
+
+ paramMap.put("FIELDMAPPINGS", fieldMappings);
+ paramMap.put("SECURITYACTIVATION", securityActivated);
+ paramMap.put("SECURITYFIELD", securityField);
+ paramMap.put("SECURITYFIELD2", securityField2);
+ paramMap.put("FIELDID", idField);
+ paramMap.put("FIELDDATE", dateField);
+ paramMap.put("FIELDCONTENT", contentField);
+ paramMap.put("FILTERCONDITION", filterCondition);
+ paramMap.put("COLLECTION", collection);
+ paramMap.put("ROWSNUMBER", rowsNumber);
+
+ }
+
+ static class PreemptiveAuth implements HttpRequestInterceptor {
+
+ private final Credentials credentials;
+
+ public PreemptiveAuth(final Credentials creds) {
+ this.credentials = creds;
+ }
+
+ @Override
+ public void process(final HttpRequest request, final HttpContext context)
throws HttpException, IOException {
+ request.addHeader(new
BasicScheme(StandardCharsets.US_ASCII).authenticate(credentials, request,
context));
+ }
+ }
+
+ protected static class CheckThread extends Thread {
+
+ protected HttpClient client;
+
+ protected String url;
+
+ protected Throwable exception = null;
+
+ protected String result = "Unknown";
+
+ public CheckThread(final HttpClient client, final String url) {
+ super();
+ setDaemon(true);
+ this.client = client;
+ this.url = url;
+ }
+
+ @Override
+ public void run() {
+ final HttpGet method = new HttpGet(url);
+ try {
+ final HttpResponse response = client.execute(method);
+ try {
+ if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
+ result = "Connection failed: " +
response.getStatusLine().getReasonPhrase();
+ return;
+ }
+ EntityUtils.consume(response.getEntity());
+ result = "Connection OK";
+ } finally {
+ EntityUtils.consume(response.getEntity());
+ method.releaseConnection();
+ }
+ } catch (final IOException ex) {
+ exception = ex;
+ }
+ }
+
+ public Throwable getException() {
+ return exception;
+ }
+
+ public String getResult() {
+ return result;
+ }
+ }
+
+ protected static class ExecuteSeedingThread extends Thread {
+
+ protected final HttpClient client;
+
+ protected final String url;
+
+ protected final XThreadStringBuffer seedBuffer;
+
+ protected Throwable exception = null;
+
+ public ExecuteSeedingThread(final HttpClient client, final String url) {
+ super();
+ setDaemon(true);
+ this.client = client;
+ this.url = url;
+ seedBuffer = new XThreadStringBuffer();
+ }
+
+ public XThreadStringBuffer getBuffer() {
+ return seedBuffer;
+ }
+ }
+}
Added:
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/solr/common_en_US.properties
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/solr/common_en_US.properties?rev=1883688&view=auto
==============================================================================
---
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/solr/common_en_US.properties
(added)
+++
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/solr/common_en_US.properties
Sat Nov 21 09:00:02 2020
@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+SolrIngester.SolrIngesterTabName=Solr Ingester
+SolrIngester.SecurityTabName=Security
+SolrIngester.ParametersTabName=Parameters
+SolrIngester.SolrAddress=Solr Address
+SolrIngester.SolrUsername=Login
+SolrIngester.SolrPassword=Password
+SolrIngester.ConnectionTimeout=Connection timeout:
+SolrIngester.SocketTimeout=Socket timeout:
+SolrIngester.SecurityActivation=Security activated
+SolrIngester.SecurityField=Security Field
+SolrIngester.SecurityField2=Security Field 2
+SolrIngester.FieldMappingTabName=Field mapping
+SolrIngester.FieldMappings=Field mappings:
+SolrIngester.MetadataFieldName=Metadata field name
+SolrIngester.FinalFieldName=Final field name
+SolrIngester.NoFieldMappingSpecified=No field mapping specified
+SolrIngester.Add=Add
+SolrIngester.AddFieldMapping=Add field mapping
+SolrIngester.Delete=Delete
+SolrIngester.DeleteFieldMapping=Delete field mapping
+SolrIngester.NoFieldNameSpecified=Please specify a field name
+SolrIngester.FieldId=ID Field
+SolrIngester.RowsNumber=Solr rows number
+SolrIngester.FieldDate=Date Field
+SolrIngester.FieldContent=Content Field
+SolrIngester.FilterCondition=Filter Condition
+SolrIngester.CollectionName=Collection Name
+SolrIngester.Parameters=Parameters
+SolrIngester.Security=Security
+SolrIngester.Delete=Delete
+SolrIngester.Add=Add
+SolrIngester.NoValueSecurityField=No value entered for Security field
+SolrIngester.NoValueCollection=No value entered for collection
+SolrIngester.NoValueIdField=No value entered for field Id
+SolrIngester.NoValueDateField=No value entered for field Date
+SolrIngester.NoValueSolrAddress=No value entered for Solr address
+SolrIngester.NoValueConnectionTimeOut= No value entered for connection timeout
+SolrIngester.NoValueSocketTimeOut=No value entered for socket timeout
\ No newline at end of file
Added:
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/solr/common_es_ES.properties
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/solr/common_es_ES.properties?rev=1883688&view=auto
==============================================================================
---
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/solr/common_es_ES.properties
(added)
+++
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/solr/common_es_ES.properties
Sat Nov 21 09:00:02 2020
@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+SolrIngester.SolrIngesterTabName=Solr Ingester
+SolrIngester.SecurityTabName=Security
+SolrIngester.ParametersTabName=Parameters
+SolrIngester.SolrUsername=Login
+SolrIngester.SolrPassword=Password
+SolrIngester.SolrAddress=Solr Address
+SolrIngester.ConnectionTimeout=Connection timeout:
+SolrIngester.SocketTimeout=Socket timeout:
+SolrIngester.SecurityActivation=Security activated
+SolrIngester.SecurityField=Security Field
+SolrIngester.SecurityField2=Security Field 2
+SolrIngester.FieldMappingTabName=Field mapping
+SolrIngester.FieldMappings=Field mappings:
+SolrIngester.MetadataFieldName=Metadata field name
+SolrIngester.FinalFieldName=Final field name
+SolrIngester.NoFieldMappingSpecified=No field mapping specified
+SolrIngester.Add=Add
+SolrIngester.AddFieldMapping=Add field mapping
+SolrIngester.Delete=Delete
+SolrIngester.DeleteFieldMapping=Delete field mapping
+SolrIngester.NoFieldNameSpecified=Please specify a field name
+SolrIngester.FieldId=ID Field
+SolrIngester.RowsNumber=Solr rows number
+SolrIngester.FieldDate=Date Field
+SolrIngester.FieldContent=Content Field
+SolrIngester.FilterCondition=Filter Condition
+SolrIngester.CollectionName=Collection Name
+SolrIngester.Parameters=Parameters
+SolrIngester.Security=Security
+SolrIngester.Delete=Delete
+SolrIngester.Add=Add
+SolrIngester.NoValueSecurityField=No value entered for Security field
+SolrIngester.NoValueCollection=No value entered for collection
+SolrIngester.NoValueIdField=No value entered for field Id
+SolrIngester.NoValueDateField=No value entered for field Date
+SolrIngester.NoValueSolrAddress=No value entered for Solr address
+SolrIngester.NoValueConnectionTimeOut= No value entered for connection timeout
+SolrIngester.NoValueSocketTimeOut=No value entered for socket timeout
\ No newline at end of file
Added:
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/solr/common_fr_FR.properties
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/solr/common_fr_FR.properties?rev=1883688&view=auto
==============================================================================
---
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/solr/common_fr_FR.properties
(added)
+++
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/solr/common_fr_FR.properties
Sat Nov 21 09:00:02 2020
@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+SolrIngester.SolrIngesterTabName=Solr Ingester
+SolrIngester.SecurityTabName=Securité
+SolrIngester.ParametersTabName=Paramètres
+SolrIngester.SolrAddress=URL Solr
+SolrIngester.SolrUsername=Login
+SolrIngester.SolrPassword=Password
+SolrIngester.ConnectionTimeout=Connection timeout:
+SolrIngester.SocketTimeout=Socket timeout:
+SolrIngester.SecurityActivation=Securité activée
+SolrIngester.SecurityField=Champ de sécurité
+SolrIngester.SecurityField2=Champ de sécurité 2
+SolrIngester.FieldMappingTabName=Correspondance des champs
+SolrIngester.FieldMappings=Correspondance des champs:
+SolrIngester.MetadataFieldName=Nom du champ de métadonnées
+SolrIngester.FinalFieldName=Champ final
+SolrIngester.NoFieldMappingSpecified=Pas de correspondance de champ spécifiée
+SolrIngester.Add=Ajouter
+SolrIngester.AddFieldMapping=Ajouter correspondance de champs
+SolrIngester.Delete=Supprimer
+SolrIngester.DeleteFieldMapping=Supprimer correspondance de champ
+SolrIngester.NoFieldNameSpecified=Spécifier un champ svp
+SolrIngester.Parameters=Paramètres
+SolrIngester.Security=Security
+SolrIngester.Delete=Delete
+SolrIngester.Add=Add
+SolrIngester.FieldId=Champ ID
+SolrIngester.RowsNumber=Solr rows number
+SolrIngester.FieldDate=Champ Date
+SolrIngester.FieldContent=Champ Contenu
+SolrIngester.FilterCondition=Condition de Filtre
+SolrIngester.CollectionName=Nom de la Collection
+SolrIngester.ParameterName=Parameter name
+SolrIngester.ParameterValue=Parameter value
+SolrIngester.NoValueSecurityField=Pas de valeur entrée pour le champ
Sécurité
+SolrIngester.NoValueCollection==Pas de valeur entrée pour la collection
+SolrIngester.NoValueIdField==Pas de valeur entrée pour le champ Id
+SolrIngester.NoValueDateField==Pas de valeur entrée pour le champ Date
+SolrIngester.NoValueSolrAddress==Pas de valeur entrée pour l'adresse Solr
+SolrIngester.NoValueConnectionTimeOut= Pas de valeur entrée pour connection
timeout
+SolrIngester.NoValueSocketTimeOut=Pas de valeur entrée pour socket timeout
\ No newline at end of file
Added:
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/solr/editConfiguration_connector.html
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/solr/editConfiguration_connector.html?rev=1883688&view=auto
==============================================================================
---
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/solr/editConfiguration_connector.html
(added)
+++
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/solr/editConfiguration_connector.html
Sat Nov 21 09:00:02 2020
@@ -0,0 +1,61 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+#if($TabName == $ResourceBundle.getString('SolrIngester.SolrIngesterTabName'))
+
+<table class="displaytable">
+ <tr><td class="separator" colspan="2"><hr/></td></tr>
+ <tr>
+ <td
class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('SolrIngester.SolrAddress'))</nobr></td>
+ <td class="value"><input name="solrAddress" type="text"
+ value="$Encoder.attributeEscape($SOLRADDRESS)" size="40" />
+ </td>
+ </tr>
+ <tr>
+ <td
class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('SolrIngester.SolrUsername'))</nobr></td>
+ <td class="value"><input name="solrUsername" type="text"
+ value="$Encoder.attributeEscape($SOLRUSERNAME)" size="40" />
+ </td>
+ </tr>
+ <tr>
+ <td
class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('SolrIngester.SolrPassword'))</nobr></td>
+ <td class="value"><input name="solrPassword" type="password"
+ value="$Encoder.attributeEscape($SOLRPASSWORD)" size="40" />
+ </td>
+ </tr>
+ <tr>
+ <td
class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('SolrIngester.ConnectionTimeout'))</nobr></td>
+ <td class="value"><input name="connectionTimeout" type="text"
+ value="$Encoder.attributeEscape($CONNECTIONTIMEOUT)" size="20" />
+ </td>
+ </tr>
+ <tr>
+ <td
class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('SolrIngester.SocketTimeout'))</nobr></td>
+ <td class="value"><input name="socketTimeout" type="text"
+ value="$Encoder.attributeEscape($SOCKETTIMEOUT)" size="20" />
+ </td>
+ </tr>
+
+</table>
+#else
+<input type="hidden" name="solrAddress" value="$SOLRADDRESS"/>
+<input type="hidden" name="solrUsername" value="$SOLRUSERNAME"/>
+<input type="hidden" name="solrPassword" value="$SOLRPASSWORD"/>
+<input type="hidden" name="connectionTimeout" value="$CONNECTIONTIMEOUT"/>
+<input type="hidden" name="socketTimeout" value="$SOCKETTIMEOUT"/>
+
+#end
\ No newline at end of file
Added:
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/solr/editConfiguration_connector.js
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/solr/editConfiguration_connector.js?rev=1883688&view=auto
==============================================================================
---
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/solr/editConfiguration_connector.js
(added)
+++
manifoldcf/branches/CONNECTORS-1653/connectors/solr/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/solr/editConfiguration_connector.js
Sat Nov 21 09:00:02 2020
@@ -0,0 +1,48 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<script type="text/javascript">
+<!--
+
+function checkConfigForSave()
+{
+ if (editconnection.solrAddress.value == "")
+ {
+
alert("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('SolrIngester.NoValueSolrAddress'))");
+
SelectTab("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('SolrIngester.SolrIngesterTabName'))");
+ eval("editconnection.solrAddress.focus()");
+ return false;
+ }
+ if (editconnection.connectionTimeout.value == "")
+ {
+
alert("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('SolrIngester.NoValueConnectionTimeOut'))");
+
SelectTab("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('SolrIngester.SolrIngesterTabName'))");
+ eval("editconnection.connectionTimeout.focus()");
+ return false;
+ }
+ if (editconnection.socketTimeout.value == "")
+ {
+
alert("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('SolrIngester.NoValueSocketTimeOut'))");
+
SelectTab("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('SolrIngester.SolrIngesterTabName'))");
+ eval("editconnection.socketTimeout.focus()");
+ return false;
+ }
+ return true;
+}
+
+//-->
+</script>
