Author: jmssiera
Date: Tue Dec 6 17:27:24 2022
New Revision: 1905801
URL: http://svn.apache.org/viewvc?rev=1905801&view=rev
Log:
solrj9 and zookeeper 3.8.0 update
Modified:
manifoldcf/branches/CONNECTORS-1740/build.xml
manifoldcf/branches/CONNECTORS-1740/connectors/solr/build.xml
manifoldcf/branches/CONNECTORS-1740/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
manifoldcf/branches/CONNECTORS-1740/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
manifoldcf/branches/CONNECTORS-1740/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
manifoldcf/branches/CONNECTORS-1740/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties
manifoldcf/branches/CONNECTORS-1740/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_es_ES.properties
manifoldcf/branches/CONNECTORS-1740/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_fr_FR.properties
manifoldcf/branches/CONNECTORS-1740/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties
manifoldcf/branches/CONNECTORS-1740/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_zh_CN.properties
manifoldcf/branches/CONNECTORS-1740/connectors/solr/pom.xml
manifoldcf/branches/CONNECTORS-1740/framework/build.xml
manifoldcf/branches/CONNECTORS-1740/framework/core/pom.xml
manifoldcf/branches/CONNECTORS-1740/pom.xml
Modified: manifoldcf/branches/CONNECTORS-1740/build.xml
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1740/build.xml?rev=1905801&r1=1905800&r2=1905801&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1740/build.xml (original)
+++ manifoldcf/branches/CONNECTORS-1740/build.xml Tue Dec 6 17:27:24 2022
@@ -31,12 +31,12 @@
<property name="javax-activation.version" value="1.2.0"/>
<property name="hsqldb.version" value="2.3.2"/>
- <property name="jetty.version" value="9.4.25.v20191220"/>
+ <property name="jetty.version" value="9.4.48.v20220622"/>
<property name="httpclient.version" value="4.5.13"/>
<property name="httpmime.version" value="4.5.13"/>
<property name="httpcore.version" value="4.4.15"/>
<property name="xerces.version" value="2.12.2"/>
- <property name="zookeeper.version" value="3.4.10"/>
+ <property name="zookeeper.version" value="3.8.0"/>
<property name="mongodb.version" value="2.14.3"/>
<property name="postgresql.version" value="42.1.3"/>
<property name="axis.version" value="1.4"/>
@@ -84,7 +84,7 @@
<property name="jakarta.annotation-api.version" value="1.3.4"/>
<property name="jakarta.jws-api.version" value="1.1.1"/>
<property name="ha-api.version" value="3.1.12"/>
- <property name="solrj.version" value="8.3.1"/>
+ <property name="solrj.version" value="9.1.0"/>
<property name="noggit.version" value="0.8"/>
<property name="commons-math3.version" value="3.6.1"/>
<property name="chemistry.version" value="1.1.0"/>
@@ -1421,10 +1421,31 @@ Use Apache Forrest version forrest-0.9-d
<target name="download-jetty">
<mkdir dir="lib"/>
<property name="jetty-package" value="org/eclipse/jetty"/>
+ <property name="jetty-http2-package" value="org/eclipse/jetty/http2"/>
<antcall target="download-via-maven"><param name="project-path"
value="${jetty-package}"/><param name="artifact-version"
value="9.2.30.v20200428"/><param name="target" value="lib"/>
<param name="artifact-name" value="jetty-jsp"/>
<param name="artifact-type" value="jar"/>
</antcall>
+ <antcall target="download-via-maven"><param name="project-path"
value="${jetty-http2-package}"/><param name="artifact-version"
value="${jetty.version}"/><param name="target" value="lib"/>
+ <param name="artifact-name" value="http2-client"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="project-path"
value="${jetty-http2-package}"/><param name="artifact-version"
value="${jetty.version}"/><param name="target" value="lib"/>
+ <param name="artifact-name" value="http2-common"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="project-path"
value="${jetty-http2-package}"/><param name="artifact-version"
value="${jetty.version}"/><param name="target" value="lib"/>
+ <param name="artifact-name" value="http2-hpack"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="project-path"
value="${jetty-http2-package}"/><param name="artifact-version"
value="${jetty.version}"/><param name="target" value="lib"/>
+ <param name="artifact-name" value="http2-http-client-transport"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="project-path"
value="${jetty-package}"/><param name="artifact-version"
value="${jetty.version}"/><param name="target" value="lib"/>
+ <param name="artifact-name" value="jetty-alpn-java-client"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
<antcall target="download-via-maven"><param name="project-path"
value="${jetty-package}"/><param name="artifact-version"
value="${jetty.version}"/><param name="target" value="lib"/>
<param name="artifact-name" value="jetty-client"/>
<param name="artifact-type" value="jar"/>
@@ -2442,6 +2463,13 @@ Use Apache Forrest version forrest-0.9-d
</antcall>
<antcall target="download-via-maven">
<param name="target" value="lib"/>
+ <param name="project-path" value="org/apache/solr"/>
+ <param name="artifact-version" value="${solrj.version}"/>
+ <param name="artifact-name" value="solr-solrj-zookeeper"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven">
+ <param name="target" value="lib"/>
<param name="project-path" value="org/noggit"/>
<param name="artifact-version" value="${noggit.version}"/>
<param name="artifact-name" value="noggit"/>
@@ -2465,6 +2493,13 @@ Use Apache Forrest version forrest-0.9-d
<param name="artifact-name" value="zookeeper"/>
<param name="artifact-type" value="jar"/>
</antcall>
+ <antcall target="download-via-maven">
+ <param name="target" value="lib"/>
+ <param name="project-path" value="org/apache/zookeeper"/>
+ <param name="artifact-version" value="${zookeeper.version}"/>
+ <param name="artifact-name" value="zookeeper-jute"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
</target>
<target name="download-commons-exec">
Modified: manifoldcf/branches/CONNECTORS-1740/connectors/solr/build.xml
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1740/connectors/solr/build.xml?rev=1905801&r1=1905800&r2=1905801&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1740/connectors/solr/build.xml (original)
+++ manifoldcf/branches/CONNECTORS-1740/connectors/solr/build.xml Tue Dec 6
17:27:24 2022
@@ -35,10 +35,6 @@
<fileset dir="../../lib">
<include name="solr-solrj*.jar"/>
<include name="commons-math3*.jar"/>
- <include name="noggit*.jar"/>
- <include name="httpmime*.jar"/>
- <include name="woodstox-core-asl*.jar"/>
- <include name="stax2-api*.jar"/>
<include name="jcl-over-slf4j*.jar"/>
</fileset>
</path>
@@ -49,10 +45,6 @@
<fileset dir="../../lib">
<include name="solr-solrj*.jar"/>
<include name="commons-math3*.jar"/>
- <include name="noggit*.jar"/>
- <include name="httpmime*.jar"/>
- <include name="woodstox-core-asl*.jar"/>
- <include name="stax2-api*.jar"/>
<include name="jcl-over-slf4j*.jar"/>
</fileset>
</copy>
Modified:
manifoldcf/branches/CONNECTORS-1740/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1740/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1905801&r1=1905800&r2=1905801&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-1740/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
(original)
+++
manifoldcf/branches/CONNECTORS-1740/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
Tue Dec 6 17:27:24 2022
@@ -37,7 +37,6 @@ import org.apache.manifoldcf.agents.inte
import org.apache.manifoldcf.agents.system.*;
import java.io.*;
-import java.net.MalformedURLException;
import java.util.*;
import java.util.regex.*;
@@ -49,10 +48,9 @@ import org.apache.http.auth.UsernamePass
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
-import org.apache.solr.client.solrj.impl.XMLResponseParser;
import org.apache.solr.common.util.ContentStreamBase;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.response.SolrPingResponse;
@@ -67,9 +65,9 @@ import org.apache.solr.common.SolrInputD
import org.apache.commons.lang.StringUtils;
-import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder;
import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder;
+import org.apache.solr.client.solrj.impl.Http2SolrClient.Builder;
/**
* Posts an input stream to SOLR
@@ -136,8 +134,8 @@ public class HttpPoster
/** Initialize the SolrCloud http poster.
*/
- public HttpPoster(String zookeeperHosts, String collection,
- int zkClientTimeout, int zkConnectTimeout,
+ public HttpPoster(List<String> zookeeperHosts, String znodePath, String
collection,
+ int socketTimeout, int connectionTimeout,
String updatePath, String removePath, String statusPath,
String allowAttributeName, String denyAttributeName, String
idAttributeName,
String originalSizeAttributeName, String modifiedDateAttributeName, String
createdDateAttributeName, String indexedDateAttributeName,
@@ -173,22 +171,24 @@ public class HttpPoster
initializeKerberos();
- try
- {
- CloudSolrClient cloudSolrServer = new CloudSolrClient.Builder()
- .withZkHost(zookeeperHosts)
- .withLBHttpSolrClient(new
ModifiedLBHttpSolrClient(HttpClientUtil.createClient(null), allowCompression))
- .build();
- cloudSolrServer.setZkClientTimeout(zkClientTimeout);
- cloudSolrServer.setZkConnectTimeout(zkConnectTimeout);
+ Optional<String> chroot = Optional.empty();
+ if(znodePath != null && !znodePath.isEmpty()) {
+ chroot = Optional.of(znodePath);
+ }
+ Builder http2SolrClientBuilder = new Http2SolrClient.Builder();
+ http2SolrClientBuilder.connectionTimeout(connectionTimeout);
+ http2SolrClientBuilder.idleTimeout(socketTimeout);
+ // CloudSolrClient does not have exceptions in its build method signature
because it catches them and encapsulate them as runtime exceptions.
+ // Thus, we need to set a try catch here to properly raise the exceptions
that may happen
+ try {
+ CloudSolrClient cloudSolrServer = new
CloudSolrClient.Builder(zookeeperHosts,
chroot).withInternalClientBuilder(http2SolrClientBuilder).build();
cloudSolrServer.setDefaultCollection(collection);
// Set the solrj instance we want to use
solrServer = cloudSolrServer;
+ } catch (Exception e) {
+ throw new ManifoldCFException(e);
}
- catch (MalformedURLException e)
- {
- throw new ManifoldCFException(e.getMessage(),e);
- }
+
}
/** Initialize the standard http poster.
@@ -242,69 +242,18 @@ public class HttpPoster
}
// Initialize standard solr-j.
-
- SSLConnectionSocketFactory myFactory;
- if (keystoreManager != null)
- {
- myFactory = new
SSLConnectionSocketFactory(keystoreManager.getSecureSocketFactory(),
NoopHostnameVerifier.INSTANCE);
- }
- else
- {
- // Use the "trust everything" one
- myFactory = new
SSLConnectionSocketFactory(KeystoreManagerFactory.getTrustingSecureSocketFactory(),NoopHostnameVerifier.INSTANCE);
- }
-
- // First, we need an HttpClient where basic auth is properly set up.
- connectionManager = new
PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory>create()
- .register("http", PlainConnectionSocketFactory.getSocketFactory())
- .register("https", myFactory)
- .build());
- connectionManager.setDefaultMaxPerRoute(1);
- connectionManager.setValidateAfterInactivity(2000);
- connectionManager.setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build());
-
- RequestConfig.Builder requestBuilder = RequestConfig.custom()
- .setCircularRedirectsAllowed(true)
- .setSocketTimeout(socketTimeout)
- .setExpectContinueEnabled(true)
- .setConnectTimeout(connectionTimeout)
- .setConnectionRequestTimeout(socketTimeout);
-
- HttpClientBuilder clientBuilder = HttpClients.custom()
- .setConnectionManager(connectionManager)
- .disableAutomaticRetries()
- .setDefaultRequestConfig(requestBuilder.build())
- .setRedirectStrategy(new LaxRedirectStrategy())
- .setRequestExecutor(new HttpRequestExecutor(socketTimeout));
-
-
+ String httpSolrServerUrl = protocol + "://" + server + ":" + port +
location;
+ Http2SolrClient.Builder solrClientBuilder = new
Http2SolrClient.Builder(httpSolrServerUrl);
+ solrClientBuilder.maxConnectionsPerHost(1);
+ // Set tiemouts
+ solrClientBuilder.connectionTimeout(connectionTimeout);
+ solrClientBuilder.idleTimeout(socketTimeout);
+ // Set auth credentials if provided
if (userID != null && userID.length() > 0 && password != null)
{
- CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
- Credentials credentials = new UsernamePasswordCredentials(userID,
password);
- if (realm != null && realm.trim().length() > 0)
- {
- final AuthScope scope = new AuthScope(AuthScope.ANY_HOST,
AuthScope.ANY_PORT, realm);
- credentialsProvider.setCredentials(scope, credentials);
- clientBuilder.addInterceptorFirst(new
PreemptiveBasicAuthInterceptor(scope));
- }
- else
- {
- credentialsProvider.setCredentials(AuthScope.ANY, credentials);
- clientBuilder.addInterceptorFirst(new
PreemptiveBasicAuthInterceptor(AuthScope.ANY));
- }
-
- clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
+ solrClientBuilder.withBasicAuthCredentials(userID, password);
}
-
- HttpClient localClient = clientBuilder.build();
-
-
- String httpSolrServerUrl = protocol + "://" + server + ":" + port +
location;
- solrServer = new ModifiedHttpSolrClient(httpSolrServerUrl, localClient,
new XMLResponseParser(), allowCompression);
+ solrServer = new Http2SolrClient.Builder(httpSolrServerUrl).build();
}
private static void initializeKerberos()
@@ -1688,7 +1637,7 @@ public class HttpPoster
* Create a new SolrPing object.
*/
public SolrPing() {
- super(METHOD.GET, "/admin/ping");
+ super(METHOD.GET, CommonParams.PING_HANDLER);
params = new ModifiableSolrParams();
}
@@ -1758,6 +1707,11 @@ public class HttpPoster
return this;
}
+ @Override
+ public String getRequestType() {
+ return SolrRequestType.ADMIN.toString();
+ }
+
}
/** See CONNECTORS-956. Make a safe lucene field name from a possibly
Modified:
manifoldcf/branches/CONNECTORS-1740/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1740/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java?rev=1905801&r1=1905800&r2=1905801&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-1740/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
(original)
+++
manifoldcf/branches/CONNECTORS-1740/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
Tue Dec 6 17:27:24 2022
@@ -18,11 +18,10 @@
*/
package org.apache.manifoldcf.agents.output.solr;
-
-/** Parameters and output data for SOLR output connector.
-*/
-public class SolrConfig
-{
+/**
+ * Parameters and output data for SOLR output connector.
+ */
+public class SolrConfig {
public static final String _rcsid = "@(#)$Id: SolrConfig.java 991374
2010-08-31 22:32:08Z kwright $";
// Configuration parameters
@@ -33,9 +32,9 @@ public class SolrConfig
public static final String SOLR_TYPE_STANDARD = "standard";
/** Type: Solr Cloud */
public static final String SOLR_TYPE_SOLRCLOUD = "solrcloud";
-
+
// SolrCloud zookeeper parameters
-
+
// Zookeeper hosts, as nodes
/** Zookeeper node */
public static final String NODE_ZOOKEEPER = "zookeeper";
@@ -43,26 +42,26 @@ public class SolrConfig
public static final String ATTR_HOST = "host";
/** Zookeeper port */
public static final String ATTR_PORT = "port";
-
+
/** Zookeeper znode path */
public static final String PARAM_ZOOKEEPER_ZNODE_PATH = "ZooKeeper znode
path";
-
+
/** ZooKeeper client timeout */
- public static final String PARAM_ZOOKEEPER_CLIENT_TIMEOUT = "ZooKeeper
client timeout";
+ public static final String PARAM_ZOOKEEPER_SOCKET_TIMEOUT = "ZooKeeper
socket timeout";
/** ZooKeeper connect timeout */
- public static final String PARAM_ZOOKEEPER_CONNECT_TIMEOUT = "ZooKeeper
connect timeout";
+ public static final String PARAM_ZOOKEEPER_CONNECTION_TIMEOUT = "ZooKeeper
connection timeout";
/** Collection name */
public static final String PARAM_COLLECTION = "Collection";
-
+
// General indexing parameters
-
+
/** Protocol */
public static final String PARAM_PROTOCOL = "Server protocol";
/** Protocol: http */
public static final String PROTOCOL_TYPE_HTTP = "http";
/** Protocol: https */
public static final String PROTOCOL_TYPE_HTTPS = "https";
-
+
/** Server name */
public static final String PARAM_SERVER = "Server name";
/** Port */
@@ -112,7 +111,7 @@ public class SolrConfig
/** Included mime types */
public static final String PARAM_INCLUDEDMIMETYPES = "Included mime types";
/** Excluded mime types */
- public static final String PARAM_EXCLUDEDMIMETYPES="Excluded mime types";
+ public static final String PARAM_EXCLUDEDMIMETYPES = "Excluded mime types";
/** Parameter describing the use of Extract Update handler */
public static final String PARAM_EXTRACTUPDATE = "Use extract update
handler";
/** Optional content field (if not using extract update handler) */
@@ -123,7 +122,7 @@ public class SolrConfig
public static final String ATTRIBUTE_NAME = "name";
/** Attribute with the argument value */
public static final String ATTRIBUTE_VALUE = "value";
-
+
// Output specification
}
