Repository: ambari Updated Branches: refs/heads/branch-2.5 379b95b63 -> 4c3ecb92e
AMBARI-21277. ADDENDUM part 3: Fail to create solr clients in Log Search / Log Feeder (oleewere) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4c3ecb92 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4c3ecb92 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4c3ecb92 Branch: refs/heads/branch-2.5 Commit: 4c3ecb92ec317e04cf7405cc38d3fb5134092d2f Parents: 379b95b Author: oleewere <[email protected]> Authored: Mon Jun 26 15:33:55 2017 +0200 Committer: oleewere <[email protected]> Committed: Mon Jun 26 15:33:55 2017 +0200 ---------------------------------------------------------------------- .../ambari-logsearch-logfeeder/pom.xml | 6 --- .../ambari/logfeeder/output/OutputSolr.java | 10 ++--- .../ambari-logsearch-portal/pom.xml | 32 ------------- .../logsearch/common/ExternalServerClient.java | 4 +- .../configurer/SolrCollectionConfigurer.java | 9 +--- .../logsearch/dao/SolrSchemaFieldDao.java | 47 +++++++------------- .../handler/CreateCollectionHandler.java | 6 +-- ambari-metrics/ambari-metrics-common/pom.xml | 4 ++ 8 files changed, 27 insertions(+), 91 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/4c3ecb92/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml b/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml index 1ad59b3..49122e8 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml +++ b/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml @@ -143,12 +143,6 @@ <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> - <exclusions> - <exclusion> - <groupId>commons-httpclient</groupId> - <artifactId>commons-httpclient</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> http://git-wip-us.apache.org/repos/asf/ambari/blob/4c3ecb92/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java index d37c388..d37a3bb 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java @@ -37,7 +37,6 @@ import org.apache.ambari.logfeeder.input.InputMarker; import org.apache.ambari.logfeeder.util.DateUtil; import org.apache.ambari.logfeeder.util.LogFeederUtil; import org.apache.commons.lang3.StringUtils; -import org.apache.http.impl.client.SystemDefaultHttpClient; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.solr.client.solrj.SolrClient; @@ -51,8 +50,6 @@ import org.apache.solr.client.solrj.response.SolrPingResponse; import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.SolrParams; public class OutputSolr extends Output { private static final Logger LOG = Logger.getLogger(OutputSolr.class); @@ -94,6 +91,7 @@ public class OutputSolr extends Output { public void init() throws Exception { super.init(); initParams(); + setupSecurity(); createOutgoingBuffer(); createSolrWorkers(); } @@ -181,10 +179,8 @@ public class OutputSolr extends Output { throw new Exception("For solr cloud property collection is mandatory"); } LOG.info("Using collection=" + collection); - setupSecurity(); - SystemDefaultHttpClient httpClient = new SystemDefaultHttpClient(); - HttpClientUtil.configureClient(httpClient, new ModifiableSolrParams((SolrParams) null)); - CloudSolrClient solrClient = new CloudSolrClient(zkConnectString, httpClient); + + CloudSolrClient solrClient = new CloudSolrClient(zkConnectString); solrClient.setDefaultCollection(collection); return solrClient; } http://git-wip-us.apache.org/repos/asf/ambari/blob/4c3ecb92/ambari-logsearch/ambari-logsearch-portal/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/pom.xml b/ambari-logsearch/ambari-logsearch-portal/pom.xml index 46b4350..a439266 100755 --- a/ambari-logsearch/ambari-logsearch-portal/pom.xml +++ b/ambari-logsearch/ambari-logsearch-portal/pom.xml @@ -38,7 +38,6 @@ <swagger.version>1.5.8</swagger.version> <spring-data-solr.version>2.0.2.RELEASE</spring-data-solr.version> <jjwt.version>0.6.0</jjwt.version> - <http-components.version>4.4.1</http-components.version> </properties> <profiles> <!-- Dev Profile Start --> @@ -524,27 +523,6 @@ <version>${spring.security.version}</version> </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - <version>${http-components.version}</version> - <exclusions> - <exclusion> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - <version>${http-components.version}</version> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpmime</artifactId> - <version>${http-components.version}</version> - </dependency> - <dependency> <groupId>org.glassfish.jersey.ext</groupId> <artifactId>jersey-spring3</artifactId> <version>2.23.2</version> @@ -559,12 +537,6 @@ <groupId>org.glassfish.jersey.connectors</groupId> <artifactId>jersey-apache-connector</artifactId> <version>${jersey.version}</version> - <exclusions> - <exclusion> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> <groupId>org.glassfish.jersey.core</groupId> @@ -656,10 +628,6 @@ <groupId>com.sun.jersey</groupId> <artifactId>jersey-server</artifactId> </exclusion> - <exclusion> - <groupId>commons-httpclient</groupId> - <artifactId>commons-httpclient</artifactId> - </exclusion> </exclusions> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/ambari/blob/4c3ecb92/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java index 99844f5..df00c15 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java @@ -20,13 +20,13 @@ package org.apache.ambari.logsearch.common; import javax.inject.Inject; import javax.inject.Named; -import javax.security.auth.login.CredentialException; import javax.ws.rs.client.Invocation; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Response; import org.apache.ambari.logsearch.conf.AuthPropsConfig; import org.apache.ambari.logsearch.util.SSLUtil; +import org.apache.commons.httpclient.auth.InvalidCredentialsException; import org.apache.log4j.Logger; import org.glassfish.jersey.client.JerseyClient; import org.glassfish.jersey.client.JerseyClientBuilder; @@ -70,7 +70,7 @@ public class ExternalServerClient { Response response = invocationBuilder.get(); if (response.getStatus() != Response.Status.OK.getStatusCode() && response.getStatus() != Response.Status.FOUND.getStatusCode()) { - throw new CredentialException(String.format("External auth failed with status code: %d, response: %s", + throw new InvalidCredentialsException(String.format("External auth failed with status code: %d, response: %s", response.getStatus(), response.readEntity(String.class))); } return response.readEntity(klass); http://git-wip-us.apache.org/repos/asf/ambari/blob/4c3ecb92/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java index aba059d..7259bf8 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java @@ -28,14 +28,9 @@ import org.apache.ambari.logsearch.handler.ReloadCollectionHandler; import org.apache.ambari.logsearch.handler.UpgradeSchemaHandler; import org.apache.ambari.logsearch.handler.UploadConfigurationHandler; import org.apache.commons.lang.StringUtils; -import org.apache.http.client.HttpClient; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.client.SystemDefaultHttpClient; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.SolrParams; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZooKeeper; @@ -128,9 +123,7 @@ public class SolrCollectionConfigurer implements SolrConfigurer { private CloudSolrClient createClient(String solrUrl, String zookeeperConnectString, String defaultCollection) { if (StringUtils.isNotEmpty(zookeeperConnectString)) { - DefaultHttpClient httpClient = new SystemDefaultHttpClient(); - HttpClientUtil.configureClient(httpClient, new ModifiableSolrParams((SolrParams) null)); - CloudSolrClient cloudSolrClient = new CloudSolrClient(zookeeperConnectString, httpClient); + CloudSolrClient cloudSolrClient = new CloudSolrClient(zookeeperConnectString); cloudSolrClient.setDefaultCollection(defaultCollection); return cloudSolrClient; } else if (StringUtils.isNotEmpty(solrUrl)) { http://git-wip-us.apache.org/repos/asf/ambari/blob/4c3ecb92/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java index 3124f35..01378c1 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java @@ -22,11 +22,9 @@ import org.apache.ambari.logsearch.common.LogSearchConstants; import org.apache.ambari.logsearch.common.LogType; import org.apache.ambari.logsearch.common.MessageEnums; import org.apache.ambari.logsearch.conf.SolrUserPropsConfig; -import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.SystemDefaultHttpClient; -import org.apache.http.util.EntityUtils; +import org.apache.http.impl.client.CloseableHttpClient; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; @@ -40,8 +38,6 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.JavaBinCodec; import org.apache.solr.common.util.NamedList; import org.codehaus.jettison.json.JSONObject; @@ -70,18 +66,18 @@ public class SolrSchemaFieldDao { @Inject private AuditSolrDao auditSolrDao; - + @Inject private SolrUserPropsConfig solrUserConfigPropsConfig; - + private int retryCount; private int skipCount; - + private Map<String, String> serviceSchemaFieldNameMap = new HashMap<>(); private Map<String, String> serviceSchemaFieldTypeMap = new HashMap<>(); private Map<String, String> auditSchemaFieldNameMap = new HashMap<>(); private Map<String, String> auditSchemaFieldTypeMap = new HashMap<>(); - + @Scheduled(fixedDelay = RETRY_SECOND * 1000) public void populateAllSchemaFields() { if (skipCount > 0) { @@ -111,7 +107,7 @@ public class SolrSchemaFieldDao { schemaRequest.setMethod(SolrRequest.METHOD.GET); schemaRequest.setPath("/schema"); schemaResponse = schemaRequest.process(solrClient); - + LOG.debug("populateSchemaFields() collection=" + solrClient.getDefaultCollection() + ", luke=" + lukeResponses + ", schema= " + schemaResponse); } catch (SolrException | SolrServerException | IOException e) { @@ -131,37 +127,26 @@ public class SolrSchemaFieldDao { } } } - + private static final String LUKE_REQUEST_URL_SUFFIX = "admin/luke?numTerms=0&wt=javabin&version=2"; - + @SuppressWarnings("unchecked") private List<LukeResponse> getLukeResponsesForCores(CloudSolrClient solrClient) { ZkStateReader zkStateReader = solrClient.getZkStateReader(); Collection<Slice> activeSlices = zkStateReader.getClusterState().getActiveSlices(solrClient.getDefaultCollection()); - + List<LukeResponse> lukeResponses = new ArrayList<>(); for (Slice slice : activeSlices) { for (Replica replica : slice.getReplicas()) { - HttpEntity httpEntity = null; - try(SystemDefaultHttpClient httpClient = new SystemDefaultHttpClient()) { - HttpClientUtil.configureClient(httpClient, new ModifiableSolrParams((SolrParams) null)); + try (CloseableHttpClient httpClient = HttpClientUtil.createClient(null)) { HttpGet request = new HttpGet(replica.getCoreUrl() + LUKE_REQUEST_URL_SUFFIX); HttpResponse response = httpClient.execute(request); - httpEntity = response.getEntity(); - NamedList<Object> lukeData = (NamedList<Object>) new JavaBinCodec(null, null).unmarshal(httpEntity.getContent()); + NamedList<Object> lukeData = (NamedList<Object>) new JavaBinCodec(null, null).unmarshal(response.getEntity().getContent()); LukeResponse lukeResponse = new LukeResponse(); lukeResponse.setResponse(lukeData); lukeResponses.add(lukeResponse); } catch (IOException e) { LOG.error("Exception during getting luke responses", e); - } finally { - if (httpEntity != null) { - try { - EntityUtils.consume(httpEntity); - } catch (IOException e) { - LOG.error("Error during consuming http entity...", e); - } - } } } } @@ -173,7 +158,7 @@ public class SolrSchemaFieldDao { try { HashMap<String, String> _schemaFieldNameMap = new HashMap<>(); HashMap<String, String> _schemaFieldTypeMap = new HashMap<>(); - + for (LukeResponse lukeResponse : lukeResponses) { for (Entry<String, FieldInfo> e : lukeResponse.getFieldInfo().entrySet()) { String name = e.getKey(); @@ -184,7 +169,7 @@ public class SolrSchemaFieldDao { } } } - + List<FieldTypeDefinition> fieldTypes = schemaResponse.getSchemaRepresentation().getFieldTypes(); for (FieldTypeDefinition fieldType : fieldTypes) { Map<String, Object> fieldAttributes = fieldType.getAttributes(); @@ -192,7 +177,7 @@ public class SolrSchemaFieldDao { String fieldTypeJson = new JSONObject(fieldAttributes).toString(); _schemaFieldTypeMap.put(name, fieldTypeJson); } - + List<Map<String, Object>> fields = schemaResponse.getSchemaRepresentation().getFields(); for (Map<String, Object> field : fields) { String name = (String) field.get("name"); @@ -202,11 +187,11 @@ public class SolrSchemaFieldDao { _schemaFieldNameMap.put(name, type); } } - + if (_schemaFieldNameMap.isEmpty() || _schemaFieldTypeMap.isEmpty()) { return; } - + synchronized (this) { schemaFieldNameMap.clear(); schemaFieldNameMap.putAll(_schemaFieldNameMap); http://git-wip-us.apache.org/repos/asf/ambari/blob/4c3ecb92/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java index 9ffe16d..752a1e1 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java @@ -23,7 +23,6 @@ import org.apache.commons.lang.StringUtils; import org.apache.http.HttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.SystemDefaultHttpClient; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.HttpClientUtil; @@ -32,8 +31,6 @@ import org.apache.solr.client.solrj.response.CollectionAdminResponse; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.SolrParams; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -173,8 +170,7 @@ public class CreateCollectionHandler implements SolrZkRequestHandler<Boolean> { private void updateMaximumNumberOfShardsPerCore(Collection<Slice> slices, SolrPropsConfig solrPropsConfig) throws IOException { String baseUrl = getRandomBaseUrl(slices); if (baseUrl != null) { - SystemDefaultHttpClient httpClient = new SystemDefaultHttpClient(); - HttpClientUtil.configureClient(httpClient, new ModifiableSolrParams((SolrParams) null)); + CloseableHttpClient httpClient = HttpClientUtil.createClient(null); HttpGet request = new HttpGet(baseUrl + String.format(MODIFY_COLLECTION_QUERY, solrPropsConfig.getCollection(), MAX_SHARDS_PER_NODE, calculateMaxShardsPerNode(solrPropsConfig))); HttpResponse response = httpClient.execute(request); http://git-wip-us.apache.org/repos/asf/ambari/blob/4c3ecb92/ambari-metrics/ambari-metrics-common/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/pom.xml b/ambari-metrics/ambari-metrics-common/pom.xml index 687b704..b52b1d9 100644 --- a/ambari-metrics/ambari-metrics-common/pom.xml +++ b/ambari-metrics/ambari-metrics-common/pom.xml @@ -108,6 +108,10 @@ <pattern>org.jboss</pattern> <shadedPattern>org.apache.hadoop.metrics2.sink.relocated.jboss</shadedPattern> </relocation> + <relocation> + <pattern>org.apache.http</pattern> + <shadedPattern>org.apache.hadoop.metrics2.sink.relocated.apache.http</shadedPattern> + </relocation> </relocations> </configuration> </execution>
