Repository: ambari Updated Branches: refs/heads/branch-2.5 e239c8cf2 -> a381a9fc4
AMBARI-21277. ADDENDUM: Fail to create solr clients in Log Search / Log Feeder (oleewere) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a381a9fc Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a381a9fc Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a381a9fc Branch: refs/heads/branch-2.5 Commit: a381a9fc40f90ac8bdaebe95c95b36f902ba4a00 Parents: e239c8c Author: oleewere <[email protected]> Authored: Tue Jun 20 21:21:05 2017 +0200 Committer: oleewere <[email protected]> Committed: Tue Jun 20 21:24:56 2017 +0200 ---------------------------------------------------------------------- .../ambari/logfeeder/output/OutputSolr.java | 8 ++-- .../ambari-logsearch-portal/pom.xml | 28 ++++++++++++ .../configurer/SolrCollectionConfigurer.java | 7 ++- .../logsearch/dao/SolrSchemaFieldDao.java | 45 +++++++++++++------- .../handler/CreateCollectionHandler.java | 4 +- 5 files changed, 71 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/a381a9fc/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java index d37a3bb..7103905 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java @@ -37,6 +37,7 @@ import org.apache.ambari.logfeeder.input.InputMarker; import org.apache.ambari.logfeeder.util.DateUtil; import org.apache.ambari.logfeeder.util.LogFeederUtil; import org.apache.commons.lang3.StringUtils; +import org.apache.http.impl.client.SystemDefaultHttpClient; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.solr.client.solrj.SolrClient; @@ -91,7 +92,6 @@ public class OutputSolr extends Output { public void init() throws Exception { super.init(); initParams(); - setupSecurity(); createOutgoingBuffer(); createSolrWorkers(); } @@ -179,8 +179,10 @@ public class OutputSolr extends Output { throw new Exception("For solr cloud property collection is mandatory"); } LOG.info("Using collection=" + collection); - - CloudSolrClient solrClient = new CloudSolrClient(zkConnectString); + setupSecurity(); + SystemDefaultHttpClient httpClient = new SystemDefaultHttpClient(); + HttpClientUtil.configureClient(httpClient, null); + CloudSolrClient solrClient = new CloudSolrClient(zkConnectString, httpClient); solrClient.setDefaultCollection(collection); return solrClient; } http://git-wip-us.apache.org/repos/asf/ambari/blob/a381a9fc/ambari-logsearch/ambari-logsearch-portal/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/pom.xml b/ambari-logsearch/ambari-logsearch-portal/pom.xml index ec0bd43..46b4350 100755 --- a/ambari-logsearch/ambari-logsearch-portal/pom.xml +++ b/ambari-logsearch/ambari-logsearch-portal/pom.xml @@ -38,6 +38,7 @@ <swagger.version>1.5.8</swagger.version> <spring-data-solr.version>2.0.2.RELEASE</spring-data-solr.version> <jjwt.version>0.6.0</jjwt.version> + <http-components.version>4.4.1</http-components.version> </properties> <profiles> <!-- Dev Profile Start --> @@ -523,6 +524,27 @@ <version>${spring.security.version}</version> </dependency> <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + <version>${http-components.version}</version> + <exclusions> + <exclusion> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpcore</artifactId> + <version>${http-components.version}</version> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpmime</artifactId> + <version>${http-components.version}</version> + </dependency> + <dependency> <groupId>org.glassfish.jersey.ext</groupId> <artifactId>jersey-spring3</artifactId> <version>2.23.2</version> @@ -537,6 +559,12 @@ <groupId>org.glassfish.jersey.connectors</groupId> <artifactId>jersey-apache-connector</artifactId> <version>${jersey.version}</version> + <exclusions> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.glassfish.jersey.core</groupId> http://git-wip-us.apache.org/repos/asf/ambari/blob/a381a9fc/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java index 7259bf8..906cfed 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java @@ -28,6 +28,9 @@ import org.apache.ambari.logsearch.handler.ReloadCollectionHandler; import org.apache.ambari.logsearch.handler.UpgradeSchemaHandler; import org.apache.ambari.logsearch.handler.UploadConfigurationHandler; import org.apache.commons.lang.StringUtils; +import org.apache.http.client.HttpClient; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.impl.client.SystemDefaultHttpClient; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer; @@ -123,7 +126,9 @@ public class SolrCollectionConfigurer implements SolrConfigurer { private CloudSolrClient createClient(String solrUrl, String zookeeperConnectString, String defaultCollection) { if (StringUtils.isNotEmpty(zookeeperConnectString)) { - CloudSolrClient cloudSolrClient = new CloudSolrClient(zookeeperConnectString); + DefaultHttpClient httpClient = new SystemDefaultHttpClient(); + HttpClientUtil.configureClient(httpClient, null); + CloudSolrClient cloudSolrClient = new CloudSolrClient(zookeeperConnectString, httpClient); cloudSolrClient.setDefaultCollection(defaultCollection); return cloudSolrClient; } else if (StringUtils.isNotEmpty(solrUrl)) { http://git-wip-us.apache.org/repos/asf/ambari/blob/a381a9fc/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java index 01378c1..26f90e3 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java @@ -22,9 +22,11 @@ import org.apache.ambari.logsearch.common.LogSearchConstants; import org.apache.ambari.logsearch.common.LogType; import org.apache.ambari.logsearch.common.MessageEnums; import org.apache.ambari.logsearch.conf.SolrUserPropsConfig; +import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.SystemDefaultHttpClient; +import org.apache.http.util.EntityUtils; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; @@ -66,18 +68,18 @@ public class SolrSchemaFieldDao { @Inject private AuditSolrDao auditSolrDao; - + @Inject private SolrUserPropsConfig solrUserConfigPropsConfig; - + private int retryCount; private int skipCount; - + private Map<String, String> serviceSchemaFieldNameMap = new HashMap<>(); private Map<String, String> serviceSchemaFieldTypeMap = new HashMap<>(); private Map<String, String> auditSchemaFieldNameMap = new HashMap<>(); private Map<String, String> auditSchemaFieldTypeMap = new HashMap<>(); - + @Scheduled(fixedDelay = RETRY_SECOND * 1000) public void populateAllSchemaFields() { if (skipCount > 0) { @@ -107,7 +109,7 @@ public class SolrSchemaFieldDao { schemaRequest.setMethod(SolrRequest.METHOD.GET); schemaRequest.setPath("/schema"); schemaResponse = schemaRequest.process(solrClient); - + LOG.debug("populateSchemaFields() collection=" + solrClient.getDefaultCollection() + ", luke=" + lukeResponses + ", schema= " + schemaResponse); } catch (SolrException | SolrServerException | IOException e) { @@ -127,26 +129,37 @@ public class SolrSchemaFieldDao { } } } - + private static final String LUKE_REQUEST_URL_SUFFIX = "admin/luke?numTerms=0&wt=javabin&version=2"; - + @SuppressWarnings("unchecked") private List<LukeResponse> getLukeResponsesForCores(CloudSolrClient solrClient) { ZkStateReader zkStateReader = solrClient.getZkStateReader(); Collection<Slice> activeSlices = zkStateReader.getClusterState().getActiveSlices(solrClient.getDefaultCollection()); - + List<LukeResponse> lukeResponses = new ArrayList<>(); for (Slice slice : activeSlices) { for (Replica replica : slice.getReplicas()) { - try (CloseableHttpClient httpClient = HttpClientUtil.createClient(null)) { + HttpEntity httpEntity = null; + try(SystemDefaultHttpClient httpClient = new SystemDefaultHttpClient()) { + HttpClientUtil.configureClient(httpClient, null); HttpGet request = new HttpGet(replica.getCoreUrl() + LUKE_REQUEST_URL_SUFFIX); HttpResponse response = httpClient.execute(request); - NamedList<Object> lukeData = (NamedList<Object>) new JavaBinCodec(null, null).unmarshal(response.getEntity().getContent()); + httpEntity = response.getEntity(); + NamedList<Object> lukeData = (NamedList<Object>) new JavaBinCodec(null, null).unmarshal(httpEntity.getContent()); LukeResponse lukeResponse = new LukeResponse(); lukeResponse.setResponse(lukeData); lukeResponses.add(lukeResponse); } catch (IOException e) { LOG.error("Exception during getting luke responses", e); + } finally { + if (httpEntity != null) { + try { + EntityUtils.consume(httpEntity); + } catch (IOException e) { + LOG.error("Error during consuming http entity...", e); + } + } } } } @@ -158,7 +171,7 @@ public class SolrSchemaFieldDao { try { HashMap<String, String> _schemaFieldNameMap = new HashMap<>(); HashMap<String, String> _schemaFieldTypeMap = new HashMap<>(); - + for (LukeResponse lukeResponse : lukeResponses) { for (Entry<String, FieldInfo> e : lukeResponse.getFieldInfo().entrySet()) { String name = e.getKey(); @@ -169,7 +182,7 @@ public class SolrSchemaFieldDao { } } } - + List<FieldTypeDefinition> fieldTypes = schemaResponse.getSchemaRepresentation().getFieldTypes(); for (FieldTypeDefinition fieldType : fieldTypes) { Map<String, Object> fieldAttributes = fieldType.getAttributes(); @@ -177,7 +190,7 @@ public class SolrSchemaFieldDao { String fieldTypeJson = new JSONObject(fieldAttributes).toString(); _schemaFieldTypeMap.put(name, fieldTypeJson); } - + List<Map<String, Object>> fields = schemaResponse.getSchemaRepresentation().getFields(); for (Map<String, Object> field : fields) { String name = (String) field.get("name"); @@ -187,11 +200,11 @@ public class SolrSchemaFieldDao { _schemaFieldNameMap.put(name, type); } } - + if (_schemaFieldNameMap.isEmpty() || _schemaFieldTypeMap.isEmpty()) { return; } - + synchronized (this) { schemaFieldNameMap.clear(); schemaFieldNameMap.putAll(_schemaFieldNameMap); http://git-wip-us.apache.org/repos/asf/ambari/blob/a381a9fc/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java index 752a1e1..b7743b1 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java @@ -23,6 +23,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.http.HttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.SystemDefaultHttpClient; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.HttpClientUtil; @@ -170,7 +171,8 @@ public class CreateCollectionHandler implements SolrZkRequestHandler<Boolean> { private void updateMaximumNumberOfShardsPerCore(Collection<Slice> slices, SolrPropsConfig solrPropsConfig) throws IOException { String baseUrl = getRandomBaseUrl(slices); if (baseUrl != null) { - CloseableHttpClient httpClient = HttpClientUtil.createClient(null); + SystemDefaultHttpClient httpClient = new SystemDefaultHttpClient(); + HttpClientUtil.configureClient(httpClient, null); HttpGet request = new HttpGet(baseUrl + String.format(MODIFY_COLLECTION_QUERY, solrPropsConfig.getCollection(), MAX_SHARDS_PER_NODE, calculateMaxShardsPerNode(solrPropsConfig))); HttpResponse response = httpClient.execute(request);
