http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java deleted file mode 100644 index da76924..0000000 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java +++ /dev/null @@ -1,313 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.ambari.logsearch.dao; - -import org.apache.ambari.logsearch.conf.SolrPropsConfig; -import org.apache.commons.lang.StringUtils; -import org.apache.http.HttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.impl.HttpClientUtil; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.response.CollectionAdminResponse; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.ZkStateReader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.ROUTER_FIELD; - -import javax.inject.Named; -import javax.ws.rs.core.Response; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; - -@Named -class SolrCollectionDao { - - private static final Logger LOG = LoggerFactory.getLogger(SolrCollectionDao.class); - - private static final int SETUP_RETRY_SECOND = 30; - private static final String MODIFY_COLLECTION_QUERY = "/admin/collections?action=MODIFYCOLLECTION&collection=%s&%s=%d"; - private static final String MAX_SHARDS_PER_NODE = "maxShardsPerNode"; - - /** - * This will try to get the collections from the Solr. Ping doesn't work if - * collection is not given - */ - boolean checkSolrStatus(CloudSolrClient cloudSolrClient) { - int waitDurationMS = 3 * 60 * 1000; - boolean status = false; - try { - long beginTimeMS = System.currentTimeMillis(); - long waitIntervalMS = 2000; - int pingCount = 0; - while (true) { - pingCount++; - try { - List<String> collectionList = getCollections(cloudSolrClient); - if (collectionList != null) { - LOG.info("checkSolrStatus(): Solr getCollections() is success. collectionList=" + collectionList); - status = true; - break; - } - } catch (Exception ex) { - LOG.error("Error while doing Solr check", ex); - } - if (System.currentTimeMillis() - beginTimeMS > waitDurationMS) { - LOG.error("Solr is not reachable even after " + (System.currentTimeMillis() - beginTimeMS) + " ms. " + - "If you are using alias, then you might have to restart LogSearch after Solr is up and running."); - break; - } else { - LOG.warn("Solr is not not reachable yet. getCollections() attempt count=" + pingCount + ". " + - "Will sleep for " + waitIntervalMS + " ms and try again."); - } - Thread.sleep(waitIntervalMS); - - } - } catch (Throwable t) { - LOG.error("Seems Solr is not up."); - } - return status; - } - - void setupCollections(final CloudSolrClient solrClient, final SolrPropsConfig solrPropsConfig) throws Exception { - boolean setupStatus = createCollectionsIfNeeded(solrClient, solrPropsConfig); - LOG.info("Setup status for " + solrPropsConfig.getCollection() + " is " + setupStatus); - if (!setupStatus) { - // Start a background thread to do setup - Thread setupThread = new Thread("setup_collection_" + solrPropsConfig.getCollection()) { - @Override - public void run() { - LOG.info("Started monitoring thread to check availability of Solr server. collection=" + solrPropsConfig.getCollection()); - int retryCount = 0; - while (true) { - try { - Thread.sleep(SETUP_RETRY_SECOND * 1000); - retryCount++; - boolean setupStatus = createCollectionsIfNeeded(solrClient, solrPropsConfig); - if (setupStatus) { - LOG.info("Setup for collection " + solrPropsConfig.getCollection() + " is successful. Exiting setup retry thread"); - break; - } - } catch (InterruptedException sleepInterrupted) { - LOG.info("Sleep interrupted while setting up collection " + solrPropsConfig.getCollection()); - break; - } catch (Exception e) { - LOG.error("Error setting up collection=" + solrPropsConfig.getCollection(), e); - } - LOG.error("Error setting collection. collection=" + solrPropsConfig.getCollection() + ", retryCount=" + retryCount); - } - } - }; - setupThread.setDaemon(true); - setupThread.start(); - } - } - - private boolean createCollectionsIfNeeded(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) { - boolean result = false; - try { - List<String> allCollectionList = getCollections(solrClient); - if (solrPropsConfig.getSplitInterval().equalsIgnoreCase("none")) { - result = createCollection(solrClient, solrPropsConfig, allCollectionList); - } else { - result = setupCollectionsWithImplicitRouting(solrClient, solrPropsConfig, allCollectionList); - } - } catch (Exception ex) { - LOG.error("Error creating collection. collectionName=" + solrPropsConfig.getCollection(), ex); - } - return result; - } - - @SuppressWarnings("unchecked") - List<String> getCollections(CloudSolrClient solrClient) throws SolrServerException, - IOException { - try { - CollectionAdminRequest.List colListReq = new CollectionAdminRequest.List(); - CollectionAdminResponse response = colListReq.process(solrClient); - if (response.getStatus() != 0) { - LOG.error("Error getting collection list from solr. response=" + response); - return null; - } - return (List<String>) response.getResponse().get("collections"); - } catch (SolrException e) { - LOG.error("getCollections() operation failed", e); - return new ArrayList<>(); - } - } - - private boolean setupCollectionsWithImplicitRouting(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig, List<String> allCollectionList) - throws Exception { - LOG.info("setupCollectionsWithImplicitRouting(). collectionName=" + solrPropsConfig.getCollection() - + ", numberOfShards=" + solrPropsConfig.getNumberOfShards()); - - // Default is true, because if the collection and shard is already there, then it will return true - boolean returnValue = true; - - List<String> shardsList = new ArrayList<String>(); - for (int i = 0; i < solrPropsConfig.getNumberOfShards(); i++) { - shardsList.add("shard" + i); - } - String shardsListStr = StringUtils.join(shardsList, ','); - - // Check if collection is already in zookeeper - if (!allCollectionList.contains(solrPropsConfig.getCollection())) { - LOG.info("Creating collection " + solrPropsConfig.getCollection() + ", shardsList=" + shardsList); - CollectionAdminRequest.Create collectionCreateRequest = new CollectionAdminRequest.Create(); - collectionCreateRequest.setCollectionName(solrPropsConfig.getCollection()); - collectionCreateRequest.setRouterName("implicit"); - collectionCreateRequest.setShards(shardsListStr); - collectionCreateRequest.setNumShards(solrPropsConfig.getNumberOfShards()); - collectionCreateRequest.setReplicationFactor(solrPropsConfig.getReplicationFactor()); - collectionCreateRequest.setConfigName(solrPropsConfig.getConfigName()); - collectionCreateRequest.setRouterField(ROUTER_FIELD); - collectionCreateRequest.setMaxShardsPerNode(solrPropsConfig.getReplicationFactor() * solrPropsConfig.getNumberOfShards()); - - CollectionAdminResponse createResponse = collectionCreateRequest.process(solrClient); - if (createResponse.getStatus() != 0) { - returnValue = false; - LOG.error("Error creating collection. collectionName=" + solrPropsConfig.getCollection() - + ", shardsList=" + shardsList +", response=" + createResponse); - } else { - LOG.info("Created collection " + solrPropsConfig.getCollection() + ", shardsList=" + shardsList); - } - } else { - LOG.info("Collection " + solrPropsConfig.getCollection() + " is already there. Will check whether it has the required shards"); - Collection<Slice> slices = getSlices(solrClient, solrPropsConfig); - Collection<String> existingShards = getShards(slices, solrPropsConfig); - if (existingShards.size() < shardsList.size()) { - try { - updateMaximumNumberOfShardsPerCore(slices, solrPropsConfig); - } catch (Throwable t) { - returnValue = false; - LOG.error(String.format("Exception during updating collection (%s)", t)); - } - } - for (String shard : shardsList) { - if (!existingShards.contains(shard)) { - try { - LOG.info("Going to add Shard " + shard + " to collection " + solrPropsConfig.getCollection()); - CollectionAdminRequest.CreateShard createShardRequest = new CollectionAdminRequest.CreateShard(); - createShardRequest.setCollectionName(solrPropsConfig.getCollection()); - createShardRequest.setShardName(shard); - CollectionAdminResponse response = createShardRequest.process(solrClient); - if (response.getStatus() != 0) { - LOG.error("Error creating shard " + shard + " in collection " + solrPropsConfig.getCollection() + ", response=" + response); - returnValue = false; - break; - } else { - LOG.info("Successfully created shard " + shard + " in collection " + solrPropsConfig.getCollection()); - } - } catch (Throwable t) { - LOG.error("Error creating shard " + shard + " in collection " + solrPropsConfig.getCollection(), t); - returnValue = false; - break; - } - } - } - } - return returnValue; - } - - private String getRandomBaseUrl(Collection<Slice> slices) { - String coreUrl = null; - if (slices != null) { - for (Slice slice : slices) { - if (!slice.getReplicas().isEmpty()) { - Replica replica = slice.getReplicas().iterator().next(); - coreUrl = replica.getStr("base_url"); - if (coreUrl != null) { - break; - } - } - } - } - return coreUrl; - } - - private void updateMaximumNumberOfShardsPerCore(Collection<Slice> slices, SolrPropsConfig solrPropsConfig) throws IOException { - String baseUrl = getRandomBaseUrl(slices); - if (baseUrl != null) { - CloseableHttpClient httpClient = HttpClientUtil.createClient(null); - HttpGet request = new HttpGet(baseUrl + String.format(MODIFY_COLLECTION_QUERY, - solrPropsConfig.getCollection(), MAX_SHARDS_PER_NODE, calculateMaxShardsPerNode(solrPropsConfig))); - HttpResponse response = httpClient.execute(request); - if (response.getStatusLine().getStatusCode() != Response.Status.OK.getStatusCode()) { - throw new IllegalStateException(String.format("Cannot update collection (%s) - increase max number of nodes per core", solrPropsConfig.getCollection())); - } - } else { - throw new IllegalStateException(String.format("Cannot get any core url for updating collection (%s)", solrPropsConfig.getCollection())); - } - } - - private Collection<Slice> getSlices(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) { - ZkStateReader reader = solrClient.getZkStateReader(); - return reader.getClusterState().getSlices(solrPropsConfig.getCollection()); - } - - private Collection<String> getShards(Collection<Slice> slices, SolrPropsConfig solrPropsConfig) { - Collection<String> list = new HashSet<>(); - for (Slice slice : slices) { - for (Replica replica : slice.getReplicas()) { - LOG.info("colName=" + solrPropsConfig.getCollection() + ", slice.name=" + slice.getName() + ", slice.state=" + slice.getState() + - ", replica.core=" + replica.getStr("core") + ", replica.state=" + replica.getStr("state")); - list.add(slice.getName()); - } - } - return list; - } - - private boolean createCollection(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig, List<String> allCollectionList) throws SolrServerException, IOException { - - if (allCollectionList.contains(solrPropsConfig.getCollection())) { - LOG.info("Collection " + solrPropsConfig.getCollection() + " is already there. Won't create it"); - return true; - } - - LOG.info("Creating collection " + solrPropsConfig.getCollection() + ", numberOfShards=" + solrPropsConfig.getNumberOfShards() + - ", replicationFactor=" + solrPropsConfig.getReplicationFactor()); - - CollectionAdminRequest.Create collectionCreateRequest = new CollectionAdminRequest.Create(); - collectionCreateRequest.setCollectionName(solrPropsConfig.getCollection()); - collectionCreateRequest.setNumShards(solrPropsConfig.getNumberOfShards()); - collectionCreateRequest.setReplicationFactor(solrPropsConfig.getReplicationFactor()); - collectionCreateRequest.setConfigName(solrPropsConfig.getConfigName()); - collectionCreateRequest.setMaxShardsPerNode(calculateMaxShardsPerNode(solrPropsConfig)); - CollectionAdminResponse createResponse = collectionCreateRequest.process(solrClient); - if (createResponse.getStatus() != 0) { - LOG.error("Error creating collection. collectionName=" + solrPropsConfig.getCollection() + ", response=" + createResponse); - return false; - } else { - LOG.info("Created collection " + solrPropsConfig.getCollection() + ", numberOfShards=" + solrPropsConfig.getNumberOfShards() + - ", replicationFactor=" + solrPropsConfig.getReplicationFactor()); - return true; - } - } - - private Integer calculateMaxShardsPerNode(SolrPropsConfig solrPropsConfig) { - return solrPropsConfig.getReplicationFactor() * solrPropsConfig.getNumberOfShards(); - } -}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java index 39b65ae..0568fd7 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java @@ -22,6 +22,9 @@ package org.apache.ambari.logsearch.dao; import org.apache.ambari.logsearch.common.LogSearchContext; import org.apache.ambari.logsearch.common.LogType; import org.apache.ambari.logsearch.common.MessageEnums; +import org.apache.ambari.logsearch.conf.SolrKerberosConfig; +import org.apache.ambari.logsearch.conf.SolrPropsConfig; +import org.apache.ambari.logsearch.conf.global.SolrCollectionState; import org.apache.ambari.logsearch.util.RESTErrorUtil; import org.apache.ambari.logsearch.util.SolrUtil; import org.apache.log4j.Logger; @@ -36,6 +39,7 @@ import org.springframework.data.solr.core.SolrCallback; import org.springframework.data.solr.core.SolrTemplate; import org.springframework.data.solr.core.query.SolrDataQuery; +import javax.inject.Inject; import java.io.IOException; public abstract class SolrDaoBase { @@ -44,6 +48,9 @@ public abstract class SolrDaoBase { private static final Logger LOG_PERFORMANCE = Logger.getLogger("org.apache.ambari.logsearch.performance"); private LogType logType; + + @Inject + private SolrKerberosConfig solrKerberosConfig; protected SolrDaoBase(LogType logType) { this.logType = logType; @@ -108,5 +115,14 @@ public abstract class SolrDaoBase { public abstract SolrTemplate getSolrTemplate(); - public abstract SolrSchemaFieldDao getSolrSchemaFieldDao(); + public abstract void setSolrTemplate(SolrTemplate solrTemplate); + + public abstract SolrCollectionState getSolrCollectionState(); + + public abstract SolrPropsConfig getSolrPropsConfig(); + + public SolrKerberosConfig getSolrKerberosConfig() { + return this.solrKerberosConfig; + } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java index e6a562e..01378c1 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java @@ -43,7 +43,6 @@ import org.apache.solr.common.util.NamedList; import org.codehaus.jettison.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.data.solr.core.SolrTemplate; import org.springframework.scheduling.annotation.Scheduled; import java.io.IOException; @@ -54,54 +53,30 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import javax.annotation.PostConstruct; import javax.inject.Inject; -import javax.inject.Named; public class SolrSchemaFieldDao { private static final Logger LOG = LoggerFactory.getLogger(SolrSchemaFieldDao.class); private static final int RETRY_SECOND = 30; - + @Inject - @Named("serviceSolrTemplate") - private SolrTemplate serviceSolrTemplate; + private ServiceLogsSolrDao serviceLogsSolrDao; @Inject - @Named("auditSolrTemplate") - private SolrTemplate auditSolrTemplate; + private AuditSolrDao auditSolrDao; @Inject private SolrUserPropsConfig solrUserConfigPropsConfig; - private CloudSolrClient serviceSolrClient; - private CloudSolrClient auditSolrClient; - private int retryCount; private int skipCount; - - private boolean serviceCollectionSetUp = false; - private boolean auditCollectionSetUp = false; private Map<String, String> serviceSchemaFieldNameMap = new HashMap<>(); private Map<String, String> serviceSchemaFieldTypeMap = new HashMap<>(); private Map<String, String> auditSchemaFieldNameMap = new HashMap<>(); private Map<String, String> auditSchemaFieldTypeMap = new HashMap<>(); - - @PostConstruct - public void init() { - this.serviceSolrClient = (CloudSolrClient) serviceSolrTemplate.getSolrClient(); - this.auditSolrClient = (CloudSolrClient) auditSolrTemplate.getSolrClient(); - } - - void serviceCollectionSetUp() { - this.serviceCollectionSetUp = true; - } - - void auditCollectionSetUp() { - this.auditCollectionSetUp = true; - } @Scheduled(fixedDelay = RETRY_SECOND * 1000) public void populateAllSchemaFields() { @@ -109,11 +84,12 @@ public class SolrSchemaFieldDao { skipCount--; return; } - - if (serviceCollectionSetUp) { + if (serviceLogsSolrDao.getSolrCollectionState().isSolrCollectionReady()) { + CloudSolrClient serviceSolrClient = (CloudSolrClient) serviceLogsSolrDao.getSolrTemplate().getSolrClient(); populateSchemaFields(serviceSolrClient, serviceSchemaFieldNameMap, serviceSchemaFieldTypeMap); } - if (auditCollectionSetUp) { + if (auditSolrDao.getSolrCollectionState().isSolrCollectionReady()) { + CloudSolrClient auditSolrClient = (CloudSolrClient) auditSolrDao.getSolrTemplate().getSolrClient(); populateSchemaFields(auditSolrClient, auditSchemaFieldNameMap, auditSchemaFieldTypeMap); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java index 106bedc..13d7e03 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java @@ -32,7 +32,11 @@ import org.apache.ambari.logsearch.common.HadoopServiceConfigHelper; import org.apache.ambari.logsearch.common.LogSearchConstants; import org.apache.ambari.logsearch.common.LogSearchContext; import org.apache.ambari.logsearch.common.LogType; +import org.apache.ambari.logsearch.conf.SolrPropsConfig; import org.apache.ambari.logsearch.conf.SolrUserPropsConfig; +import org.apache.ambari.logsearch.conf.global.SolrCollectionState; +import org.apache.ambari.logsearch.configurer.LogfeederFilterConfigurer; +import org.apache.ambari.logsearch.configurer.SolrCollectionConfigurer; import org.apache.ambari.logsearch.model.common.LogFeederDataMap; import org.apache.ambari.logsearch.model.common.LogfeederFilterData; import org.apache.solr.client.solrj.SolrQuery; @@ -66,12 +70,13 @@ public class UserConfigSolrDao extends SolrDaoBase { private SolrUserPropsConfig solrUserConfig; @Inject - private SolrCollectionDao solrCollectionDao; - - @Inject @Named("userConfigSolrTemplate") private SolrTemplate userConfigSolrTemplate; + @Inject + @Named("solrUserConfigState") + private SolrCollectionState solrUserConfigState; + public UserConfigSolrDao() { super(LogType.SERVICE); } @@ -81,6 +86,11 @@ public class UserConfigSolrDao extends SolrDaoBase { return userConfigSolrTemplate; } + @Override + public void setSolrTemplate(SolrTemplate solrTemplate) { + this.userConfigSolrTemplate = solrTemplate; + } + @PostConstruct public void postConstructor() { String solrUrl = solrUserConfig.getSolrUrl(); @@ -88,24 +98,14 @@ public class UserConfigSolrDao extends SolrDaoBase { String collection = solrUserConfig.getCollection(); try { - solrCollectionDao.checkSolrStatus(getSolrClient()); - solrCollectionDao.setupCollections(getSolrClient(), solrUserConfig); - intializeLogFeederFilter(); - + new SolrCollectionConfigurer(this).start(); + new LogfeederFilterConfigurer(this).start(); } catch (Exception e) { LOG.error("error while connecting to Solr for history logs : solrUrl=" + solrUrl + ", zkConnectString=" + zkConnectString + ", collection=" + collection, e); } } - private void intializeLogFeederFilter() { - try { - getUserFilter(); - } catch (SolrServerException | IOException e) { - LOG.error("not able to save logfeeder filter while initialization", e); - } - } - public void saveUserFilter(LogFeederDataMap logfeederFilterWrapper) throws SolrException, SolrServerException, IOException { String filterName = LogSearchConstants.LOGFEEDER_FILTER_NAME; String json = JSONUtil.objToJson(logfeederFilterWrapper); @@ -179,7 +179,12 @@ public class UserConfigSolrDao extends SolrDaoBase { } @Override - public SolrSchemaFieldDao getSolrSchemaFieldDao() { - throw new UnsupportedOperationException(); + public SolrCollectionState getSolrCollectionState() { + return solrUserConfigState; + } + + @Override + public SolrPropsConfig getSolrPropsConfig() { + return solrUserConfig; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java index 0fc9852..caf0636 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java @@ -115,4 +115,11 @@ public class DocConstants { public static final String GET_USER_FILTER_BY_ID_OD = "Get user filter by id"; public static final String GET_ALL_USER_NAMES_OD = "Get all user names"; } + + public class StatusOperationDescriptions { + public static final String STATUS_OD = "Get statuses for collections (not health state - show true if something already done)"; + public static final String SERVICE_LOGS_STATUS_OD = "Get statuses for service log collection (not health state - show true if something already done)"; + public static final String AUDIT_LOGS_STATUS_OD = "Get statuses for collections (not health state - show true if something already done)"; + public static final String USER_CONFIG_STATUS_OD = "Get statuses for userconfig collection (not health state - show true if something already done)"; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ACLHandler.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ACLHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ACLHandler.java new file mode 100644 index 0000000..fde176f --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ACLHandler.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.handler; + +import org.apache.ambari.logsearch.conf.SolrPropsConfig; +import org.apache.commons.collections.CollectionUtils; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.SolrZooKeeper; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Stat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +public class ACLHandler implements SolrZkRequestHandler<Boolean> { + + private static final Logger LOG = LoggerFactory.getLogger(ACLHandler.class); + + @Override + public Boolean handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception { + List<ACL> aclsToSetList = solrPropsConfig.getZkAcls(); + if (CollectionUtils.isNotEmpty(aclsToSetList)) { + LOG.info("Setting acls for '{}' collection...", solrPropsConfig.getCollection()); + SolrZkClient zkClient = solrClient.getZkStateReader().getZkClient(); + SolrZooKeeper solrZooKeeper = zkClient.getSolrZooKeeper(); + String collectionPath = String.format("/collections/%s", solrPropsConfig.getCollection()); + String configsPath = String.format("/configs/%s", solrPropsConfig.getConfigName()); + List<ACL> collectionAcls = solrZooKeeper.getACL(collectionPath, new Stat()); + if (isRefreshAclsNeeded(aclsToSetList, collectionAcls)) { + LOG.info("Acls differs for {}, update acls.", collectionPath); + setRecursivelyOn(solrZooKeeper, collectionPath, aclsToSetList); + } + List<ACL> configsAcls = solrZooKeeper.getACL(configsPath, new Stat()); + if (isRefreshAclsNeeded(aclsToSetList, configsAcls)) { + LOG.info("Acls differs for {}, update acls.", configsPath); + setRecursivelyOn(solrZooKeeper, configsPath, aclsToSetList); + } + } + return true; + } + + private boolean isRefreshAclsNeeded(List<ACL> acls, List<ACL> newAcls) { + boolean result = false; + if (acls != null) { + if (acls.size() != newAcls.size()) { + return true; + } + result = aclDiffers(acls, newAcls); + if (!result) { + result = aclDiffers(newAcls, acls); + } + } + return result; + } + + private boolean aclDiffers(List<ACL> aclList1, List<ACL> aclList2) { + for (ACL acl : aclList1) { + for (ACL newAcl : aclList2) { + if (acl.getId() != null && acl.getId().getId().equals(newAcl.getId().getId()) + && acl.getPerms() != newAcl.getPerms()) { + LOG.info("ACL for '{}' differs: '{}' on znode, should be '{}'", + acl.getId().getId(), acl.getPerms(), newAcl.getPerms()); + return true; + } + } + } + return false; + } + + private void setRecursivelyOn(SolrZooKeeper solrZooKeeper, String node, List<ACL> acls) + throws KeeperException, InterruptedException { + solrZooKeeper.setACL(node, acls, -1); + for (String child : solrZooKeeper.getChildren(node, null)) { + String path = node.endsWith("/") ? node + child : node + "/" + child; + setRecursivelyOn(solrZooKeeper, path, acls); + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java new file mode 100644 index 0000000..752a1e1 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.handler; + +import org.apache.ambari.logsearch.conf.SolrPropsConfig; +import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.HttpClientUtil; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.response.CollectionAdminResponse; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkStateReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.core.Response; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; + +import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.ROUTER_FIELD; + +public class CreateCollectionHandler implements SolrZkRequestHandler<Boolean> { + + private static final Logger LOG = LoggerFactory.getLogger(CreateCollectionHandler.class); + + private static final String MODIFY_COLLECTION_QUERY = "/admin/collections?action=MODIFYCOLLECTION&collection=%s&%s=%d"; + private static final String MAX_SHARDS_PER_NODE = "maxShardsPerNode"; + + private List<String> allCollectionList; + + public CreateCollectionHandler(List<String> allCollectionList) { + this.allCollectionList = allCollectionList; + } + + @Override + public Boolean handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception { + boolean result; + if (solrPropsConfig.getSplitInterval().equalsIgnoreCase("none")) { + result = createCollection(solrClient, solrPropsConfig, this.allCollectionList); + } else { + result = setupCollectionsWithImplicitRouting(solrClient, solrPropsConfig, this.allCollectionList); + } + return result; + } + + private boolean setupCollectionsWithImplicitRouting(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig, List<String> allCollectionList) + throws Exception { + LOG.info("setupCollectionsWithImplicitRouting(). collectionName=" + solrPropsConfig.getCollection() + + ", numberOfShards=" + solrPropsConfig.getNumberOfShards()); + + // Default is true, because if the collection and shard is already there, then it will return true + boolean returnValue = true; + + List<String> shardsList = new ArrayList<String>(); + for (int i = 0; i < solrPropsConfig.getNumberOfShards(); i++) { + shardsList.add("shard" + i); + } + String shardsListStr = StringUtils.join(shardsList, ','); + + // Check if collection is already in zookeeper + if (!allCollectionList.contains(solrPropsConfig.getCollection())) { + LOG.info("Creating collection " + solrPropsConfig.getCollection() + ", shardsList=" + shardsList); + CollectionAdminRequest.Create collectionCreateRequest = new CollectionAdminRequest.Create(); + collectionCreateRequest.setCollectionName(solrPropsConfig.getCollection()); + collectionCreateRequest.setRouterName("implicit"); + collectionCreateRequest.setShards(shardsListStr); + collectionCreateRequest.setNumShards(solrPropsConfig.getNumberOfShards()); + collectionCreateRequest.setReplicationFactor(solrPropsConfig.getReplicationFactor()); + collectionCreateRequest.setConfigName(solrPropsConfig.getConfigName()); + collectionCreateRequest.setRouterField(ROUTER_FIELD); + collectionCreateRequest.setMaxShardsPerNode(solrPropsConfig.getReplicationFactor() * solrPropsConfig.getNumberOfShards()); + + CollectionAdminResponse createResponse = collectionCreateRequest.process(solrClient); + if (createResponse.getStatus() != 0) { + returnValue = false; + LOG.error("Error creating collection. collectionName=" + solrPropsConfig.getCollection() + + ", shardsList=" + shardsList +", response=" + createResponse); + } else { + LOG.info("Created collection " + solrPropsConfig.getCollection() + ", shardsList=" + shardsList); + } + } else { + LOG.info("Collection " + solrPropsConfig.getCollection() + " is already there. Will check whether it has the required shards"); + Collection<Slice> slices = getSlices(solrClient, solrPropsConfig); + Collection<String> existingShards = getShards(slices, solrPropsConfig); + if (existingShards.size() < shardsList.size()) { + try { + updateMaximumNumberOfShardsPerCore(slices, solrPropsConfig); + } catch (Throwable t) { + returnValue = false; + LOG.error(String.format("Exception during updating collection (%s)", t)); + } + } + for (String shard : shardsList) { + if (!existingShards.contains(shard)) { + try { + LOG.info("Going to add Shard " + shard + " to collection " + solrPropsConfig.getCollection()); + CollectionAdminRequest.CreateShard createShardRequest = new CollectionAdminRequest.CreateShard(); + createShardRequest.setCollectionName(solrPropsConfig.getCollection()); + createShardRequest.setShardName(shard); + CollectionAdminResponse response = createShardRequest.process(solrClient); + if (response.getStatus() != 0) { + LOG.error("Error creating shard " + shard + " in collection " + solrPropsConfig.getCollection() + ", response=" + response); + returnValue = false; + break; + } else { + LOG.info("Successfully created shard " + shard + " in collection " + solrPropsConfig.getCollection()); + } + } catch (Throwable t) { + LOG.error("Error creating shard " + shard + " in collection " + solrPropsConfig.getCollection(), t); + returnValue = false; + break; + } + } + } + } + return returnValue; + } + + private boolean createCollection(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig, List<String> allCollectionList) throws SolrServerException, IOException { + + if (allCollectionList.contains(solrPropsConfig.getCollection())) { + LOG.info("Collection " + solrPropsConfig.getCollection() + " is already there. Won't create it"); + return true; + } + + LOG.info("Creating collection " + solrPropsConfig.getCollection() + ", numberOfShards=" + solrPropsConfig.getNumberOfShards() + + ", replicationFactor=" + solrPropsConfig.getReplicationFactor()); + + CollectionAdminRequest.Create collectionCreateRequest = new CollectionAdminRequest.Create(); + collectionCreateRequest.setCollectionName(solrPropsConfig.getCollection()); + collectionCreateRequest.setNumShards(solrPropsConfig.getNumberOfShards()); + collectionCreateRequest.setReplicationFactor(solrPropsConfig.getReplicationFactor()); + collectionCreateRequest.setConfigName(solrPropsConfig.getConfigName()); + collectionCreateRequest.setMaxShardsPerNode(calculateMaxShardsPerNode(solrPropsConfig)); + CollectionAdminResponse createResponse = collectionCreateRequest.process(solrClient); + if (createResponse.getStatus() != 0) { + LOG.error("Error creating collection. collectionName=" + solrPropsConfig.getCollection() + ", response=" + createResponse); + return false; + } else { + LOG.info("Created collection " + solrPropsConfig.getCollection() + ", numberOfShards=" + solrPropsConfig.getNumberOfShards() + + ", replicationFactor=" + solrPropsConfig.getReplicationFactor()); + return true; + } + } + + private void updateMaximumNumberOfShardsPerCore(Collection<Slice> slices, SolrPropsConfig solrPropsConfig) throws IOException { + String baseUrl = getRandomBaseUrl(slices); + if (baseUrl != null) { + CloseableHttpClient httpClient = HttpClientUtil.createClient(null); + HttpGet request = new HttpGet(baseUrl + String.format(MODIFY_COLLECTION_QUERY, + solrPropsConfig.getCollection(), MAX_SHARDS_PER_NODE, calculateMaxShardsPerNode(solrPropsConfig))); + HttpResponse response = httpClient.execute(request); + if (response.getStatusLine().getStatusCode() != Response.Status.OK.getStatusCode()) { + throw new IllegalStateException(String.format("Cannot update collection (%s) - increase max number of nodes per core", solrPropsConfig.getCollection())); + } + } else { + throw new IllegalStateException(String.format("Cannot get any core url for updating collection (%s)", solrPropsConfig.getCollection())); + } + } + + private Collection<Slice> getSlices(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) { + ZkStateReader reader = solrClient.getZkStateReader(); + return reader.getClusterState().getSlices(solrPropsConfig.getCollection()); + } + + private Collection<String> getShards(Collection<Slice> slices, SolrPropsConfig solrPropsConfig) { + Collection<String> list = new HashSet<>(); + for (Slice slice : slices) { + for (Replica replica : slice.getReplicas()) { + LOG.info("colName=" + solrPropsConfig.getCollection() + ", slice.name=" + slice.getName() + ", slice.state=" + slice.getState() + + ", replica.core=" + replica.getStr("core") + ", replica.state=" + replica.getStr("state")); + list.add(slice.getName()); + } + } + return list; + } + + private String getRandomBaseUrl(Collection<Slice> slices) { + String coreUrl = null; + if (slices != null) { + for (Slice slice : slices) { + if (!slice.getReplicas().isEmpty()) { + Replica replica = slice.getReplicas().iterator().next(); + coreUrl = replica.getStr("base_url"); + if (coreUrl != null) { + break; + } + } + } + } + return coreUrl; + } + + private Integer calculateMaxShardsPerNode(SolrPropsConfig solrPropsConfig) { + return solrPropsConfig.getReplicationFactor() * solrPropsConfig.getNumberOfShards(); + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ListCollectionHandler.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ListCollectionHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ListCollectionHandler.java new file mode 100644 index 0000000..124ce40 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ListCollectionHandler.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.handler; + +import org.apache.ambari.logsearch.conf.SolrPropsConfig; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.response.CollectionAdminResponse; +import org.apache.solr.common.SolrException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +public class ListCollectionHandler implements SolrZkRequestHandler<List<String>> { + + private static final Logger LOG = LoggerFactory.getLogger(ListCollectionHandler.class); + + @Override + public List<String> handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception { + try { + CollectionAdminRequest.List colListReq = new CollectionAdminRequest.List(); + CollectionAdminResponse response = colListReq.process(solrClient); + if (response.getStatus() != 0) { + LOG.error("Error getting collection list from solr. response=" + response); + return null; + } + return (List<String>) response.getResponse().get("collections"); + } catch (SolrException e) { + LOG.error("getCollections() operation failed", e); + return new ArrayList<>(); + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ReloadCollectionHandler.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ReloadCollectionHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ReloadCollectionHandler.java new file mode 100644 index 0000000..52f3366 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ReloadCollectionHandler.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.handler; + +import org.apache.ambari.logsearch.conf.SolrPropsConfig; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ReloadCollectionHandler implements SolrZkRequestHandler<Boolean> { + + private static final Logger LOG = LoggerFactory.getLogger(ReloadCollectionHandler.class); + + @Override + public Boolean handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception { + boolean result = false; + try { + LOG.info("Reload collection - '{}'", solrPropsConfig.getCollection()); + CollectionAdminRequest.Reload reloadCollectionRequest = new CollectionAdminRequest.Reload(); + reloadCollectionRequest.setCollectionName(solrPropsConfig.getCollection()); + reloadCollectionRequest.process(solrClient); + result = true; + } catch (Exception e) { + LOG.error(String.format("Reload collection ('%s') failed.", solrPropsConfig.getCollection()), e); + } + return result; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/SolrZkRequestHandler.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/SolrZkRequestHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/SolrZkRequestHandler.java new file mode 100644 index 0000000..85ae6cb --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/SolrZkRequestHandler.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.handler; + +import org.apache.ambari.logsearch.conf.SolrPropsConfig; +import org.apache.solr.client.solrj.impl.CloudSolrClient; + +interface SolrZkRequestHandler<T> { + T handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception; +} http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/UploadConfigurationHandler.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/UploadConfigurationHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/UploadConfigurationHandler.java new file mode 100644 index 0000000..23defea --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/UploadConfigurationHandler.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.handler; + +import org.apache.ambari.logsearch.conf.SolrPropsConfig; +import org.apache.commons.io.FileUtils; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.ZkConfigManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.File; +import java.io.IOException; +import java.nio.file.FileSystems; +import java.nio.file.Paths; +import java.util.UUID; + +public class UploadConfigurationHandler implements SolrZkRequestHandler<Boolean> { + + private static final Logger LOG = LoggerFactory.getLogger(UploadConfigurationHandler.class); + + private static final String SCHEMA_FILE = "managed-schema"; + private static final String SOLR_CONFIG_FILE = "solrconfig.xml"; + + private File configSetFolder; + + public UploadConfigurationHandler(File configSetFolder) { + this.configSetFolder = configSetFolder; + } + + @Override + public Boolean handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception { + boolean reloadCollectionNeeded = false; + String separator = FileSystems.getDefault().getSeparator(); + String downloadFolderLocation = String.format("%s%s%s%s%s", System.getProperty("java.io.tmpdir"), separator, + UUID.randomUUID().toString(), separator, solrPropsConfig.getConfigName()); + solrClient.connect(); + SolrZkClient zkClient = solrClient.getZkStateReader().getZkClient(); + File tmpDir = new File(downloadFolderLocation); + try { + ZkConfigManager zkConfigManager = new ZkConfigManager(zkClient); + boolean configExists = zkConfigManager.configExists(solrPropsConfig.getConfigName()); + if (configExists) { + LOG.info("Config set exists for '{}' collection. Refreshing it if needed...", solrPropsConfig.getCollection()); + if (!tmpDir.mkdirs()) { + LOG.error("Cannot create directories for '{}'", tmpDir.getAbsolutePath()); + } + zkConfigManager.downloadConfigDir(solrPropsConfig.getConfigName(), Paths.get(downloadFolderLocation)); + File[] listOfFiles = configSetFolder.listFiles(); + if (listOfFiles != null) { + for (File file : listOfFiles) { + if (file.getName().equals(SOLR_CONFIG_FILE) || file.getName().equals(SCHEMA_FILE)) { // TODO: try to find an another solution to reload schema + if (!FileUtils.contentEquals(file, new File(String.format("%s%s%s", downloadFolderLocation, separator, file.getName())))){ + LOG.info("One of the local solr config file differs ('{}'), upload config set to zookeeper", file.getName()); + zkConfigManager.uploadConfigDir(configSetFolder.toPath(), solrPropsConfig.getConfigName()); + reloadCollectionNeeded = true; + break; + } + } + } + } + } else { + LOG.info("Config set does not exist for '{}' collection. Uploading it to zookeeper...", solrPropsConfig.getCollection()); + File[] listOfFiles = configSetFolder.listFiles(); + if (listOfFiles != null) { + zkConfigManager.uploadConfigDir(configSetFolder.toPath(), solrPropsConfig.getConfigName()); + } + } + } catch (Exception e) { + throw new RuntimeException(String.format("Cannot upload configurations to zk. (collection: %s, config set folder: %s)", + solrPropsConfig.getCollection(), solrPropsConfig.getConfigSetFolder()), e); + } finally { + if (tmpDir.exists()) { + try { + FileUtils.deleteDirectory(tmpDir); + } catch (IOException e){ + LOG.error("Cannot delete temp directory.", e); + } + } + } + return reloadCollectionNeeded; + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/StatusResource.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/StatusResource.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/StatusResource.java new file mode 100644 index 0000000..8cc6b94 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/StatusResource.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.rest; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.apache.ambari.logsearch.conf.global.SolrCollectionState; +import org.springframework.context.annotation.Scope; + +import javax.inject.Inject; +import javax.inject.Named; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.ambari.logsearch.doc.DocConstants.StatusOperationDescriptions.AUDIT_LOGS_STATUS_OD; +import static org.apache.ambari.logsearch.doc.DocConstants.StatusOperationDescriptions.SERVICE_LOGS_STATUS_OD; +import static org.apache.ambari.logsearch.doc.DocConstants.StatusOperationDescriptions.STATUS_OD; +import static org.apache.ambari.logsearch.doc.DocConstants.StatusOperationDescriptions.USER_CONFIG_STATUS_OD; + +@Api(value = "status", description = "Status Operations") +@Path("status") +@Named +@Scope("request") +public class StatusResource { + + @Inject + @Named("solrServiceLogsState") + private SolrCollectionState solrServiceLogsState; + + @Inject + @Named("solrAuditLogsState") + private SolrCollectionState solrAuditLogsState; + + @Inject + @Named("solrUserConfigState") + private SolrCollectionState solrUserConfigState; + + @GET + @Produces({"application/json"}) + @ApiOperation(STATUS_OD) + public Map<String, SolrCollectionState> getStatus() { + Map<String, SolrCollectionState> response = new HashMap<>(); + response.put("serviceLogs", solrServiceLogsState); + response.put("auditLogs", solrAuditLogsState); + response.put("userConfig", solrUserConfigState); + return response; + } + + @GET + @Path("/servicelogs") + @Produces({"application/json"}) + @ApiOperation(SERVICE_LOGS_STATUS_OD) + public SolrCollectionState getServiceLogStatus() { + return solrServiceLogsState; + } + + @GET + @Path("/auditlogs") + @Produces({"application/json"}) + @ApiOperation(AUDIT_LOGS_STATUS_OD) + public SolrCollectionState getSolrAuditLogsStatus() { + return solrAuditLogsState; + } + + @GET + @Path("/userconfig") + @Produces({"application/json"}) + @ApiOperation(USER_CONFIG_STATUS_OD) + public SolrCollectionState getSolrUserConfigStatus() { + return solrUserConfigState; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/RESTErrorUtil.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/RESTErrorUtil.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/RESTErrorUtil.java index 532428b..b1a53f2 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/RESTErrorUtil.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/RESTErrorUtil.java @@ -41,14 +41,18 @@ public class RESTErrorUtil { return createRESTException(response, HttpServletResponse.SC_BAD_REQUEST); } - public static WebApplicationException createRESTException(String errorMessage, MessageEnums messageEnum) { - List<MessageData> messageList = new ArrayList<MessageData>(); + public static VResponse createMessageResponse(String errorMessage, MessageEnums messageEnum) { + List<MessageData> messageList = new ArrayList<>(); messageList.add(messageEnum.getMessage()); - VResponse response = new VResponse(); response.setStatusCode(VResponse.STATUS_ERROR); response.setMsgDesc(errorMessage); response.setMessageList(messageList); + return response; + } + + public static WebApplicationException createRESTException(String errorMessage, MessageEnums messageEnum) { + VResponse response = createMessageResponse(errorMessage, messageEnum); WebApplicationException webAppEx = createRESTException(response); logger.error("Operation error. response=" + response, webAppEx); return webAppEx; http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/AbstractLogsearchGlobalStateFilter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/AbstractLogsearchGlobalStateFilter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/AbstractLogsearchGlobalStateFilter.java new file mode 100644 index 0000000..4d529fb --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/AbstractLogsearchGlobalStateFilter.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.web.filters; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.ambari.logsearch.common.MessageEnums; +import org.apache.ambari.logsearch.common.VResponse; +import org.apache.ambari.logsearch.conf.SolrPropsConfig; +import org.apache.ambari.logsearch.conf.global.SolrCollectionState; +import org.apache.ambari.logsearch.util.RESTErrorUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.security.web.util.matcher.RequestMatcher; + +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +abstract public class AbstractLogsearchGlobalStateFilter implements Filter { + + private static final Logger LOG = LoggerFactory.getLogger(AbstractLogsearchGlobalStateFilter.class); + + protected static final String ZNODE_NOT_READY_MSG = "ZNode is not available for %s. (connection string: %s, endpoint: %s)"; + protected static final String ZK_CONFIG_NOT_READY_MSG = "Collection configuration has not uploaded yet for %s. (configuration name: %s, collection name: %s, endpoint: %s)"; + protected static final String SOLR_COLLECTION_NOT_READY_MSG = "Solr has not accessible yet for %s collection. (endpoint: %s)"; + + private SolrCollectionState solrCollectionState; + private RequestMatcher requestMatcher; + private SolrPropsConfig solrPropsConfig; + + public AbstractLogsearchGlobalStateFilter(RequestMatcher requestMatcher, SolrCollectionState state, SolrPropsConfig solrPropsConfig) { + this.requestMatcher = requestMatcher; + this.solrCollectionState = state; + this.solrPropsConfig = solrPropsConfig; + } + + @Override + public void init(FilterConfig filterConfig) throws ServletException { + } + + @Override + public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { + HttpServletRequest request = (HttpServletRequest) servletRequest; + if (requestMatcher.matches(request)) { + VResponse errorResponse = getErrorResponse(solrCollectionState, solrPropsConfig, request); + if (errorResponse != null) { + LOG.info("{} request is filtered out: {}", request.getRequestURL(), errorResponse.getMsgDesc()); + HttpServletResponse resp = (HttpServletResponse) servletResponse; + resp.setStatus(500); + resp.setContentType("application/json"); + resp.getWriter().print(createStringFromErrorMessageObject(errorResponse)); + return; + } + } + filterChain.doFilter(servletRequest, servletResponse); + } + + @Override + public void destroy() { + } + + /** + * Fill the error message with data in case of any condition based on collection state. + * If return value is null, that means there were no any errors + * @param solrCollectionState object to store solr state (e.g. : zookeeper/solr collection availability) + * @param solrPropsConfig object to store details for solr collection (e.g. : collection name, connection string) + */ + public abstract VResponse getErrorResponse(SolrCollectionState solrCollectionState, SolrPropsConfig solrPropsConfig, HttpServletRequest request); + + private String createStringFromErrorMessageObject(VResponse responseObject) { + try { + ObjectMapper mapper = new ObjectMapper(); + return mapper.writeValueAsString(responseObject); + } catch (Exception e) { + throw RESTErrorUtil.createRESTException("Cannot parse response object on backend", MessageEnums.ERROR_CREATING_OBJECT); + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchAuditLogsStateFilter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchAuditLogsStateFilter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchAuditLogsStateFilter.java new file mode 100644 index 0000000..117fdd4 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchAuditLogsStateFilter.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.web.filters; + +import org.apache.ambari.logsearch.common.MessageEnums; +import org.apache.ambari.logsearch.common.VResponse; +import org.apache.ambari.logsearch.conf.SolrPropsConfig; +import org.apache.ambari.logsearch.conf.global.SolrCollectionState; +import org.apache.ambari.logsearch.util.RESTErrorUtil; +import org.springframework.security.web.util.matcher.RequestMatcher; + +import javax.servlet.http.HttpServletRequest; + +public class LogsearchAuditLogsStateFilter extends AbstractLogsearchGlobalStateFilter { + + public LogsearchAuditLogsStateFilter(RequestMatcher requestMatcher, SolrCollectionState state, SolrPropsConfig solrPropsConfig) { + super(requestMatcher, state, solrPropsConfig); + } + + @Override + public VResponse getErrorResponse(SolrCollectionState solrCollectionState, SolrPropsConfig solrPropsConfig, HttpServletRequest request) { + String requestUri = request.getRequestURI(); + if (!solrCollectionState.isZnodeReady()) { + return RESTErrorUtil.createMessageResponse(String.format(ZNODE_NOT_READY_MSG, + "audit logs", solrPropsConfig.getZkConnectString(), requestUri), MessageEnums.ZNODE_NOT_READY); + } else if (!solrCollectionState.isConfigurationUploaded()) { + return RESTErrorUtil.createMessageResponse(String.format(ZK_CONFIG_NOT_READY_MSG, "audit logs", + solrPropsConfig.getConfigName(), solrPropsConfig.getCollection(), requestUri), MessageEnums.ZK_CONFIG_NOT_READY); + } else if (!solrCollectionState.isSolrCollectionReady()) { + return RESTErrorUtil.createMessageResponse(String.format(SOLR_COLLECTION_NOT_READY_MSG, + solrPropsConfig.getCollection(), requestUri), MessageEnums.SOLR_COLLECTION_NOT_READY); + } + return null; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchServiceLogsStateFilter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchServiceLogsStateFilter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchServiceLogsStateFilter.java new file mode 100644 index 0000000..74b30e1 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchServiceLogsStateFilter.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.web.filters; + +import org.apache.ambari.logsearch.common.MessageEnums; +import org.apache.ambari.logsearch.common.VResponse; +import org.apache.ambari.logsearch.conf.SolrPropsConfig; +import org.apache.ambari.logsearch.conf.global.SolrCollectionState; +import org.apache.ambari.logsearch.util.RESTErrorUtil; +import org.springframework.security.web.util.matcher.RequestMatcher; + +import javax.servlet.http.HttpServletRequest; + +public class LogsearchServiceLogsStateFilter extends AbstractLogsearchGlobalStateFilter { + + public LogsearchServiceLogsStateFilter(RequestMatcher requestMatcher, SolrCollectionState state, SolrPropsConfig solrPropsConfig) { + super(requestMatcher, state, solrPropsConfig); + } + + @Override + public VResponse getErrorResponse(SolrCollectionState solrCollectionState, SolrPropsConfig solrPropsConfig, HttpServletRequest request) { + String requestUri = request.getRequestURI(); + if (!solrCollectionState.isZnodeReady()) { + return RESTErrorUtil.createMessageResponse(String.format(ZNODE_NOT_READY_MSG, + "service logs", solrPropsConfig.getZkConnectString(), requestUri), MessageEnums.ZNODE_NOT_READY); + } else if (!solrCollectionState.isConfigurationUploaded()) { + return RESTErrorUtil.createMessageResponse(String.format(ZK_CONFIG_NOT_READY_MSG, "service logs", + solrPropsConfig.getConfigName(), solrPropsConfig.getCollection(), requestUri), MessageEnums.ZK_CONFIG_NOT_READY); + } else if (!solrCollectionState.isSolrCollectionReady()) { + return RESTErrorUtil.createMessageResponse(String.format(SOLR_COLLECTION_NOT_READY_MSG, + solrPropsConfig.getCollection(), requestUri), MessageEnums.SOLR_COLLECTION_NOT_READY); + } + return null; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchUserConfigStateFilter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchUserConfigStateFilter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchUserConfigStateFilter.java new file mode 100644 index 0000000..037bed0 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchUserConfigStateFilter.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.web.filters; + +import org.apache.ambari.logsearch.common.MessageEnums; +import org.apache.ambari.logsearch.common.VResponse; +import org.apache.ambari.logsearch.conf.SolrPropsConfig; +import org.apache.ambari.logsearch.conf.global.SolrCollectionState; +import org.apache.ambari.logsearch.util.RESTErrorUtil; +import org.springframework.security.web.util.matcher.RequestMatcher; + +import javax.servlet.http.HttpServletRequest; + +public class LogsearchUserConfigStateFilter extends AbstractLogsearchGlobalStateFilter { + + + public LogsearchUserConfigStateFilter(RequestMatcher requestMatcher, SolrCollectionState state, SolrPropsConfig solrPropsConfig) { + super(requestMatcher, state, solrPropsConfig); + } + + @Override + public VResponse getErrorResponse(SolrCollectionState solrCollectionState, SolrPropsConfig solrPropsConfig, HttpServletRequest request) { + String requestUri = request.getRequestURI(); + if (!solrCollectionState.isZnodeReady()) { + return RESTErrorUtil.createMessageResponse(String.format(ZNODE_NOT_READY_MSG, + "userconfig", solrPropsConfig.getZkConnectString(), requestUri), MessageEnums.ZNODE_NOT_READY); + } else if (!solrCollectionState.isConfigurationUploaded()) { + return RESTErrorUtil.createMessageResponse(String.format(ZK_CONFIG_NOT_READY_MSG, "userconfig", + solrPropsConfig.getConfigName(), solrPropsConfig.getCollection(), requestUri), MessageEnums.ZK_CONFIG_NOT_READY); + } else if (!solrCollectionState.isSolrCollectionReady()) { + return RESTErrorUtil.createMessageResponse(String.format(SOLR_COLLECTION_NOT_READY_MSG, + solrPropsConfig.getCollection(), requestUri), MessageEnums.SOLR_COLLECTION_NOT_READY); + } + return null; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Utils.js ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Utils.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Utils.js index 85c8245..15322ef 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Utils.js +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Utils.js @@ -411,8 +411,15 @@ define(['require', try { if (!errorShown) { errorShown = true; + var errorMessage = "Some issues on server, Please try again later." + if (error != null && error.responseText != null) { + var errorObj = JSON.parse(error.responseText); + if (errorObj.hasOwnProperty('msgDesc')) { + errorMessage = errorObj.msgDesc; + } + } Utils.notifyError({ - content: "Some issue on server, Please try again later." + content: errorMessage }); setTimeout(function() { errorShown = false; http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java index fa6162a..6bb5a77 100644 --- a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java +++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java @@ -20,7 +20,6 @@ package org.apache.ambari.logsearch.solr; import org.apache.ambari.logsearch.solr.commands.CheckConfigZkCommand; import org.apache.ambari.logsearch.solr.commands.CreateCollectionCommand; -import org.apache.ambari.logsearch.solr.commands.CreateSaslUsersZkCommand; import org.apache.ambari.logsearch.solr.commands.CreateShardCommand; import org.apache.ambari.logsearch.solr.commands.CreateSolrZnodeZkCommand; import org.apache.ambari.logsearch.solr.commands.DownloadConfigZkCommand; @@ -34,7 +33,6 @@ import org.apache.ambari.logsearch.solr.commands.SetClusterPropertyZkCommand; import org.apache.ambari.logsearch.solr.commands.UploadConfigZkCommand; import org.apache.ambari.logsearch.solr.commands.CheckZnodeZkCommand; import org.apache.ambari.logsearch.solr.util.ShardUtils; -import org.apache.commons.lang.StringUtils; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.SolrZkClient; @@ -129,18 +127,6 @@ public class AmbariSolrCloudClient { } /** - * Add sasl user (only if not exist on the znode). - */ - public String addSaslUsers() throws Exception { - LOG.info("Add sasl user to znode: {}", this.saslUsers); - String newUsers = new CreateSaslUsersZkCommand(getRetryTimes(), getInterval()).run(this); - if (StringUtils.isNotEmpty(newUsers)) { - LOG.info("New sasl users added to znode: {}", newUsers); - } - return newUsers; - } - - /** * Create a znode only if it does not exist. Return 0 code if it exists. */ public void createZnode() throws Exception { http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CreateSaslUsersZkCommand.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CreateSaslUsersZkCommand.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CreateSaslUsersZkCommand.java deleted file mode 100644 index b144b41..0000000 --- a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CreateSaslUsersZkCommand.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.ambari.logsearch.solr.commands; - -import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient; -import org.apache.ambari.logsearch.solr.util.AclUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.SolrZooKeeper; -import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.data.ACL; -import org.apache.zookeeper.data.Id; -import org.apache.zookeeper.data.Stat; - -import java.util.ArrayList; -import java.util.List; -import java.util.Set; - -public class CreateSaslUsersZkCommand extends AbstractZookeeperRetryCommand<String> { - - public CreateSaslUsersZkCommand(int maxRetries, int interval) { - super(maxRetries, interval); - } - - @Override - protected String executeZkCommand(AmbariSolrCloudClient client, SolrZkClient zkClient, SolrZooKeeper solrZooKeeper) throws Exception { - List<ACL> acls = solrZooKeeper.getACL(client.getZnode(), new Stat()); - List<String> newUsers = new ArrayList<>(); - Set<String> existingUsers = AclUtils.getUsersFromAclData(acls); - String saslUsers = client.getSaslUsers(); - if (StringUtils.isNotEmpty(saslUsers)) { - String[] saslUserNames = saslUsers.split(","); - for (String saslUser : saslUserNames) { - if (!existingUsers.contains(saslUser)) { - acls.add(new ACL(ZooDefs.Perms.ALL, new Id("sasl", saslUser))); - newUsers.add(saslUser); - } - } - } - acls = AclUtils.updatePermissionForScheme(acls, "world", ZooDefs.Perms.READ); - solrZooKeeper.setACL(client.getZnode(), acls, -1); - return StringUtils.join(newUsers, ","); - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SecureSolrZNodeZkCommand.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SecureSolrZNodeZkCommand.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SecureSolrZNodeZkCommand.java index faa21d4..31ad5d3 100644 --- a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SecureSolrZNodeZkCommand.java +++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SecureSolrZNodeZkCommand.java @@ -23,6 +23,8 @@ import org.apache.ambari.logsearch.solr.util.AclUtils; import org.apache.commons.lang.StringUtils; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.SolrZooKeeper; +import org.apache.solr.common.cloud.ZkConfigManager; +import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Id; @@ -52,9 +54,13 @@ public class SecureSolrZNodeZkCommand extends AbstractZookeeperRetryCommand<Bool String configsPath = String.format("%s/%s", zNode, "configs"); String collectionsPath = String.format("%s/%s", zNode, "collections"); - List<String> exlustePaths = Arrays.asList(configsPath, collectionsPath); + String aliasesPath = String.format("%s/%s", zNode, "aliases.json"); // TODO: protect this later somehow + List<String> excludePaths = Arrays.asList(configsPath, collectionsPath, aliasesPath); - AclUtils.setRecursivelyOn(client.getSolrZkClient().getSolrZooKeeper(), zNode, newAclList, exlustePaths); + createZnodeIfNeeded(configsPath, client.getSolrZkClient()); + createZnodeIfNeeded(collectionsPath, client.getSolrZkClient()); + + AclUtils.setRecursivelyOn(client.getSolrZkClient().getSolrZooKeeper(), zNode, newAclList, excludePaths); List<ACL> commonConfigAcls = new ArrayList<>(); commonConfigAcls.addAll(saslUserList); @@ -71,4 +77,11 @@ public class SecureSolrZNodeZkCommand extends AbstractZookeeperRetryCommand<Bool return true; } + + private void createZnodeIfNeeded(String configsPath, SolrZkClient zkClient) throws KeeperException, InterruptedException { + if (!zkClient.exists(configsPath, true)) { + LOG.info("'{}' does not exist. Creating it ...", configsPath); + zkClient.makePath(configsPath, true); + } + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/util/AclUtils.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/util/AclUtils.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/util/AclUtils.java index 8e0132d..fc25c49 100644 --- a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/util/AclUtils.java +++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/util/AclUtils.java @@ -34,36 +34,6 @@ import java.util.Set; public class AclUtils { - public static Set<String> getUsersFromAclData(List<ACL> acls) { - Set<String> result = new HashSet<>(); - if (!acls.isEmpty()) { - for (ACL acl : acls) { - String username = ""; - String id = acl.getId().getId(); - String[] splitted = id.split(":"); - if (splitted.length > 1) { - username = splitted[0]; - } else { - username = id; - } - result.add(username); - } - } - return result; - } - - public static List<ACL> updatePermissionForScheme(List<ACL> acls, String scheme, int permission) { - List<ACL> aclResult = new ArrayList<>(); - if (!acls.isEmpty()) { - for (ACL acl : acls) { - int permissionToAdd = scheme.equals(acl.getId().getScheme()) ? permission : acl.getPerms(); - acl.setPerms(permissionToAdd); - aclResult.add(acl); - } - } - return aclResult; - } - public static List<ACL> mergeAcls(List<ACL> originalAcls, List<ACL> updateAcls) { Map<String, ACL> aclMap = new HashMap<>(); List<ACL> acls = new ArrayList<>(); http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/docker/bin/start.sh ---------------------------------------------------------------------- diff --git a/ambari-logsearch/docker/bin/start.sh b/ambari-logsearch/docker/bin/start.sh index 81c9e66..4c60981 100644 --- a/ambari-logsearch/docker/bin/start.sh +++ b/ambari-logsearch/docker/bin/start.sh @@ -81,12 +81,6 @@ function start_solr() { } function start_logsearch() { - echo "Upload configuration sets ..." - - $ZKCLI -zkhost localhost:9983 -cmd upconfig -confdir $LOGSEARCH_SERVER_PATH/solr_configsets/audit_logs/conf -confname audit_logs - $ZKCLI -zkhost localhost:9983 -cmd upconfig -confdir $LOGSEARCH_SERVER_PATH/solr_configsets/hadoop_logs/conf -confname hadoop_logs - $ZKCLI -zkhost localhost:9983 -cmd upconfig -confdir $LOGSEARCH_SERVER_PATH/solr_configsets/history/conf -confname history - $LOGSEARCH_SERVER_PATH/run.sh touch /var/log/ambari-logsearch-portal/logsearch-app.log }
