http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudCLI.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudCLI.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudCLI.java index 1e3cca5..6035c86 100644 --- a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudCLI.java +++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudCLI.java @@ -37,19 +37,31 @@ public class AmbariSolrCloudCLI { private static final Logger LOG = LoggerFactory.getLogger(AmbariSolrCloudCLI.class); - private static final int ZK_CLIENT_TIMEOUT = 10000; - private static final int ZK_CLIENT_CONNECT_TIMEOUT = 10000; + private static final int ZK_CLIENT_TIMEOUT = 15000; + private static final int ZK_CLIENT_CONNECT_TIMEOUT = 15000; private static final String CREATE_COLLECTION_COMMAND = "create-collection"; private static final String UPLOAD_CONFIG_COMMAND = "upload-config"; private static final String DOWNLOAD_CONFIG_COMMAND = "download-config"; private static final String CONFIG_CHECK_COMMAND = "check-config"; private static final String CREATE_SHARD_COMMAND = "create-shard"; + private static final String CREATE_ZNODE = "create-znode"; + private static final String SET_CLUSTER_PROP = "cluster-prop"; + private static final String SETUP_KERBEROS = "setup-kerberos"; + private static final String SETUP_KERBEROS_PLUGIN = "setup-kerberos-plugin"; + private static final String CREATE_SASL_USERS = "create-sasl-users"; + private static final String CHECK_ZNODE = "check-znode"; private static final String CMD_LINE_SYNTAX = "\n./solrCloudCli.sh --create-collection -z host1:2181,host2:2181/ambari-solr -c collection -cs conf_set" + "\n./solrCloudCli.sh --upload-config -z host1:2181,host2:2181/ambari-solr -d /tmp/myconfig_dir -cs config_set" + "\n./solrCloudCli.sh --download-config -z host1:2181,host2:2181/ambari-solr -cs config_set -d /tmp/myonfig_dir" + "\n./solrCloudCli.sh --check-config -z host1:2181,host2:2181/ambari-solr -cs config_set" - + "\n./solrCloudCli.sh --create-shard -z host1:2181,host2:2181/ambari-solr -c collection -sn myshard\n"; + + "\n./solrCloudCli.sh --create-shard -z host1:2181,host2:2181/ambari-solr -c collection -sn myshard" + + "\n./solrCloudCli.sh --create-znode -z host1:2181,host2:2181 -zn /ambari-solr" + + "\n./solrCloudCli.sh --check-znode -z host1:2181,host2:2181 -zn /ambari-solr" + + "\n./solrCloudCli.sh --cluster-prop -z host1:2181,host2:2181/ambari-solr -cpn urlScheme -cpn http" + + "\n./solrCloudCli.sh --create-sasl-users -z host1:2181,host2:2181 -zn /ambari-solr -csu logsearch,atlas,ranger" + + "\n./solrCloudCli.sh --setup-kerberos -z host1:2181,host2:2181 --secure -zn /ambari-solr-secure -cfz /ambari-solr-unsecure -jf /etc/path/my_jaas.conf" + + "\n./solrCloudCli.sh --setup-kerberos-plugin -z host1:2181,host2:2181 -zn /ambari-solr\n"; public static void main(String[] args) { Options options = new Options(); @@ -82,11 +94,41 @@ public class AmbariSolrCloudCLI { .desc("Check configuration exists in Zookeeper (command)") .build(); + final Option checkZnodeOption = Option.builder("chz") + .longOpt(CHECK_ZNODE) + .desc("Check znode exists in Zookeeper (command)") + .build(); + final Option createShardOption = Option.builder("csh") .longOpt(CREATE_SHARD_COMMAND) .desc("Create shard in Solr (command)") .build(); + final Option setClusterPropOption = Option.builder("cp") + .longOpt(SET_CLUSTER_PROP) + .desc("Set cluster property (command)") + .build(); + + final Option createZnodeOption = Option.builder("cz") + .longOpt(CREATE_ZNODE) + .desc("Create Znode (command)") + .build(); + + final Option setupKerberosPluginOption = Option.builder("skp") + .longOpt(SETUP_KERBEROS_PLUGIN) + .desc("Setup kerberos plugin in security.json (command)") + .build(); + + final Option setupKerberosOption = Option.builder("sk") + .longOpt(SETUP_KERBEROS) + .desc("Setup kerberos (command)") + .build(); + + final Option createSaslUsersOption = Option.builder("csu") + .longOpt(CREATE_SASL_USERS) + .desc("Create sasl users") + .build(); + final Option shardNameOption = Option.builder("sn") .longOpt("shard-name") .desc("Name of the shard for create-shard command") @@ -101,11 +143,18 @@ public class AmbariSolrCloudCLI { final Option zkConnectStringOption = Option.builder("z") .longOpt("zookeeper-connect-string") - .desc("Zookeeper quorum [and a Znode]") + .desc("Zookeeper quorum [and Znode (optional)]") .numberOfArgs(1) .argName("host:port,host:port[/ambari-solr]") .build(); + final Option znodeOption = Option.builder("zn") + .longOpt("znode") + .desc("Zookeeper ZNode") + .numberOfArgs(1) + .argName("/ambari-solr") + .build(); + final Option collectionOption = Option.builder("c") .longOpt("collection") .desc("Collection name") @@ -229,6 +278,39 @@ public class AmbariSolrCloudCLI { .argName("trust store type") .build(); + final Option propNameOption = Option.builder("cpn") + .longOpt("property-name") + .desc("Cluster property name") + .numberOfArgs(1) + .argName("cluster prop name") + .build(); + + final Option propValueOption = Option.builder("cpv") + .longOpt("property-value") + .desc("Cluster property value") + .numberOfArgs(1) + .argName("cluster prop value") + .build(); + + final Option copyFromZnodeOption = Option.builder("cfz") + .longOpt("copy-from-znode") + .desc("Copy-from-znode") + .numberOfArgs(1) + .argName("/ambari-solr-secure") + .build(); + + final Option saslUsersOption = Option.builder("su") + .longOpt("sasl-users") + .desc("Sasl users (comma separated list)") + .numberOfArgs(1) + .argName("atlas,ranger,logsearch-solr") + .build(); + + final Option secureOption = Option.builder("sec") + .longOpt("secure") + .desc("Flag for enable/disable kerberos (with --setup-kerberos or --setup-kerberos-plugin)") + .build(); + options.addOption(helpOption); options.addOption(retryOption); options.addOption(intervalOption); @@ -255,7 +337,18 @@ public class AmbariSolrCloudCLI { options.addOption(trustStoreLocationOption); options.addOption(trustStorePasswordOption); options.addOption(trustStoreTypeOption); - + options.addOption(setClusterPropOption); + options.addOption(propNameOption); + options.addOption(propValueOption); + options.addOption(createZnodeOption); + options.addOption(znodeOption); + options.addOption(setupKerberosOption); + options.addOption(secureOption); + options.addOption(copyFromZnodeOption); + options.addOption(createSaslUsersOption); + options.addOption(saslUsersOption); + options.addOption(checkZnodeOption); + options.addOption(setupKerberosPluginOption); try { CommandLineParser cmdLineParser = new DefaultParser(); @@ -281,9 +374,28 @@ public class AmbariSolrCloudCLI { } else if (cli.hasOption("chc")) { command = CONFIG_CHECK_COMMAND; validateRequiredOptions(cli, command, zkConnectStringOption, configSetOption); + } else if (cli.hasOption("cp")) { + command = SET_CLUSTER_PROP; + validateRequiredOptions(cli, command, zkConnectStringOption, propNameOption, propValueOption); + } else if (cli.hasOption("cz")) { + command = CREATE_ZNODE; + validateRequiredOptions(cli, command, zkConnectStringOption, znodeOption); + } else if (cli.hasOption("sk")) { + command = SETUP_KERBEROS; + validateRequiredOptions(cli, command, zkConnectStringOption, copyFromZnodeOption); + } else if (cli.hasOption("csu")){ + command = CREATE_SASL_USERS; + validateRequiredOptions(cli, command, zkConnectStringOption, znodeOption, jaasFileOption); + } else if (cli.hasOption("chz")){ + command = CHECK_ZNODE; + validateRequiredOptions(cli, command, zkConnectStringOption, znodeOption); + } else if (cli.hasOption("skp")) { + command = SETUP_KERBEROS_PLUGIN; + validateRequiredOptions(cli, command, zkConnectStringOption, znodeOption); } else { List<String> commands = Arrays.asList(CREATE_COLLECTION_COMMAND, CREATE_SHARD_COMMAND, UPLOAD_CONFIG_COMMAND, - DOWNLOAD_CONFIG_COMMAND, CONFIG_CHECK_COMMAND); + DOWNLOAD_CONFIG_COMMAND, CONFIG_CHECK_COMMAND, SET_CLUSTER_PROP, CREATE_ZNODE, SETUP_KERBEROS, + CHECK_ZNODE, CREATE_SASL_USERS, SETUP_KERBEROS_PLUGIN); helpFormatter.printHelp(CMD_LINE_SYNTAX, options); exit(1, String.format("One of the supported commands is required (%s)", StringUtils.join(commands, "|"))); } @@ -308,7 +420,12 @@ public class AmbariSolrCloudCLI { String trustStoreLocation = cli.hasOption("tsl") ? cli.getOptionValue("tsl") : null; String trustStorePassword = cli.hasOption("tsp") ? cli.getOptionValue("tsp") : null; String trustStoreType = cli.hasOption("tst") ? cli.getOptionValue("tst") : null; - + String clusterPropName = cli.hasOption("cpn") ? cli.getOptionValue("cpn") : null; + String clusterPropValue = cli.hasOption("cpv") ? cli.getOptionValue("cpv") : null; + String znode = cli.hasOption("zn") ? cli.getOptionValue("zn") : null; + boolean isSecure = cli.hasOption("sec"); + String copyFromZnode = cli.hasOption("cfz") ? cli.getOptionValue("cfz") : null; + String saslUsers = cli.hasOption("su") ? cli.getOptionValue("su") : null; AmbariSolrCloudClientBuilder clientBuilder = new AmbariSolrCloudClientBuilder() .withZkConnectString(zkConnectString) @@ -329,7 +446,13 @@ public class AmbariSolrCloudCLI { .withKeyStoreType(keyStoreType) .withTrustStoreLocation(trustStoreLocation) .withTrustStorePassword(trustStorePassword) - .withTrustStoreType(trustStoreType); + .withTrustStoreType(trustStoreType) + .withClusterPropName(clusterPropName) + .withClusterPropValue(clusterPropValue) + .withZnode(znode) + .withCopyFromZnode(copyFromZnode) + .withSecure(isSecure) + .withSaslUsers(saslUsers); AmbariSolrCloudClient solrCloudClient = null; switch (command) { @@ -361,10 +484,41 @@ public class AmbariSolrCloudCLI { case CREATE_SHARD_COMMAND: solrCloudClient = clientBuilder .withSolrCloudClient() - .withSolrZkClient(ZK_CLIENT_TIMEOUT, ZK_CLIENT_CONNECT_TIMEOUT) .build(); solrCloudClient.createShard(shardName); break; + case SET_CLUSTER_PROP: + solrCloudClient = clientBuilder.build(); + solrCloudClient.setClusterProp(); + break; + case CREATE_ZNODE: + solrCloudClient = clientBuilder.build(); + solrCloudClient.createZnode(); + break; + case SETUP_KERBEROS: + solrCloudClient = clientBuilder.build(); + if (solrCloudClient.isSecure()) { + solrCloudClient.enableKerberos(); + } else { + solrCloudClient.disableKerberos(); + } + break; + case CREATE_SASL_USERS: + solrCloudClient = clientBuilder.build(); + solrCloudClient.addSaslUsers(); + break; + case CHECK_ZNODE: + solrCloudClient = clientBuilder.build(); + boolean znodeExists = solrCloudClient.isZnodeExists(znode); + if (!znodeExists) { + exit(1, String.format("'%s' znode does not exist. Solr is responsible to create the ZNode, " + + "check Solr started successfully or not", znode)); + } + break; + case SETUP_KERBEROS_PLUGIN: + solrCloudClient = clientBuilder.build(); + solrCloudClient.setupKerberosPlugin(); + break; default: throw new AmbariSolrCloudClientException(String.format("Not found command: '%s'", command)); }
http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java index 33e94d0..20096fe 100644 --- a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java +++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java @@ -19,14 +19,25 @@ package org.apache.ambari.logsearch.solr; import org.apache.ambari.logsearch.solr.commands.CheckConfigZkCommand; +import org.apache.ambari.logsearch.solr.commands.CopyZnodeZkCommand; import org.apache.ambari.logsearch.solr.commands.CreateCollectionCommand; +import org.apache.ambari.logsearch.solr.commands.CreateSaslUsersZkCommand; import org.apache.ambari.logsearch.solr.commands.CreateShardCommand; +import org.apache.ambari.logsearch.solr.commands.CreateSolrZnodeZkCommand; import org.apache.ambari.logsearch.solr.commands.DownloadConfigZkCommand; +import org.apache.ambari.logsearch.solr.commands.EnableKerberosPluginSolrZkCommand; import org.apache.ambari.logsearch.solr.commands.GetShardsCommand; import org.apache.ambari.logsearch.solr.commands.GetSolrHostsCommand; +import org.apache.ambari.logsearch.solr.commands.GetStateFileZkCommand; import org.apache.ambari.logsearch.solr.commands.ListCollectionCommand; +import org.apache.ambari.logsearch.solr.commands.SecureZNodeZkCommand; +import org.apache.ambari.logsearch.solr.commands.SetClusterPropertyZkCommand; +import org.apache.ambari.logsearch.solr.commands.UpdateStateFileZkCommand; import org.apache.ambari.logsearch.solr.commands.UploadConfigZkCommand; +import org.apache.ambari.logsearch.solr.commands.CheckZnodeZkCommand; +import org.apache.ambari.logsearch.solr.domain.AmbariSolrState; import org.apache.ambari.logsearch.solr.util.ShardUtils; +import org.apache.commons.lang.StringUtils; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.SolrZkClient; @@ -57,7 +68,13 @@ public class AmbariSolrCloudClient { private final String routerName; private final String routerField; private final boolean splitting; - private String jaasFile; + private final String jaasFile; + private final String znode; + private final String saslUsers; + private final String propName; + private final String propValue; + private final boolean secure; + private final String copyFromZnode; public AmbariSolrCloudClient(AmbariSolrCloudClientBuilder builder) { this.zkConnectString = builder.zkConnectString; @@ -75,6 +92,12 @@ public class AmbariSolrCloudClient { this.routerName = builder.routerName; this.routerField = builder.routerField; this.splitting = builder.splitting; + this.znode = builder.znode; + this.saslUsers = builder.saslUsers; + this.propName = builder.propName; + this.propValue = builder.propValue; + this.secure = builder.secure; + this.copyFromZnode = builder.copyFromZnode; } /** @@ -102,6 +125,93 @@ public class AmbariSolrCloudClient { } /** + * Set cluster property in clusterprops.json. + */ + public void setClusterProp() throws Exception { + LOG.info("Set cluster prop: '{}'", this.getPropName()); + String newPropValue = new SetClusterPropertyZkCommand(getRetryTimes(), getInterval()).run(this); + LOG.info("Set cluster prop '{}' successfully to '{}'", this.getPropName(), newPropValue); + } + + /** + * Add sasl user (only if not exist on the znode). + */ + public String addSaslUsers() throws Exception { + LOG.info("Add sasl user to znode: {}", this.saslUsers); + String newUsers = new CreateSaslUsersZkCommand(getRetryTimes(), getInterval()).run(this); + if (StringUtils.isNotEmpty(newUsers)) { + LOG.info("New sasl users added to znode: {}", newUsers); + } + return newUsers; + } + + /** + * Create a znode only if it does not exist. Return 0 code if it exists. + */ + public void createZnode() throws Exception { + boolean znodeExists = isZnodeExists(this.znode); + if (znodeExists) { + LOG.info("Znode '{}' already exists.", this.znode); + } else { + LOG.info("Znode '{}' does not exist. Creating...", this.znode); + String newZnode = new CreateSolrZnodeZkCommand(getRetryTimes(), getInterval()).run(this); + LOG.info("Znode '{}' is created successfully.", newZnode); + } + } + + /** + * Check znode exists or not based on the zookeeper connect string. + * E.g.: localhost:2181 and znode: /ambari-solr, checks existance of localhost:2181/ambari-solr + */ + public boolean isZnodeExists(String znode) throws Exception { + LOG.info("Check '{}' znode exists or not", znode); + boolean result = new CheckZnodeZkCommand(getRetryTimes(), getInterval(), znode).run(this); + if (result) { + LOG.info("'{}' znode exists", znode); + } else { + LOG.info("'{}' znode does not exist", znode); + } + return result; + } + + public void setupKerberosPlugin() throws Exception { + LOG.info("Setup kerberos plugin in security.json"); + new EnableKerberosPluginSolrZkCommand(getRetryTimes(), getInterval()).run(this); + LOG.info("KerberosPlugin is set in security.json"); + } + + public void enableKerberos() throws Exception { + LOG.info("Trying to enable kerberos from solr-client"); + AmbariSolrState actualState = new GetStateFileZkCommand(getRetryTimes(), getInterval(), this.copyFromZnode).run(this); + if (AmbariSolrState.UNSECURE.equals(actualState)) { + LOG.info("Enable kerberos. Copy unsecure znode ('{}') content to secure ('{}')", this.znode, this.copyFromZnode); + new CopyZnodeZkCommand(getRetryTimes(), getInterval()).run(this); + LOG.info("Copying contents from unsecure znode to secure znode has done."); + setupKerberosPlugin(); + new SecureZNodeZkCommand(getRetryTimes(), getInterval()).run(this); + LOG.info("Set world:anyone permission to READ only."); + AmbariSolrState state = new UpdateStateFileZkCommand(getRetryTimes(), getInterval(), this.copyFromZnode).run(this); + LOG.info("Enabling kerberos successful, state: {}", state); + } else { + LOG.info("It is not needed to enable kerberos."); + } + } + + public void disableKerberos() throws Exception { + LOG.info("Trying to disable kerberos from solr-client"); + AmbariSolrState actualState = new GetStateFileZkCommand(getRetryTimes(), getInterval(), this.znode).run(this); + if (AmbariSolrState.SECURE.equals(actualState)) { + LOG.info("Disable kerberos. Copy secure znode ('{}') content to unsecure ('{}')"); + new CopyZnodeZkCommand(getRetryTimes(), getInterval()).run(this); + LOG.info("Copying contents from secure znode to unsecure znode has done."); + AmbariSolrState state = new UpdateStateFileZkCommand(getRetryTimes(), getInterval(), this.znode).run(this); + LOG.info("Set new state: {}", state); + } else { + LOG.info("It is not needed to disable kerberos."); + } + } + + /** * Upload config set to zookeeper */ public String uploadConfiguration() throws Exception { @@ -231,4 +341,28 @@ public class AmbariSolrCloudClient { public String getJaasFile() { return jaasFile; } + + public String getSaslUsers() { + return saslUsers; + } + + public String getZnode() { + return znode; + } + + public String getPropName() { + return propName; + } + + public String getPropValue() { + return propValue; + } + + public boolean isSecure() { + return secure; + } + + public String getCopyFromZnode() { + return copyFromZnode; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClientBuilder.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClientBuilder.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClientBuilder.java index de90859..bad622b 100644 --- a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClientBuilder.java +++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClientBuilder.java @@ -47,6 +47,12 @@ public class AmbariSolrCloudClientBuilder { SolrZkClient solrZkClient; boolean splitting; String jaasFile; + String znode; + String saslUsers; + String propName; + String propValue; + boolean secure; + String copyFromZnode; public AmbariSolrCloudClient build() { return new AmbariSolrCloudClient(this); @@ -170,6 +176,36 @@ public class AmbariSolrCloudClientBuilder { return this; } + public AmbariSolrCloudClientBuilder withSaslUsers(String saslUsers) { + this.saslUsers = saslUsers; + return this; + } + + public AmbariSolrCloudClientBuilder withZnode(String znode) { + this.znode = znode; + return this; + } + + public AmbariSolrCloudClientBuilder withClusterPropName(String clusterPropName) { + this.propName = clusterPropName; + return this; + } + + public AmbariSolrCloudClientBuilder withClusterPropValue(String clusterPropValue) { + this.propValue = clusterPropValue; + return this; + } + + public AmbariSolrCloudClientBuilder withSecure(boolean isSecure) { + this.secure = isSecure; + return this; + } + + public AmbariSolrCloudClientBuilder withCopyFromZnode(String copyFromZnode) { + this.copyFromZnode = copyFromZnode; + return this; + } + private void setupSecurity(String jaasFile) { if (jaasFile != null) { System.setProperty("java.security.auth.login.config", jaasFile); http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/AbstractStateFileZkCommand.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/AbstractStateFileZkCommand.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/AbstractStateFileZkCommand.java new file mode 100644 index 0000000..664cd88 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/AbstractStateFileZkCommand.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.solr.commands; + +import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient; +import org.apache.ambari.logsearch.solr.domain.AmbariSolrState;; +import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.map.ObjectMapper; + +public abstract class AbstractStateFileZkCommand extends AbstractZookeeperRetryCommand<AmbariSolrState>{ + + public static final String STATE_FILE = "ambari-solr-state.json"; + public static final String STATE_FIELD = "ambari_solr_security_state"; + + public AbstractStateFileZkCommand(int maxRetries, int interval) { + super(maxRetries, interval); + } + + public AmbariSolrState getStateFromJson(AmbariSolrCloudClient client, String fileName) throws Exception { + byte[] data = client.getSolrZkClient().getData(fileName, null, null, true); + String input = new String(data); + ObjectMapper mapper = new ObjectMapper(); + JsonNode rootNode = mapper.readValue(input.getBytes(), JsonNode.class); + return AmbariSolrState.valueOf(rootNode.get(STATE_FIELD).asText()); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/AbstractZookeeperConfigCommand.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/AbstractZookeeperConfigCommand.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/AbstractZookeeperConfigCommand.java new file mode 100644 index 0000000..031f5f7 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/AbstractZookeeperConfigCommand.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.solr.commands; + +import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.SolrZooKeeper; +import org.apache.solr.common.cloud.ZkConfigManager; + +public abstract class AbstractZookeeperConfigCommand<RESPONSE> extends AbstractZookeeperRetryCommand<RESPONSE> { + + public AbstractZookeeperConfigCommand(int maxRetries, int interval) { + super(maxRetries, interval); + } + + protected abstract RESPONSE executeZkConfigCommand(ZkConfigManager zkConfigManager, AmbariSolrCloudClient client) + throws Exception; + + @Override + protected RESPONSE executeZkCommand(AmbariSolrCloudClient client, SolrZkClient zkClient, SolrZooKeeper solrZooKeeper) throws Exception { + ZkConfigManager zkConfigManager = createZkConfigManager(zkClient); + return executeZkConfigCommand(zkConfigManager, client); + } + + protected ZkConfigManager createZkConfigManager(SolrZkClient zkClient) { + return new ZkConfigManager(zkClient); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/AbstractZookeeperRetryCommand.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/AbstractZookeeperRetryCommand.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/AbstractZookeeperRetryCommand.java index c9e8558..9995ea6 100644 --- a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/AbstractZookeeperRetryCommand.java +++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/AbstractZookeeperRetryCommand.java @@ -20,7 +20,7 @@ package org.apache.ambari.logsearch.solr.commands; import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient; import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZkConfigManager; +import org.apache.solr.common.cloud.SolrZooKeeper; public abstract class AbstractZookeeperRetryCommand<RESPONSE> extends AbstractRetryCommand<RESPONSE> { @@ -28,17 +28,13 @@ public abstract class AbstractZookeeperRetryCommand<RESPONSE> extends AbstractRe super(maxRetries, interval); } - protected abstract RESPONSE executeZkCommand(ZkConfigManager zkConfigManager, AmbariSolrCloudClient client) + protected abstract RESPONSE executeZkCommand(AmbariSolrCloudClient client, SolrZkClient zkClient, SolrZooKeeper solrZooKeeper) throws Exception; @Override public RESPONSE createAndProcessRequest(AmbariSolrCloudClient client) throws Exception { SolrZkClient zkClient = client.getSolrZkClient(); - ZkConfigManager zkConfigManager = createZkConfigManager(zkClient); - return executeZkCommand(zkConfigManager, client); - } - - protected ZkConfigManager createZkConfigManager(SolrZkClient zkClient) { - return new ZkConfigManager(zkClient); + SolrZooKeeper solrZooKeeper = zkClient.getSolrZooKeeper(); + return executeZkCommand(client, zkClient, solrZooKeeper); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CheckConfigZkCommand.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CheckConfigZkCommand.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CheckConfigZkCommand.java index 819395c..182f049 100644 --- a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CheckConfigZkCommand.java +++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CheckConfigZkCommand.java @@ -21,14 +21,14 @@ package org.apache.ambari.logsearch.solr.commands; import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient; import org.apache.solr.common.cloud.ZkConfigManager; -public class CheckConfigZkCommand extends AbstractZookeeperRetryCommand<Boolean> { +public class CheckConfigZkCommand extends AbstractZookeeperConfigCommand<Boolean> { public CheckConfigZkCommand(int maxRetries, int interval) { super(maxRetries, interval); } @Override - protected Boolean executeZkCommand(ZkConfigManager zkConfigManager, AmbariSolrCloudClient client) throws Exception { + protected Boolean executeZkConfigCommand(ZkConfigManager zkConfigManager, AmbariSolrCloudClient client) throws Exception { return zkConfigManager.configExists(client.getConfigSet()); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CheckZnodeZkCommand.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CheckZnodeZkCommand.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CheckZnodeZkCommand.java new file mode 100644 index 0000000..6e82cc3 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CheckZnodeZkCommand.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.solr.commands; + +import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient; +import org.apache.ambari.logsearch.solr.AmbariSolrCloudClientException; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.SolrZooKeeper; +import org.apache.zookeeper.KeeperException; + +public class CheckZnodeZkCommand extends AbstractZookeeperRetryCommand<Boolean> { + + private String znode; + + public CheckZnodeZkCommand(int maxRetries, int interval, String znode) { + super(maxRetries, interval); + this.znode = znode; + } + + @Override + protected Boolean executeZkCommand(AmbariSolrCloudClient client, SolrZkClient zkClient, SolrZooKeeper solrZooKeeper) throws Exception { + try { + return zkClient.exists(this.znode, false); + } catch (KeeperException e) { + throw new AmbariSolrCloudClientException("Exception during checking znode, " + + "Check zookeeper servers are running (n+1/2) or zookeeper quorum has established or not.", e); + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CopyZnodeZkCommand.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CopyZnodeZkCommand.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CopyZnodeZkCommand.java new file mode 100644 index 0000000..a202930 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CopyZnodeZkCommand.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.solr.commands; + +import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.SolrZooKeeper; +import org.apache.zookeeper.CreateMode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +public class CopyZnodeZkCommand extends AbstractZookeeperRetryCommand<Boolean> { + + private static final Logger LOG = LoggerFactory.getLogger(CopyZnodeZkCommand.class); + + public CopyZnodeZkCommand(int maxRetries, int interval) { + super(maxRetries, interval); + } + + @Override + protected Boolean executeZkCommand(AmbariSolrCloudClient client, SolrZkClient zkClient, SolrZooKeeper solrZooKeeper) + throws Exception { + String znode = client.getZnode(); + String znodeCopyFromZnode = client.getCopyFromZnode(); + List<String> children = solrZooKeeper.getChildren(znodeCopyFromZnode, true); + children = filterRootChildren(children); + copyConent(znodeCopyFromZnode, znode, children, zkClient); + return true; + } + + private void copyConent(String srcParentPath, String destParentPath, List<String> children, SolrZkClient zkClient) + throws Exception { + if (!children.isEmpty()) { + for (String child : children) { + String srcPath = String.format("%s/%s", srcParentPath, child); + String destPath = String.format("%s/%s", destParentPath, child); + byte[] data = zkClient.getData(srcPath, null, null, true); + if (zkClient.exists(destPath, true)) { + zkClient.setData(destPath, data, true); + } else { + zkClient.create(destPath, data, CreateMode.PERSISTENT, true); + } + LOG.info("Copy file from '{}' to '{}'", srcPath, destPath); + copyConent(srcPath, destPath, zkClient.getChildren(srcPath, null, true), zkClient); + } + } + } + + private List<String> filterRootChildren(List<String> children) { + List<String> filteredResult = new ArrayList<>(); + if (!children.isEmpty()) { + for (String child : children) { + if (!child.equals("security.json") && !child.equals(AbstractStateFileZkCommand.STATE_FILE)){ + filteredResult.add(child); + } + } + } + return filteredResult; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CreateSaslUsersZkCommand.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CreateSaslUsersZkCommand.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CreateSaslUsersZkCommand.java new file mode 100644 index 0000000..1c25b9f --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CreateSaslUsersZkCommand.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.solr.commands; + +import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient; +import org.apache.ambari.logsearch.solr.util.AclUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.SolrZooKeeper; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Id; +import org.apache.zookeeper.data.Stat; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +public class CreateSaslUsersZkCommand extends AbstractZookeeperRetryCommand<String> { + + public CreateSaslUsersZkCommand(int maxRetries, int interval) { + super(maxRetries, interval); + } + + @Override + protected String executeZkCommand(AmbariSolrCloudClient client, SolrZkClient zkClient, SolrZooKeeper solrZooKeeper) throws Exception { + List<ACL> acls = solrZooKeeper.getACL(client.getZnode(), new Stat()); + List<String> newUsers = new ArrayList<>(); + Set<String> existingUsers = AclUtils.getUsersFromAclData(acls); + String saslUsers = client.getSaslUsers(); + if (StringUtils.isNotEmpty(saslUsers)) { + String[] saslUserNames = saslUsers.split(","); + for (String saslUser : saslUserNames) { + if (!existingUsers.contains(saslUser)) { + //solrZooKeeper.addAuthInfo("sasl", saslUser.getBytes(StandardCharsets.UTF_8)); + acls.add(new ACL(ZooDefs.Perms.ALL, new Id("sasl", saslUser))); + newUsers.add(saslUser); + } + } + } + solrZooKeeper.setACL(client.getZnode(), acls, -1); + return StringUtils.join(newUsers, ","); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CreateSolrZnodeZkCommand.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CreateSolrZnodeZkCommand.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CreateSolrZnodeZkCommand.java new file mode 100644 index 0000000..cd95e38 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CreateSolrZnodeZkCommand.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.solr.commands; + +import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient; +import org.apache.ambari.logsearch.solr.AmbariSolrCloudClientException; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.SolrZooKeeper; +import org.apache.zookeeper.KeeperException; + +public class CreateSolrZnodeZkCommand extends AbstractZookeeperRetryCommand<String> { + + public CreateSolrZnodeZkCommand(int maxRetries, int interval) { + super(maxRetries, interval); + } + + @Override + protected String executeZkCommand(AmbariSolrCloudClient client, SolrZkClient zkClient, SolrZooKeeper solrZooKeeper) throws Exception { + try { + zkClient.makePath(client.getZnode(), true); + return client.getZnode(); + } catch (KeeperException e) { + throw new AmbariSolrCloudClientException("Cannot create ZNode, check zookeeper servers are running (n+1/2), or zookeeper quorum has established or not.",e); + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/DownloadConfigZkCommand.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/DownloadConfigZkCommand.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/DownloadConfigZkCommand.java index 2a0fa31..9da4e53 100644 --- a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/DownloadConfigZkCommand.java +++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/DownloadConfigZkCommand.java @@ -19,22 +19,29 @@ package org.apache.ambari.logsearch.solr.commands; import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient; +import org.apache.ambari.logsearch.solr.AmbariSolrCloudClientException; import org.apache.solr.common.cloud.ZkConfigManager; +import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; -public class DownloadConfigZkCommand extends AbstractZookeeperRetryCommand<String> { +public class DownloadConfigZkCommand extends AbstractZookeeperConfigCommand<String> { public DownloadConfigZkCommand(int maxRetries, int interval) { super(maxRetries, interval); } @Override - protected String executeZkCommand(ZkConfigManager zkConfigManager, AmbariSolrCloudClient client) throws Exception { + protected String executeZkConfigCommand(ZkConfigManager zkConfigManager, AmbariSolrCloudClient client) throws Exception { Path configDir = Paths.get(client.getConfigDir()); String configSet = client.getConfigSet(); - zkConfigManager.downloadConfigDir(configSet, configDir); - return configDir.toString(); + try { + zkConfigManager.downloadConfigDir(configSet, configDir); + return configDir.toString(); + } catch (IOException e){ + throw new AmbariSolrCloudClientException("Error downloading configuration set, check Solr Znode has started or not " + + "(starting Solr (for Log Search) is responsible to create the Znode)" ,e); + } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/EnableKerberosPluginSolrZkCommand.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/EnableKerberosPluginSolrZkCommand.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/EnableKerberosPluginSolrZkCommand.java new file mode 100644 index 0000000..fe97994 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/EnableKerberosPluginSolrZkCommand.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.solr.commands; + +import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.SolrZooKeeper; +import org.apache.zookeeper.CreateMode; + +import java.nio.charset.StandardCharsets; + +public class EnableKerberosPluginSolrZkCommand extends AbstractZookeeperRetryCommand<String> { + + private static final String SECURITY_JSON = "/security.json"; + private static final String SECURE_CONTENT = "{\"authentication\":{\"class\": \"org.apache.solr.security.KerberosPlugin\"}}"; + private static final String UNSECURE_CONTENT = "{}"; + + public EnableKerberosPluginSolrZkCommand(int maxRetries, int interval) { + super(maxRetries, interval); + } + + @Override + protected String executeZkCommand(AmbariSolrCloudClient client, SolrZkClient zkClient, SolrZooKeeper solrZooKeeper) throws Exception { + String result = ""; + String filePath = client.getZnode() + SECURITY_JSON; + String fileContent = getFileContent(zkClient, filePath); + if (client.isSecure()) { + if (!fileContent.equals(SECURE_CONTENT)) { + putFileContent(zkClient, filePath, SECURE_CONTENT); + } + result = SECURE_CONTENT; + } else { + if (!fileContent.equals(UNSECURE_CONTENT)) { + putFileContent(zkClient, filePath, UNSECURE_CONTENT); + } + result = UNSECURE_CONTENT; + } + return result; + } + + private void putFileContent(SolrZkClient zkClient, String fileName, String content) throws Exception { + if (zkClient.exists(fileName, true)) { + zkClient.setData(fileName, content.getBytes(StandardCharsets.UTF_8), true); + } else { + zkClient.create(fileName, content.getBytes(StandardCharsets.UTF_8), CreateMode.PERSISTENT, true); + } + } + + private String getFileContent(SolrZkClient zkClient, String fileName) throws Exception { + String result; + if (zkClient.exists(fileName, true)) { + byte[] data = zkClient.getData(fileName, null, null, true); + result = new String(data, StandardCharsets.UTF_8); + } else { + result = UNSECURE_CONTENT; + } + return result; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/GetStateFileZkCommand.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/GetStateFileZkCommand.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/GetStateFileZkCommand.java new file mode 100644 index 0000000..d04c89d --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/GetStateFileZkCommand.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.solr.commands; + +import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient; +import org.apache.ambari.logsearch.solr.domain.AmbariSolrState; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.SolrZooKeeper; + +public class GetStateFileZkCommand extends AbstractStateFileZkCommand { + private String unsecureZnode; + + public GetStateFileZkCommand(int maxRetries, int interval, String unsecureZnode) { + super(maxRetries, interval); + this.unsecureZnode = unsecureZnode; + } + + @Override + protected AmbariSolrState executeZkCommand(AmbariSolrCloudClient client, SolrZkClient zkClient, SolrZooKeeper solrZooKeeper) throws Exception { + AmbariSolrState result = AmbariSolrState.UNSECURE; + String stateFile = String.format("%s/%s", unsecureZnode, AbstractStateFileZkCommand.STATE_FILE); + if (zkClient.exists(stateFile, true)) { + result = getStateFromJson(client, stateFile); + } + return result; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SecureZNodeZkCommand.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SecureZNodeZkCommand.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SecureZNodeZkCommand.java new file mode 100644 index 0000000..cdc3ebf --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SecureZNodeZkCommand.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.solr.commands; + +import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient; +import org.apache.ambari.logsearch.solr.util.AclUtils; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.SolrZooKeeper; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Stat; + +import java.util.List; + +public class SecureZNodeZkCommand extends AbstractZookeeperRetryCommand<Boolean> { + + public SecureZNodeZkCommand(int maxRetries, int interval) { + super(maxRetries, interval); + } + + @Override + protected Boolean executeZkCommand(AmbariSolrCloudClient client, SolrZkClient zkClient, SolrZooKeeper solrZooKeeper) throws Exception { + List<ACL> acls = solrZooKeeper.getACL(client.getZnode(), new Stat()); + boolean isSecure = client.isSecure(); + boolean isPermissionDiffers = AclUtils.isPermissionDiffersForScheme(acls, "world", ZooDefs.Perms.READ); + if (isSecure && isPermissionDiffers) { + acls = AclUtils.updatePermissionForScheme(acls, "world", ZooDefs.Perms.READ); + solrZooKeeper.setACL(client.getZnode(), acls, -1); + } + return isSecure; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SetClusterPropertyZkCommand.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SetClusterPropertyZkCommand.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SetClusterPropertyZkCommand.java new file mode 100644 index 0000000..575d807 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SetClusterPropertyZkCommand.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.solr.commands; + +import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.SolrZooKeeper; +import org.apache.solr.common.cloud.ZkStateReader; + +public class SetClusterPropertyZkCommand extends AbstractZookeeperRetryCommand<String>{ + + public SetClusterPropertyZkCommand(int maxRetries, int interval) { + super(maxRetries, interval); + } + + @Override + protected String executeZkCommand(AmbariSolrCloudClient client, SolrZkClient zkClient, SolrZooKeeper solrZooKeeper) throws Exception { + String propertyName = client.getPropName(); + String propertyValue = client.getPropValue(); + ZkStateReader reader = new ZkStateReader(zkClient); + reader.setClusterProperty(propertyName, propertyValue); + return propertyValue; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/UpdateStateFileZkCommand.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/UpdateStateFileZkCommand.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/UpdateStateFileZkCommand.java new file mode 100644 index 0000000..dbb6d79 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/UpdateStateFileZkCommand.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.solr.commands; + +import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient; +import org.apache.ambari.logsearch.solr.domain.AmbariSolrState; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.SolrZooKeeper; +import org.apache.zookeeper.CreateMode; +import org.codehaus.jackson.map.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; + +public class UpdateStateFileZkCommand extends AbstractStateFileZkCommand { + + private static final Logger LOG = LoggerFactory.getLogger(UpdateStateFileZkCommand.class); + + private String unsecureZnode; + + public UpdateStateFileZkCommand(int maxRetries, int interval, String unsecureZnode) { + super(maxRetries, interval); + this.unsecureZnode = unsecureZnode; + } + + @Override + protected AmbariSolrState executeZkCommand(AmbariSolrCloudClient client, SolrZkClient zkClient, SolrZooKeeper solrZooKeeper) throws Exception { + boolean secure = client.isSecure(); + String stateFile = String.format("%s/%s", unsecureZnode, AbstractStateFileZkCommand.STATE_FILE); + AmbariSolrState result = null; + if (secure) { + LOG.info("Update state file in secure mode."); + updateStateFile(client, zkClient, AmbariSolrState.SECURE, stateFile); + result = AmbariSolrState.SECURE; + } else { + LOG.info("Update state file in unsecure mode."); + updateStateFile(client, zkClient, AmbariSolrState.UNSECURE, stateFile); + result = AmbariSolrState.UNSECURE; + } + return result; + } + + private void updateStateFile(AmbariSolrCloudClient client, SolrZkClient zkClient, AmbariSolrState stateToUpdate, + String stateFile) throws Exception { + if (!zkClient.exists(stateFile, true)) { + LOG.info("State file does not exits. Initializing it as '{}'", stateToUpdate); + zkClient.create(stateFile, createStateJson(stateToUpdate).getBytes(StandardCharsets.UTF_8), + CreateMode.PERSISTENT, true); + } else { + AmbariSolrState stateOnSecure = getStateFromJson(client, stateFile); + if (stateToUpdate.equals(stateOnSecure)) { + LOG.info("State file is in '{}' mode. No update.", stateOnSecure); + } else { + LOG.info("State file is in '{}' mode. Updating it to '{}'", stateOnSecure, stateToUpdate); + zkClient.setData(stateFile, createStateJson(stateToUpdate).getBytes(StandardCharsets.UTF_8), true); + } + } + } + + private String createStateJson(AmbariSolrState state) throws Exception { + Map<String, String> secStateMap = new HashMap<>(); + secStateMap.put(AbstractStateFileZkCommand.STATE_FIELD, state.toString()); + return new ObjectMapper().writeValueAsString(secStateMap); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/UploadConfigZkCommand.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/UploadConfigZkCommand.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/UploadConfigZkCommand.java index 466f8ca..14f4654 100644 --- a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/UploadConfigZkCommand.java +++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/UploadConfigZkCommand.java @@ -19,19 +19,21 @@ package org.apache.ambari.logsearch.solr.commands; import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient; +import org.apache.ambari.logsearch.solr.AmbariSolrCloudClientException; import org.apache.solr.common.cloud.ZkConfigManager; +import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; -public class UploadConfigZkCommand extends AbstractZookeeperRetryCommand<String> { +public class UploadConfigZkCommand extends AbstractZookeeperConfigCommand<String> { public UploadConfigZkCommand(int maxRetries, int interval) { super(maxRetries, interval); } @Override - protected String executeZkCommand(ZkConfigManager zkConfigManager, AmbariSolrCloudClient client) throws Exception { + protected String executeZkConfigCommand(ZkConfigManager zkConfigManager, AmbariSolrCloudClient client) throws Exception { Path configDir = Paths.get(client.getConfigDir()); String configSet = client.getConfigSet(); zkConfigManager.uploadConfigDir(configDir, configSet); http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/domain/AmbariSolrState.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/domain/AmbariSolrState.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/domain/AmbariSolrState.java new file mode 100644 index 0000000..577fefb --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/domain/AmbariSolrState.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.solr.domain; + +/** + * Enum state values for storing security status in unsecure znode + */ +public enum AmbariSolrState { + SECURE, UNSECURE +} http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/util/AclUtils.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/util/AclUtils.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/util/AclUtils.java new file mode 100644 index 0000000..23e5fb3 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/util/AclUtils.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.solr.util; + +import org.apache.zookeeper.data.ACL; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class AclUtils { + + public static Set<String> getUsersFromAclData(List<ACL> acls) { + Set<String> result = new HashSet<>(); + if (!acls.isEmpty()) { + for (ACL acl : acls) { + String username = ""; + String id = acl.getId().getId(); + String[] splitted = id.split(":"); + if (splitted.length > 1) { + username = splitted[0]; + } else { + username = id; + } + result.add(username); + } + } + return result; + } + + public static List<ACL> updatePermissionForScheme(List<ACL> acls, String scheme, int permission) { + List<ACL> aclResult = new ArrayList<>(); + if (!acls.isEmpty()) { + for (ACL acl : acls) { + int permissionToAdd = scheme.equals(acl.getId().getScheme()) ? permission : acl.getPerms(); + acl.setPerms(permissionToAdd); + aclResult.add(acl); + } + } + return aclResult; + } + + public static boolean isPermissionDiffersForScheme(List<ACL> acls, String scheme, int permission) { + boolean result = false; + if (!acls.isEmpty()) { + for (ACL acl : acls) { + if (scheme.equals(acl.getId().getScheme()) && acl.getPerms() == permission) { + result = true; + } + } + } + return result; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-logsearch/ambari-logsearch-solr-client/src/main/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/resources/log4j.properties b/ambari-logsearch/ambari-logsearch-solr-client/src/main/resources/log4j.properties index e8fedce..e8dca12 100644 --- a/ambari-logsearch/ambari-logsearch-solr-client/src/main/resources/log4j.properties +++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/resources/log4j.properties @@ -16,16 +16,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -solr.client.log.dir=/var/log/ambari-logsearch-solr-client - -log4j.rootLogger=INFO,file,stdout,stderr - -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.File=${solr.client.log.dir}/solr-client.log -log4j.appender.file.MaxFileSize=80MB -log4j.appender.file.MaxBackupIndex=60 -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.layout.ConversionPattern=%d{DATE} %5p [%t] %c{1}:%L - %m%n +log4j.rootLogger=INFO,stdout,stderr log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.Threshold=INFO http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py index d75e87d..8c57860 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py @@ -20,10 +20,12 @@ limitations under the License. from resource_management.core.resources.system import Directory, File from resource_management.core.source import StaticFile, InlineTemplate -from resource_management.libraries.resources.properties_file import PropertiesFile +from resource_management.core.exceptions import Fail from resource_management.libraries.functions.format import format -from resource_management.libraries.resources.template_config import TemplateConfig +from resource_management.libraries.functions.decorator import retry from resource_management.libraries.functions import solr_cloud_util +from resource_management.libraries.resources.properties_file import PropertiesFile +from resource_management.libraries.resources.template_config import TemplateConfig def metadata(type='server'): @@ -107,18 +109,19 @@ def metadata(type='server'): if type == 'server' and params.search_backend_solr and params.has_logsearch_solr: solr_cloud_util.setup_solr_client(params.config) + check_znode() + jaasFile=params.atlas_jaas_file if params.security_enabled else None + upload_conf_set('basic_configs', jaasFile) - upload_conf_set('basic_configs') - - create_collection('vertex_index', 'basic_configs') - create_collection('edge_index', 'basic_configs') - create_collection('fulltext_index', 'basic_configs') + create_collection('vertex_index', 'basic_configs', jaasFile) + create_collection('edge_index', 'basic_configs', jaasFile) + create_collection('fulltext_index', 'basic_configs', jaasFile) if params.security_enabled: TemplateConfig(format(params.atlas_jaas_file), owner=params.metadata_user) -def upload_conf_set(config_set): +def upload_conf_set(config_set, jaasFile): import params solr_cloud_util.upload_configuration_to_zk( @@ -130,12 +133,12 @@ def upload_conf_set(config_set): java64_home=params.java64_home, user=params.metadata_user, solrconfig_content=InlineTemplate(params.metadata_solrconfig_content), + jaas_file=jaasFile, retry=30, interval=5) -def create_collection(collection, config_set): +def create_collection(collection, config_set, jaasFile): import params - jaasFile=params.atlas_jaas_file if params.security_enabled else None solr_cloud_util.create_collection( zookeeper_quorum=params.zookeeper_quorum, solr_znode=params.logsearch_solr_znode, @@ -146,3 +149,12 @@ def create_collection(collection, config_set): jaas_file=jaasFile, shards=params.atlas_solr_shards, replication_factor = params.logsearch_solr_replication_factor) + +@retry(times=10, sleep_time=5, err_class=Fail) +def check_znode(): + import params + solr_cloud_util.check_znode( + zookeeper_quorum=params.zookeeper_quorum, + solr_znode=params.logsearch_solr_znode, + java64_home=params.java64_home, + user=params.metadata_user) http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-env.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-env.xml index 8b272f5..e458931 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-env.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-env.xml @@ -160,7 +160,7 @@ </property> <property> <name>logsearch_solr_minmem</name> - <value>512</value> + <value>1024</value> <display-name>Log Search Solr Minimum Heap Size</display-name> <description>Solr minimum heap size e.g.512m</description> <value-attributes> @@ -174,7 +174,7 @@ </property> <property> <name>logsearch_solr_maxmem</name> - <value>1024</value> + <value>2048</value> <display-name>Log Search Solr Maximum Heap Size</display-name> <description>Solr maximum heap size e.g. 512m</description> <value-attributes> http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/service_check.py index ba3f460..30a1cda 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/service_check.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/service_check.py @@ -28,7 +28,7 @@ class LogSearchServiceCheck(Script): try: Execute(params.smoke_logsearch_cmd, user=params.logsearch_user, - tries=10, try_sleep=5) + tries=10, try_sleep=5, timeout=10) Logger.info('Log Search Server up and running') except: Logger.error('Log Search Server not running') http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py index d656012..af762ec 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py @@ -115,7 +115,7 @@ def setup_logsearch(): def upload_conf_set(config_set, solrconfig_content = None): import params - + jaas_file = params.logsearch_jaas_file if params.security_enabled else None solr_cloud_util.upload_configuration_to_zk( zookeeper_quorum=params.zookeeper_quorum, solr_znode=params.logsearch_solr_znode, @@ -125,4 +125,5 @@ def upload_conf_set(config_set, solrconfig_content = None): java64_home=params.java64_home, user=params.logsearch_solr_user, solrconfig_content= solrconfig_content, + jaas_file=jaas_file, retry=30, interval=5) http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch_solr.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch_solr.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch_solr.py index ff10165..42f34a0 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch_solr.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch_solr.py @@ -19,7 +19,7 @@ limitations under the License. from resource_management.core.exceptions import Fail from resource_management.core.source import InlineTemplate, Template -from resource_management.core.resources.system import Directory, Execute, File +from resource_management.core.resources.system import Directory, File from resource_management.libraries.functions.decorator import retry from resource_management.libraries.functions.format import format from resource_management.libraries.functions import solr_cloud_util @@ -78,22 +78,36 @@ def setup_logsearch_solr(name = None): owner=params.logsearch_solr_user, group=params.user_group ) - zk_cli_prefix = format('export JAVA_HOME={java64_home}; {cloud_scripts}/zkcli.sh -zkhost {zookeeper_quorum}') - create_ambari_solr_znode(zk_cli_prefix) + jaas_file = params.logsearch_solr_jaas_file if params.security_enabled else None url_scheme = 'https' if params.logsearch_solr_ssl_enabled else 'http' - Execute(format('{zk_cli_prefix}{logsearch_solr_znode} -cmd clusterprop -name urlScheme -val {url_scheme}'), - user=params.logsearch_solr_user) + + create_ambari_solr_znode() if params.security_enabled: File(format("{logsearch_solr_jaas_file}"), content=Template("logsearch_solr_jaas.conf.j2"), owner=params.logsearch_solr_user) - security_content = '\'{"authentication":{"class": "org.apache.solr.security.KerberosPlugin"}}\'' - else: - security_content = '\'{}\'' - Execute(format('{zk_cli_prefix} -cmd put {logsearch_solr_znode}/security.json ') + security_content, - user=params.logsearch_solr_user) + + solr_cloud_util.set_cluster_prop( + zookeeper_quorum=params.zookeeper_quorum, + solr_znode=params.logsearch_solr_znode, + java64_home=params.java64_home, + user=params.logsearch_solr_user, + prop_name="urlScheme", + prop_value=url_scheme, + jaas_file=jaas_file + ) + + solr_cloud_util.setup_kerberos_plugin( + zookeeper_quorum=params.zookeeper_quorum, + solr_znode=params.logsearch_solr_znode, + user=params.logsearch_solr_user, + jaas_file=jaas_file, + java64_home=params.java64_home, + secure=params.security_enabled + ) + elif name == 'client': solr_cloud_util.setup_solr_client(params.config) @@ -106,9 +120,10 @@ def setup_logsearch_solr(name = None): raise Fail('Nor client or server were selected to install.') @retry(times=30, sleep_time=5, err_class=Fail) -def create_ambari_solr_znode(zk_cli_prefix): +def create_ambari_solr_znode(): import params - Execute(format('{zk_cli_prefix} -cmd makepath {logsearch_solr_znode}'), - not_if=format("{zk_cli_prefix}{logsearch_solr_znode} -cmd list"), - user=params.logsearch_solr_user - ) \ No newline at end of file + solr_cloud_util.create_znode( + zookeeper_quorum=params.zookeeper_quorum, + solr_znode=params.logsearch_solr_znode, + java64_home=params.java64_home, + user=params.logsearch_solr_user) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py index 84026d8..7079626 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py @@ -26,6 +26,7 @@ from resource_management.libraries.resources.xml_config import XmlConfig from resource_management.libraries.resources.modify_properties_file import ModifyPropertiesFile from resource_management.libraries.resources.properties_file import PropertiesFile from resource_management.core.exceptions import Fail +from resource_management.libraries.functions.decorator import retry from resource_management.libraries.functions.format import format from resource_management.libraries.functions.is_empty import is_empty from resource_management.core.utils import PasswordString @@ -556,12 +557,13 @@ def create_core_site_xml(conf_dir): def setup_ranger_audit_solr(): import params - + jaas_file = params.solr_jaas_file if params.security_enabled else None if params.security_enabled and params.stack_supports_ranger_kerberos: File(format("{solr_jaas_file}"), content=Template("ranger_solr_jass_conf.j2"), owner=params.unix_user ) + check_znode() solr_cloud_util.upload_configuration_to_zk( zookeeper_quorum = params.zookeeper_quorum, @@ -571,6 +573,7 @@ def setup_ranger_audit_solr(): tmp_dir = params.tmp_dir, java64_home = params.java_home, user = params.unix_user, + jaas_file=jaas_file, retry=30, interval=5) solr_cloud_util.create_collection( @@ -590,4 +593,13 @@ def setup_ranger_admin_passwd_change(): if params.admin_password != params.default_admin_password: cmd = format('ambari-python-wrap {ranger_home}/db_setup.py -changepassword {admin_username} {default_admin_password!p} {admin_password!p}') Logger.info('Updating admin password') - Execute(cmd, environment={'JAVA_HOME': params.java_home, 'RANGER_ADMIN_HOME': params.ranger_home}, user=params.unix_user) \ No newline at end of file + Execute(cmd, environment={'JAVA_HOME': params.java_home, 'RANGER_ADMIN_HOME': params.ranger_home}, user=params.unix_user) + +@retry(times=10, sleep_time=5, err_class=Fail) +def check_znode(): + import params + solr_cloud_util.check_znode( + zookeeper_quorum=params.zookeeper_quorum, + solr_znode=params.solr_znode, + java64_home=params.java_home, + user=params.unix_user) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py index 0da6e4d..9e1c7f7 100644 --- a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py +++ b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py @@ -152,6 +152,7 @@ class TestMetadataServer(RMFTestCase): mode=0664, content = '' ) + self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181 --znode /logsearch --check-znode --retry 5 --interval 10') self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/logsearch --download-config --config-dir /tmp/solr_config_basic_configs_0.[0-9]* --config-set basic_configs --retry 30 --interval 5') self.assertResourceCalledRegexp('^File$', '^/tmp/solr_config_basic_configs_0.[0-9]*', content=InlineTemplate(self.getConfig()['configurations']['atlas-solrconfig']['content']), http://git-wip-us.apache.org/repos/asf/ambari/blob/0e73da6e/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_solr.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_solr.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_solr.py index 0ef54f3..6319be8 100644 --- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_solr.py +++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_solr.py @@ -99,16 +99,12 @@ class TestSolr(RMFTestCase): group='hadoop', content = Template('zoo.cfg.j2') ) - self.assertResourceCalled('Execute', 'export JAVA_HOME=/usr/jdk64/jdk1.7.0_45; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:2181 -cmd makepath /logsearch', - not_if = "export JAVA_HOME=/usr/jdk64/jdk1.7.0_45; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:2181/logsearch -cmd list", - user = "solr" - ) - self.assertResourceCalled('Execute', 'export JAVA_HOME=/usr/jdk64/jdk1.7.0_45; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:2181/logsearch -cmd clusterprop -name urlScheme -val http', - user = "solr" - ) - self.assertResourceCalled('Execute', 'export JAVA_HOME=/usr/jdk64/jdk1.7.0_45; /usr/lib/ambari-logsearch-solr/server/scripts/cloud-scripts/zkcli.sh -zkhost c6401.ambari.apache.org:2181 -cmd put /logsearch/security.json \'{}\'', - user = "solr" - ) + self.assertResourceCalled('Execute', 'export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181 --znode /logsearch --create-znode --retry 5 --interval 10', + user = "solr") + self.assertResourceCalled('Execute', 'export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/logsearch --cluster-prop --property-name urlScheme --property-value http', + user = "solr") + self.assertResourceCalled('Execute', 'export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181 --znode /logsearch --setup-kerberos-plugin', + user = "solr") def test_configure_default(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logsearch_solr.py",
