Repository: incubator-slider Updated Branches: refs/heads/develop 95cf1759e -> b1785a60c
SLIDER-840 clear sensitive properties from client install, add env conf type for download, fix tests for secure mode Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b1785a60 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b1785a60 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b1785a60 Branch: refs/heads/develop Commit: b1785a60c524ec555ea872f8d8e2839ec4aa345e Parents: 95cf175 Author: Billie Rinaldi <[email protected]> Authored: Thu Apr 2 12:32:38 2015 -0700 Committer: Billie Rinaldi <[email protected]> Committed: Thu Apr 2 13:00:08 2015 -0700 ---------------------------------------------------------------------- app-packages/accumulo/appConfig-default.json | 1 + .../accumulo/appConfig-secured-default.json | 2 +- .../accumulo/appConfig-ssl-default.json | 1 + .../accumulo/configuration/accumulo-env.xml | 15 +++++++ .../accumulo/package/files/accumulo-slider.py | 18 +------- .../accumulo/package/scripts/accumulo_client.py | 47 +++++++++++++++++--- .../accumulo/package/scripts/client_params.py | 3 +- app-packages/accumulo/package/scripts/params.py | 2 +- .../funtest/accumulo/AccumuloBasicIT.groovy | 10 ++--- .../funtest/accumulo/AccumuloScriptIT.groovy | 2 - .../core/registry/docstore/ConfigFormat.java | 1 + .../PublishedConfigurationOutputter.java | 29 ++++++++++-- 12 files changed, 92 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b1785a60/app-packages/accumulo/appConfig-default.json ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/appConfig-default.json b/app-packages/accumulo/appConfig-default.json index e6e8149..fcaaeb4 100644 --- a/app-packages/accumulo/appConfig-default.json +++ b/app-packages/accumulo/appConfig-default.json @@ -38,6 +38,7 @@ "site.accumulo-site.general.security.credential.provider.paths": "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks", "site.accumulo-site.instance.rpc.ssl.enabled": "false", "site.accumulo-site.instance.rpc.ssl.clientAuth": "false", + "site.accumulo-site.instance.rpc.sasl.enabled": "false", "site.accumulo-site.general.kerberos.keytab": "${accumulo.keytab}", "site.accumulo-site.general.kerberos.principal": "${accumulo.principal}", http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b1785a60/app-packages/accumulo/appConfig-secured-default.json ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/appConfig-secured-default.json b/app-packages/accumulo/appConfig-secured-default.json index 40c38c0..df44f37 100644 --- a/app-packages/accumulo/appConfig-secured-default.json +++ b/app-packages/accumulo/appConfig-secured-default.json @@ -38,10 +38,10 @@ "site.accumulo-site.general.security.credential.provider.paths": "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks", "site.accumulo-site.instance.rpc.ssl.enabled": "false", "site.accumulo-site.instance.rpc.ssl.clientAuth": "false", + "site.accumulo-site.instance.rpc.sasl.enabled": "true", "site.accumulo-site.general.kerberos.keytab": "${AGENT_WORK_ROOT}/keytabs/${USER_NAME}.ACCUMULO.service.keytab", "site.accumulo-site.general.kerberos.principal": "${USER_NAME}/[email protected]", - "site.accumulo-site.instance.rpc.sasl.enabled": "true", "site.accumulo-site.instance.security.authenticator": "org.apache.accumulo.server.security.handler.KerberosAuthenticator", "site.accumulo-site.instance.security.authorizor": "org.apache.accumulo.server.security.handler.KerberosAuthorizor", "site.accumulo-site.instance.security.permissionHandler": "org.apache.accumulo.server.security.handler.KerberosPermissionHandler", http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b1785a60/app-packages/accumulo/appConfig-ssl-default.json ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/appConfig-ssl-default.json b/app-packages/accumulo/appConfig-ssl-default.json index 5f213c2..7e84fc1 100644 --- a/app-packages/accumulo/appConfig-ssl-default.json +++ b/app-packages/accumulo/appConfig-ssl-default.json @@ -42,6 +42,7 @@ "site.accumulo-site.general.security.credential.provider.paths": "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks", "site.accumulo-site.instance.rpc.ssl.enabled": "true", "site.accumulo-site.instance.rpc.ssl.clientAuth": "true", + "site.accumulo-site.instance.rpc.sasl.enabled": "false", "site.accumulo-site.general.kerberos.keytab": "${accumulo.keytab}", "site.accumulo-site.general.kerberos.principal": "${accumulo.principal}", "site.accumulo-site.rpc.javax.net.ssl.keyStore": "${AGENT_WORK_ROOT}/secstores/keystore-${COMPONENT_NAME}.p12", http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b1785a60/app-packages/accumulo/configuration/accumulo-env.xml ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/configuration/accumulo-env.xml b/app-packages/accumulo/configuration/accumulo-env.xml index b993a5c..a778c89 100644 --- a/app-packages/accumulo/configuration/accumulo-env.xml +++ b/app-packages/accumulo/configuration/accumulo-env.xml @@ -75,6 +75,21 @@ export HADOOP_PREFIX=${@//site/accumulo-env/hadoop_prefix} export HADOOP_CONF_DIR=${@//site/accumulo-env/hadoop_conf_dir} export JAVA_HOME=${@//site/accumulo-env/java_home} export ZOOKEEPER_HOME=${@//site/accumulo-env/zookeeper_home} +export ACCUMULO_GENERAL_OPTS="-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -Djava.net.preferIPv4Stack=true" +export ACCUMULO_OTHER_OPTS="-Xmx${@//site/accumulo-env/other_heapsize} -Xms${@//site/accumulo-env/other_heapsize}" +# what do when the JVM runs out of heap memory +export ACCUMULO_KILL_CMD='kill -9 %p' + </value> + </property> + <property> + <name>server_content</name> + <description>This is the template for a server accumulo-env.sh file</description> + <value> +#! /usr/bin/env bash +export HADOOP_PREFIX=${@//site/accumulo-env/hadoop_prefix} +export HADOOP_CONF_DIR=${@//site/accumulo-env/hadoop_conf_dir} +export JAVA_HOME=${@//site/accumulo-env/java_home} +export ZOOKEEPER_HOME=${@//site/accumulo-env/zookeeper_home} export ACCUMULO_LOG_DIR=${@//site/global/app_log_dir} export ACCUMULO_TSERVER_OPTS="-Xmx${@//site/accumulo-env/tserver_heapsize} -Xms${@//site/accumulo-env/tserver_heapsize}" export ACCUMULO_MASTER_OPTS="-Xmx${@//site/accumulo-env/master_heapsize} -Xms${@//site/accumulo-env/master_heapsize}" http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b1785a60/app-packages/accumulo/package/files/accumulo-slider.py ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/package/files/accumulo-slider.py b/app-packages/accumulo/package/files/accumulo-slider.py index e685293..aa871f8 100644 --- a/app-packages/accumulo/package/files/accumulo-slider.py +++ b/app-packages/accumulo/package/files/accumulo-slider.py @@ -17,11 +17,7 @@ import os import sys -import json -import glob -import tempfile import subprocess -import shutil SLIDER_DIR = os.getenv('SLIDER_HOME', None) if SLIDER_DIR == None or (not os.path.exists(SLIDER_DIR)): @@ -88,7 +84,6 @@ def get_all_conf(): client_file = os.path.join(confdir, 'client.conf') site_file = os.path.join(confdir, 'accumulo-site.xml') env_file = os.path.join(confdir, 'accumulo-env.sh') - env_json = os.path.join(confdir, 'accumulo-env.json') if os.path.exists(client_file): os.remove(client_file) @@ -96,21 +91,10 @@ def get_all_conf(): os.remove(site_file) if os.path.exists(env_file): os.remove(env_file) - if os.path.exists(env_json): - os.remove(env_json) get_conf("client", "properties", client_file) get_conf("accumulo-site", "xml", site_file) - get_conf("accumulo-env", "json", env_json) - - infile = open(env_json) - outfile = open(env_file, 'w') - try: - content = json.load(infile) - outfile.write(content['content']) - finally: - outfile.close() - infile.close() + get_conf("accumulo-env", "env", env_file) def get_conf(confname, fileformat, destfile): if os.path.exists(destfile): http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b1785a60/app-packages/accumulo/package/scripts/accumulo_client.py ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/package/scripts/accumulo_client.py b/app-packages/accumulo/package/scripts/accumulo_client.py index cd11269..0680e52 100644 --- a/app-packages/accumulo/package/scripts/accumulo_client.py +++ b/app-packages/accumulo/package/scripts/accumulo_client.py @@ -19,6 +19,7 @@ limitations under the License. """ from resource_management import * +import json class AccumuloClient(Script): def check_provider_contains(self, provider, alias): @@ -47,17 +48,45 @@ class AccumuloClient(Script): content=StaticFile("accumulo-slider.py"), mode=0755 ) - TemplateConfig(format("{conf_dir}/accumulo-slider-env.sh"), + TemplateConfig(format("{client_conf}/accumulo-slider-env.sh"), mode=0755 ) if client_params.app_name: Logger.info("Creating configs for app %s" % client_params.app_name) - Directory(client_params.conf_dir, - content=format("{conf_dir}/templates")) - Execute( format("{bin_dir}/accumulo-slider " - "--appconf {client_root}/conf --app {app_name} getconf ")) + Directory(client_params.conf_download_dir) + Execute( format("SLIDER_CONF_DIR={slider_conf_dir} " + "{slider_home_dir}/bin/slider registry --getconf " + "accumulo-env --format env " + "--dest {conf_download_dir}/accumulo-env.sh " + "--name {app_name}")) + Execute( format("SLIDER_CONF_DIR={slider_conf_dir} " + "{slider_home_dir}/bin/slider registry --getconf " + "accumulo-site --format json " + "--dest {conf_download_dir}/accumulo-site.xml " + "--name {app_name}")) + with open(format("{conf_download_dir}/accumulo-site.xml"),"r") as fp: + site = json.load(fp) + sensitive_props = ["instance.secret", + "general.security.credential.provider.paths", + "rpc.javax.net.ssl.keyStorePassword", + "rpc.javax.net.ssl.trustStorePassword", + "monitor.ssl.keyStorePassword", + "monitor.ssl.trustStorePassword", + "trace.password", + "trace.token.property.password"] + for prop in sensitive_props: + site.pop(prop, None) + XmlConfig( "accumulo-site.xml", + conf_dir = client_params.conf_download_dir, + configurations=site, + mode=0644) + Execute( format("SLIDER_CONF_DIR={slider_conf_dir} " + "{slider_home_dir}/bin/slider registry --getconf " + "client --format properties " + "--dest {conf_download_dir}/client.conf " + "--name {app_name}")) configs = {} - with open(format("{client_root}/conf/client.conf"),"r") as fp: + with open(format("{conf_download_dir}/client.conf"),"r") as fp: content = fp.readlines() for line in content: index = line.find("=") @@ -74,7 +103,7 @@ class AccumuloClient(Script): configs['rpc.javax.net.ssl.keyStoreType'] = client_params.store_type configs['rpc.javax.net.ssl.trustStore'] = client_params.truststore_path configs['rpc.javax.net.ssl.trustStoreType'] = client_params.store_type - PropertiesFile(format("{client_root}/conf/client.conf"), + PropertiesFile(format("{conf_download_dir}/client.conf"), properties = configs ) Execute( format("SLIDER_CONF_DIR={slider_conf_dir} " @@ -87,6 +116,10 @@ class AccumuloClient(Script): "--truststore {truststore_path} " "--name {app_name} --alias {truststore_alias} " "--provider {credential_provider}")) + Directory(client_params.client_conf, + content=format("{client_conf}/templates")) + Directory(client_params.client_conf, + content=client_params.conf_download_dir) else: Logger.info("No app name provided, leaving client install unconfigured") http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b1785a60/app-packages/accumulo/package/scripts/client_params.py ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/package/scripts/client_params.py b/app-packages/accumulo/package/scripts/client_params.py index 70d3d9a..3b87e5a 100644 --- a/app-packages/accumulo/package/scripts/client_params.py +++ b/app-packages/accumulo/package/scripts/client_params.py @@ -25,7 +25,8 @@ config = Script.get_config() client_root = config['configurations']['global']['client_root'] bin_dir = os.path.join(client_root, 'bin') -conf_dir = os.path.join(client_root, 'conf') +client_conf = os.path.join(client_root, 'conf') +conf_download_dir = os.path.join(client_conf, 'download') app_name = None if 'app_name' in config['configurations']['global']: http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b1785a60/app-packages/accumulo/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/package/scripts/params.py b/app-packages/accumulo/package/scripts/params.py index fc3ba45..8dbb7f8 100644 --- a/app-packages/accumulo/package/scripts/params.py +++ b/app-packages/accumulo/package/scripts/params.py @@ -41,7 +41,7 @@ tserver_heapsize = config['configurations']['accumulo-env']['tserver_heapsize'] monitor_heapsize = config['configurations']['accumulo-env']['monitor_heapsize'] gc_heapsize = config['configurations']['accumulo-env']['gc_heapsize'] other_heapsize = config['configurations']['accumulo-env']['other_heapsize'] -env_sh_template = config['configurations']['accumulo-env']['content'] +env_sh_template = config['configurations']['accumulo-env']['server_content'] # accumulo local directory structure accumulo_root = config['configurations']['global']['app_root'] http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b1785a60/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy index 2d137be..f0504e4 100644 --- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy +++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy @@ -139,13 +139,9 @@ class AccumuloBasicIT extends AccumuloAgentCommandTestBase { def path = buildClusterPath(getClusterName()) assert !clusterFS.exists(path) - SliderShell shell = slider(EXIT_SUCCESS, - [ - ACTION_CREATE, getClusterName(), - ARG_TEMPLATE, APP_TEMPLATE, - ARG_RESOURCES, APP_RESOURCE, "<", - sysprop("test.app.resources.dir") + "/test_password_file" - ]) + SliderShell shell = createTemplatedSliderApplication(getClusterName(), + APP_TEMPLATE, APP_RESOURCE, + ["<", sysprop("test.app.resources.dir") + "/test_password_file"]) logShell(shell) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b1785a60/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloScriptIT.groovy ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloScriptIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloScriptIT.groovy index be18609..0d908cc 100644 --- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloScriptIT.groovy +++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloScriptIT.groovy @@ -98,10 +98,8 @@ class AccumuloScriptIT extends AccumuloBasicIT { accumulo("version") accumulo("classpath") - accumulo("admin checkTablets") accumulo("admin listInstances") accumulo("admin ping") - accumulo("admin volumes") String zookeepers = SLIDER_CONFIG.get( RegistryConstants.KEY_REGISTRY_ZK_QUORUM, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b1785a60/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java index ded62f7..12581d7 100644 --- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java +++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java @@ -23,6 +23,7 @@ public enum ConfigFormat { JSON("json"), PROPERTIES("properties"), XML("xml"), + ENV("env"), // YAML("yaml"); ; ConfigFormat(String suffix) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b1785a60/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java index ea15f39..15ac207 100644 --- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java +++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java @@ -100,6 +100,8 @@ public abstract class PublishedConfigurationOutputter { return new PropertiesOutputter(owner); case JSON: return new JsonOutputter(owner); + case ENV: + return new EnvOutputter(owner); default: throw new RuntimeException("Unsupported format :" + format); } @@ -169,7 +171,28 @@ public abstract class PublishedConfigurationOutputter { return owner.asJson(); } } - - - + + + public static class EnvOutputter extends PublishedConfigurationOutputter { + + public EnvOutputter(PublishedConfiguration owner) { + super(owner); + } + + @Override + public void save(File dest) throws IOException { + FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8); + } + + @Override + public String asString() throws IOException { + if (!owner.entries.containsKey("content")) { + throw new IOException("Configuration has no content field and cannot " + + "be retrieved as type 'env'"); + } + return owner.entries.get("content"); + } + } + + }
