Repository: ambari Updated Branches: refs/heads/branch-2.5 6fed7e276 -> fa3b473c8
AMBARI-19645 Log Search should use Credential Store API to store keystore/truststore passwords - ambari side (mgergely) Change-Id: Ieba7ed3e0a0a7440a07131913d1949d8b5f1579b Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fa3b473c Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fa3b473c Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fa3b473c Branch: refs/heads/branch-2.5 Commit: fa3b473c8b1ddc257abaf227bb090f94d80576c5 Parents: 6fed7e2 Author: Miklos Gergely <[email protected]> Authored: Thu Feb 9 13:37:55 2017 +0100 Committer: Miklos Gergely <[email protected]> Committed: Thu Feb 9 13:37:55 2017 +0100 ---------------------------------------------------------------------- .../0.5.0/configuration/logfeeder-env.xml | 2 ++ .../0.5.0/configuration/logsearch-env.xml | 2 ++ .../LOGSEARCH/0.5.0/metainfo.xml | 6 +++++ .../LOGSEARCH/0.5.0/package/scripts/params.py | 22 ++++++++------- .../0.5.0/package/scripts/setup_logfeeder.py | 28 ++++++-------------- .../0.5.0/package/scripts/setup_logsearch.py | 22 ++++++--------- .../stacks/2.4/LOGSEARCH/test_logfeeder.py | 23 ++-------------- .../stacks/2.4/LOGSEARCH/test_logsearch.py | 19 +++---------- 8 files changed, 45 insertions(+), 79 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/fa3b473c/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml index 508ef4e..e308479 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml @@ -90,6 +90,7 @@ <description>Password to open the trust store file.</description> <value-attributes> <type>password</type> + <keystore>true</keystore> </value-attributes> <on-ambari-upgrade add="true"/> </property> @@ -115,6 +116,7 @@ <description>Password to open the key store file.</description> <value-attributes> <type>password</type> + <keystore>true</keystore> </value-attributes> <on-ambari-upgrade add="true"/> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/fa3b473c/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml index 10b21be..f1e871d 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml @@ -135,6 +135,7 @@ <description>Password to open the trust store file.</description> <value-attributes> <type>password</type> + <keystore>true</keystore> </value-attributes> <on-ambari-upgrade add="true"/> </property> @@ -160,6 +161,7 @@ <description>Password to open the key store file.</description> <value-attributes> <type>password</type> + <keystore>true</keystore> </value-attributes> <on-ambari-upgrade add="true"/> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/fa3b473c/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml index 2b95b78..245a94d 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml @@ -24,6 +24,12 @@ <comment>Log aggregation, analysis, and visualization for Ambari managed services. This service is <b>Technical Preview</b>.</comment> <version>0.5.0</version> + + <credential-store> + <supported>true</supported> + <enabled>true</enabled> + </credential-store> + <components> <component> http://git-wip-us.apache.org/repos/asf/ambari/blob/fa3b473c/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py index 08c0a7b..fecd802 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py @@ -18,6 +18,8 @@ See the License for the specific language governing permissions and limitations under the License. """ + +import os from ambari_commons.constants import AMBARI_SUDO_BINARY from logsearch_config_aggregator import get_logfeeder_metadata, get_logsearch_metadata, get_logsearch_meta_configs from resource_management.libraries.functions.default import default @@ -54,7 +56,6 @@ security_enabled = status_params.security_enabled logsearch_server_conf = "/etc/ambari-logsearch-portal/conf" logsearch_server_keys_folder = logsearch_server_conf + "/keys" logsearch_logfeeder_conf = "/etc/ambari-logsearch-logfeeder/conf" -logsearch_logfeeder_keys_folder = logsearch_logfeeder_conf + "/keys" logsearch_config_set_dir = format("{logsearch_server_conf}/solr_configsets") @@ -168,6 +169,14 @@ logsearch_debug_enabled = str(config['configurations']['logsearch-env']["logsear logsearch_debug_port = config['configurations']['logsearch-env']["logsearch_debug_port"] logsearch_app_max_memory = config['configurations']['logsearch-env']['logsearch_app_max_memory'] +logsearch_keystore_location = config['configurations']['logsearch-env']['logsearch_keystore_location'] +logsearch_keystore_type = config['configurations']['logsearch-env']['logsearch_keystore_type'] +logsearch_truststore_location = config['configurations']['logsearch-env']['logsearch_truststore_location'] +logsearch_truststore_type = config['configurations']['logsearch-env']['logsearch_truststore_type'] + +logsearch_env_config = dict(config['configurations']['logsearch-env']) +logsearch_env_jceks_file = os.path.join(logsearch_server_conf, 'logsearch.jceks') + #Logsearch log4j properties logsearch_log_maxfilesize = default('/configurations/logsearch-log4j/logsearch_log_maxfilesize',10) logsearch_log_maxbackupindex = default('/configurations/logsearch-log4j/logsearch_log_maxbackupindex',10) @@ -296,19 +305,14 @@ solr_audit_logs_enable = default('/configurations/logfeeder-env/logfeeder_solr_a logfeeder_env_content = config['configurations']['logfeeder-env']['content'] logfeeder_log4j_content = config['configurations']['logfeeder-log4j']['content'] -logsearch_keystore_location = config['configurations']['logsearch-env']['logsearch_keystore_location'] -logsearch_keystore_password = config['configurations']['logsearch-env']['logsearch_keystore_password'] -logsearch_keystore_type = config['configurations']['logsearch-env']['logsearch_keystore_type'] -logsearch_truststore_location = config['configurations']['logsearch-env']['logsearch_truststore_location'] -logsearch_truststore_password = config['configurations']['logsearch-env']['logsearch_truststore_password'] -logsearch_truststore_type = config['configurations']['logsearch-env']['logsearch_truststore_type'] logfeeder_keystore_location = config['configurations']['logfeeder-env']['logfeeder_keystore_location'] -logfeeder_keystore_password = config['configurations']['logfeeder-env']['logfeeder_keystore_password'] logfeeder_keystore_type = config['configurations']['logfeeder-env']['logfeeder_keystore_type'] logfeeder_truststore_location = config['configurations']['logfeeder-env']['logfeeder_truststore_location'] -logfeeder_truststore_password = config['configurations']['logfeeder-env']['logfeeder_truststore_password'] logfeeder_truststore_type = config['configurations']['logfeeder-env']['logfeeder_truststore_type'] +logfeeder_env_config = dict(config['configurations']['logfeeder-env']) +logfeeder_env_jceks_file = os.path.join(logsearch_logfeeder_conf, 'logfeeder.jceks') + logfeeder_ambari_config_content = config['configurations']['logfeeder-ambari-config']['content'] logfeeder_output_config_content = config['configurations']['logfeeder-output-config']['content'] http://git-wip-us.apache.org/repos/asf/ambari/blob/fa3b473c/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py index a04618f..6952c2c 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py @@ -22,6 +22,7 @@ from resource_management.core.resources.system import Directory, File from resource_management.libraries.functions.format import format from resource_management.core.source import InlineTemplate, Template from resource_management.libraries.resources.properties_file import PropertiesFile +from resource_management.libraries.functions.security_commons import update_credential_provider_path, HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME def setup_logfeeder(): import params @@ -39,31 +40,18 @@ def setup_logfeeder(): recursive_ownership=True ) - Directory(params.logsearch_logfeeder_keys_folder, - cd_access='a', - mode=0755, - owner=params.logsearch_user, - group=params.user_group) - - File(format("{logsearch_logfeeder_keys_folder}/ks_pass.txt"), - content=params.logfeeder_keystore_password, - mode=0600, - owner=params.logsearch_user, - group=params.user_group - ) - - File(format("{logsearch_logfeeder_keys_folder}/ts_pass.txt"), - content=params.logfeeder_truststore_password, - mode=0600, - owner=params.logsearch_user, - group=params.user_group - ) - File(params.logfeeder_log, mode=0644, content='' ) + params.logfeeder_env_config = update_credential_provider_path(params.logfeeder_env_config, + 'logfeeder-env', + params.logfeeder_env_jceks_file, + params.logsearch_user, + params.user_group + ) + params.logfeeder_properties[HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME] = 'jceks://file' + params.logfeeder_env_jceks_file PropertiesFile(format("{logsearch_logfeeder_conf}/logfeeder.properties"), properties = params.logfeeder_properties ) http://git-wip-us.apache.org/repos/asf/ambari/blob/fa3b473c/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py index 08d3d9d..ba91e20 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py @@ -21,6 +21,7 @@ from resource_management.core.resources.system import Directory, Execute, File from resource_management.libraries.functions.format import format from resource_management.core.source import InlineTemplate, Template from resource_management.libraries.resources.properties_file import PropertiesFile +from resource_management.libraries.functions.security_commons import update_credential_provider_path, HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME def setup_logsearch(): @@ -49,20 +50,6 @@ def setup_logsearch(): owner=params.logsearch_user, group=params.user_group) - File(format("{logsearch_server_keys_folder}/ks_pass.txt"), - content=params.logsearch_keystore_password, - mode=0600, - owner=params.logsearch_user, - group=params.user_group - ) - - File(format("{logsearch_server_keys_folder}/ts_pass.txt"), - content=params.logsearch_truststore_password, - mode=0600, - owner=params.logsearch_user, - group=params.user_group - ) - File(params.logsearch_log, mode=0644, owner=params.logsearch_user, @@ -70,6 +57,13 @@ def setup_logsearch(): content='' ) + params.logsearch_env_config = update_credential_provider_path(params.logsearch_env_config, + 'logsearch-env', + params.logsearch_env_jceks_file, + params.logsearch_user, + params.user_group + ) + params.logsearch_properties[HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME] = 'jceks://file' + params.logsearch_env_jceks_file PropertiesFile(format("{logsearch_server_conf}/logsearch.properties"), properties=params.logsearch_properties ) http://git-wip-us.apache.org/repos/asf/ambari/blob/fa3b473c/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py index 1c79c5c..00e8e1f 100644 --- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py +++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py @@ -55,33 +55,14 @@ class TestLogFeeder(RMFTestCase): cd_access='a', mode=0755 ) - self.assertResourceCalled('Directory', '/etc/ambari-logsearch-logfeeder/conf/keys', - owner = 'logsearch', - group = 'hadoop', - cd_access = 'a', - mode = 0755 - ) - - self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/keys/ks_pass.txt', - owner='logsearch', - group='hadoop', - mode=0600, - content='bigdata' - ) - - self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/keys/ts_pass.txt', - owner='logsearch', - group='hadoop', - mode=0600, - content='bigdata' - ) self.assertResourceCalled('File', '/var/log/ambari-logsearch-logfeeder/logfeeder.out', mode=0644, content='' ) self.assertResourceCalled('PropertiesFile', '/etc/ambari-logsearch-logfeeder/conf/logfeeder.properties', - properties={'logfeeder.checkpoint.folder': '/etc/ambari-logsearch-logfeeder/conf/checkpoints', + properties={'hadoop.security.credential.provider.path': 'jceks://file/etc/ambari-logsearch-logfeeder/conf/logfeeder-env.jceks', + 'logfeeder.checkpoint.folder': '/etc/ambari-logsearch-logfeeder/conf/checkpoints', 'logfeeder.config.files': 'output.config.json,input.config-ambari.json,global.config.json,input.config-logsearch.json,input.config-zookeeper.json', 'logfeeder.metrics.collector.hosts': '', 'logfeeder.metrics.collector.path': '/ws/v1/timeline/metrics', http://git-wip-us.apache.org/repos/asf/ambari/blob/fa3b473c/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py index f63cd42..380151c 100644 --- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py +++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py @@ -70,20 +70,6 @@ class TestLogSearch(RMFTestCase): mode = 0755 ) - self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/keys/ks_pass.txt', - owner='logsearch', - group='hadoop', - mode=0600, - content='bigdata' - ) - - self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/keys/ts_pass.txt', - owner='logsearch', - group='hadoop', - mode=0600, - content='bigdata' - ) - self.assertResourceCalled('File', '/var/log/ambari-logsearch-portal/logsearch.out', owner = 'logsearch', group = 'hadoop', @@ -91,7 +77,8 @@ class TestLogSearch(RMFTestCase): content = '' ) self.assertResourceCalled('PropertiesFile', '/etc/ambari-logsearch-portal/conf/logsearch.properties', - properties = {'logsearch.audit.logs.split.interval.mins': '1', + properties = {'hadoop.security.credential.provider.path': 'jceks://file/etc/ambari-logsearch-portal/conf/logsearch-env.jceks', + 'logsearch.audit.logs.split.interval.mins': '1', 'logsearch.auth.external_auth.enabled': 'false', 'logsearch.auth.external_auth.host_url': 'http://c6401.ambari.apache.org:8080', 'logsearch.auth.external_auth.login_url': '/api/v1/users/$USERNAME/privileges?fields=*', @@ -152,6 +139,8 @@ class TestLogSearch(RMFTestCase): self.assertResourceCalled('Execute', ('chmod', '-R', 'ugo+r', '/etc/ambari-logsearch-portal/conf/solr_configsets'), sudo = True ) + + def test_configure_default(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logsearch.py",
