Repository: ambari Updated Branches: refs/heads/trunk 2437a799a -> af675f47d
AMBARI-10884. Addition of security to Atlas service definition (Jon Maron via smohanty) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/af675f47 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/af675f47 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/af675f47 Branch: refs/heads/trunk Commit: af675f47df28dec850cd603ee6ac2520bf444d1c Parents: 2437a79 Author: Sumit Mohanty <[email protected]> Authored: Thu May 14 12:40:42 2015 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Thu May 14 12:40:42 2015 -0700 ---------------------------------------------------------------------- .../common-services/ATLAS/0.1.0.2.3/alerts.json | 9 +- .../ATLAS/0.1.0.2.3/kerberos.json | 26 ++-- .../ATLAS/0.1.0.2.3/package/scripts/metadata.py | 9 ++ .../package/scripts/metadata_server.py | 73 +++++++++-- .../ATLAS/0.1.0.2.3/package/scripts/params.py | 19 ++- .../0.1.0.2.3/package/scripts/service_check.py | 35 ++---- .../0.1.0.2.3/package/scripts/status_params.py | 3 +- .../stacks/2.3/ATLAS/test_metadata_server.py | 120 ++++++++++--------- 8 files changed, 184 insertions(+), 110 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/af675f47/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/alerts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/alerts.json b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/alerts.json index 7202950..5c4c06e 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/alerts.json +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/alerts.json @@ -37,10 +37,13 @@ "source": { "type": "WEB", "uri": { - "http": "{{metadata-env/metadata_port}}", + "http": "{{hostname}}:{{metadata-env/metadata_port}}", + "https": "{{hostname}}:21443", + "https_property": "{{application-properties/enableTLS}}", + "https_property_value": "true", "default_port": 21000, - "kerberos_keytab": "{{metadata-runtime.properties/*.metadata.http.authentication.keytab}}", - "kerberos_principal": "{{metadata-runtime.properties/*.metadata.http.authentication.principal}}" + "kerberos_keytab": "{{application-properties/http_authentication_kerberos_keytab}}", + "kerberos_principal": "{{application-properties/http_authentication_kerberos_principal}}" }, "reporting": { "ok": { http://git-wip-us.apache.org/repos/asf/ambari/blob/af675f47/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/kerberos.json b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/kerberos.json index 210f414..b57f565 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/kerberos.json +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/kerberos.json @@ -2,30 +2,28 @@ "services": [ { "name": "ATLAS", - "identities": [ - { - "name": "/spnego" - } - ], "configurations": [ { - "metadata-runtime.properties": { - "*.metadata.authentication.type": "kerberos", - "*.metadata.http.authentication.type": "kerberos", - "*.dfs.namenode.kerberos.principal": "nn/_HOST@${realm}" + "application-properties": { + "authentication_method": "kerberos", + "http_authentication_enabled": "true", + "http_authentication_type": "kerberos" } } ], + "auth_to_local_properties" : [ + "application-properties/http_authentication_kerberos_name_rules" + ], "components": [ { "name": "ATLAS_SERVER", "identities": [ { - "name": "metadata_server", + "name": "atlas", "principal": { "value": "atlas/_HOST@${realm}", "type" : "service", - "configuration": "application-properties/*.metadata.authentication.principal", + "configuration": "application-properties/authentication_principal", "local_username" : "${metadata-env/metadata_user}" }, "keytab": { @@ -38,17 +36,17 @@ "name": "${cluster-env/user_group}", "access": "" }, - "configuration": "application-properties/*.metadata.authentication.keytab" + "configuration": "application-properties/authentication_keytab" } }, { "name": "/spnego", "principal": { "value": "HTTP/_HOST@${realm}", - "configuration": "application-properties/*.metadata.http.authentication.principal" + "configuration": "application-properties/http_authentication_kerberos_principal" }, "keytab": { - "configuration": "application-properties/*.metadata.http.authentication.keytab" + "configuration": "application-properties/http_authentication_kerberos_keytab" } } ] http://git-wip-us.apache.org/repos/asf/ambari/blob/af675f47/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py index 61cdec4..a4274ca 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py @@ -17,6 +17,8 @@ See the License for the specific language governing permissions and limitations under the License. """ +import os +import shutil from resource_management import Directory, Fail, Logger, File, \ InlineTemplate, StaticFile from resource_management.libraries.functions import format @@ -65,6 +67,13 @@ def metadata(): recursive=True ) + metadata_war_file = format('{params.metadata_home}/server/webapp/metadata.war') + if not os.path.isfile(metadata_war_file): + raise Fail("Unable to copy {0} because it does not exist".format(metadata_war_file)) + + Logger.info("Copying {0} to {1}".format(metadata_war_file, params.expanded_war_dir)) + shutil.copy2(metadata_war_file, params.expanded_war_dir) + File(format('{conf_dir}/application.properties'), content=InlineTemplate(params.application_properties_content), mode=0644, http://git-wip-us.apache.org/repos/asf/ambari/blob/af675f47/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py index 2484315..c07961d 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py @@ -16,13 +16,12 @@ See the License for the specific language governing permissions and limitations under the License. """ -import os -import shutil - from metadata import metadata -from resource_management import Execute, check_process_status, Script, Fail, \ - Logger +from resource_management import Execute, check_process_status, Script from resource_management.libraries.functions import format +from resource_management.libraries.functions.security_commons import build_expectations, \ + get_params_from_filesystem, validate_security_config_properties, \ + FILE_TYPE_PROPERTIES class MetadataServer(Script): @@ -31,7 +30,6 @@ class MetadataServer(Script): def install(self, env): self.install_packages(env) - self.configure(env) def configure(self, env): import params @@ -46,13 +44,7 @@ class MetadataServer(Script): def start(self, env, rolling_restart=False): import params env.set_params(params) - - metadata_war_file = format('{params.metadata_home}/server/webapp/metadata.war') - if not os.path.isfile(metadata_war_file): - raise Fail("Unable to copy {0} because it does not exist".format(metadata_war_file)) - - Logger.info("Copying {0} to {1}".format(metadata_war_file, params.expanded_war_dir)) - shutil.copy2(metadata_war_file, params.expanded_war_dir) + self.configure(env) daemon_cmd = format('source {params.conf_dir}/metadata-env.sh ; {params.metadata_start_script} --port {params.metadata_port}') no_op_test = format('ls {params.pid_file} >/dev/null 2>&1 && ps -p `cat {params.pid_file}` >/dev/null 2>&1') @@ -76,5 +68,60 @@ class MetadataServer(Script): env.set_params(status_params) check_process_status(status_params.pid_file) + def security_status(self, env): + import status_params + + env.set_params(status_params) + + props_value_check = {'metadata.authentication.method': 'kerberos', + 'metadata.http.authentication.enabled': 'true', + 'metadata.http.authentication.type': 'kerberos'} + props_empty_check = ['metadata.authentication.principal', + 'metadata.authentication.keytab', + 'metadata.http.authentication.kerberos.principal', + 'metadata.http.authentication.kerberos.keytab'] + props_read_check = ['metadata.authentication.keytab', + 'metadata.http.authentication.kerberos.keytab'] + atlas_site_expectations = build_expectations('application-properties', + props_value_check, + props_empty_check, + props_read_check) + + atlas_expectations = {} + atlas_expectations.update(atlas_site_expectations) + + security_params = get_params_from_filesystem(status_params.conf_dir, + {'application.properties': FILE_TYPE_PROPERTIES}) + result_issues = validate_security_config_properties(security_params, atlas_expectations) + if not result_issues: # If all validations passed successfully + try: + # Double check the dict before calling execute + if ( 'application-properties' not in security_params + or 'metadata.authentication.keytab' not in security_params['application-properties'] + or 'metadata.authentication.principal' not in security_params['application-properties']): + self.put_structured_out({"securityState": "UNSECURED"}) + self.put_structured_out( + {"securityIssuesFound": "Atlas service keytab file or principal are not set property."}) + return + + if ( 'application-properties' not in security_params + or 'metadata.http.authentication.kerberos.keytab' not in security_params['application-properties'] + or 'metadata.http.authentication.kerberos.principal' not in security_params['application-properties']): + self.put_structured_out({"securityState": "UNSECURED"}) + self.put_structured_out( + {"securityIssuesFound": "HTTP Authentication keytab file or principal are not set property."}) + return + + self.put_structured_out({"securityState": "SECURED_KERBEROS"}) + except Exception as e: + self.put_structured_out({"securityState": "ERROR"}) + self.put_structured_out({"securityStateErrorInfo": str(e)}) + else: + issues = [] + for cf in result_issues: + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) + self.put_structured_out({"securityState": "UNSECURED"}) + if __name__ == "__main__": MetadataServer().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/af675f47/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py index 8401a5d..577c5ac 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py @@ -69,6 +69,9 @@ metadata_host = config['hostname'] application_properties = config['configurations']['application-properties'] for key, value in application_properties.iteritems(): + # fix the multi-line property + if (key == 'http_authentication_kerberos_name_rules'): + value = ' \\ \n'.join(value.splitlines()) globals()[key] = value metadata_env_content = config['configurations']['metadata-env']['content'] @@ -77,4 +80,18 @@ application_properties_content = config['configurations']['application-propertie metadata_opts = config['configurations']['metadata-env']['metadata_opts'] metadata_classpath = config['configurations']['metadata-env']['metadata_classpath'] data_dir = config['configurations']['metadata-env']['metadata_data_dir'] -expanded_war_dir = os.environ['METADATA_EXPANDED_WEBAPP_DIR'] if 'METADATA_EXPANDED_WEBAPP_DIR' in os.environ else '/var/lib/atlas/server/webapp' \ No newline at end of file +expanded_war_dir = os.environ['METADATA_EXPANDED_WEBAPP_DIR'] if 'METADATA_EXPANDED_WEBAPP_DIR' in os.environ else '/var/lib/atlas/server/webapp' + +# smoke test +smoke_test_user = config['configurations']['cluster-env']['smokeuser'] +smoke_test_password = 'smoke' +smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] +smokeuser_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] + +kinit_path_local = status_params.kinit_path_local + +security_check_status_file = format('{log_dir}/security_check.status') +if security_enabled: + smoke_cmd = format('curl --negotiate -u : -b ~/cookiejar.txt -c ~/cookiejar.txt -s -o /dev/null -w "%{{http_code}}" http://{metadata_host}:{metadata_port}/') +else: + smoke_cmd = format('curl -s -o /dev/null -w "%{{http_code}}" http://{metadata_host}:{metadata_port}/') http://git-wip-us.apache.org/repos/asf/ambari/blob/af675f47/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/service_check.py index 1ecb795..194a3fb 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/service_check.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/service_check.py @@ -16,42 +16,29 @@ See the License for the specific language governing permissions and limitations under the License. """ -import httplib -import socket -import time -from resource_management import Script, Logger, ComponentIsNotRunning, Fail +from resource_management import Script, Logger, ComponentIsNotRunning, \ + Execute from resource_management.libraries.functions import format class AtlasServiceCheck(Script): - ATLAS_CONNECT_TRIES = 5 - ATLAS_CONNECT_TIMEOUT = 10 def service_check(self, env): import params env.set_params(params) - for i in xrange(0, self.ATLAS_CONNECT_TRIES): - try: - conn = httplib.HTTPConnection(params.metadata_host, - int(params.metadata_port)) - conn.request("GET", format("http://{params.metadata_host}:{params.metadata_port}/";)) - except (httplib.HTTPException, socket.error) as ex: - if i < self.ATLAS_CONNECT_TRIES - 1: - time.sleep(self.ATLAS_CONNECT_TIMEOUT) - Logger.info("Connection failed. Next retry in %s seconds." - % (self.ATLAS_CONNECT_TIMEOUT)) - continue - else: - raise Fail("Service check has failed.") - - resp = conn.getresponse() - if resp.status == 200 : + if params.security_enabled: + Execute(format("{kinit_path_local} -kt {smokeuser_keytab} {smokeuser_principal}"), + user=params.metadata_user) + + try: + Execute(params.smoke_cmd, user=params.metadata_user, tries = 5, + try_sleep = 10) Logger.info('Atlas server up and running') - else: + except: Logger.debug('Atlas server not running') - raise ComponentIsNotRunning() + if __name__ == "__main__": AtlasServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/af675f47/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py index aa7f614..db295c4 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py @@ -20,6 +20,7 @@ limitations under the License. import os from resource_management import Script from resource_management.libraries.functions import get_kinit_path, format +from resource_management.libraries.functions.default import default config = Script.get_config() @@ -32,5 +33,5 @@ metadata_user = config['configurations']['metadata-env']['metadata_user'] # Security related/required params hostname = config['hostname'] security_enabled = config['configurations']['cluster-env']['security_enabled'] -kinit_path_local = get_kinit_path() +kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) tmp_dir = Script.get_tmp_dir() http://git-wip-us.apache.org/repos/asf/ambari/blob/af675f47/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py index 6f1377b..9db21f8 100644 --- a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py +++ b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py @@ -28,7 +28,70 @@ class TestMetadataServer(RMFTestCase): COMMON_SERVICES_PACKAGE_DIR = "ATLAS/0.1.0.2.3/package" STACK_VERSION = "2.3" - def test_configure_default(self): + def configureResourcesCalled(self): + self.assertResourceCalled('Directory', '/var/run/atlas', + owner='atlas', + group='hadoop', + recursive=True, + cd_access='a', + mode=0755 + ) + self.assertResourceCalled('Directory', '/etc/atlas/conf', + owner='atlas', + group='hadoop', + recursive=True, + cd_access='a', + mode=0755 + ) + self.assertResourceCalled('Directory', '/var/log/atlas', + owner='atlas', + group='hadoop', + recursive=True, + cd_access='a', + mode=0755 + ) + self.assertResourceCalled('Directory', '/var/lib/atlas/data', + owner='atlas', + group='hadoop', + recursive=True, + cd_access='a', + mode=0644 + ) + self.assertResourceCalled('Directory', '/var/lib/atlas/server/webapp', + owner='atlas', + group='hadoop', + recursive=True, + cd_access='a', + mode=0644 + ) + self.assertResourceCalled('File', + '/etc/atlas/conf/application.properties', + content=InlineTemplate( + self.getConfig()['configurations'][ + 'application-properties']['content']), + owner='atlas', + group='hadoop', + mode=0644, + ) + self.assertResourceCalled('File', '/etc/atlas/conf/metadata-env.sh', + content=InlineTemplate( + self.getConfig()['configurations'][ + 'metadata-env']['content']), + owner='atlas', + group='hadoop', + mode=0755, + ) + self.assertResourceCalled('File', '/etc/atlas/conf/log4j.xml', + content=StaticFile('log4j.xml'), + owner='atlas', + group='hadoop', + mode=0644, + ) + + @patch("shutil.copy2", new = MagicMock()) + @patch("os.path.isfile") + def test_configure_default(self, isfile_mock): + isfile_mock.return_value = True self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/metadata_server.py", classname = "MetadataServer", @@ -38,59 +101,7 @@ class TestMetadataServer(RMFTestCase): target = RMFTestCase.TARGET_COMMON_SERVICES ) - self.assertResourceCalled('Directory', '/var/run/atlas', - owner = 'atlas', - group = 'hadoop', - recursive = True, - cd_access = 'a', - mode=0755 - ) - self.assertResourceCalled('Directory', '/etc/atlas/conf', - owner = 'atlas', - group = 'hadoop', - recursive = True, - cd_access = 'a', - mode=0755 - ) - self.assertResourceCalled('Directory', '/var/log/atlas', - owner = 'atlas', - group = 'hadoop', - recursive = True, - cd_access = 'a', - mode=0755 - ) - self.assertResourceCalled('Directory', '/var/lib/atlas/data', - owner = 'atlas', - group = 'hadoop', - recursive = True, - cd_access = 'a', - mode=0644 - ) - self.assertResourceCalled('Directory', '/var/lib/atlas/server/webapp', - owner = 'atlas', - group = 'hadoop', - recursive = True, - cd_access = 'a', - mode=0644 - ) - self.assertResourceCalled('File', '/etc/atlas/conf/application.properties', - content = InlineTemplate(self.getConfig()['configurations']['application-properties']['content']), - owner = 'atlas', - group = 'hadoop', - mode = 0644, - ) - self.assertResourceCalled('File', '/etc/atlas/conf/metadata-env.sh', - content = InlineTemplate(self.getConfig()['configurations']['metadata-env']['content']), - owner = 'atlas', - group = 'hadoop', - mode = 0755, - ) - self.assertResourceCalled('File', '/etc/atlas/conf/log4j.xml', - content = StaticFile('log4j.xml'), - owner = 'atlas', - group = 'hadoop', - mode = 0644, - ) + self.configureResourcesCalled() self.assertNoMoreResources() @patch("shutil.copy2", new = MagicMock()) @@ -105,6 +116,7 @@ class TestMetadataServer(RMFTestCase): hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES ) + self.configureResourcesCalled() self.assertResourceCalled('Execute', 'source /etc/atlas/conf/metadata-env.sh ; /usr/hdp/current/atlas-server/bin/metadata_start.py --port 21000', not_if = 'ls /var/run/atlas/metadata.pid >/dev/null 2>&1 && ps -p `cat /var/run/atlas/metadata.pid` >/dev/null 2>&1', user = 'atlas',
