Repository: ambari Updated Branches: refs/heads/trunk 0a9101e97 -> 260419bb8
AMBARI-14921 - Atlas Integration : Support Atlas / Sqoop integration Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/260419bb Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/260419bb Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/260419bb Branch: refs/heads/trunk Commit: 260419bb848ce962bbc238379221096eb2a31a58 Parents: 0a9101e Author: tbeerbower <tbeerbo...@hortonworks.com> Authored: Thu Feb 18 11:22:03 2016 -0500 Committer: tbeerbower <tbeerbo...@hortonworks.com> Committed: Thu Feb 18 12:15:55 2016 -0500 ---------------------------------------------------------------------- .../1.4.4.2.0/configuration/sqoop-site.xml | 43 ++++++++++++++++++++ .../SQOOP/1.4.4.2.0/metainfo.xml | 6 +++ .../1.4.4.2.0/package/scripts/params_linux.py | 20 ++++++++- .../SQOOP/1.4.4.2.0/package/scripts/sqoop.py | 26 ++++++++++++ .../python/stacks/2.0.6/SQOOP/test_sqoop.py | 12 ++++++ .../python/stacks/2.0.6/configs/default.json | 5 +++ .../python/stacks/2.0.6/configs/secured.json | 5 +++ .../test/python/stacks/2.3/configs/default.json | 5 +++ 8 files changed, 121 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/260419bb/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml new file mode 100644 index 0000000..dd299fe --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml @@ -0,0 +1,43 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <property> + <name>atlas.cluster.name</name> + <value>{{cluster_name}}</value> + <depends-on> + <property> + <type>application-properties</type> + <name>atlas.cluster.name</name> + </property> + </depends-on> + </property> + <property> + <name>sqoop.job.data.publish.class</name> + <value>{{job_data_publish_class}}</value> + <depends-on> + <property> + <type>application-properties</type> + <name>atlas.cluster.name</name> + </property> + </depends-on> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/260419bb/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml index b5db91b..2c9d881 100644 --- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml @@ -55,6 +55,11 @@ </commandScript> <configFiles> <configFile> + <type>xml</type> + <fileName>squoop-site.xml</fileName> + <dictionaryName>squoop-site</dictionaryName> + </configFile> + <configFile> <type>env</type> <fileName>sqoop-env.sh</fileName> <dictionaryName>sqoop-env</dictionaryName> @@ -89,6 +94,7 @@ <configuration-dependencies> <config-type>sqoop-env</config-type> + <config-type>sqoop-site</config-type> </configuration-dependencies> </service> </services> http://git-wip-us.apache.org/repos/asf/ambari/blob/260419bb/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py index 3a3f93e..0006078 100644 --- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py @@ -21,6 +21,7 @@ from resource_management.libraries.functions.version import format_hdp_stack_ver from resource_management.libraries.functions.default import default from resource_management.libraries.functions.get_kinit_path import get_kinit_path from resource_management.libraries.script import Script +import os # a map of the Ambari role to the component name # for use with /usr/hdp/current/<component> @@ -31,6 +32,9 @@ SERVER_ROLE_DIRECTORY_MAP = { component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "SQOOP") config = Script.get_config() + +cluster_name = config['clusterName'] + ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] stack_name = default("/hostLevelParams/stack_name", None) @@ -103,4 +107,18 @@ if "jdbc_drivers" in config['configurations']['sqoop-env']: continue sqoop_jdbc_drivers_dict[jdbc_jar_name] = jdbc_symlink_name sqoop_jdbc_drivers_name_dict[jdbc_jar_name] = jdbc_driver_name -jdk_location = config['hostLevelParams']['jdk_location'] \ No newline at end of file +jdk_location = config['hostLevelParams']['jdk_location'] + +job_data_publish_class = '' + +######################################################## +############# Atlas related params ##################### +######################################################## + +atlas_hosts = default('/clusterHostInfo/atlas_server_hosts', []) +has_atlas = len(atlas_hosts) > 0 + +if has_atlas: + atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else '/usr/hdp/current/atlas-server' + atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf' + job_data_publish_class = 'org.apache.atlas.sqoop.hook.SqoopHook' \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/260419bb/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py index 66a5ee4..392bff7 100644 --- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py @@ -19,6 +19,7 @@ limitations under the License. from resource_management.core.source import InlineTemplate, DownloadSource from resource_management.libraries.functions import format +from resource_management.libraries.resources.xml_config import XmlConfig from resource_management.core.resources.system import File, Link, Directory from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from ambari_commons import OSConst @@ -46,6 +47,31 @@ def sqoop(type=None): group = params.user_group, create_parents = True ) + + configs = {} + configs.update(params.config['configurations']['sqoop-site']) + + XmlConfig("sqoop-site.xml", + conf_dir = params.sqoop_conf_dir, + configurations = configs, + configuration_attributes=params.config['configuration_attributes']['sqoop-site'], + owner = params.sqoop_user, + group = params.user_group + ) + + if params.has_atlas: + Link(params.sqoop_conf_dir + "/application.properties", + to = params.atlas_conf_dir + "/application.properties" + ) + + atlas_sqoop_hook_dir = params.atlas_home_dir + "/hook/sqoop" + src_files = os.listdir(atlas_sqoop_hook_dir) + for file_name in src_files: + atlas_sqoop_hook_file_name = os.path.join(atlas_sqoop_hook_dir, file_name) + sqoop_lib_file_name = os.path.join(params.sqoop_lib, file_name) + if (os.path.isfile(atlas_sqoop_hook_file_name)): + Link(sqoop_lib_file_name, to = atlas_sqoop_hook_file_name) + File(format("{sqoop_conf_dir}/sqoop-env.sh"), owner=params.sqoop_user, group = params.user_group, http://git-wip-us.apache.org/repos/asf/ambari/blob/260419bb/ambari-server/src/test/python/stacks/2.0.6/SQOOP/test_sqoop.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/SQOOP/test_sqoop.py b/ambari-server/src/test/python/stacks/2.0.6/SQOOP/test_sqoop.py index 5a10305..e12db4d 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/SQOOP/test_sqoop.py +++ b/ambari-server/src/test/python/stacks/2.0.6/SQOOP/test_sqoop.py @@ -39,6 +39,12 @@ class TestSqoop(RMFTestCase): create_parents = True, owner = 'sqoop', group = 'hadoop',) + self.assertResourceCalled('XmlConfig', 'sqoop-site.xml', + owner = 'sqoop', + group = 'hadoop', + conf_dir = '/usr/lib/sqoop/conf', + configurations = self.getConfig()['configurations']['sqoop-site'], + configuration_attributes = self.getConfig()['configuration_attributes']['sqoop-site']) self.assertResourceCalled('File', '/usr/lib/sqoop/conf/sqoop-env.sh', owner = 'sqoop', group = 'hadoop', @@ -89,6 +95,12 @@ class TestSqoop(RMFTestCase): group = 'hadoop', create_parents = True, ) + self.assertResourceCalled('XmlConfig', 'sqoop-site.xml', + owner = 'sqoop', + group = 'hadoop', + conf_dir = '/usr/lib/sqoop/conf', + configurations = self.getConfig()['configurations']['sqoop-site'], + configuration_attributes = self.getConfig()['configuration_attributes']['sqoop-site']) self.assertResourceCalled('File', '/usr/lib/sqoop/conf/sqoop-env.sh', content = InlineTemplate(self.getConfig()['configurations']['sqoop-env']['content']), owner = 'sqoop', http://git-wip-us.apache.org/repos/asf/ambari/blob/260419bb/ambari-server/src/test/python/stacks/2.0.6/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json index 3d3987b..adf12ad 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json @@ -40,6 +40,10 @@ "taskId": 152, "public_hostname": "c6401.ambari.apache.org", "configurations": { + "sqoop-site": { + "atlas.cluster.name": "c1", + "sqoop.job.data.publish.class": "org.apache.atlas.sqoop.hook.SqoopHook" + }, "mapred-site": { "mapreduce.jobhistory.address": "c6402.ambari.apache.org:10020", "mapreduce.cluster.administrators": " hadoop", @@ -877,6 +881,7 @@ "ams-hbase-log4j": {}, "ams-site": {}, "ams-ssl-server": {}, + "sqoop-site": {}, "yarn-site": { "final": { "yarn.nodemanager.disk-health-checker.min-healthy-disks": "true", http://git-wip-us.apache.org/repos/asf/ambari/blob/260419bb/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json index d673973..e17559d 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json @@ -37,6 +37,10 @@ "taskId": 186, "public_hostname": "c6401.ambari.apache.org", "configurations": { + "sqoop-site": { + "atlas.cluster.name": "c1", + "sqoop.job.data.publish.class": "org.apache.atlas.sqoop.hook.SqoopHook" + }, "mapred-site": { "mapreduce.jobhistory.address": "c6402.ambari.apache.org:10020", "mapreduce.jobhistory.webapp.spnego-keytab-file": "/etc/security/keytabs/spnego.service.keytab", @@ -765,6 +769,7 @@ } }, "configuration_attributes": { + "sqoop-site": {}, "yarn-site": { "final": { "yarn.nodemanager.disk-health-checker.min-healthy-disks": "true", http://git-wip-us.apache.org/repos/asf/ambari/blob/260419bb/ambari-server/src/test/python/stacks/2.3/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/configs/default.json b/ambari-server/src/test/python/stacks/2.3/configs/default.json index a36455c..4a72f08 100644 --- a/ambari-server/src/test/python/stacks/2.3/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.3/configs/default.json @@ -33,6 +33,10 @@ "slider-client": { "slider.yarn.queue": "default" }, + "sqoop-site": { + "atlas.cluster.name": "c1", + "sqoop.job.data.publish.class": "org.apache.atlas.sqoop.hook.SqoopHook" + }, "mahout-env": { "mahout_user": "mahout" }, @@ -214,6 +218,7 @@ } }, "configuration_attributes": { + "sqoop-site": {}, "yarn-site": { "final": { "yarn.nodemanager.disk-health-checker.min-healthy-disks": "true",