Repository: ambari Updated Branches: refs/heads/trunk 2f218e7ea -> 4a3797d80
AMBARI-11357. Spark History Server in HDP 2.3 no longer needs tez.tar.gz, which now needs to be uploaded to HDFS during Tez Service Check (alejandro) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4a3797d8 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4a3797d8 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4a3797d8 Branch: refs/heads/trunk Commit: 4a3797d8012c3bc0365ddd4e16bd12c1ac49672b Parents: 2f218e7 Author: Alejandro Fernandez <[email protected]> Authored: Fri May 22 18:52:12 2015 -0700 Committer: Alejandro Fernandez <[email protected]> Committed: Mon May 25 01:30:08 2015 -0700 ---------------------------------------------------------------------- .../package/scripts/job_history_server.py | 9 +- .../1.2.0.2.2/package/scripts/spark_service.py | 10 +- .../SPARK/1.3.1.2.3/metainfo.xml | 152 +++++++++++++++++++ .../0.4.0.2.1/package/scripts/service_check.py | 14 +- .../stacks/HDP/2.3/services/SPARK/metainfo.xml | 4 + .../stacks/2.2/SPARK/test_job_history_server.py | 11 -- 6 files changed, 181 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/4a3797d8/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py index 106b8cc..4b0bbfa 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py @@ -78,9 +78,12 @@ class JobHistoryServer(Script): conf_select.select(params.stack_name, "spark", params.version) hdp_select.select("spark-historyserver", params.version) - resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user) - if resource_created: - params.HdfsResource(None, action="execute") + # Spark 1.3.1.2.3, and higher, which was included in HDP 2.3, does not have a dependency on Tez, so it does not + # need to copy the tarball, otherwise, copy it. + if compare_versions(format_hdp_stack_version(params.version), '2.3.0.0') < 0: + resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user) + if resource_created: + params.HdfsResource(None, action="execute") if __name__ == "__main__": JobHistoryServer().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/4a3797d8/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py index 834e208..8d758e3 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py @@ -20,6 +20,7 @@ limitations under the License. from resource_management.libraries.script.script import Script from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.functions.version import compare_versions from resource_management.libraries.functions.copy_tarball import copy_to_hdfs from resource_management.libraries.functions import format from resource_management.core.resources.system import File, Execute @@ -32,9 +33,12 @@ def spark_service(action): spark_kinit_cmd = format("{kinit_path_local} -kt {spark_kerberos_keytab} {spark_principal}; ") Execute(spark_kinit_cmd, user=params.spark_user) - resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user) - if resource_created: - params.HdfsResource(None, action="execute") + # Spark 1.3.1.2.3, and higher, which was included in HDP 2.3, does not have a dependency on Tez, so it does not + # need to copy the tarball, otherwise, copy it. + if params.hdp_stack_version and compare_versions(params.hdp_stack_version, '2.3.0.0') < 0: + resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user) + if resource_created: + params.HdfsResource(None, action="execute") no_op_test = format( 'ls {spark_history_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_history_server_pid_file}` >/dev/null 2>&1') http://git-wip-us.apache.org/repos/asf/ambari/blob/4a3797d8/ambari-server/src/main/resources/common-services/SPARK/1.3.1.2.3/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.3.1.2.3/metainfo.xml b/ambari-server/src/main/resources/common-services/SPARK/1.3.1.2.3/metainfo.xml new file mode 100644 index 0000000..7929bf6 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SPARK/1.3.1.2.3/metainfo.xml @@ -0,0 +1,152 @@ +<?xml version="1.0"?> +<!--Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>SPARK</name> + <extends>common-services/SPARK/1.2.0.2.2</extends> + <displayName>Spark</displayName> + <comment>Apache Spark is a fast and general engine for large-scale data processing.</comment> + <version>1.3.1.2.3</version> + <components> + <component> + <name>SPARK_JOBHISTORYSERVER</name> + <displayName>Spark History Server</displayName> + <category>MASTER</category> + <cardinality>1</cardinality> + <versionAdvertised>true</versionAdvertised> + <dependencies> + <dependency> + <name>TEZ/TEZ_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>HDFS/HDFS_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>YARN/YARN_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> + <commandScript> + <script>scripts/job_history_server.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + <component> + <name>SPARK_CLIENT</name> + <displayName>Spark Client</displayName> + <category>CLIENT</category> + <cardinality>1+</cardinality> + <versionAdvertised>true</versionAdvertised> + <dependencies> + <dependency> + <name>HDFS/HDFS_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>YARN/YARN_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> + <commandScript> + <script>scripts/spark_client.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + </components> + + <osSpecifics> + <osSpecific> + <osFamily>redhat6,suse11</osFamily> + <packages> + <package> + <name>spark_2_2_*</name> + </package> + <package> + <name>spark_2_2_*-python</name> + </package> + </packages> + </osSpecific> + <osSpecific> + <osFamily>debian7,ubuntu12,ubuntu14</osFamily> + <packages> + <package> + <name>spark-2-2-.*</name> + </package> + <package> + <name>spark-2-2-.*-python</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <configuration-dependencies> + <config-type>spark-defaults</config-type> + <config-type>spark-env</config-type> + <config-type>spark-log4j-properties</config-type> + <config-type>spark-metrics-properties</config-type> + <config-type>spark-javaopts-properties</config-type> + </configuration-dependencies> + + <commandScript> + <script>scripts/service_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + + <requiredServices> + <service>YARN</service> + </requiredServices> + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/4a3797d8/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/service_check.py index 98973ae..dcfcd33 100644 --- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/service_check.py +++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/service_check.py @@ -18,12 +18,19 @@ limitations under the License. Ambari Agent """ - -from resource_management import * +from resource_management.libraries.script import Script +from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.resources.execute_hadoop import ExecuteHadoop +from resource_management.libraries.functions import format from resource_management.libraries.functions.version import compare_versions +from resource_management.libraries.functions.copy_tarball import copy_to_hdfs +from resource_management.core.resources.system import File, Execute + from ambari_commons import OSConst from ambari_commons.os_family_impl import OsFamilyImpl +from resource_management.core.logger import Logger + class TezServiceCheck(Script): pass @@ -54,6 +61,9 @@ class TezServiceCheckLinux(TezServiceCheck): source = format("{tmp_dir}/sample-tez-test"), ) + if params.hdp_stack_version and compare_versions(params.hdp_stack_version, '2.2.0.0') >= 0: + copy_to_hdfs("tez", params.user_group, params.hdfs_user) + params.HdfsResource(None, action = "execute") ExecuteHadoop(wordcount_command, http://git-wip-us.apache.org/repos/asf/ambari/blob/4a3797d8/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml index 205ccc9..6e9f593 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml @@ -48,6 +48,10 @@ </packages> </osSpecific> </osSpecifics> + + <requiredServices> + <service>YARN</service> + </requiredServices> </service> </services> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/4a3797d8/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py b/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py index dbefa5c..8e91481 100644 --- a/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py +++ b/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py @@ -264,17 +264,6 @@ class TestJobHistoryServer(RMFTestCase): mocks_dict = mocks_dict) self.assertResourceCalled('Execute', 'hdp-select set spark-historyserver {0}'.format(version)) - - copy_to_hdfs_mock.assert_called_with("tez", "hadoop", "hdfs") - self.assertResourceCalled('HdfsResource', None, - security_enabled = False, - hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', - keytab = UnknownConfigurationMock(), - kinit_path_local = '/usr/bin/kinit', - user = 'hdfs', - action = ['execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', - hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', - ) self.assertNoMoreResources() self.assertEquals(2, mocks_dict['call'].call_count)
