AMBARI-10225: [WinTP2] Merge HDPWIN TEZ package scripts to common services (jluniya)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/194df087 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/194df087 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/194df087 Branch: refs/heads/trunk Commit: 194df087131fe9a254e6c0e2c47499305bc72075 Parents: e4686db Author: Jayush Luniya <jlun...@hortonworks.com> Authored: Fri Mar 27 08:54:00 2015 -0700 Committer: Jayush Luniya <jlun...@hortonworks.com> Committed: Fri Mar 27 08:54:00 2015 -0700 ---------------------------------------------------------------------- .../TEZ/0.4.0.2.1/package/scripts/params.py | 58 ++------------- .../0.4.0.2.1/package/scripts/params_linux.py | 76 ++++++++++++++++++++ .../0.4.0.2.1/package/scripts/params_windows.py | 30 ++++++++ .../0.4.0.2.1/package/scripts/service_check.py | 14 ++++ .../TEZ/0.4.0.2.1/package/scripts/tez.py | 21 +++--- .../TEZ/0.4.0.2.1/package/scripts/tez_client.py | 26 +++++-- .../2.1/services/TEZ/package/scripts/params.py | 30 -------- .../2.1/services/TEZ/package/scripts/tez.py | 31 -------- .../services/TEZ/package/scripts/tez_client.py | 42 ----------- 9 files changed, 155 insertions(+), 173 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/194df087/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params.py index b58bcd6..a10c1d4 100644 --- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params.py @@ -17,59 +17,9 @@ See the License for the specific language governing permissions and limitations under the License. """ +from ambari_commons import OSCheck -from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions -from resource_management import * - -# server configurations -config = Script.get_config() -tmp_dir = Script.get_tmp_dir() - -stack_name = default("/hostLevelParams/stack_name", None) - -# This is expected to be of the form #.#.#.# -stack_version_unformatted = str(config['hostLevelParams']['stack_version']) -hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) - -# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade -version = default("/commandParams/version", None) - -if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0: - hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin" - path_to_tez_examples_jar = "/usr/hdp/{hdp_version}/tez/tez-examples*.jar" +if OSCheck.is_windows_family(): + from params_windows import * else: - hadoop_bin_dir = "/usr/bin" - path_to_tez_examples_jar = "/usr/lib/tez/tez-mapreduce-examples*.jar" -hadoop_conf_dir = "/etc/hadoop/conf" - -kinit_path_local = functions.get_kinit_path() -security_enabled = config['configurations']['cluster-env']['security_enabled'] -smokeuser = config['configurations']['cluster-env']['smokeuser'] -smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] -smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] -hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] -hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] -hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] - -config_dir_prefix = "/etc/tez" -config_dir = format("{config_dir_prefix}/conf") - -hadoop_home = '/usr' -java64_home = config['hostLevelParams']['java_home'] - -tez_user = config['configurations']['tez-env']['tez_user'] -user_group = config['configurations']['cluster-env']['user_group'] -tez_env_sh_template = config['configurations']['tez-env']['content'] - -import functools -# Create partial functions with common arguments for every HdfsDirectory call -# to create hdfs directory we need to call params.HdfsDirectory in code -HdfsDirectory = functools.partial( - HdfsDirectory, - conf_dir=hadoop_conf_dir, - hdfs_user=hdfs_user, - security_enabled=security_enabled, - keytab=hdfs_user_keytab, - kinit_path_local=kinit_path_local, - bin_dir=hadoop_bin_dir -) + from params_linux import * http://git-wip-us.apache.org/repos/asf/ambari/blob/194df087/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py new file mode 100644 index 0000000..3245ff0 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions +from resource_management import * + +# server configurations +config = Script.get_config() +tmp_dir = Script.get_tmp_dir() + +stack_name = default("/hostLevelParams/stack_name", None) + +# This is expected to be of the form #.#.#.# +stack_version_unformatted = str(config['hostLevelParams']['stack_version']) +hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) + +# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade +version = default("/commandParams/version", None) + +if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0: + hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin" + path_to_tez_examples_jar = "/usr/hdp/{hdp_version}/tez/tez-examples*.jar" +else: + hadoop_bin_dir = "/usr/bin" + path_to_tez_examples_jar = "/usr/lib/tez/tez-mapreduce-examples*.jar" +hadoop_conf_dir = "/etc/hadoop/conf" + +kinit_path_local = functions.get_kinit_path() +security_enabled = config['configurations']['cluster-env']['security_enabled'] +smokeuser = config['configurations']['cluster-env']['smokeuser'] +smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] +hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] +hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] + +config_dir_prefix = "/etc/tez" +config_dir = format("{config_dir_prefix}/conf") + +hadoop_home = '/usr' +java64_home = config['hostLevelParams']['java_home'] + +tez_user = config['configurations']['tez-env']['tez_user'] +user_group = config['configurations']['cluster-env']['user_group'] +tez_env_sh_template = config['configurations']['tez-env']['content'] + +import functools +# Create partial functions with common arguments for every HdfsDirectory call +# to create hdfs directory we need to call params.HdfsDirectory in code +HdfsDirectory = functools.partial( + HdfsDirectory, + conf_dir=hadoop_conf_dir, + hdfs_user=hdfs_user, + security_enabled=security_enabled, + keytab=hdfs_user_keytab, + kinit_path_local=kinit_path_local, + bin_dir=hadoop_bin_dir +) + http://git-wip-us.apache.org/repos/asf/ambari/blob/194df087/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py new file mode 100644 index 0000000..db34c9f --- /dev/null +++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py @@ -0,0 +1,30 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +import os + +config = Script.get_config() +tez_user = "hadoop" +tez_home_dir = None +tez_conf_dir = "conf" + +if os.environ.has_key("TEZ_HOME"): + tez_home_dir = os.environ["TEZ_HOME"] + tez_conf_dir = os.path.join(tez_home_dir, "conf") http://git-wip-us.apache.org/repos/asf/ambari/blob/194df087/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/service_check.py index 434b505..35ae5b6 100644 --- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/service_check.py +++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/service_check.py @@ -22,8 +22,14 @@ Ambari Agent from resource_management import * from resource_management.libraries.functions.version import compare_versions from resource_management.libraries.functions.dynamic_variable_interpretation import copy_tarballs_to_hdfs +from ambari_commons import OSConst +from ambari_commons.os_family_impl import OsFamilyImpl class TezServiceCheck(Script): + pass + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class TezServiceCheckLinux(TezServiceCheck): def service_check(self, env): import params env.set_params(params) @@ -91,6 +97,14 @@ class TezServiceCheck(Script): ) +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class TezServiceCheckWindows(TezServiceCheck): + def service_check(self, env): + import params + env.set_params(params) + smoke_cmd = os.path.join(params.hdp_root,"Run-SmokeTests.cmd") + service = "TEZ" + Execute(format("cmd /C {smoke_cmd} {service}"), logoutput=True, user=params.tez_user) if __name__ == "__main__": http://git-wip-us.apache.org/repos/asf/ambari/blob/194df087/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez.py index bb8b7c2..ceafa56 100644 --- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez.py +++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez.py @@ -20,7 +20,10 @@ Ambari Agent """ from resource_management import * +from ambari_commons import OSConst +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) def tez(): import params @@ -48,14 +51,14 @@ def tez(): ) -def tez_TemplateConfig(name): +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def tez(): import params - - if not isinstance(name, list): - name = [name] - - for x in name: - TemplateConfig(format("{config_dir}/{x}"), - owner = params.tez_user - ) + XmlConfig("tez-site.xml", + conf_dir=params.tez_conf_dir, + configurations=params.config['configurations']['tez-site'], + owner=params.tez_user, + mode="f", + configuration_attributes=params.config['configuration_attributes']['tez-site'] + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/194df087/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py index 00375d7..9746d13 100644 --- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py +++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py @@ -22,8 +22,20 @@ Ambari Agent import sys from resource_management import * from tez import tez +from ambari_commons import OSConst +from ambari_commons.os_family_impl import OsFamilyImpl class TezClient(Script): + def configure(self, env): + import params + env.set_params(params) + tez() + + def status(self, env): + raise ClientComponentHasNoStatus() + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class TezClientLinux(TezClient): def get_stack_to_component(self): return {"HDP": "hadoop-client"} @@ -39,14 +51,14 @@ class TezClient(Script): self.install_packages(env) self.configure(env) - def configure(self, env): - import params - env.set_params(params) - tez() - - def status(self, env): - raise ClientComponentHasNoStatus() +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class TezClientWindows(TezClient): + def install(self, env): + import params + if params.tez_home_dir is None: + self.install_packages(env) + self.configure(env) if __name__ == "__main__": TezClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/194df087/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/params.py deleted file mode 100644 index db34c9f..0000000 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/params.py +++ /dev/null @@ -1,30 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management import * -import os - -config = Script.get_config() -tez_user = "hadoop" -tez_home_dir = None -tez_conf_dir = "conf" - -if os.environ.has_key("TEZ_HOME"): - tez_home_dir = os.environ["TEZ_HOME"] - tez_conf_dir = os.path.join(tez_home_dir, "conf") http://git-wip-us.apache.org/repos/asf/ambari/blob/194df087/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/tez.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/tez.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/tez.py deleted file mode 100644 index d9f5a38..0000000 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/tez.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management import * -import sys - -def tez(): - import params - XmlConfig("tez-site.xml", - conf_dir=params.tez_conf_dir, - configurations=params.config['configurations']['tez-site'], - owner=params.tez_user, - mode="f", - configuration_attributes=params.config['configuration_attributes']['tez-site'] - ) http://git-wip-us.apache.org/repos/asf/ambari/blob/194df087/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/tez_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/tez_client.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/tez_client.py deleted file mode 100644 index 89dd6bb..0000000 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/tez_client.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -import sys -from resource_management import * -from tez import tez - -class TezClient(Script): - def install(self, env): - import params - if params.tez_home_dir is None: - self.install_packages(env) - self.configure(env) - - def configure(self, env): - import params - env.set_params(params) - tez() - - def status(self, env): - raise ClientComponentHasNoStatus() - -if __name__ == "__main__": - TezClient().execute()
