Repository: ambari Updated Branches: refs/heads/trunk f72e0bdb1 -> 7be96532b
AMBARI-10105: Merge HDPWIN SQOOP package scripts to common services (jluniya) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b8e6923d Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b8e6923d Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b8e6923d Branch: refs/heads/trunk Commit: b8e6923d97fc89f4161172a6e9066deed0f2429d Parents: f72e0bd Author: Jayush Luniya <[email protected]> Authored: Tue Mar 31 14:13:02 2015 -0700 Committer: Jayush Luniya <[email protected]> Committed: Tue Mar 31 14:13:02 2015 -0700 ---------------------------------------------------------------------- .../SQOOP/1.4.4.2.0/package/scripts/params.py | 43 ++------------ .../1.4.4.2.0/package/scripts/params_linux.py | 61 ++++++++++++++++++++ .../1.4.4.2.0/package/scripts/params_windows.py | 30 ++++++++++ .../1.4.4.2.0/package/scripts/service_check.py | 21 ++++++- .../SQOOP/1.4.4.2.0/package/scripts/sqoop.py | 15 ++++- .../1.4.4.2.0/package/scripts/sqoop_client.py | 36 +++++++----- .../services/SQOOP/package/scripts/params.py | 32 ---------- .../SQOOP/package/scripts/service_check.py | 34 ----------- .../2.1/services/SQOOP/package/scripts/sqoop.py | 27 --------- .../SQOOP/package/scripts/sqoop_client.py | 41 ------------- 10 files changed, 149 insertions(+), 191 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b8e6923d/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params.py index e12acbe..f4931a5 100644 --- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params.py @@ -17,44 +17,9 @@ limitations under the License. """ -from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions -from resource_management.libraries.functions.default import default -from resource_management import * +from ambari_commons import OSCheck -config = Script.get_config() - -stack_name = default("/hostLevelParams/stack_name", None) - -stack_version_unformatted = str(config['hostLevelParams']['stack_version']) -hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) - -# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade -version = default("/commandParams/version", None) - -#hadoop params -if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0: - sqoop_conf_dir = '/etc/sqoop/conf' - sqoop_lib = '/usr/hdp/current/sqoop-client/lib' - hadoop_home = '/usr/hdp/current/hbase-client' - hbase_home = '/usr/hdp/current/hbase-client' - hive_home = '/usr/hdp/current/hive-client' - sqoop_bin_dir = '/usr/hdp/current/sqoop-client/bin/' +if OSCheck.is_windows_family(): + from params_windows import * else: - sqoop_conf_dir = "/usr/lib/sqoop/conf" - sqoop_lib = "/usr/lib/sqoop/lib" - hadoop_home = '/usr/lib/hadoop' - hbase_home = "/usr/lib/hbase" - hive_home = "/usr/lib/hive" - sqoop_bin_dir = "/usr/bin" - -zoo_conf_dir = "/etc/zookeeper" -security_enabled = config['configurations']['cluster-env']['security_enabled'] -smokeuser = config['configurations']['cluster-env']['smokeuser'] -smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] -user_group = config['configurations']['cluster-env']['user_group'] -sqoop_env_sh_template = config['configurations']['sqoop-env']['content'] - -sqoop_user = config['configurations']['sqoop-env']['sqoop_user'] - -smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] -kinit_path_local = functions.get_kinit_path() + from params_linux import * http://git-wip-us.apache.org/repos/asf/ambari/blob/b8e6923d/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py new file mode 100644 index 0000000..20246ab --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py @@ -0,0 +1,61 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions.get_kinit_path import get_kinit_path +from resource_management.libraries.script import Script + +config = Script.get_config() + +stack_name = default("/hostLevelParams/stack_name", None) + +stack_version_unformatted = str(config['hostLevelParams']['stack_version']) +hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) + +# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade +version = default("/commandParams/version", None) + +#hadoop params +if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0: + sqoop_conf_dir = '/etc/sqoop/conf' + sqoop_lib = '/usr/hdp/current/sqoop-client/lib' + hadoop_home = '/usr/hdp/current/hbase-client' + hbase_home = '/usr/hdp/current/hbase-client' + hive_home = '/usr/hdp/current/hive-client' + sqoop_bin_dir = '/usr/hdp/current/sqoop-client/bin/' +else: + sqoop_conf_dir = "/usr/lib/sqoop/conf" + sqoop_lib = "/usr/lib/sqoop/lib" + hadoop_home = '/usr/lib/hadoop' + hbase_home = "/usr/lib/hbase" + hive_home = "/usr/lib/hive" + sqoop_bin_dir = "/usr/bin" + +zoo_conf_dir = "/etc/zookeeper" +security_enabled = config['configurations']['cluster-env']['security_enabled'] +smokeuser = config['configurations']['cluster-env']['smokeuser'] +smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] +user_group = config['configurations']['cluster-env']['user_group'] +sqoop_env_sh_template = config['configurations']['sqoop-env']['content'] + +sqoop_user = config['configurations']['sqoop-env']['sqoop_user'] + +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] +kinit_path_local = get_kinit_path() http://git-wip-us.apache.org/repos/asf/ambari/blob/b8e6923d/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_windows.py new file mode 100644 index 0000000..e4b71f1 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_windows.py @@ -0,0 +1,30 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.script import Script +import os + +config = Script.get_config() + +sqoop_user = "sqoop" + +hdp_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"], "..")) +sqoop_env_cmd_template = config['configurations']['sqoop-env']['content'] +sqoop_home_dir = os.environ["SQOOP_HOME"] +sqoop_conf_dir = os.path.join(sqoop_home_dir, "conf") \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/b8e6923d/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/service_check.py index 140862c..426315f 100644 --- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/service_check.py +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/service_check.py @@ -19,10 +19,18 @@ limitations under the License. """ -from resource_management import * - +from resource_management.libraries.script.script import Script +from resource_management.core.resources import Execute +from resource_management.libraries.functions import format +from ambari_commons.os_family_impl import OsFamilyImpl +from ambari_commons import OSConst +import os class SqoopServiceCheck(Script): + pass + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class SqoopServiceCheckDefault(SqoopServiceCheck): def get_stack_to_component(self): return {"HDP": "sqoop-server"} @@ -40,5 +48,14 @@ class SqoopServiceCheck(Script): logoutput = True ) +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class SqoopServiceCheckWindows(SqoopServiceCheck): + def service_check(self, env): + import params + env.set_params(params) + smoke_cmd = os.path.join(params.hdp_root,"Run-SmokeTests.cmd") + service = "SQOOP" + Execute(format("cmd /C {smoke_cmd} {service}"), logoutput=True) + if __name__ == "__main__": SqoopServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/b8e6923d/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py index 99ad575..e929a8c 100644 --- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py @@ -17,9 +17,22 @@ limitations under the License. """ -from resource_management import * +from resource_management.core.source import InlineTemplate +from resource_management.libraries.functions import format +from resource_management.core.resources.system import File, Link, Directory +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from ambari_commons import OSConst import os + +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def sqoop(type=None): + import params + File(os.path.join(params.sqoop_conf_dir, "sqoop-env.cmd"), + content=InlineTemplate(params.sqoop_env_cmd_template) + ) + +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) def sqoop(type=None): import params Link(params.sqoop_lib + "/mysql-connector-java.jar", http://git-wip-us.apache.org/repos/asf/ambari/blob/b8e6923d/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop_client.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop_client.py index d836513..dad0514 100644 --- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop_client.py +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop_client.py @@ -18,25 +18,15 @@ limitations under the License. """ -import sys -from resource_management import * +from resource_management.core.exceptions import ClientComponentHasNoStatus +from resource_management.core.resources.system import Execute +from resource_management.libraries.script.script import Script from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version - from sqoop import sqoop - +from ambari_commons.os_family_impl import OsFamilyImpl +from ambari_commons import OSConst class SqoopClient(Script): - - def get_stack_to_component(self): - return {"HDP": "sqoop-client"} - - def pre_rolling_restart(self, env): - import params - env.set_params(params) - - if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: - Execute(format("hdp-select set sqoop-client {version}")) - def install(self, env): self.install_packages(env) self.configure(env) @@ -49,5 +39,21 @@ class SqoopClient(Script): def status(self, env): raise ClientComponentHasNoStatus() +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class SqoopClientDefault(SqoopClient): + def get_stack_to_component(self): + return {"HDP": "sqoop-client"} + + def pre_rolling_restart(self, env): + import params + env.set_params(params) + + if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: + Execute(format("hdp-select set sqoop-client {version}")) + +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class SqoopClientWindows(SqoopClient): + pass + if __name__ == "__main__": SqoopClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/b8e6923d/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/params.py deleted file mode 100644 index 8d1d59c..0000000 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/params.py +++ /dev/null @@ -1,32 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management import * -import os - -config = Script.get_config() -sqoop_user = "sqoop" -sqoop_home_dir = None -sqoop_conf_dir = "conf" - -hdp_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"], "..")) -sqoop_env_cmd_template = config['configurations']['sqoop-env']['content'] -if os.environ.has_key("SQOOP_HOME"): - sqoop_home_dir = os.environ["SQOOP_HOME"] - sqoop_conf_dir = os.path.join(sqoop_home_dir, "conf") http://git-wip-us.apache.org/repos/asf/ambari/blob/b8e6923d/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/service_check.py deleted file mode 100644 index 18c0f15..0000000 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/service_check.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - - -from resource_management import * - - -class SqoopServiceCheck(Script): - def service_check(self, env): - import params - env.set_params(params) - smoke_cmd = os.path.join(params.hdp_root,"Run-SmokeTests.cmd") - service = "SQOOP" - Execute(format("cmd /C {smoke_cmd} {service}"), logoutput=True) - -if __name__ == "__main__": - SqoopServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/b8e6923d/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/sqoop.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/sqoop.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/sqoop.py deleted file mode 100644 index f32c20e..0000000 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/sqoop.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management import * -import os - -def sqoop(): - import params - File(os.path.join(params.sqoop_conf_dir, "sqoop-env.cmd"), - content=InlineTemplate(params.sqoop_env_cmd_template) - ) http://git-wip-us.apache.org/repos/asf/ambari/blob/b8e6923d/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/sqoop_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/sqoop_client.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/sqoop_client.py deleted file mode 100644 index 1aa5e01..0000000 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/sqoop_client.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import sys -from resource_management import * -from sqoop import sqoop - -class SqoopClient(Script): - def install(self, env): - import params - if params.sqoop_home_dir is None: - self.install_packages(env) - self.configure(env) - - def configure(self, env): - import params - env.set_params(params) - sqoop() - - def status(self, env): - raise ClientComponentHasNoStatus() - -if __name__ == "__main__": - SqoopClient().execute()
