Updated Branches: refs/heads/trunk 95ba3b452 -> d2ebacc09
AMBARI-3931. Sqoop on HDP2. Using resource management lib (Eugene Chekanskiy via dlysnichenko) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d2ebacc0 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d2ebacc0 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d2ebacc0 Branch: refs/heads/trunk Commit: d2ebacc09e91ebdbf9b2442864214b28af5ffff6 Parents: 95ba3b4 Author: Lisnichenko Dmitro <[email protected]> Authored: Fri Nov 29 18:33:27 2013 +0200 Committer: Lisnichenko Dmitro <[email protected]> Committed: Fri Nov 29 18:33:27 2013 +0200 ---------------------------------------------------------------------- .../HDP/2.0._/services/SQOOP/metainfo.xml | 46 ++++++++++++++--- .../services/SQOOP/package/scripts/__init__.py | 21 ++++++++ .../services/SQOOP/package/scripts/params.py | 40 +++++++++++++++ .../SQOOP/package/scripts/service_check.py | 39 ++++++++++++++ .../services/SQOOP/package/scripts/sqoop.py | 54 ++++++++++++++++++++ .../SQOOP/package/scripts/sqoop_client.py | 40 +++++++++++++++ .../SQOOP/package/templates/sqoop-env.sh.j2 | 36 +++++++++++++ 7 files changed, 268 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/d2ebacc0/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/metainfo.xml index 9a50700..0070c34 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/metainfo.xml @@ -16,15 +16,45 @@ limitations under the License. --> <metainfo> - <user>root</user> - <comment>Tool for transferring bulk data between Apache Hadoop and structured data stores such as relational databases</comment> - <version>1.4.4.2.0.6.0</version> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>SQOOP</name> + <comment>Tool for transferring bulk data between Apache Hadoop and + structured data stores such as relational databases + </comment> + <version>1.4.4.2.0.6.0</version> - <components> + <components> <component> - <name>SQOOP</name> - <category>CLIENT</category> + <name>SQOOP</name> + <category>CLIENT</category> + <commandScript> + <script>scripts/sqoop_client.py</script> + <scriptType>PYTHON</scriptType> + </commandScript> </component> - </components> - + </components> + <osSpecifics> + <osSpecific> + <osType>any</osType> + <packages> + <package> + <type>rpm</type> + <name>sqoop</name> + </package> + <package> + <type>rpm</type> + <name>mysql-connector-java</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + <commandScript> + <script>scripts/service_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + </service> + </services> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/d2ebacc0/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/__init__.py b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/__init__.py new file mode 100644 index 0000000..b80fda6 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/__init__.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python2.6 +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" http://git-wip-us.apache.org/repos/asf/ambari/blob/d2ebacc0/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/params.py new file mode 100644 index 0000000..559ee81 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/params.py @@ -0,0 +1,40 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management import * + +config = Script.get_config() + +security_enabled = config['configurations']['global']['security_enabled'] +smokeuser = config['configurations']['global']['smokeuser'] +user_group = config['configurations']['global']['user_group'] + +sqoop_conf_dir = "/usr/lib/sqoop/conf" +hbase_home = "/usr" +hive_home = "/usr" +zoo_conf_dir = "/etc/zookeeper" +sqoop_lib = "/usr/lib/sqoop/lib" +sqoop_user = "sqoop" + +keytab_path = default("keytab_path","/etc/security/keytabs") + +smoke_user_keytab = "/etc/security/keytabs/smokeuser.headless.keytab" # smoke_user_keytab +kinit_path_local = functions.get_kinit_path([default('kinit_path_local'),"/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) http://git-wip-us.apache.org/repos/asf/ambari/blob/d2ebacc0/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/service_check.py new file mode 100644 index 0000000..88ac32c --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/service_check.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python2.6 +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + + +from resource_management import * + + +class SqoopServiceCheck(Script): + def service_check(self, env): + import params + env.set_params(params) + if params.security_enabled: + Execute(format("{kinit_path_local} -kt {smoke_user_keytab} {smoke_test_user}")) + Execute("sqoop version", + user = params.smokeuser, + logoutput = True + ) + +if __name__ == "__main__": + SqoopServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/d2ebacc0/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/sqoop.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/sqoop.py b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/sqoop.py new file mode 100644 index 0000000..b4f24b5 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/sqoop.py @@ -0,0 +1,54 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management import * +import sys + +def sqoop(type=None): + import params + Link(params.sqoop_lib + "/mysql-connector-java.jar", + to = '/usr/share/java/mysql-connector-java.jar' + ) + Directory(params.sqoop_conf_dir, + owner = params.sqoop_user, + group = params.user_group + ) + sqoop_TemplateConfig("sqoop-env.sh") + File (params.sqoop_conf_dir + "/sqoop-env-template.sh", + owner = params.sqoop_user, + group = params.user_group + ) + File (params.sqoop_conf_dir + "/sqoop-site-template.xml", + owner = params.sqoop_user, + group = params.user_group + ) + File (params.sqoop_conf_dir + "/sqoop-site.xml", + owner = params.sqoop_user, + group = params.user_group + ) + pass + +def sqoop_TemplateConfig(name, tag=None): + import params + TemplateConfig( format("{sqoop_conf_dir}/{name}"), + owner = params.sqoop_user, + template_tag = tag + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/d2ebacc0/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/sqoop_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/sqoop_client.py b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/sqoop_client.py new file mode 100644 index 0000000..90dfecf --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/sqoop_client.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python2.6 +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +import sys +from resource_management import * + +from sqoop import sqoop + + +class SqoopClient(Script): + def install(self, env): + self.install_packages(env) + self.configure(env) + + def configure(self, env): + import params + env.set_params(params) + sqoop(type='client') + +if __name__ == "__main__": + SqoopClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/d2ebacc0/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/templates/sqoop-env.sh.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/templates/sqoop-env.sh.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/templates/sqoop-env.sh.j2 new file mode 100644 index 0000000..90cbc75 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/templates/sqoop-env.sh.j2 @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# included in all the hadoop scripts with source command +# should not be executable directly +# also should not be passed any arguments, since we need original $* + +# Set Hadoop-specific environment variables here. + +#Set path to where bin/hadoop is available +#Set path to where bin/hadoop is available +export HADOOP_HOME=${HADOOP_HOME:-/usr/lib/hadoop} + +#set the path to where bin/hbase is available +export HBASE_HOME=${HBASE_HOME:-/usr/lib/hbase} + +#Set the path to where bin/hive is available +export HIVE_HOME=${HIVE_HOME:-/usr/lib/hive} + +#Set the path for where zookeper config dir is +export ZOOCFGDIR=${ZOOCFGDIR:-/etc/zookeeper/conf} + +# add libthrift in hive to sqoop class path first so hive imports work +export SQOOP_USER_CLASSPATH="`ls ${HIVE_HOME}/lib/libthrift-*.jar 2> /dev/null`:${SQOOP_USER_CLASSPATH}"
