Repository: ambari Updated Branches: refs/heads/trunk 47b845fb9 -> b33391236
AMBARI-21023. HDP 3.0 TP - create service definition for Sqoop with configs, kerberos, widgets, etc.(vbrodetsky) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b3339123 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b3339123 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b3339123 Branch: refs/heads/trunk Commit: b33391236b385e2e4620031b9773261bce8efdf6 Parents: 47b845f Author: Vitaly Brodetskyi <[email protected]> Authored: Tue May 16 22:16:59 2017 +0300 Committer: Vitaly Brodetskyi <[email protected]> Committed: Tue May 16 22:16:59 2017 +0300 ---------------------------------------------------------------------- .../sqoop-atlas-application.properties.xml | 47 +++++++ .../SQOOP/1.4.4.3.0/configuration/sqoop-env.xml | 87 ++++++++++++ .../1.4.4.3.0/configuration/sqoop-site.xml | 38 ++++++ .../SQOOP/1.4.4.3.0/kerberos.json | 20 +++ .../SQOOP/1.4.4.3.0/metainfo.xml | 115 ++++++++++++++++ .../SQOOP/1.4.4.3.0/package/scripts/__init__.py | 19 +++ .../SQOOP/1.4.4.3.0/package/scripts/params.py | 27 ++++ .../1.4.4.3.0/package/scripts/params_linux.py | 135 +++++++++++++++++++ .../1.4.4.3.0/package/scripts/params_windows.py | 30 +++++ .../1.4.4.3.0/package/scripts/service_check.py | 62 +++++++++ .../SQOOP/1.4.4.3.0/package/scripts/sqoop.py | 124 +++++++++++++++++ .../1.4.4.3.0/package/scripts/sqoop_client.py | 66 +++++++++ .../SQOOP/1.4.4.3.0/role_command_order.json | 6 + .../stacks/HDP/3.0/services/SQOOP/metainfo.xml | 27 ++++ 14 files changed, 803 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-atlas-application.properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-atlas-application.properties.xml b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-atlas-application.properties.xml new file mode 100644 index 0000000..1364776 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-atlas-application.properties.xml @@ -0,0 +1,47 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <!-- + These two security properties will only be read if the cluster is Kerberized, but ok to add them even without Kerberos. + It's important to have at least one property in this config file so it gets added while merging configs during EU/RU + from an earlier stack to HDP 2.5+. + Also, it allows a fresh install with Sqoop to expose this config type in the UI. + --> + <property> + <name>atlas.jaas.KafkaClient.option.useTicketCache</name> + <value>true</value> + <description> + Set this to "true" if you want the TGT to be obtained from the ticket cache. + Set this option to "false" if you do not want this module to use the ticket cache. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>atlas.jaas.KafkaClient.option.renewTicket</name> + <value>true</value> + <description> + Set this to "true" if you want the TGT to renew the ticket when it expires. + Set this option to "false" if you do not want this module to renew tickets. + </description> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-env.xml b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-env.xml new file mode 100644 index 0000000..6e16ab33 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-env.xml @@ -0,0 +1,87 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_adding_forbidden="true"> + <!-- sqoop-env.sh --> + <property> + <name>sqoop.atlas.hook</name> + <value>false</value> + <display-name>Enable Atlas Hook</display-name> + <description>Enable Atlas Hook</description> + <value-attributes> + <type>boolean</type> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + <depends-on> + <property> + <type>application-properties</type> + <name>atlas.rest.address</name> + </property> + </depends-on> + </property> + <property> + <name>content</name> + <display-name>sqoop-env template</display-name> + <description>This is the jinja template for sqoop-env.sh file</description> + <value> +# Set Hadoop-specific environment variables here. + +#Set path to where bin/hadoop is available +#Set path to where bin/hadoop is available +export HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}} + +#set the path to where bin/hbase is available +export HBASE_HOME=${HBASE_HOME:-{{hbase_home}}} + +#Set the path to where bin/hive is available +export HIVE_HOME=${HIVE_HOME:-{{hive_home}}} + +#Set the path for where zookeper config dir is +export ZOOCFGDIR=${ZOOCFGDIR:-/etc/zookeeper/conf} + +# add libthrift in hive to sqoop class path first so hive imports work +export SQOOP_USER_CLASSPATH="`ls ${HIVE_HOME}/lib/libthrift-*.jar 2> /dev/null`:${SQOOP_USER_CLASSPATH}" + </value> + <value-attributes> + <type>content</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>sqoop_user</name> + <display-name>Sqoop User</display-name> + <description>User to run Sqoop as</description> + <property-type>USER</property-type> + <value>sqoop</value> + <value-attributes> + <type>user</type> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>jdbc_drivers</name> + <description>Comma separated list of additional JDBC drivers class names</description> + <value> </value> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-site.xml b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-site.xml new file mode 100644 index 0000000..389550d --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-site.xml @@ -0,0 +1,38 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <!-- This property was valid in HDP 2.3 and 2.4, but removed in HDP 2.5 --> + <property> + <name>sqoop.job.data.publish.class</name> + <on-ambari-upgrade add="false"/> + <depends-on> + <property> + <type>sqoop-env</type> + <name>sqoop.atlas.hook</name> + </property> + <property> + <type>application-properties</type> + <name>atlas.rest.address</name> + </property> + </depends-on> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/kerberos.json b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/kerberos.json new file mode 100644 index 0000000..de12e7c --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/kerberos.json @@ -0,0 +1,20 @@ +{ + "services": [ + { + "name": "SQOOP", + "configurations": [ + { + "sqoop-atlas-application.properties": { + "atlas.jaas.KafkaClient.option.useTicketCache": "true", + "atlas.jaas.KafkaClient.option.renewTicket": "true" + } + } + ], + "components": [ + { + "name": "SQOOP" + } + ] + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/metainfo.xml b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/metainfo.xml new file mode 100644 index 0000000..999d93a --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/metainfo.xml @@ -0,0 +1,115 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>SQOOP</name> + <displayName>Sqoop</displayName> + <comment>Tool for transferring bulk data between Apache Hadoop and + structured data stores such as relational databases + </comment> + <version>1.4.4.3.0</version> + + <components> + <component> + <name>SQOOP</name> + <displayName>Sqoop Client</displayName> + <category>CLIENT</category> + <cardinality>1+</cardinality> + <versionAdvertised>true</versionAdvertised> + <dependencies> + <dependency> + <name>HDFS/HDFS_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> + <commandScript> + <script>scripts/sqoop_client.py</script> + <scriptType>PYTHON</scriptType> + </commandScript> + <configFiles> + <configFile> + <type>xml</type> + <fileName>sqoop-site.xml</fileName> + <dictionaryName>sqoop-site</dictionaryName> + </configFile> + <configFile> + <type>env</type> + <fileName>sqoop-env.sh</fileName> + <dictionaryName>sqoop-env</dictionaryName> + </configFile> + </configFiles> + </component> + </components> + <osSpecifics> + <osSpecific> + <osFamily>any</osFamily> + <packages> + <package> + <name>mysql-connector-java</name> + <skipUpgrade>true</skipUpgrade> + <condition>should_install_mysql_connector</condition> + </package> + </packages> + </osSpecific> + <osSpecific> + <osFamily>redhat7,amazon2015,redhat6,suse11,suse12</osFamily> + <packages> + <package> + <name>sqoop_${stack_version}</name> + </package> + </packages> + </osSpecific> + <osSpecific> + <osFamily>debian7,ubuntu12,ubuntu14,ubuntu16</osFamily> + <packages> + <package> + <name>sqoop-${stack_version}</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + <commandScript> + <script>scripts/service_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + + <requiredServices> + <service>HDFS</service> + </requiredServices> + + <configuration-dependencies> + <config-type>sqoop-env</config-type> + <config-type>sqoop-site</config-type> + <config-type>application.properties</config-type> + </configuration-dependencies> + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/__init__.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/__init__.py new file mode 100644 index 0000000..5561e10 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/__init__.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params.py new file mode 100644 index 0000000..61573ee --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params.py @@ -0,0 +1,27 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from ambari_commons import OSCheck +from resource_management.libraries.functions.default import default + +if OSCheck.is_windows_family(): + from params_windows import * +else: + from params_linux import * + + http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_linux.py new file mode 100644 index 0000000..c1138b3 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_linux.py @@ -0,0 +1,135 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import os + +from resource_management.libraries.functions.version import format_stack_version +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions.get_kinit_path import get_kinit_path +from resource_management.libraries.script import Script +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.expect import expect +from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster + + +# a map of the Ambari role to the component name +# for use with <stack-root>/current/<component> +SERVER_ROLE_DIRECTORY_MAP = { + 'SQOOP' : 'sqoop-client' +} + +component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "SQOOP") + +config = Script.get_config() +stack_root = Script.get_stack_root() + +# Needed since this is an Atlas Hook service. +cluster_name = config['clusterName'] + +ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] + +stack_name = default("/hostLevelParams/stack_name", None) + +stack_version_unformatted = config['hostLevelParams']['stack_version'] +stack_version_formatted = format_stack_version(stack_version_unformatted) + +agent_stack_retry_on_unavailability = config['hostLevelParams']['agent_stack_retry_on_unavailability'] +agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int) + +# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade +version = default("/commandParams/version", None) + +# default hadoop params +sqoop_conf_dir = "/usr/lib/sqoop/conf" +sqoop_lib = "/usr/lib/sqoop/lib" +hadoop_home = '/usr/lib/hadoop' +hbase_home = "/usr/lib/hbase" +hive_home = "/usr/lib/hive" +sqoop_bin_dir = "/usr/bin" +zoo_conf_dir = "/etc/zookeeper" + +# For stack versions supporting rolling upgrade +if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted): + sqoop_conf_dir = format("{stack_root}/current/sqoop-client/conf") + sqoop_lib = format("{stack_root}/current/sqoop-client/lib") + hadoop_home = format("{stack_root}/current/hadoop-client") + hbase_home = format("{stack_root}/current/hbase-client") + hive_home = format("{stack_root}/current/hive-client") + sqoop_bin_dir = format("{stack_root}/current/sqoop-client/bin/") + zoo_conf_dir = format("{stack_root}/current/zookeeper-client/conf") + +security_enabled = config['configurations']['cluster-env']['security_enabled'] +smokeuser = config['configurations']['cluster-env']['smokeuser'] +smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] +user_group = config['configurations']['cluster-env']['user_group'] +sqoop_env_sh_template = config['configurations']['sqoop-env']['content'] + +sqoop_user = config['configurations']['sqoop-env']['sqoop_user'] + +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] +kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) +#JDBC driver jar name +sqoop_jdbc_drivers_dict = [] +sqoop_jdbc_drivers_name_dict = {} +sqoop_jdbc_drivers_to_remove = {} +if "jdbc_drivers" in config['configurations']['sqoop-env']: + sqoop_jdbc_drivers = config['configurations']['sqoop-env']['jdbc_drivers'].split(',') + + for driver_name in sqoop_jdbc_drivers: + previous_jdbc_jar_name = None + driver_name = driver_name.strip() + if driver_name and not driver_name == '': + if driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver": + jdbc_name = default("/hostLevelParams/custom_mssql_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None) + jdbc_driver_name = "mssql" + elif driver_name == "com.mysql.jdbc.Driver": + jdbc_name = default("/hostLevelParams/custom_mysql_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None) + jdbc_driver_name = "mysql" + elif driver_name == "org.postgresql.Driver": + jdbc_name = default("/hostLevelParams/custom_postgres_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None) + jdbc_driver_name = "postgres" + elif driver_name == "oracle.jdbc.driver.OracleDriver": + jdbc_name = default("/hostLevelParams/custom_oracle_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None) + jdbc_driver_name = "oracle" + elif driver_name == "org.hsqldb.jdbc.JDBCDriver": + jdbc_name = default("/hostLevelParams/custom_hsqldb_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_hsqldb_jdbc_name", None) + jdbc_driver_name = "hsqldb" + else: + continue + sqoop_jdbc_drivers_dict.append(jdbc_name) + sqoop_jdbc_drivers_to_remove[jdbc_name] = previous_jdbc_jar_name + sqoop_jdbc_drivers_name_dict[jdbc_name] = jdbc_driver_name +jdk_location = config['hostLevelParams']['jdk_location'] + + +######################################################## +############# Atlas related params ##################### +######################################################## +#region Atlas Hooks +sqoop_atlas_application_properties = default('/configurations/sqoop-atlas-application.properties', {}) +enable_atlas_hook = default('/configurations/sqoop-env/sqoop.atlas.hook', False) +atlas_hook_filename = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties') +#endregion http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_windows.py new file mode 100644 index 0000000..f930765 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_windows.py @@ -0,0 +1,30 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.script import Script +import os + +config = Script.get_config() + +sqoop_user = "sqoop" + +stack_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"], "..")) +sqoop_env_cmd_template = config['configurations']['sqoop-env']['content'] +sqoop_home_dir = os.environ["SQOOP_HOME"] +sqoop_conf_dir = os.path.join(sqoop_home_dir, "conf") \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/service_check.py new file mode 100644 index 0000000..bb503f5 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/service_check.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + + +from resource_management.libraries.script.script import Script +from resource_management.core.resources import Execute +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions import format +from ambari_commons.os_family_impl import OsFamilyImpl +from ambari_commons import OSConst +import os + +class SqoopServiceCheck(Script): + pass + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class SqoopServiceCheckDefault(SqoopServiceCheck): + + def get_component_name(self): + return "sqoop-server" + + def service_check(self, env): + import params + env.set_params(params) + if params.security_enabled: + Execute(format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal}"), + user = params.smokeuser, + ) + Execute("sqoop version", + user = params.smokeuser, + path = params.sqoop_bin_dir, + logoutput = True + ) + +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class SqoopServiceCheckWindows(SqoopServiceCheck): + def service_check(self, env): + import params + env.set_params(params) + smoke_cmd = os.path.join(params.stack_root,"Run-SmokeTests.cmd") + service = "SQOOP" + Execute(format("cmd /C {smoke_cmd} {service}"), logoutput=True) + +if __name__ == "__main__": + SqoopServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/sqoop.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/sqoop.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/sqoop.py new file mode 100644 index 0000000..436402c --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/sqoop.py @@ -0,0 +1,124 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +# Python Imports +import os + +# Local Imports +from resource_management.core.source import InlineTemplate, DownloadSource +from resource_management.libraries.functions import format +from resource_management.libraries.functions.get_config import get_config +from resource_management.libraries.resources.xml_config import XmlConfig +from resource_management.core.resources.system import File, Link, Directory +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from ambari_commons import OSConst +from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster, setup_atlas_hook, setup_atlas_jar_symlinks +from ambari_commons.constants import SERVICE + + +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def sqoop(type=None): + import params + File(os.path.join(params.sqoop_conf_dir, "sqoop-env.cmd"), + content=InlineTemplate(params.sqoop_env_cmd_template) + ) + +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +def sqoop(type=None): + import params + Link(params.sqoop_lib + "/mysql-connector-java.jar", + to = '/usr/share/java/mysql-connector-java.jar' + ) + + jdbc_connector() + + Directory(params.sqoop_conf_dir, + owner = params.sqoop_user, + group = params.user_group, + create_parents = True + ) + + configs = {} + sqoop_site_config = get_config('sqoop-site') + if sqoop_site_config: + configs.update(sqoop_site_config) + + XmlConfig("sqoop-site.xml", + conf_dir = params.sqoop_conf_dir, + configurations = configs, + configuration_attributes=params.config['configuration_attributes']['sqoop-site'], + owner = params.sqoop_user, + group = params.user_group + ) + + # Generate atlas-application.properties.xml file and symlink the hook jars + if params.enable_atlas_hook: + atlas_hook_filepath = os.path.join(params.sqoop_conf_dir, params.atlas_hook_filename) + setup_atlas_hook(SERVICE.SQOOP, params.sqoop_atlas_application_properties, atlas_hook_filepath, params.sqoop_user, params.user_group) + setup_atlas_jar_symlinks("sqoop", params.sqoop_lib) + + + File(format("{sqoop_conf_dir}/sqoop-env.sh"), + owner=params.sqoop_user, + group = params.user_group, + content=InlineTemplate(params.sqoop_env_sh_template) + ) + update_config_permissions(["sqoop-env-template.sh", + "sqoop-site-template.xml", + "sqoop-site.xml"]) + pass + +def update_config_permissions(names): + import params + for filename in names: + full_filename = os.path.join(params.sqoop_conf_dir, filename) + File(full_filename, + owner = params.sqoop_user, + group = params.user_group, + only_if = format("test -e {full_filename}") + ) + +def jdbc_connector(): + import params + from urllib2 import HTTPError + from resource_management import Fail + for jar_name in params.sqoop_jdbc_drivers_dict: + if not jar_name or 'mysql' in jar_name: + continue + downloaded_custom_connector = format("{sqoop_lib}/{jar_name}") + custom_connector_to_remove = format("{sqoop_lib}/" + str(params.sqoop_jdbc_drivers_to_remove[jar_name])) + jdbc_driver_label = params.sqoop_jdbc_drivers_name_dict[jar_name] + driver_curl_source = format("{jdk_location}/{jar_name}") + environment = { + "no_proxy": format("{ambari_server_hostname}") + } + try: + if custom_connector_to_remove and os.path.isfile(custom_connector_to_remove): + File(custom_connector_to_remove, action='delete') + + File(downloaded_custom_connector, + content = DownloadSource(driver_curl_source), + mode = 0644, + ) + except HTTPError: + error_string = format("Could not download {driver_curl_source}\n\ + Please upload jdbc driver to server by run command:\n\ + ambari-server setup --jdbc-db={jdbc_driver_label} --jdbc-driver=<PATH TO DRIVER>\n\ + at {ambari_server_hostname}") + raise Fail(error_string) + http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/sqoop_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/sqoop_client.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/sqoop_client.py new file mode 100644 index 0000000..d420fab --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/sqoop_client.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core.exceptions import ClientComponentHasNoStatus +from resource_management.core.resources.system import Execute +from resource_management.libraries.functions.default import default +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from sqoop import sqoop +from ambari_commons.os_family_impl import OsFamilyImpl +from ambari_commons import OSConst + +class SqoopClient(Script): + def install(self, env): + self.install_packages(env) + self.configure(env) + + def configure(self, env): + import params + env.set_params(params) + sqoop(type='client') + + def status(self, env): + raise ClientComponentHasNoStatus() + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class SqoopClientDefault(SqoopClient): + def get_component_name(self): + return "sqoop-client" + + def pre_upgrade_restart(self, env, upgrade_type=None): + import params + env.set_params(params) + + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): + conf_select.select(params.stack_name, "sqoop", params.version) + stack_select.select("sqoop-client", params.version) + + +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class SqoopClientWindows(SqoopClient): + pass + +if __name__ == "__main__": + SqoopClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/role_command_order.json b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/role_command_order.json new file mode 100644 index 0000000..f9a6cb4 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/role_command_order.json @@ -0,0 +1,6 @@ +{ + "general_deps" : { + "_comment" : "dependencies for SQOOP", + "SQOOP_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"] + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/stacks/HDP/3.0/services/SQOOP/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/SQOOP/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/SQOOP/metainfo.xml new file mode 100644 index 0000000..757bce5 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/SQOOP/metainfo.xml @@ -0,0 +1,27 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>SQOOP</name> + <version>1.4.4.3.0</version> + <extends>common-services/SQOOP/1.4.4.3.0</extends> + </service> + </services> +</metainfo>
