http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/params.py deleted file mode 100644 index 174c92c..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/params.py +++ /dev/null @@ -1,78 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -from resource_management import * -import status_params - -# server configurations -config = Script.get_config() -tmp_dir = Script.get_tmp_dir() - -hcat_user = config['configurations']['hive-env']['hcat_user'] -webhcat_user = config['configurations']['hive-env']['webhcat_user'] -webhcat_env_sh_template = config['configurations']['webhcat-env']['content'] - -config_dir = '/etc/hcatalog/conf' - -templeton_log_dir = config['configurations']['hive-env']['hcat_log_dir'] -templeton_pid_dir = status_params.templeton_pid_dir - -pid_file = status_params.pid_file - -hadoop_conf_dir = config['configurations']['webhcat-site']['templeton.hadoop.conf.dir'] -templeton_jar = config['configurations']['webhcat-site']['templeton.jar'] - -hadoop_home = '/usr' -user_group = config['configurations']['cluster-env']['user_group'] - -webhcat_server_host = config['clusterHostInfo']['webhcat_server_host'] - -webhcat_apps_dir = "/apps/webhcat" -smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] -smokeuser = config['configurations']['cluster-env']['smokeuser'] -security_enabled = config['configurations']['cluster-env']['security_enabled'] -kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) - -#hdfs directories -webhcat_apps_dir = "/apps/webhcat" -hcat_hdfs_user_dir = format("/user/{hcat_user}") -hcat_hdfs_user_mode = 0755 -webhcat_hdfs_user_dir = format("/user/{webhcat_user}") -webhcat_hdfs_user_mode = 0755 -#for create_hdfs_directory -hostname = config["hostname"] -security_param = "true" if security_enabled else "false" -hadoop_conf_dir = "/etc/hadoop/conf" -hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] -hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] -hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] -kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) -import functools -#create partial functions with common arguments for every HdfsDirectory call -#to create hdfs directory we need to call params.HdfsDirectory in code -HdfsDirectory = functools.partial( - HdfsDirectory, - conf_dir=hadoop_conf_dir, - hdfs_user=hdfs_user, - security_enabled = security_enabled, - keytab = hdfs_user_keytab, - kinit_path_local = kinit_path_local -)
http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/service_check.py deleted file mode 100644 index b6101ac..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/service_check.py +++ /dev/null @@ -1,44 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" -from resource_management import * - -class WebHCatServiceCheck(Script): - def service_check(self, env): - import params - env.set_params(params) - - File(format("{tmp_dir}/templetonSmoke.sh"), - content= StaticFile('templetonSmoke.sh'), - mode=0755 - ) - - cmd = format("sh {tmp_dir}/templetonSmoke.sh {webhcat_server_host[0]} {smokeuser} {smokeuser_keytab}" - " {security_param} {kinit_path_local}", - smokeuser_keytab=params.smoke_user_keytab if params.security_enabled else "no_keytab") - - Execute(cmd, - tries=3, - try_sleep=5, - path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', - logoutput=True) - -if __name__ == "__main__": - WebHCatServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/status_params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/status_params.py deleted file mode 100644 index 23823e6..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/status_params.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management import * - -config = Script.get_config() - -templeton_pid_dir = config['configurations']['hive-env']['hcat_pid_dir'] -pid_file = format('{templeton_pid_dir}/webhcat.pid') http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat.py deleted file mode 100644 index 037cdb5..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat.py +++ /dev/null @@ -1,107 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" -from resource_management import * - - -def webhcat(): - import params - - if params.hcat_hdfs_user_dir != params.webhcat_hdfs_user_dir: - params.HdfsDirectory(params.hcat_hdfs_user_dir, - action="create_delayed", - owner=params.hcat_user, - mode=params.hcat_hdfs_user_mode - ) - params.HdfsDirectory(params.webhcat_hdfs_user_dir, - action="create_delayed", - owner=params.webhcat_user, - mode=params.webhcat_hdfs_user_mode - ) - params.HdfsDirectory(params.webhcat_apps_dir, - action="create_delayed", - owner=params.webhcat_user, - mode=0755 - ) - params.HdfsDirectory(None, action="create") - Directory(params.templeton_pid_dir, - owner=params.webhcat_user, - mode=0755, - group=params.user_group, - recursive=True) - - Directory(params.templeton_log_dir, - owner=params.webhcat_user, - mode=0755, - group=params.user_group, - recursive=True) - - Directory(params.config_dir, - owner=params.webhcat_user, - group=params.user_group) - - XmlConfig("webhcat-site.xml", - conf_dir=params.config_dir, - configurations=params.config['configurations']['webhcat-site'], - configuration_attributes=params.config['configuration_attributes']['webhcat-site'], - owner=params.webhcat_user, - group=params.user_group, - ) - - File(format("{config_dir}/webhcat-env.sh"), - owner=params.webhcat_user, - group=params.user_group, - content=InlineTemplate(params.webhcat_env_sh_template) - ) - - if params.security_enabled: - kinit_if_needed = format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_user};") - else: - kinit_if_needed = "" - - if kinit_if_needed: - Execute(kinit_if_needed, - user=params.webhcat_user, - path='/bin' - ) - - CopyFromLocal('/usr/lib/hadoop/contrib/streaming/hadoop-streaming*.jar', - owner=params.webhcat_user, - mode=0755, - dest_dir=params.webhcat_apps_dir, - kinnit_if_needed=kinit_if_needed, - hdfs_user=params.hdfs_user - ) - - CopyFromLocal('/usr/share/HDP-webhcat/pig.tar.gz', - owner=params.webhcat_user, - mode=0755, - dest_dir=params.webhcat_apps_dir, - kinnit_if_needed=kinit_if_needed, - hdfs_user=params.hdfs_user - ) - - CopyFromLocal('/usr/share/HDP-webhcat/hive.tar.gz', - owner=params.webhcat_user, - mode=0755, - dest_dir=params.webhcat_apps_dir, - kinnit_if_needed=kinit_if_needed, - hdfs_user=params.hdfs_user - ) http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat_server.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat_server.py deleted file mode 100644 index 2111fa4..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat_server.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" -import sys -from resource_management import * - -from webhcat import webhcat -from webhcat_service import webhcat_service - -class WebHCatServer(Script): - def install(self, env): - self.install_packages(env) - def configure(self, env): - import params - env.set_params(params) - webhcat() - - def start(self, env): - import params - env.set_params(params) - self.configure(env) # FOR SECURITY - webhcat_service(action = 'start') - - def stop(self, env): - import params - env.set_params(params) - - webhcat_service(action = 'stop') - - def status(self, env): - import status_params - env.set_params(status_params) - check_process_status(status_params.pid_file) - -if __name__ == "__main__": - WebHCatServer().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat_service.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat_service.py deleted file mode 100644 index 57ec17a..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat_service.py +++ /dev/null @@ -1,40 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" -from resource_management import * - -def webhcat_service(action='start'): - import params - - cmd = format('env HADOOP_HOME={hadoop_home} /usr/lib/hcatalog/sbin/webhcat_server.sh') - - if action == 'start': - demon_cmd = format('{cmd} start') - no_op_test = format('ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1') - Execute(demon_cmd, - user=params.webhcat_user, - not_if=no_op_test - ) - elif action == 'stop': - demon_cmd = format('{cmd} stop') - Execute(demon_cmd, - user=params.webhcat_user - ) - Execute(format('rm -f {pid_file}')) http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/1.3/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/1.3/role_command_order.json index a05324f..5a75a8e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3/role_command_order.json +++ b/ambari-server/src/main/resources/stacks/HDP/1.3/role_command_order.json @@ -20,10 +20,8 @@ "WEBHCAT_SERVER-START"], "MAPREDUCE_SERVICE_CHECK-SERVICE_CHECK": ["JOBTRACKER-START", "TASKTRACKER-START"], "OOZIE_SERVICE_CHECK-SERVICE_CHECK": ["OOZIE_SERVER-START"], - "WEBHCAT_SERVICE_CHECK-SERVICE_CHECK": ["WEBHCAT_SERVER-START"], "HBASE_SERVICE_CHECK-SERVICE_CHECK": ["HBASE_MASTER-START", "HBASE_REGIONSERVER-START"], - "HIVE_SERVICE_CHECK-SERVICE_CHECK": ["HIVE_SERVER-START", "HIVE_METASTORE-START"], - "HCAT_SERVICE_CHECK-SERVICE_CHECK": ["HIVE_SERVER-START"], + "HIVE_SERVICE_CHECK-SERVICE_CHECK": ["HIVE_SERVER-START", "HIVE_METASTORE-START","WEBHCAT_SERVER-START"], "PIG_SERVICE_CHECK-SERVICE_CHECK": ["JOBTRACKER-START", "TASKTRACKER-START"], "SQOOP_SERVICE_CHECK-SERVICE_CHECK": ["JOBTRACKER-START", "TASKTRACKER-START"], "ZOOKEEPER_SERVICE_CHECK-SERVICE_CHECK": ["ZOOKEEPER_SERVER-START"], http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/1.3/services/HIVE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3/services/HIVE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/1.3/services/HIVE/metainfo.xml index bf0daf5..ca181f3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3/services/HIVE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/1.3/services/HIVE/metainfo.xml @@ -23,10 +23,5 @@ <comment>Data warehouse system for ad-hoc queries & analysis of large datasets and table & storage management service</comment> <version>0.11.0.1.3</version> </service> - <service> - <name>HCATALOG</name> - <comment>HCATALOG</comment> - <version>0.11.0.1.3</version> - </service> </services> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/2.0.6/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/role_command_order.json index 6b32edf..a92fa1d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/role_command_order.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/role_command_order.json @@ -21,10 +21,8 @@ "WEBHCAT_SERVER-START", "FLUME_HANDLER-START"], "MAPREDUCE_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"], "OOZIE_SERVICE_CHECK-SERVICE_CHECK": ["OOZIE_SERVER-START", "MAPREDUCE2_SERVICE_CHECK-SERVICE_CHECK"], - "WEBHCAT_SERVICE_CHECK-SERVICE_CHECK": ["WEBHCAT_SERVER-START"], "HBASE_SERVICE_CHECK-SERVICE_CHECK": ["HBASE_MASTER-START", "HBASE_REGIONSERVER-START"], - "HIVE_SERVICE_CHECK-SERVICE_CHECK": ["HIVE_SERVER-START", "HIVE_METASTORE-START"], - "HCAT_SERVICE_CHECK-SERVICE_CHECK": ["HIVE_SERVER-START"], + "HIVE_SERVICE_CHECK-SERVICE_CHECK": ["HIVE_SERVER-START", "HIVE_METASTORE-START", "WEBHCAT_SERVER-START"], "PIG_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"], "SQOOP_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"], "ZOOKEEPER_SERVICE_CHECK-SERVICE_CHECK": ["ZOOKEEPER_SERVER-START"], http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/configuration/webhcat-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/configuration/webhcat-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/configuration/webhcat-env.xml new file mode 100644 index 0000000..14a473f --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/configuration/webhcat-env.xml @@ -0,0 +1,54 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + <!-- webhcat-env.sh --> + <property> + <name>content</name> + <description>webhcat-env.sh content</description> + <value> +# The file containing the running pid +PID_FILE={{webhcat_pid_file}} + +TEMPLETON_LOG_DIR={{templeton_log_dir}}/ + + +WEBHCAT_LOG_DIR={{templeton_log_dir}}/ + +# The console error log +ERROR_LOG={{templeton_log_dir}}/webhcat-console-error.log + +# The console log +CONSOLE_LOG={{templeton_log_dir}}/webhcat-console.log + +#TEMPLETON_JAR=templeton_jar_name + +#HADOOP_PREFIX=hadoop_prefix + +#HCAT_PREFIX=hive_prefix + +# Set HADOOP_HOME to point to a specific hadoop install directory +export HADOOP_HOME={{hadoop_home}} + </value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/configuration/webhcat-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/configuration/webhcat-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/configuration/webhcat-site.xml new file mode 100644 index 0000000..0523dab --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/configuration/webhcat-site.xml @@ -0,0 +1,138 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> + +<!-- The default settings for Templeton. --> +<!-- Edit templeton-site.xml to change settings for your local --> +<!-- install. --> + +<configuration supports_final="true"> + + <property> + <name>templeton.port</name> + <value>50111</value> + <description>The HTTP port for the main server.</description> + </property> + + <property> + <name>templeton.hadoop.conf.dir</name> + <value>/etc/hadoop/conf</value> + <description>The path to the Hadoop configuration.</description> + </property> + + <property> + <name>templeton.jar</name> + <value>/usr/lib/hcatalog/share/webhcat/svr/webhcat.jar</value> + <description>The path to the Templeton jar file.</description> + </property> + + <property> + <name>templeton.libjars</name> + <value>/usr/lib/zookeeper/zookeeper.jar</value> + <description>Jars to add the the classpath.</description> + </property> + + + <property> + <name>templeton.hadoop</name> + <value>/usr/bin/hadoop</value> + <description>The path to the Hadoop executable.</description> + </property> + + <property> + <name>templeton.pig.archive</name> + <value>hdfs:///apps/webhcat/pig.tar.gz</value> + <description>The path to the Pig archive.</description> + </property> + + <property> + <name>templeton.pig.path</name> + <value>pig.tar.gz/pig/bin/pig</value> + <description>The path to the Pig executable.</description> + </property> + + <property> + <name>templeton.hcat</name> + <value>/usr/bin/hcat</value> + <description>The path to the hcatalog executable.</description> + </property> + + <property> + <name>templeton.hive.archive</name> + <value>hdfs:///apps/webhcat/hive.tar.gz</value> + <description>The path to the Hive archive.</description> + </property> + + <property> + <name>templeton.hive.home</name> + <value>hive.tar.gz/hive</value> + <description>The path to the Hive home within the tar. Has no effect if templeton.hive.archive is not set.</description> + </property> + + <property> + <name>templeton.hcat.home</name> + <value>hive.tar.gz/hive/hcatalog</value> + <description>The path to the HCat home within the tar. Has no effect if templeton.hive.archive is not set.</description> + </property> + + <property> + <name>templeton.hive.path</name> + <value>hive.tar.gz/hive/bin/hive</value> + <description>The path to the Hive executable.</description> + </property> + + <property> + <name>templeton.hive.properties</name> + <value>hive.metastore.local=false, hive.metastore.uris=thrift://localhost:9933, hive.metastore.sasl.enabled=false</value> + <description>Properties to set when running hive.</description> + </property> + + + <property> + <name>templeton.zookeeper.hosts</name> + <value>localhost:2181</value> + <description>ZooKeeper servers, as comma separated host:port pairs</description> + </property> + + <property> + <name>templeton.storage.class</name> + <value>org.apache.hive.hcatalog.templeton.tool.ZooKeeperStorage</value> + <description>The class to use as storage</description> + </property> + + <property> + <name>templeton.override.enabled</name> + <value>false</value> + <description> + Enable the override path in templeton.override.jars + </description> + </property> + + <property> + <name>templeton.streaming.jar</name> + <value>hdfs:///apps/webhcat/hadoop-streaming.jar</value> + <description>The hdfs path to the Hadoop streaming jar file.</description> + </property> + + <property> + <name>templeton.exec.timeout</name> + <value>60000</value> + <description>Time out for templeton api</description> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/metainfo.xml index 31bfe44..483a66f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/metainfo.xml @@ -75,7 +75,55 @@ <scriptType>PYTHON</scriptType> </commandScript> </component> - + <component> + <name>WEBHCAT_SERVER</name> + <displayName>WebHCat Server</displayName> + <category>MASTER</category> + <cardinality>1</cardinality> + <dependencies> + <dependency> + <name>HDFS/HDFS_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>ZOOKEEPER/ZOOKEEPER_SERVER</name> + <scope>cluster</scope> + <auto-deploy> + <enabled>true</enabled> + <co-locate>HIVE/WEBHCAT_SERVER</co-locate> + </auto-deploy> + </dependency> + <dependency> + <name>ZOOKEEPER/ZOOKEEPER_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>YARN/YARN_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> + <commandScript> + <script>scripts/webhcat_server.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> <component> <name>MYSQL_SERVER</name> <displayName>MySQL Server</displayName> @@ -119,6 +167,37 @@ </configFile> </configFiles> </component> + <component> + <name>HCAT</name> + <displayName>HCat</displayName> + <category>CLIENT</category> + <commandScript> + <script>scripts/hcat_client.py</script> + <scriptType>PYTHON</scriptType> + </commandScript> + <configFiles> + <configFile> + <type>xml</type> + <fileName>hive-site.xml</fileName> + <dictionaryName>hive-site</dictionaryName> + </configFile> + <configFile> + <type>env</type> + <fileName>hive-env.sh</fileName> + <dictionaryName>hive-env</dictionaryName> + </configFile> + <configFile> + <type>env</type> + <fileName>hive-log4j.properties</fileName> + <dictionaryName>hive-log4j</dictionaryName> + </configFile> + <configFile> + <type>env</type> + <fileName>hive-exec-log4j.properties</fileName> + <dictionaryName>hive-exec-log4j</dictionaryName> + </configFile> + </configFiles> + </component> </components> <osSpecifics> @@ -129,6 +208,15 @@ <name>hive</name> </package> <package> + <name>hcatalog</name> + </package> + <package> + <name>webhcat-tar-hive</name> + </package> + <package> + <name>webhcat-tar-pig</name> + </package> + <package> <name>mysql-connector-java</name> </package> </packages> @@ -176,80 +264,9 @@ <config-type>hive-log4j</config-type> <config-type>hive-exec-log4j</config-type> <config-type>hive-env</config-type> + <config-type>webhcat-site</config-type> + <config-type>webhcat-env</config-type> </configuration-dependencies> </service> - - <service> - <name>HCATALOG</name> - <displayName>HCatalog</displayName> - <comment>A table and storage management layer for Hadoop that enables users with different data processing tools - to more easily read and write data on the grid. - </comment> - <version>0.12.0.2.0.6.0</version> - <components> - <component> - <name>HCAT</name> - <displayName>HCat</displayName> - <category>CLIENT</category> - <commandScript> - <script>scripts/hcat_client.py</script> - <scriptType>PYTHON</scriptType> - </commandScript> - <configFiles> - <configFile> - <type>xml</type> - <fileName>hive-site.xml</fileName> - <dictionaryName>hive-site</dictionaryName> - </configFile> - <configFile> - <type>env</type> - <fileName>hive-env.sh</fileName> - <dictionaryName>hive-env</dictionaryName> - </configFile> - <configFile> - <type>env</type> - <fileName>hive-log4j.properties</fileName> - <dictionaryName>hive-log4j</dictionaryName> - </configFile> - <configFile> - <type>env</type> - <fileName>hive-exec-log4j.properties</fileName> - <dictionaryName>hive-exec-log4j</dictionaryName> - </configFile> - </configFiles> - </component> - </components> - <osSpecifics> - <osSpecific> - <osFamily>any</osFamily> - <packages> - <package> - <name>hcatalog</name> - </package> - </packages> - </osSpecific> - </osSpecifics> - <commandScript> - <script>scripts/service_check.py</script> - <scriptType>PYTHON</scriptType> - <timeout>300</timeout> - </commandScript> - - <requiredServices> - <service>HIVE</service> - </requiredServices> - - <configuration-dependencies> - <config-type>hive-site</config-type> - <config-type>hive-env</config-type> - </configuration-dependencies> - <excluded-config-types> - <config-type>hive-env</config-type> - <config-type>hive-site</config-type> - <config-type>hive-exec-log4j</config-type> - <config-type>hive-log4j</config-type> - </excluded-config-types> - </service> - </services> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/files/templetonSmoke.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/files/templetonSmoke.sh b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/files/templetonSmoke.sh new file mode 100644 index 0000000..2d07b8b --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/files/templetonSmoke.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# + +export ttonhost=$1 +export smoke_test_user=$2 +export smoke_user_keytab=$3 +export security_enabled=$4 +export kinit_path_local=$5 +export ttonurl="http://${ttonhost}:50111/templeton/v1" + +if [[ $security_enabled == "true" ]]; then + kinitcmd="${kinit_path_local} -kt ${smoke_user_keytab} ${smoke_test_user}; " +else + kinitcmd="" +fi + +export no_proxy=$ttonhost +cmd="${kinitcmd}curl --negotiate -u : -s -w 'http_code <%{http_code}>' $ttonurl/status 2>&1" +retVal=`su - ${smoke_test_user} -c "$cmd"` +httpExitCode=`echo $retVal |sed 's/.*http_code <\([0-9]*\)>.*/\1/'` + +if [[ "$httpExitCode" -ne "200" ]] ; then + echo "Templeton Smoke Test (status cmd): Failed. : $retVal" + export TEMPLETON_EXIT_CODE=1 + exit 1 +fi + +exit 0 + +#try hcat ddl command +echo "user.name=${smoke_test_user}&exec=show databases;" /tmp/show_db.post.txt +cmd="${kinitcmd}curl --negotiate -u : -s -w 'http_code <%{http_code}>' -d \@${destdir}/show_db.post.txt $ttonurl/ddl 2>&1" +retVal=`su - ${smoke_test_user} -c "$cmd"` +httpExitCode=`echo $retVal |sed 's/.*http_code <\([0-9]*\)>.*/\1/'` + +if [[ "$httpExitCode" -ne "200" ]] ; then + echo "Templeton Smoke Test (ddl cmd): Failed. : $retVal" + export TEMPLETON_EXIT_CODE=1 + exit 1 +fi + +# NOT SURE?? SUHAS +if [[ $security_enabled == "true" ]]; then + echo "Templeton Pig Smoke Tests not run in secure mode" + exit 0 +fi + +#try pig query +outname=${smoke_test_user}.`date +"%M%d%y"`.$$; +ttonTestOutput="/tmp/idtest.${outname}.out"; +ttonTestInput="/tmp/idtest.${outname}.in"; +ttonTestScript="idtest.${outname}.pig" + +echo "A = load '$ttonTestInput' using PigStorage(':');" > /tmp/$ttonTestScript +echo "B = foreach A generate \$0 as id; " >> /tmp/$ttonTestScript +echo "store B into '$ttonTestOutput';" >> /tmp/$ttonTestScript + +#copy pig script to hdfs +su - ${smoke_test_user} -c "hadoop dfs -copyFromLocal /tmp/$ttonTestScript /tmp/$ttonTestScript" + +#copy input file to hdfs +su - ${smoke_test_user} -c "hadoop dfs -copyFromLocal /etc/passwd $ttonTestInput" + +#create, copy post args file +echo -n "user.name=${smoke_test_user}&file=/tmp/$ttonTestScript" > /tmp/pig_post.txt + +#submit pig query +cmd="curl -s -w 'http_code <%{http_code}>' -d \@${destdir}/pig_post.txt $ttonurl/pig 2>&1" +retVal=`su - ${smoke_test_user} -c "$cmd"` +httpExitCode=`echo $retVal |sed 's/.*http_code <\([0-9]*\)>.*/\1/'` +if [[ "$httpExitCode" -ne "200" ]] ; then + echo "Templeton Smoke Test (pig cmd): Failed. : $retVal" + export TEMPLETON_EXIT_CODE=1 + exit 1 +fi + +exit 0 http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py index a38c12a..fd8945f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py @@ -36,6 +36,7 @@ if rpm_version is not None: hadoop_conf_dir = format("/usr/hdp/{rpm_version}/etc/hadoop/conf") hadoop_bin_dir = format("/usr/hdp/{rpm_version}/hadoop/bin") hadoop_home = format('/usr/hdp/{rpm_version}/hadoop') + hadoop_streeming_jars = format("/usr/hdp/{rpm_version}/hadoop-mapreduce/hadoop-streaming-*.jar") hive_conf_dir = format('/usr/hdp/{rpm_version}/etc/hive/conf') hive_client_conf_dir = format('/usr/hdp/{rpm_version}/etc/hive/conf') hive_server_conf_dir = format('/usr/hdp/{rpm_version}/etc/hive/conf.server') @@ -46,16 +47,21 @@ if rpm_version is not None: if str(hdp_stack_version).startswith('2.0'): hcat_conf_dir = format('/usr/hdp/{rpm_version}/etc/hcatalog/conf') + config_dir = format('/usr/hdp/{rpm_version}/etc/hcatalog/conf') hcat_lib = format('/usr/hdp/{rpm_version}/hive/hcatalog/share/hcatalog') + webhcat_bin_dir = format('/usr/hdp/{rpm_version}/hive/hcatalog/sbin') # for newer versions else: hcat_conf_dir = format('/usr/hdp/{rpm_version}/etc/hive-hcatalog/conf') + config_dir = format('/usr/hdp/{rpm_version}/etc/hive-webhcat/conf') hcat_lib = format('/usr/hdp/{rpm_version}/hive/hive-hcatalog/share/hcatalog') + webhcat_bin_dir = format('/usr/hdp/{rpm_version}/hive/hive-hcatalog/sbin') else: hadoop_conf_dir = "/etc/hadoop/conf" hadoop_bin_dir = "/usr/bin" hadoop_home = '/usr' + hadoop_streeming_jars = '/usr/lib/hadoop-mapreduce/hadoop-streaming-*.jar' hive_conf_dir = "/etc/hive/conf" hive_bin = '/usr/lib/hive/bin' hive_lib = '/usr/lib/hive/lib/' @@ -66,11 +72,15 @@ else: if str(hdp_stack_version).startswith('2.0'): hcat_conf_dir = '/etc/hcatalog/conf' + config_dir = '/etc/hcatalog/conf' hcat_lib = '/usr/lib/hcatalog/share/hcatalog' + webhcat_bin_dir = '/usr/lib/hcatalog/sbin' # for newer versions else: hcat_conf_dir = '/etc/hive-hcatalog/conf' + config_dir = '/etc/hive-webhcat/conf' hcat_lib = '/usr/lib/hive-hcatalog/share/hcatalog' + webhcat_bin_dir = '/usr/lib/hive-hcatalog/sbin' execute_path = os.environ['PATH'] + os.pathsep + hive_bin hive_metastore_user_name = config['configurations']['hive-site']['javax.jdo.option.ConnectionUserName'] @@ -202,9 +212,7 @@ hive_apps_whs_dir = config['configurations']['hive-site']["hive.metastore.wareho #for create_hdfs_directory hostname = config["hostname"] hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] -hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] -kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) # Tez libraries tez_lib_uris = default("/configurations/tez-site/tez.lib.uris", None) @@ -228,6 +236,38 @@ if os.path.exists(mysql_jdbc_driver_jar): else: hive_exclude_packages = [] +######################################################## +########### WebHCat related params ##################### +######################################################## + +if str(config['hostLevelParams']['stack_version']).startswith('2.0'): + config_dir = '/etc/hcatalog/conf' + webhcat_bin_dir = '/usr/lib/hcatalog/sbin' +# for newer versions +else: + config_dir = '/etc/hive-webhcat/conf' + webhcat_bin_dir = '/usr/lib/hive-hcatalog/sbin' + +webhcat_env_sh_template = config['configurations']['webhcat-env']['content'] +templeton_log_dir = config['configurations']['hive-env']['hcat_log_dir'] +templeton_pid_dir = status_params.hcat_pid_dir + +webhcat_pid_file = status_params.webhcat_pid_file + +templeton_jar = config['configurations']['webhcat-site']['templeton.jar'] + + +webhcat_server_host = config['clusterHostInfo']['webhcat_server_host'] + +webhcat_apps_dir = "/apps/webhcat" + +hcat_hdfs_user_dir = format("/user/{hcat_user}") +hcat_hdfs_user_mode = 0755 +webhcat_hdfs_user_dir = format("/user/{webhcat_user}") +webhcat_hdfs_user_mode = 0755 +#for create_hdfs_directory +security_param = "true" if security_enabled else "false" + import functools #create partial functions with common arguments for every HdfsDirectory call #to create hdfs directory we need to call params.HdfsDirectory in code http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/service_check.py index 09ba1bf..d7b10eb 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/service_check.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/service_check.py @@ -23,6 +23,7 @@ import socket import sys from hcat_service_check import hcat_service_check +from webhcat_service_check import webhcat_service_check class HiveServiceCheck(Script): def service_check(self, env): @@ -42,6 +43,7 @@ class HiveServiceCheck(Script): sys.exit(1) hcat_service_check() + webhcat_service_check() if __name__ == "__main__": HiveServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/status_params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/status_params.py index d03d76b..e6f2514 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/status_params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/status_params.py @@ -28,6 +28,7 @@ hive_pid = 'hive-server.pid' hive_metastore_pid = 'hive.pid' hcat_pid_dir = config['configurations']['hive-env']['hcat_pid_dir'] #hcat_pid_dir +webhcat_pid_file = format('{hcat_pid_dir}/webhcat.pid') if System.get_instance().os_family == "suse" or System.get_instance().os_family == "ubuntu": daemon_name = 'mysql' http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat.py new file mode 100644 index 0000000..c56ae5f --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat.py @@ -0,0 +1,112 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +from resource_management import * +import sys + + +def webhcat(): + import params + + params.HdfsDirectory(params.webhcat_apps_dir, + action="create_delayed", + owner=params.webhcat_user, + mode=0755 + ) + if params.hcat_hdfs_user_dir != params.webhcat_hdfs_user_dir: + params.HdfsDirectory(params.hcat_hdfs_user_dir, + action="create_delayed", + owner=params.hcat_user, + mode=params.hcat_hdfs_user_mode + ) + params.HdfsDirectory(params.webhcat_hdfs_user_dir, + action="create_delayed", + owner=params.webhcat_user, + mode=params.webhcat_hdfs_user_mode + ) + params.HdfsDirectory(None, action="create") + + Directory(params.templeton_pid_dir, + owner=params.webhcat_user, + mode=0755, + group=params.user_group, + recursive=True) + + Directory(params.templeton_log_dir, + owner=params.webhcat_user, + mode=0755, + group=params.user_group, + recursive=True) + + Directory(params.config_dir, + owner=params.webhcat_user, + group=params.user_group) + + XmlConfig("webhcat-site.xml", + conf_dir=params.config_dir, + configurations=params.config['configurations']['webhcat-site'], + configuration_attributes=params.config['configuration_attributes']['webhcat-site'], + owner=params.webhcat_user, + group=params.user_group, + ) + + File(format("{config_dir}/webhcat-env.sh"), + owner=params.webhcat_user, + group=params.user_group, + content=InlineTemplate(params.webhcat_env_sh_template) + ) + + if params.security_enabled: + kinit_if_needed = format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name};") + else: + kinit_if_needed = "" + + if kinit_if_needed: + Execute(kinit_if_needed, + user=params.webhcat_user, + path='/bin' + ) + + CopyFromLocal(params.hadoop_streeming_jars, + owner=params.webhcat_user, + mode=0755, + dest_dir=params.webhcat_apps_dir, + kinnit_if_needed=kinit_if_needed, + hdfs_user=params.hdfs_user, + hadoop_conf_dir=params.hadoop_conf_dir + ) + + CopyFromLocal('/usr/share/HDP-webhcat/pig.tar.gz', + owner=params.webhcat_user, + mode=0755, + dest_dir=params.webhcat_apps_dir, + kinnit_if_needed=kinit_if_needed, + hdfs_user=params.hdfs_user, + hadoop_conf_dir=params.hadoop_conf_dir + ) + + CopyFromLocal('/usr/share/HDP-webhcat/hive.tar.gz', + owner=params.webhcat_user, + mode=0755, + dest_dir=params.webhcat_apps_dir, + kinnit_if_needed=kinit_if_needed, + hdfs_user=params.hdfs_user, + hadoop_conf_dir=params.hadoop_conf_dir + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat_server.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat_server.py new file mode 100644 index 0000000..088cb41 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat_server.py @@ -0,0 +1,53 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +import sys +from resource_management import * + +from webhcat import webhcat +from webhcat_service import webhcat_service + +class WebHCatServer(Script): + def install(self, env): + self.install_packages(env) + def configure(self, env): + import params + env.set_params(params) + webhcat() + + def start(self, env): + import params + env.set_params(params) + self.configure(env) # FOR SECURITY + webhcat_service(action = 'start') + + def stop(self, env): + import params + env.set_params(params) + + webhcat_service(action = 'stop') + + def status(self, env): + import status_params + env.set_params(status_params) + check_process_status(status_params.webhcat_pid_file) + +if __name__ == "__main__": + WebHCatServer().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat_service.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat_service.py new file mode 100644 index 0000000..41fb529 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat_service.py @@ -0,0 +1,40 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +from resource_management import * + +def webhcat_service(action='start'): + import params + + cmd = format('env HADOOP_HOME={hadoop_home} {webhcat_bin_dir}/webhcat_server.sh') + + if action == 'start': + demon_cmd = format('{cmd} start') + no_op_test = format('ls {webhcat_pid_file} >/dev/null 2>&1 && ps `cat {webhcat_pid_file}` >/dev/null 2>&1') + Execute(demon_cmd, + user=params.webhcat_user, + not_if=no_op_test + ) + elif action == 'stop': + demon_cmd = format('{cmd} stop') + Execute(demon_cmd, + user=params.webhcat_user + ) + Execute(format('rm -f {webhcat_pid_file}')) http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat_service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat_service_check.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat_service_check.py new file mode 100644 index 0000000..8d15e47 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat_service_check.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * + +def webhcat_service_check(): + import params + File(format("{tmp_dir}/templetonSmoke.sh"), + content= StaticFile('templetonSmoke.sh'), + mode=0755 + ) + + cmd = format("{tmp_dir}/templetonSmoke.sh {webhcat_server_host[0]} {smokeuser} {smokeuser_keytab}" + " {security_param} {kinit_path_local}", + smokeuser_keytab=params.smoke_user_keytab if params.security_enabled else "no_keytab") + + Execute(cmd, + tries=3, + try_sleep=5, + path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', + logoutput=True) + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/metainfo.xml index bebc7d6..a72b7b4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/metainfo.xml @@ -59,7 +59,7 @@ </auto-deploy> </dependency> <dependency> - <name>HCATALOG/HCAT</name> + <name>HIVE/HCAT</name> <scope>host</scope> <auto-deploy> <enabled>true</enabled> http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-servicegroups.cfg.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-servicegroups.cfg.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-servicegroups.cfg.j2 index 00f0740..aee9b15 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-servicegroups.cfg.j2 +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-servicegroups.cfg.j2 @@ -54,12 +54,6 @@ define servicegroup { alias OOZIE Checks } {% endif %} -{% if hostgroup_defs['webhcat-server'] %} -define servicegroup { - servicegroup_name WEBHCAT - alias WEBHCAT Checks -} -{% endif %} {% if hostgroup_defs['nagios-server'] %} define servicegroup { servicegroup_name NAGIOS @@ -72,7 +66,7 @@ define servicegroup { alias GANGLIA Checks } {% endif %} -{% if hostgroup_defs['hiveserver'] %} +{% if hostgroup_defs['hiveserver'] or hostgroup_defs['webhcat-server'] %} define servicegroup { servicegroup_name HIVE alias HIVE Checks http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-services.cfg.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-services.cfg.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-services.cfg.j2 index 045e9ad..8e92efc 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-services.cfg.j2 +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-services.cfg.j2 @@ -729,7 +729,7 @@ define service { hostgroup_name webhcat-server use hadoop-service service_description WEBHCAT::WebHCat Server status - servicegroups WEBHCAT + servicegroups HIVE {% if security_enabled %} check_command check_templeton_status!{{ templeton_port }}!v1!{{ str(security_enabled).lower() }}!{{ nagios_keytab_path }}!{{ nagios_principal_name }}!{{ kinit_path_local }} {% else %} http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/configuration/webhcat-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/configuration/webhcat-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/configuration/webhcat-env.xml deleted file mode 100644 index 1dba691..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/configuration/webhcat-env.xml +++ /dev/null @@ -1,54 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> - -<configuration> - <!-- webhcat-env.sh --> - <property> - <name>content</name> - <description>webhcat-env.sh content</description> - <value> -# The file containing the running pid -PID_FILE={{pid_file}} - -TEMPLETON_LOG_DIR={{templeton_log_dir}}/ - - -WEBHCAT_LOG_DIR={{templeton_log_dir}}/ - -# The console error log -ERROR_LOG={{templeton_log_dir}}/webhcat-console-error.log - -# The console log -CONSOLE_LOG={{templeton_log_dir}}/webhcat-console.log - -#TEMPLETON_JAR=templeton_jar_name - -#HADOOP_PREFIX=hadoop_prefix - -#HCAT_PREFIX=hive_prefix - -# Set HADOOP_HOME to point to a specific hadoop install directory -export HADOOP_HOME={{hadoop_home}} - </value> - </property> - -</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/configuration/webhcat-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/configuration/webhcat-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/configuration/webhcat-site.xml deleted file mode 100644 index 0523dab..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/configuration/webhcat-site.xml +++ /dev/null @@ -1,138 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. ---> - -<!-- The default settings for Templeton. --> -<!-- Edit templeton-site.xml to change settings for your local --> -<!-- install. --> - -<configuration supports_final="true"> - - <property> - <name>templeton.port</name> - <value>50111</value> - <description>The HTTP port for the main server.</description> - </property> - - <property> - <name>templeton.hadoop.conf.dir</name> - <value>/etc/hadoop/conf</value> - <description>The path to the Hadoop configuration.</description> - </property> - - <property> - <name>templeton.jar</name> - <value>/usr/lib/hcatalog/share/webhcat/svr/webhcat.jar</value> - <description>The path to the Templeton jar file.</description> - </property> - - <property> - <name>templeton.libjars</name> - <value>/usr/lib/zookeeper/zookeeper.jar</value> - <description>Jars to add the the classpath.</description> - </property> - - - <property> - <name>templeton.hadoop</name> - <value>/usr/bin/hadoop</value> - <description>The path to the Hadoop executable.</description> - </property> - - <property> - <name>templeton.pig.archive</name> - <value>hdfs:///apps/webhcat/pig.tar.gz</value> - <description>The path to the Pig archive.</description> - </property> - - <property> - <name>templeton.pig.path</name> - <value>pig.tar.gz/pig/bin/pig</value> - <description>The path to the Pig executable.</description> - </property> - - <property> - <name>templeton.hcat</name> - <value>/usr/bin/hcat</value> - <description>The path to the hcatalog executable.</description> - </property> - - <property> - <name>templeton.hive.archive</name> - <value>hdfs:///apps/webhcat/hive.tar.gz</value> - <description>The path to the Hive archive.</description> - </property> - - <property> - <name>templeton.hive.home</name> - <value>hive.tar.gz/hive</value> - <description>The path to the Hive home within the tar. Has no effect if templeton.hive.archive is not set.</description> - </property> - - <property> - <name>templeton.hcat.home</name> - <value>hive.tar.gz/hive/hcatalog</value> - <description>The path to the HCat home within the tar. Has no effect if templeton.hive.archive is not set.</description> - </property> - - <property> - <name>templeton.hive.path</name> - <value>hive.tar.gz/hive/bin/hive</value> - <description>The path to the Hive executable.</description> - </property> - - <property> - <name>templeton.hive.properties</name> - <value>hive.metastore.local=false, hive.metastore.uris=thrift://localhost:9933, hive.metastore.sasl.enabled=false</value> - <description>Properties to set when running hive.</description> - </property> - - - <property> - <name>templeton.zookeeper.hosts</name> - <value>localhost:2181</value> - <description>ZooKeeper servers, as comma separated host:port pairs</description> - </property> - - <property> - <name>templeton.storage.class</name> - <value>org.apache.hive.hcatalog.templeton.tool.ZooKeeperStorage</value> - <description>The class to use as storage</description> - </property> - - <property> - <name>templeton.override.enabled</name> - <value>false</value> - <description> - Enable the override path in templeton.override.jars - </description> - </property> - - <property> - <name>templeton.streaming.jar</name> - <value>hdfs:///apps/webhcat/hadoop-streaming.jar</value> - <description>The hdfs path to the Hadoop streaming jar file.</description> - </property> - - <property> - <name>templeton.exec.timeout</name> - <value>60000</value> - <description>Time out for templeton api</description> - </property> - -</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/metainfo.xml deleted file mode 100644 index 5f493a6..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/metainfo.xml +++ /dev/null @@ -1,110 +0,0 @@ -<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<metainfo> - <schemaVersion>2.0</schemaVersion> - <services> - <service> - <name>WEBHCAT</name> - <displayName>WebHCat</displayName> - <comment>Provides a REST-like web API for HCatalog and related Hadoop components.</comment> - <version>0.12.0.2.0</version> - <components> - <component> - <name>WEBHCAT_SERVER</name> - <displayName>WebHCat Server</displayName> - <category>MASTER</category> - <cardinality>1</cardinality> - <dependencies> - <dependency> - <name>HDFS/HDFS_CLIENT</name> - <scope>host</scope> - <auto-deploy> - <enabled>true</enabled> - </auto-deploy> - </dependency> - <dependency> - <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name> - <scope>host</scope> - <auto-deploy> - <enabled>true</enabled> - </auto-deploy> - </dependency> - <dependency> - <name>ZOOKEEPER/ZOOKEEPER_SERVER</name> - <scope>cluster</scope> - <auto-deploy> - <enabled>true</enabled> - <co-locate>WEBHCAT/WEBHCAT_SERVER</co-locate> - </auto-deploy> - </dependency> - <dependency> - <name>ZOOKEEPER/ZOOKEEPER_CLIENT</name> - <scope>host</scope> - <auto-deploy> - <enabled>true</enabled> - </auto-deploy> - </dependency> - <dependency> - <name>YARN/YARN_CLIENT</name> - <scope>host</scope> - <auto-deploy> - <enabled>true</enabled> - </auto-deploy> - </dependency> - </dependencies> - <commandScript> - <script>scripts/webhcat_server.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - </component> - </components> - <osSpecifics> - <osSpecific> - <osFamily>any</osFamily> - <packages> - <package> - <name>hcatalog</name> - </package> - <package> - <name>webhcat-tar-hive</name> - </package> - <package> - <name>webhcat-tar-pig</name> - </package> - </packages> - </osSpecific> - </osSpecifics> - <commandScript> - <script>scripts/service_check.py</script> - <scriptType>PYTHON</scriptType> - <timeout>300</timeout> - </commandScript> - - <requiredServices> - <service>HIVE</service> - <service>ZOOKEEPER</service> - </requiredServices> - - <configuration-dependencies> - <config-type>webhcat-site</config-type> - <config-type>webhcat-env</config-type> - </configuration-dependencies> - </service> - </services> -</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/files/templetonSmoke.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/files/templetonSmoke.sh b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/files/templetonSmoke.sh deleted file mode 100644 index 2d07b8b..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/files/templetonSmoke.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env bash -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# - -export ttonhost=$1 -export smoke_test_user=$2 -export smoke_user_keytab=$3 -export security_enabled=$4 -export kinit_path_local=$5 -export ttonurl="http://${ttonhost}:50111/templeton/v1" - -if [[ $security_enabled == "true" ]]; then - kinitcmd="${kinit_path_local} -kt ${smoke_user_keytab} ${smoke_test_user}; " -else - kinitcmd="" -fi - -export no_proxy=$ttonhost -cmd="${kinitcmd}curl --negotiate -u : -s -w 'http_code <%{http_code}>' $ttonurl/status 2>&1" -retVal=`su - ${smoke_test_user} -c "$cmd"` -httpExitCode=`echo $retVal |sed 's/.*http_code <\([0-9]*\)>.*/\1/'` - -if [[ "$httpExitCode" -ne "200" ]] ; then - echo "Templeton Smoke Test (status cmd): Failed. : $retVal" - export TEMPLETON_EXIT_CODE=1 - exit 1 -fi - -exit 0 - -#try hcat ddl command -echo "user.name=${smoke_test_user}&exec=show databases;" /tmp/show_db.post.txt -cmd="${kinitcmd}curl --negotiate -u : -s -w 'http_code <%{http_code}>' -d \@${destdir}/show_db.post.txt $ttonurl/ddl 2>&1" -retVal=`su - ${smoke_test_user} -c "$cmd"` -httpExitCode=`echo $retVal |sed 's/.*http_code <\([0-9]*\)>.*/\1/'` - -if [[ "$httpExitCode" -ne "200" ]] ; then - echo "Templeton Smoke Test (ddl cmd): Failed. : $retVal" - export TEMPLETON_EXIT_CODE=1 - exit 1 -fi - -# NOT SURE?? SUHAS -if [[ $security_enabled == "true" ]]; then - echo "Templeton Pig Smoke Tests not run in secure mode" - exit 0 -fi - -#try pig query -outname=${smoke_test_user}.`date +"%M%d%y"`.$$; -ttonTestOutput="/tmp/idtest.${outname}.out"; -ttonTestInput="/tmp/idtest.${outname}.in"; -ttonTestScript="idtest.${outname}.pig" - -echo "A = load '$ttonTestInput' using PigStorage(':');" > /tmp/$ttonTestScript -echo "B = foreach A generate \$0 as id; " >> /tmp/$ttonTestScript -echo "store B into '$ttonTestOutput';" >> /tmp/$ttonTestScript - -#copy pig script to hdfs -su - ${smoke_test_user} -c "hadoop dfs -copyFromLocal /tmp/$ttonTestScript /tmp/$ttonTestScript" - -#copy input file to hdfs -su - ${smoke_test_user} -c "hadoop dfs -copyFromLocal /etc/passwd $ttonTestInput" - -#create, copy post args file -echo -n "user.name=${smoke_test_user}&file=/tmp/$ttonTestScript" > /tmp/pig_post.txt - -#submit pig query -cmd="curl -s -w 'http_code <%{http_code}>' -d \@${destdir}/pig_post.txt $ttonurl/pig 2>&1" -retVal=`su - ${smoke_test_user} -c "$cmd"` -httpExitCode=`echo $retVal |sed 's/.*http_code <\([0-9]*\)>.*/\1/'` -if [[ "$httpExitCode" -ne "200" ]] ; then - echo "Templeton Smoke Test (pig cmd): Failed. : $retVal" - export TEMPLETON_EXIT_CODE=1 - exit 1 -fi - -exit 0 http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/__init__.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/__init__.py deleted file mode 100644 index 35de4bb..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/params.py deleted file mode 100644 index f37ac27..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/params.py +++ /dev/null @@ -1,102 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -from resource_management import * -import status_params - -# server configurations -config = Script.get_config() -tmp_dir = Script.get_tmp_dir() - -#RPM versioning support -rpm_version = default("/configurations/hadoop-env/rpm_version", None) - -#hadoop params -hdp_stack_version = config['hostLevelParams']['stack_version'] -if rpm_version is not None: - hadoop_bin_dir = format("/usr/hdp/{rpm_version}/hadoop/bin") - hadoop_home = format('/usr/hdp/{rpm_version}/hadoop') - hadoop_streeming_jars = format("/usr/hdp/{rpm_version}/hadoop-mapreduce/hadoop-streaming-*.jar") - if str(hdp_stack_version).startswith('2.0'): - config_dir = format('/usr/hdp/{rpm_version}/etc/hcatalog/conf') - webhcat_bin_dir = format('/usr/hdp/{rpm_version}/hive/hcatalog/sbin') - # for newer versions - else: - config_dir = format('/usr/hdp/{rpm_version}/etc/hive-webhcat/conf') - webhcat_bin_dir = format('/usr/hdp/{rpm_version}/hive/hive-hcatalog/sbin') -else: - hadoop_bin_dir = "/usr/bin" - hadoop_home = '/usr' - hadoop_streeming_jars = '/usr/lib/hadoop-mapreduce/hadoop-streaming-*.jar' - if str(hdp_stack_version).startswith('2.0'): - config_dir = '/etc/hcatalog/conf' - webhcat_bin_dir = '/usr/lib/hcatalog/sbin' - # for newer versions - else: - config_dir = '/etc/hive-webhcat/conf' - webhcat_bin_dir = '/usr/lib/hive-hcatalog/sbin' - -hcat_user = config['configurations']['hive-env']['hcat_user'] -webhcat_user = config['configurations']['hive-env']['webhcat_user'] - -webhcat_env_sh_template = config['configurations']['webhcat-env']['content'] -templeton_log_dir = config['configurations']['hive-env']['hcat_log_dir'] -templeton_pid_dir = status_params.templeton_pid_dir - -pid_file = status_params.pid_file - -hadoop_conf_dir = config['configurations']['webhcat-site']['templeton.hadoop.conf.dir'] -templeton_jar = config['configurations']['webhcat-site']['templeton.jar'] - -user_group = config['configurations']['cluster-env']['user_group'] - -webhcat_server_host = config['clusterHostInfo']['webhcat_server_host'] - -webhcat_apps_dir = "/apps/webhcat" -smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] -smokeuser = config['configurations']['cluster-env']['smokeuser'] -security_enabled = config['configurations']['cluster-env']['security_enabled'] -kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) - -hcat_hdfs_user_dir = format("/user/{hcat_user}") -hcat_hdfs_user_mode = 0755 -webhcat_hdfs_user_dir = format("/user/{webhcat_user}") -webhcat_hdfs_user_mode = 0755 -webhcat_apps_dir = "/apps/webhcat" -#for create_hdfs_directory -hostname = config["hostname"] -security_param = "true" if security_enabled else "false" -hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] -hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] -hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] -kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) -import functools -#create partial functions with common arguments for every HdfsDirectory call -#to create hdfs directory we need to call params.HdfsDirectory in code -HdfsDirectory = functools.partial( - HdfsDirectory, - conf_dir=hadoop_conf_dir, - hdfs_user=hdfs_user, - security_enabled = security_enabled, - keytab = hdfs_user_keytab, - kinit_path_local = kinit_path_local, - bin_dir = hadoop_bin_dir -) http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/service_check.py deleted file mode 100644 index 0e3c0f0..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/service_check.py +++ /dev/null @@ -1,45 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" -from resource_management import * - -class WebHCatServiceCheck(Script): - def service_check(self, env): - import params - - env.set_params(params) - - File(format("{tmp_dir}/templetonSmoke.sh"), - content= StaticFile('templetonSmoke.sh'), - mode=0755 - ) - - cmd = format("{tmp_dir}/templetonSmoke.sh {webhcat_server_host[0]} {smokeuser} {smokeuser_keytab}" - " {security_param} {kinit_path_local}", - smokeuser_keytab=params.smoke_user_keytab if params.security_enabled else "no_keytab") - - Execute(cmd, - tries=3, - try_sleep=5, - path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', - logoutput=True) - -if __name__ == "__main__": - WebHCatServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/601014ed/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/status_params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/status_params.py deleted file mode 100644 index 23823e6..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/status_params.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management import * - -config = Script.get_config() - -templeton_pid_dir = config['configurations']['hive-env']['hcat_pid_dir'] -pid_file = format('{templeton_pid_dir}/webhcat.pid')