Updated Branches: refs/heads/trunk 0776b7f77 -> 0d30fb95e
AMBARI-4167. Oozie on HDP1. Using resource management lib (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0d30fb95 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0d30fb95 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0d30fb95 Branch: refs/heads/trunk Commit: 0d30fb95ef4723f7cfe7f924a8cc6705b5237d97 Parents: 0776b7f Author: Andrew Onischuk <[email protected]> Authored: Tue Dec 24 11:42:01 2013 -0800 Committer: Andrew Onischuk <[email protected]> Committed: Tue Dec 24 11:42:33 2013 -0800 ---------------------------------------------------------------------- .../HDP/1.3._/services/OOZIE/metainfo.xml | 69 +++++++++++--- .../services/OOZIE/package/files/oozieSmoke.sh | 93 ++++++++++++++++++ .../OOZIE/package/files/wrap_ooziedb.sh | 31 ++++++ .../services/OOZIE/package/scripts/oozie.py | 99 ++++++++++++++++++++ .../OOZIE/package/scripts/oozie_client.py | 33 +++++++ .../OOZIE/package/scripts/oozie_server.py | 45 +++++++++ .../OOZIE/package/scripts/oozie_service.py | 45 +++++++++ .../services/OOZIE/package/scripts/params.py | 64 +++++++++++++ .../OOZIE/package/scripts/service_check.py | 47 ++++++++++ .../OOZIE/package/scripts/status_params.py | 26 +++++ .../OOZIE/package/templates/oozie-env.sh.j2 | 64 +++++++++++++ .../package/templates/oozie-log4j.properties.j2 | 74 +++++++++++++++ 12 files changed, 676 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/0d30fb95/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/metainfo.xml index e4af208..1492cd4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/metainfo.xml @@ -16,23 +16,64 @@ limitations under the License. --> <metainfo> - <user>root</user> - <comment>System for workflow coordination and execution of Apache Hadoop jobs. This also includes the installation of the optional Oozie Web Console which relies on and will install the <a target="_blank" href="http://www.sencha.com/products/extjs/license/">ExtJS</a> Library.</comment> - <version>3.3.2.1.3.3.0</version> - - <components> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>OOZIE</name> + <comment>System for workflow coordination and execution of Apache Hadoop jobs. This also includes the installation of the optional Oozie Web Console which relies on and will install the <a target="_blank" href="http://www.sencha.com/products/extjs/license/">ExtJS</a> Library. + </comment> + <version>3.3.2.1.3.3.0</version> + <components> <component> - <name>OOZIE_SERVER</name> - <category>MASTER</category> + <name>OOZIE_SERVER</name> + <category>MASTER</category> + <commandScript> + <script>scripts/oozie_server.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> </component> <component> - <name>OOZIE_CLIENT</name> - <category>CLIENT</category> + <name>OOZIE_CLIENT</name> + <category>CLIENT</category> + <commandScript> + <script>scripts/oozie_client.py</script> + <scriptType>PYTHON</scriptType> + </commandScript> </component> - </components> - <configuration-dependencies> - <config-type>global</config-type> - <config-type>oozie-site</config-type> - </configuration-dependencies> + </components> + + <osSpecifics> + <osSpecific> + <osType>any</osType> + <packages> + <package> + <type>rpm</type> + <name>oozie.noarch</name> + </package> + <package> + <type>rpm</type> + <name>oozie-client.noarch</name> + </package> + <package> + <type>rpm</type> + <name>extjs-2.2-1</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <commandScript> + <script>scripts/service_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + + <configuration-dependencies> + <config-type>global</config-type> + <config-type>oozie-site</config-type> + </configuration-dependencies> + </service> + </services> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/0d30fb95/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/files/oozieSmoke.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/files/oozieSmoke.sh b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/files/oozieSmoke.sh new file mode 100644 index 0000000..2446544 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/files/oozieSmoke.sh @@ -0,0 +1,93 @@ +#!/bin/sh +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +function getValueFromField { + xmllint $1 | grep "<name>$2</name>" -C 2 | grep '<value>' | cut -d ">" -f2 | cut -d "<" -f1 + return $? +} + +function checkOozieJobStatus { + local job_id=$1 + local num_of_tries=$2 + #default num_of_tries to 10 if not present + num_of_tries=${num_of_tries:-10} + local i=0 + local rc=1 + local cmd="source ${oozie_conf_dir}/oozie-env.sh ; /usr/bin/oozie job -oozie ${OOZIE_SERVER} -info $job_id" + su - ${smoke_test_user} -c "$cmd" + while [ $i -lt $num_of_tries ] ; do + cmd_output=`su - ${smoke_test_user} -c "$cmd"` + (IFS='';echo $cmd_output) + act_status=$(IFS='';echo $cmd_output | grep ^Status | cut -d':' -f2 | sed 's| ||g') + echo "workflow_status=$act_status" + if [ "RUNNING" == "$act_status" ]; then + #increment the couner and get the status again after waiting for 15 secs + sleep 15 + (( i++ )) + elif [ "SUCCEEDED" == "$act_status" ]; then + rc=0; + break; + else + rc=1 + break; + fi + done + return $rc +} + +export oozie_conf_dir=$1 +export hadoop_conf_dir=$2 +export smoke_test_user=$3 +export security_enabled=$4 +export smoke_user_keytab=$5 +export kinit_path_local=$6 + +export OOZIE_EXIT_CODE=0 +export JOBTRACKER=`getValueFromField ${hadoop_conf_dir}/mapred-site.xml mapred.job.tracker` +export NAMENODE=`getValueFromField ${hadoop_conf_dir}/core-site.xml fs.default.name` +export OOZIE_SERVER=`getValueFromField ${oozie_conf_dir}/oozie-site.xml oozie.base.url | tr '[:upper:]' '[:lower:]'` +export OOZIE_EXAMPLES_DIR=`rpm -ql oozie-client | grep 'oozie-examples.tar.gz$' | xargs dirname` +cd $OOZIE_EXAMPLES_DIR + +tar -zxf oozie-examples.tar.gz +sed -i "s|nameNode=hdfs://localhost:8020|nameNode=$NAMENODE|g" examples/apps/map-reduce/job.properties +sed -i "s|nameNode=hdfs://localhost:9000|nameNode=$NAMENODE|g" examples/apps/map-reduce/job.properties +sed -i "s|jobTracker=localhost:8021|jobTracker=$JOBTRACKER|g" examples/apps/map-reduce/job.properties +sed -i "s|jobTracker=localhost:9001|jobTracker=$JOBTRACKER|g" examples/apps/map-reduce/job.properties +sed -i "s|oozie.wf.application.path=hdfs://localhost:9000|oozie.wf.application.path=$NAMENODE|g" examples/apps/map-reduce/job.properties + +if [[ $security_enabled == "true" ]]; then + kinitcmd="${kinit_path_local} -kt ${smoke_user_keytab} ${smoke_test_user}; " +else + kinitcmd="" +fi + +su - ${smoke_test_user} -c "hadoop dfs -rmr examples" +su - ${smoke_test_user} -c "hadoop dfs -rmr input-data" +su - ${smoke_test_user} -c "hadoop dfs -copyFromLocal $OOZIE_EXAMPLES_DIR/examples examples" +su - ${smoke_test_user} -c "hadoop dfs -copyFromLocal $OOZIE_EXAMPLES_DIR/examples/input-data input-data" + +cmd="${kinitcmd}source ${oozie_conf_dir}/oozie-env.sh ; /usr/bin/oozie job -oozie $OOZIE_SERVER -config $OOZIE_EXAMPLES_DIR/examples/apps/map-reduce/job.properties -run" +job_info=`su - ${smoke_test_user} -c "$cmd" | grep "job:"` +job_id="`echo $job_info | cut -d':' -f2`" +checkOozieJobStatus "$job_id" +OOZIE_EXIT_CODE="$?" +exit $OOZIE_EXIT_CODE http://git-wip-us.apache.org/repos/asf/ambari/blob/0d30fb95/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/files/wrap_ooziedb.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/files/wrap_ooziedb.sh b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/files/wrap_ooziedb.sh new file mode 100644 index 0000000..97a513c --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/files/wrap_ooziedb.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +OUT=`cd /var/tmp/oozie && /usr/lib/oozie/bin/ooziedb.sh "$@" 2>&1` +EC=$? +echo $OUT +GRVAR=`echo ${OUT} | grep -o "java.lang.Exception: DB schema exists"` +if [ ${EC} -ne 0 ] && [ -n "$GRVAR" ] +then + exit 0 +else + exit $EC +fi http://git-wip-us.apache.org/repos/asf/ambari/blob/0d30fb95/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/oozie.py b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/oozie.py new file mode 100644 index 0000000..91da7ae --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/oozie.py @@ -0,0 +1,99 @@ +from resource_management import * + +def oozie(is_server=False + ): + import params + + XmlConfig( "oozie-site.xml", + conf_dir = params.conf_dir, + configurations = params.config['configurations']['oozie-site'], + owner = params.oozie_user, + group = params.user_group, + mode = 0664 + ) + + Directory( params.conf_dir, + owner = params.oozie_user, + group = params.user_group + ) + + TemplateConfig( format("{conf_dir}/oozie-env.sh"), + owner = params.oozie_user + ) + + TemplateConfig( format("{conf_dir}/oozie-log4j.properties"), + owner = params.oozie_user + ) + + if params.jdbc_driver_name == "com.mysql.jdbc.Driver" or params.jdbc_driver_name == "oracle.jdbc.driver.OracleDriver": + Execute(format("/bin/sh -c 'cd /usr/lib/ambari-agent/ &&\ + curl -kf --retry 5 {jdk_location}{check_db_connection_jar_name}\ + -o {check_db_connection_jar_name}'"), + not_if = format("[ -f {check_db_connection_jar} ]") + ) + + oozie_ownership( ) + + if is_server: + oozie_server_specific( ) + +def oozie_ownership( +): + import params + + File ( format("{conf_dir}/adminusers.txt"), + owner = params.oozie_user, + group = params.user_group + ) + + File ( format("{conf_dir}/hadoop-config.xml"), + owner = params.oozie_user, + group = params.user_group + ) + + File ( format("{conf_dir}/oozie-default.xml"), + owner = params.oozie_user, + group = params.user_group + ) + + Directory ( format("{conf_dir}/action-conf"), + owner = params.oozie_user, + group = params.user_group + ) + + File ( format("{conf_dir}/action-conf/hive.xml"), + owner = params.oozie_user, + group = params.user_group + ) + +def oozie_server_specific( +): + import params + + oozie_server_directorties = [params.oozie_pid_dir, params.oozie_log_dir, params.oozie_tmp_dir, params.oozie_data_dir, params.oozie_lib_dir, params.oozie_webapps_dir] + Directory( oozie_server_directorties, + owner = params.oozie_user, + mode = 0755, + recursive = True + ) + + cmd1 = "cd /usr/lib/oozie && tar -xvf oozie-sharelib.tar.gz" + cmd2 = format("cd /usr/lib/oozie && mkdir -p {oozie_tmp_dir}") + + # this is different for HDP2 + cmd3 = format("cd /usr/lib/oozie && chown {oozie_user}:{user_group} {oozie_tmp_dir}") + if params.jdbc_driver_name=="com.mysql.jdbc.Driver" or params.jdbc_driver_name=="oracle.jdbc.driver.OracleDriver": + cmd3 += format(" && mkdir -p {oozie_libext_dir} && cp {jdbc_driver_jar} {oozie_libext_dir}") + + # this is different for HDP2 + cmd4 = format("cd {oozie_tmp_dir} && /usr/lib/oozie/bin/oozie-setup.sh -hadoop 0.20.200 {hadoop_jar_location} -extjs {ext_js_path} {jar_option} {jar_path}") + + no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1") + Execute( [cmd1, cmd2, cmd3], + not_if = no_op_test + ) + Execute( cmd4, + user = params.oozie_user, + not_if = no_op_test + ) + http://git-wip-us.apache.org/repos/asf/ambari/blob/0d30fb95/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/oozie_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/oozie_client.py b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/oozie_client.py new file mode 100644 index 0000000..1d5db39 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/oozie_client.py @@ -0,0 +1,33 @@ +import sys +from resource_management import * + +from oozie import oozie +from oozie_service import oozie_service + + +class OozieClient(Script): + def install(self, env): + self.install_packages(env) + self.configure(env) + + def configure(self, env): + import params + env.set_params(params) + + oozie(is_server=False) + + def status(self, env): + raise ClientComponentHasNoStatus() + +def main(): + command_type = sys.argv[1] if len(sys.argv)>1 else "install" + print "Running "+command_type + command_data_file = '/root/workspace/Oozie/input.json' + basedir = '/root/workspace/Oozie/main' + sys.argv = ["", command_type, command_data_file, basedir] + + OozieClient().execute() + +if __name__ == "__main__": + #main() + OozieClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/0d30fb95/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/oozie_server.py b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/oozie_server.py new file mode 100644 index 0000000..0c4c46b --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/oozie_server.py @@ -0,0 +1,45 @@ +import sys +from resource_management import * + +from oozie import oozie +from oozie_service import oozie_service + + +class OozieServer(Script): + def install(self, env): + self.install_packages(env) + self.configure(env) + + def configure(self, env): + import params + env.set_params(params) + + oozie(is_server=True) + + def start(self, env): + import params + env.set_params(params) + oozie_service(action='start') + + def stop(self, env): + import params + env.set_params(params) + oozie_service(action='stop') + + def status(self, env): + import status_params + env.set_params(status_params) + check_process_status(status_params.pid_file) + +def main(): + command_type = sys.argv[1] if len(sys.argv)>1 else "start" + print "Running "+command_type + command_data_file = '/root/workspace/Oozie/input.json' + basedir = '/root/workspace/Oozie/main' + sys.argv = ["", command_type, command_data_file, basedir] + + OozieServer().execute() + +if __name__ == "__main__": + #main() + OozieServer().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/0d30fb95/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/oozie_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/oozie_service.py b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/oozie_service.py new file mode 100644 index 0000000..1d8767c --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/oozie_service.py @@ -0,0 +1,45 @@ +from resource_management import * + +def oozie_service(action = 'start'): # 'start' or 'stop' + import params + + kinit_if_needed = format("{kinit_path_local} -kt {oozie_keytab} {oozie_principal};") if params.security_enabled else "" + no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1") + + if action == 'start': + start_cmd = format("cd {oozie_tmp_dir} && /usr/lib/oozie/bin/oozie-start.sh") + + if params.jdbc_driver_name == "com.mysql.jdbc.Driver" or params.jdbc_driver_name == "oracle.jdbc.driver.OracleDriver": + db_connection_check_command = format("{java_home}/bin/java -cp {check_db_connection_jar}:{jdbc_driver_jar} org.apache.ambari.server.DBConnectionVerification {oozie_jdbc_connection_url} {oozie_metastore_user_name} {oozie_metastore_user_passwd} {jdbc_driver_name}") + else: + db_connection_check_command = None + + cmd1 = format("cd {oozie_tmp_dir} && /usr/lib/oozie/bin/ooziedb.sh create -sqlfile oozie.sql -run") + cmd2 = format("{kinit_if_needed} hadoop dfs -put /usr/lib/oozie/share {oozie_hdfs_user_dir} ; hadoop dfs -chmod -R 755 {oozie_hdfs_user_dir}/share") + + if db_connection_check_command: + Execute( db_connection_check_command) + + Execute( cmd1, + user = params.oozie_user, + not_if = no_op_test, + ignore_failures = True + ) + + Execute( cmd2, + user = params.oozie_user, + not_if = format("{kinit_if_needed} hadoop dfs -ls /user/oozie/share | awk 'BEGIN {{count=0;}} /share/ {{count++}} END {{if (count > 0) {{exit 0}} else {{exit 1}}}}'") + ) + + Execute( start_cmd, + user = params.oozie_user, + not_if = no_op_test, + ) + elif action == 'stop': + stop_cmd = format("su - {oozie_user} -c 'cd {oozie_tmp_dir} && /usr/lib/oozie/bin/oozie-stop.sh' && rm -f {pid_file}") + Execute( stop_cmd, + only_if = no_op_test + ) + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/0d30fb95/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/params.py new file mode 100644 index 0000000..85e587c --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/params.py @@ -0,0 +1,64 @@ +from resource_management import * +import status_params + +# server configurations +config = Script.get_config() + +oozie_user = config['configurations']['global']['oozie_user'] +smokeuser = config['configurations']['global']['smokeuser'] +conf_dir = "/etc/oozie/conf" +hadoop_conf_dir = "/etc/hadoop/conf" +user_group = config['configurations']['global']['user_group'] +jdk_location = config['hostLevelParams']['jdk_location'] +check_db_connection_jar_name = "DBConnectionVerification.jar" +check_db_connection_jar = format("/usr/lib/ambari-agent/{check_db_connection_jar_name}") +hadoop_prefix = "/usr" +oozie_tmp_dir = "/var/tmp/oozie" +oozie_hdfs_user_dir = format("/user/{oozie_user}") +oozie_pid_dir = status_params.oozie_pid_dir +pid_file = status_params.pid_file +hadoop_jar_location = "/usr/lib/hadoop/" +# for HDP2 it's "/usr/share/HDP-oozie/ext-2.2.zip" +ext_js_path = "/usr/share/HDP-oozie/ext.zip" +oozie_libext_dir = "/usr/lib/oozie/libext" +lzo_enabled = config['configurations']['global']['lzo_enabled'] +security_enabled = config['configurations']['global']['security_enabled'] + +kinit_path_local = get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) +oozie_service_keytab = config['configurations']['oozie-site']['oozie.service.HadoopAccessorService.keytab.file'] +oozie_principal = config['configurations']['oozie-site']['oozie.service.HadoopAccessorService.kerberos.principal'] +smokeuser_keytab = config['configurations']['global']['smokeuser_keytab'] +oozie_keytab = config['configurations']['global']['oozie_keytab'] + +oracle_driver_jar_name = "ojdbc6.jar" +java_share_dir = "/usr/share/java" + +java_home = config['configurations']['global']['java64_home'] +oozie_metastore_user_name = config['configurations']['oozie-site']['oozie.service.JPAService.jdbc.username'] +oozie_metastore_user_passwd = default("/configurations/oozie-site/oozie.service.JPAService.jdbc.password","") +oozie_jdbc_connection_url = default("/configurations/oozie-site/oozie.service.JPAService.jdbc.url", "") +oozie_log_dir = config['configurations']['global']['oozie_log_dir'] +oozie_data_dir = config['configurations']['global']['oozie_data_dir'] +oozie_lib_dir = "/var/lib/oozie/" +oozie_webapps_dir = "/var/lib/oozie/oozie-server/webapps/" + +jdbc_driver_name = default("/configurations/oozie-site/oozie.service.JPAService.jdbc.driver", "") + +if jdbc_driver_name == "com.mysql.jdbc.Driver": + jdbc_driver_jar = "/usr/share/java/mysql-connector-java.jar" +elif jdbc_driver_name == "oracle.jdbc.driver.OracleDriver": + jdbc_driver_jar = "/usr/share/java/ojdbc6.jar" +else: + jdbc_driver_jar = "" + +if lzo_enabled or jdbc_driver_name: + jar_option = "-jars" +else: + jar_option = "" + +lzo_jar_suffix = "/usr/lib/hadoop/lib/hadoop-lzo-0.5.0.jar" if lzo_enabled else "" + +if lzo_enabled and jdbc_driver_name: + jar_path = format("{lzo_jar_suffix}:{jdbc_driver_jar}") +else: + jar_path = "{lzo_jar_suffix}{jdbc_driver_jar}" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/0d30fb95/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/service_check.py new file mode 100644 index 0000000..7dbfc87 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/service_check.py @@ -0,0 +1,47 @@ +from resource_management import * + +class OozieServiceCheck(Script): + def service_check(self, env): + import params + env.set_params(params) + + # on HDP2 this file is different + smoke_test_file_name = 'oozieSmoke.sh' + + oozie_smoke_shell_file( smoke_test_file_name) + +def oozie_smoke_shell_file( + file_name +): + import params + + File( format("/tmp/{file_name}"), + content = StaticFile(file_name), + mode = 0755 + ) + + if params.security_enabled: + sh_cmd = format("sh /tmp/{file_name} {conf_dir} {hadoop_conf_dir} {smokeuser} {security_enabled} {smokeuser_keytab} {kinit_path_local}") + else: + sh_cmd = format("sh /tmp/{file_name} {conf_dir} {hadoop_conf_dir} {smokeuser} {security_enabled}") + + Execute( format("/tmp/{file_name}"), + command = sh_cmd, + tries = 3, + try_sleep = 5, + logoutput = True + ) + +def main(): + import sys + command_type = 'service_check' + command_data_file = '/root/workspace/Oozie/input.json' + basedir = '/root/workspace/Oozie/main' + sys.argv = ["", command_type, command_data_file, basedir] + + OozieServiceCheck().execute() + +if __name__ == "__main__": + OozieServiceCheck().execute() + #main() + http://git-wip-us.apache.org/repos/asf/ambari/blob/0d30fb95/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/status_params.py b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/status_params.py new file mode 100644 index 0000000..c44fcf4 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/scripts/status_params.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * + +config = Script.get_config() + +oozie_pid_dir = config['configurations']['global']['oozie_pid_dir'] +pid_file = format("{oozie_pid_dir}/oozie.pid") http://git-wip-us.apache.org/repos/asf/ambari/blob/0d30fb95/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/templates/oozie-env.sh.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/templates/oozie-env.sh.j2 b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/templates/oozie-env.sh.j2 new file mode 100644 index 0000000..270a1a8 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/templates/oozie-env.sh.j2 @@ -0,0 +1,64 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +#Set JAVA HOME +export JAVA_HOME={{java_home}} + +# Set Oozie specific environment variables here. + +# Settings for the Embedded Tomcat that runs Oozie +# Java System properties for Oozie should be specified in this variable +# +# export CATALINA_OPTS= + +# Oozie configuration file to load from Oozie configuration directory +# +# export OOZIE_CONFIG_FILE=oozie-site.xml + +# Oozie logs directory +# +export OOZIE_LOG={{oozie_log_dir}} + +# Oozie pid directory +# +export CATALINA_PID={{pid_file}} + +#Location of the data for oozie +export OOZIE_DATA={{oozie_data_dir}} + +# Oozie Log4J configuration file to load from Oozie configuration directory +# +# export OOZIE_LOG4J_FILE=oozie-log4j.properties + +# Reload interval of the Log4J configuration file, in seconds +# +# export OOZIE_LOG4J_RELOAD=10 + +# The port Oozie server runs +# +# export OOZIE_HTTP_PORT=11000 + +# The host name Oozie server runs on +# +# export OOZIE_HTTP_HOSTNAME=`hostname -f` + +# The base URL for callback URLs to Oozie +# +# export OOZIE_BASE_URL="http://${OOZIE_HTTP_HOSTNAME}:${OOZIE_HTTP_PORT}/oozie"; +export JAVA_LIBRARY_PATH=/usr/lib/hadoop/lib/native/Linux-amd64-64 http://git-wip-us.apache.org/repos/asf/ambari/blob/0d30fb95/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/templates/oozie-log4j.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/templates/oozie-log4j.properties.j2 b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/templates/oozie-log4j.properties.j2 new file mode 100644 index 0000000..e4a2662 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/OOZIE/package/templates/oozie-log4j.properties.j2 @@ -0,0 +1,74 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. +# + +# If the Java System property 'oozie.log.dir' is not defined at Oozie start up time +# XLogService sets its value to '${oozie.home}/logs' + +log4j.appender.oozie=org.apache.log4j.DailyRollingFileAppender +log4j.appender.oozie.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.oozie.File=${oozie.log.dir}/oozie.log +log4j.appender.oozie.Append=true +log4j.appender.oozie.layout=org.apache.log4j.PatternLayout +log4j.appender.oozie.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n + +log4j.appender.oozieops=org.apache.log4j.DailyRollingFileAppender +log4j.appender.oozieops.DatePattern='.'yyyy-MM-dd +log4j.appender.oozieops.File=${oozie.log.dir}/oozie-ops.log +log4j.appender.oozieops.Append=true +log4j.appender.oozieops.layout=org.apache.log4j.PatternLayout +log4j.appender.oozieops.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n + +log4j.appender.oozieinstrumentation=org.apache.log4j.DailyRollingFileAppender +log4j.appender.oozieinstrumentation.DatePattern='.'yyyy-MM-dd +log4j.appender.oozieinstrumentation.File=${oozie.log.dir}/oozie-instrumentation.log +log4j.appender.oozieinstrumentation.Append=true +log4j.appender.oozieinstrumentation.layout=org.apache.log4j.PatternLayout +log4j.appender.oozieinstrumentation.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n + +log4j.appender.oozieaudit=org.apache.log4j.DailyRollingFileAppender +log4j.appender.oozieaudit.DatePattern='.'yyyy-MM-dd +log4j.appender.oozieaudit.File=${oozie.log.dir}/oozie-audit.log +log4j.appender.oozieaudit.Append=true +log4j.appender.oozieaudit.layout=org.apache.log4j.PatternLayout +log4j.appender.oozieaudit.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n + +log4j.appender.openjpa=org.apache.log4j.DailyRollingFileAppender +log4j.appender.openjpa.DatePattern='.'yyyy-MM-dd +log4j.appender.openjpa.File=${oozie.log.dir}/oozie-jpa.log +log4j.appender.openjpa.Append=true +log4j.appender.openjpa.layout=org.apache.log4j.PatternLayout +log4j.appender.openjpa.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n + +log4j.logger.openjpa=INFO, openjpa +log4j.logger.oozieops=INFO, oozieops +log4j.logger.oozieinstrumentation=ALL, oozieinstrumentation +log4j.logger.oozieaudit=ALL, oozieaudit +log4j.logger.org.apache.oozie=INFO, oozie +log4j.logger.org.apache.hadoop=WARN, oozie +log4j.logger.org.mortbay=WARN, oozie +log4j.logger.org.hsqldb=WARN, oozie +log4j.logger.org.apache.hadoop.security.authentication.server=INFO, oozie
