AMBARI-21060. HDP 3.0 TP - create service definition for Oozie with configs, kerberos, widgets, etc.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cdc18ecb Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cdc18ecb Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cdc18ecb Branch: refs/heads/trunk Commit: cdc18ecb9d06ef6810962fb3742d75781ff8e831 Parents: 74972de Author: Vitaly Brodetskyi <[email protected]> Authored: Mon May 22 15:41:25 2017 +0300 Committer: Vitaly Brodetskyi <[email protected]> Committed: Mon May 22 15:42:16 2017 +0300 ---------------------------------------------------------------------- .../common-services/OOZIE/4.2.0.3.0/alerts.json | 45 ++ .../OOZIE/4.2.0.3.0/configuration/oozie-env.xml | 255 +++++++++ .../4.2.0.3.0/configuration/oozie-log4j.xml | 149 ++++++ .../4.2.0.3.0/configuration/oozie-site.xml | 254 +++++++++ .../OOZIE/4.2.0.3.0/kerberos.json | 70 +++ .../OOZIE/4.2.0.3.0/metainfo.xml | 203 ++++++++ .../package/alerts/alert_check_oozie_server.py | 244 +++++++++ .../4.2.0.3.0/package/files/oozieSmoke2.sh | 84 +++ .../files/prepareOozieHdfsDirectories.sh | 42 ++ .../4.2.0.3.0/package/files/wrap_ooziedb.sh | 31 ++ .../scripts/check_oozie_server_status.py | 38 ++ .../OOZIE/4.2.0.3.0/package/scripts/oozie.py | 516 +++++++++++++++++++ .../4.2.0.3.0/package/scripts/oozie_client.py | 78 +++ .../4.2.0.3.0/package/scripts/oozie_server.py | 163 ++++++ .../package/scripts/oozie_server_upgrade.py | 237 +++++++++ .../4.2.0.3.0/package/scripts/oozie_service.py | 188 +++++++ .../OOZIE/4.2.0.3.0/package/scripts/params.py | 39 ++ .../4.2.0.3.0/package/scripts/params_linux.py | 374 ++++++++++++++ .../4.2.0.3.0/package/scripts/params_windows.py | 34 ++ .../4.2.0.3.0/package/scripts/service_check.py | 140 +++++ .../4.2.0.3.0/package/scripts/status_params.py | 65 +++ .../package/templates/adminusers.txt.j2 | 28 + .../templates/input.config-oozie.json.j2 | 48 ++ .../package/templates/oozie-log4j.properties.j2 | 93 ++++ .../4.2.0.3.0/package/templates/oozie.conf.j2 | 35 ++ .../package/templates/zkmigrator_jaas.conf.j2 | 26 + .../OOZIE/4.2.0.3.0/quicklinks/quicklinks.json | 45 ++ .../OOZIE/4.2.0.3.0/role_command_order.json | 9 + .../OOZIE/4.2.0.3.0/themes/theme.json | 116 +++++ .../stacks/HDP/3.0/services/OOZIE/metainfo.xml | 27 + 30 files changed, 3676 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/cdc18ecb/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/alerts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/alerts.json b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/alerts.json new file mode 100644 index 0000000..a1d267f --- /dev/null +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/alerts.json @@ -0,0 +1,45 @@ +{ + "OOZIE": { + "service": [], + "OOZIE_SERVER": [ + { + "name": "oozie_server_webui", + "label": "Oozie Server Web UI", + "description": "This host-level alert is triggered if the Oozie server Web UI is unreachable.", + "interval": 1, + "scope": "ANY", + "source": { + "type": "WEB", + "uri": { + "http": "{{oozie-site/oozie.base.url}}/?user.name={{oozie-env/oozie_user}}", + "kerberos_keytab": "{{cluster-env/smokeuser_keytab}}", + "kerberos_principal": "{{cluster-env/smokeuser_principal_name}}", + "connection_timeout": 5.0 + }, + "reporting": { + "ok": { + "text": "HTTP {0} response in {2:.3f}s" + }, + "warning":{ + "text": "HTTP {0} response from {1} in {2:.3f}s ({3})" + }, + "critical": { + "text": "Connection failed to {1} ({3})" + } + } + } + }, + { + "name": "oozie_server_status", + "label": "Oozie Server Status", + "description": "This host-level alert is triggered if the Oozie server cannot be determined to be up and responding to client requests.", + "interval": 1, + "scope": "ANY", + "source": { + "type": "SCRIPT", + "path": "OOZIE/4.0.0.2.0/package/alerts/alert_check_oozie_server.py" + } + } + ] + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/cdc18ecb/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/configuration/oozie-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/configuration/oozie-env.xml b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/configuration/oozie-env.xml new file mode 100644 index 0000000..0f67356 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/configuration/oozie-env.xml @@ -0,0 +1,255 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_adding_forbidden="true"> + <property> + <name>oozie_user</name> + <display-name>Oozie User</display-name> + <value>oozie</value> + <property-type>USER</property-type> + <description>Oozie User.</description> + <value-attributes> + <type>user</type> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>oozie_admin_users</name> + <value>{oozie_user}, oozie-admin</value> + <description>Oozie admin users.</description> + <on-ambari-upgrade add="false"/> + </property> + + <property> + <name>oozie_data_dir</name> + <value>/hadoop/oozie/data</value> + <display-name>Oozie Data Dir</display-name> + <description>Data directory in which the Oozie DB exists</description> + <value-attributes> + <type>directory</type> + <empty-value-valid>true</empty-value-valid> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>oozie_log_dir</name> + <value>/var/log/oozie</value> + <display-name>Oozie Log Dir</display-name> + <description>Directory for oozie logs</description> + <value-attributes> + <type>directory</type> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>oozie_tmp_dir</name> + <value>/var/tmp/oozie</value> + <display-name>Oozie Tmp Dir</display-name> + <description>Directory for oozie temporary files</description> + <value-attributes> + <type>directory</type> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>oozie_pid_dir</name> + <value>/var/run/oozie</value> + <display-name>Oozie PID Dir</display-name> + <description>Directory in which the pid files for oozie reside.</description> + <value-attributes> + <type>directory</type> + <editable-only-at-install>true</editable-only-at-install> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>oozie_admin_port</name> + <value>11001</value> + <display-name>Oozie Server Admin Port</display-name> + <description>The admin port Oozie server runs.</description> + <value-attributes> + <overridable>false</overridable> + <type>int</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>oozie_heapsize</name> + <value>2048</value> + <description>Oozie heap size.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>oozie_permsize</name> + <value>256</value> + <description>Oozie permanent generation size.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>oozie_user_nofile_limit</name> + <value>32000</value> + <description>Max open files limit setting for OOZIE user.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>oozie_user_nproc_limit</name> + <value>16000</value> + <description>Max number of processes limit setting for OOZIE user.</description> + <on-ambari-upgrade add="false"/> + </property> + <!-- oozie-env.sh --> + + <property> + <name>service_check_job_name</name> + <value>no-op</value> + <description> + Job name from Oozie examples that will be executed at each Oozie service check action. + </description> + <on-ambari-upgrade add="false"/> + </property> + + + + <property> + <name>content</name> + <display-name>oozie-env template</display-name> + <description>This is the jinja template for oozie-env.sh file</description> + <value> +#!/bin/bash + +if [ -d "/usr/lib/bigtop-tomcat" ]; then + export OOZIE_CONFIG=${OOZIE_CONFIG:-{{conf_dir}}} + export CATALINA_BASE=${CATALINA_BASE:-{{oozie_server_dir}}} + export CATALINA_TMPDIR=${CATALINA_TMPDIR:-/var/tmp/oozie} + export OOZIE_CATALINA_HOME=/usr/lib/bigtop-tomcat +fi + +#Set JAVA HOME +export JAVA_HOME={{java_home}} + +export JRE_HOME=${JAVA_HOME} + +# Set Oozie specific environment variables here. + +# Settings for the Embedded Tomcat that runs Oozie +# Java System properties for Oozie should be specified in this variable +# +{% if java_version < 8 %} +export CATALINA_OPTS="$CATALINA_OPTS -Xmx{{oozie_heapsize}} -XX:MaxPermSize={{oozie_permsize}}" +{% else %} +export CATALINA_OPTS="$CATALINA_OPTS -Xmx{{oozie_heapsize}}" +{% endif %} +# Oozie configuration file to load from Oozie configuration directory +# +# export OOZIE_CONFIG_FILE=oozie-site.xml + +# Oozie logs directory +# +export OOZIE_LOG={{oozie_log_dir}} + +# Oozie pid directory +# +export CATALINA_PID={{pid_file}} + +#Location of the data for oozie +export OOZIE_DATA={{oozie_data_dir}} + +# Oozie Log4J configuration file to load from Oozie configuration directory +# +# export OOZIE_LOG4J_FILE=oozie-log4j.properties + +# Reload interval of the Log4J configuration file, in seconds +# +# export OOZIE_LOG4J_RELOAD=10 + +# The port Oozie server runs +# +export OOZIE_HTTP_PORT={{oozie_server_port}} + +# The admin port Oozie server runs +# +export OOZIE_ADMIN_PORT={{oozie_server_admin_port}} + +# The host name Oozie server runs on +# +# export OOZIE_HTTP_HOSTNAME=`hostname -f` + +# The base URL for callback URLs to Oozie +# +# export OOZIE_BASE_URL="http://${OOZIE_HTTP_HOSTNAME}:${OOZIE_HTTP_PORT}/oozie"; +export JAVA_LIBRARY_PATH={{hadoop_lib_home}}/native/Linux-{{architecture}}-64 + +# At least 1 minute of retry time to account for server downtime during +# upgrade/downgrade +export OOZIE_CLIENT_OPTS="${OOZIE_CLIENT_OPTS} -Doozie.connection.retry.count=5 " + +{% if sqla_db_used or lib_dir_available %} +export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:{{jdbc_libs_dir}}" +export JAVA_LIBRARY_PATH="$JAVA_LIBRARY_PATH:{{jdbc_libs_dir}}" +{% endif %} + +# Set Hadoop-related properties +export HADOOP_OPTS="-Dhdp.version=${HDP_VERSION} ${HADOOP_OPTS}" + </value> + <value-attributes> + <type>content</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>oozie_database</name> + <value>New Derby Database</value> + <display-name>Oozie Database</display-name> + <description>Oozie Server Database.</description> + <value-attributes> + <overridable>false</overridable> + <type>value-list</type> + <entries> + <entry> + <value>New Derby Database</value> + <label>New Derby</label> + </entry> + <entry> + <value>Existing MySQL / MariaDB Database</value> + <label>Existing MySQL / MariaDB</label> + </entry> + <entry> + <value>Existing PostgreSQL Database</value> + <label>Existing PostgreSQL</label> + </entry> + <entry> + <value>Existing Oracle Database</value> + <label>Existing Oracle</label> + </entry> + <entry> + <value>Existing SQL Anywhere Database</value> + <label>Existing SQL Anywhere</label> + </entry> + </entries> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/cdc18ecb/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/configuration/oozie-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/configuration/oozie-log4j.xml b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/configuration/oozie-log4j.xml new file mode 100644 index 0000000..005cc0e --- /dev/null +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/configuration/oozie-log4j.xml @@ -0,0 +1,149 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="false"> + + <property> + <name>oozie_log_maxhistory</name> + <value>720</value> + <description>The number of hours for which log files will be retained</description> + <display-name>Oozie Log: # Hours of Log Rentention</display-name> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>content</name> + <display-name>oozie-log4j template</display-name> + <description>Custom log4j.properties</description> + <value> +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. +# + +# If the Java System property 'oozie.log.dir' is not defined at Oozie start up time +# XLogService sets its value to '${oozie.home}/logs' + +# The appender that Oozie uses must be named 'oozie' (i.e. log4j.appender.oozie) + +# Using the RollingFileAppender with the OozieRollingPolicy will roll the log file every hour and retain up to MaxHistory number of +# log files. If FileNamePattern ends with ".gz" it will create gzip files. +log4j.appender.oozie=org.apache.log4j.rolling.RollingFileAppender +log4j.appender.oozie.RollingPolicy=org.apache.oozie.util.OozieRollingPolicy +log4j.appender.oozie.File=${oozie.log.dir}/oozie.log +log4j.appender.oozie.Append=true +log4j.appender.oozie.layout=org.apache.log4j.PatternLayout +log4j.appender.oozie.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - SERVER[${oozie.instance.id}] %m%n +# The FileNamePattern must end with "-%d{yyyy-MM-dd-HH}.gz" or "-%d{yyyy-MM-dd-HH}" and also start with the +# value of log4j.appender.oozie.File +log4j.appender.oozie.RollingPolicy.FileNamePattern=${log4j.appender.oozie.File}-%d{yyyy-MM-dd-HH} +# The MaxHistory controls how many log files will be retained (720 hours / 24 hours per day = 30 days); -1 to disable +log4j.appender.oozie.RollingPolicy.MaxHistory={{oozie_log_maxhistory}} + + + +log4j.appender.oozieError=org.apache.log4j.rolling.RollingFileAppender +log4j.appender.oozieError.RollingPolicy=org.apache.oozie.util.OozieRollingPolicy +log4j.appender.oozieError.File=${oozie.log.dir}/oozie-error.log +log4j.appender.oozieError.Append=true +log4j.appender.oozieError.layout=org.apache.log4j.PatternLayout +log4j.appender.oozieError.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - SERVER[${oozie.instance.id}] %m%n +# The FileNamePattern must end with "-%d{yyyy-MM-dd-HH}.gz" or "-%d{yyyy-MM-dd-HH}" and also start with the +# value of log4j.appender.oozieError.File +log4j.appender.oozieError.RollingPolicy.FileNamePattern=${log4j.appender.oozieError.File}-%d{yyyy-MM-dd-HH} +# The MaxHistory controls how many log files will be retained (720 hours / 24 hours per day = 30 days); -1 to disable +log4j.appender.oozieError.RollingPolicy.MaxHistory=720 +log4j.appender.oozieError.filter.1 = org.apache.log4j.varia.LevelMatchFilter +log4j.appender.oozieError.filter.1.levelToMatch = WARN +log4j.appender.oozieError.filter.2 = org.apache.log4j.varia.LevelMatchFilter +log4j.appender.oozieError.filter.2.levelToMatch = ERROR +log4j.appender.oozieError.filter.3 = org.apache.log4j.varia.LevelMatchFilter +log4j.appender.oozieError.filter.3.levelToMatch = FATAL +log4j.appender.oozieError.filter.4 = org.apache.log4j.varia.DenyAllFilter + + + +# Uncomment the below two lines to use the DailyRollingFileAppender instead +# The DatePattern must end with either "dd" or "HH" +#log4j.appender.oozie=org.apache.log4j.DailyRollingFileAppender +#log4j.appender.oozie.DatePattern='.'yyyy-MM-dd-HH + +log4j.appender.oozieops=org.apache.log4j.DailyRollingFileAppender +log4j.appender.oozieops.DatePattern='.'yyyy-MM-dd +log4j.appender.oozieops.File=${oozie.log.dir}/oozie-ops.log +log4j.appender.oozieops.Append=true +log4j.appender.oozieops.layout=org.apache.log4j.PatternLayout +log4j.appender.oozieops.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n + +log4j.appender.oozieinstrumentation=org.apache.log4j.DailyRollingFileAppender +log4j.appender.oozieinstrumentation.DatePattern='.'yyyy-MM-dd +log4j.appender.oozieinstrumentation.File=${oozie.log.dir}/oozie-instrumentation.log +log4j.appender.oozieinstrumentation.Append=true +log4j.appender.oozieinstrumentation.layout=org.apache.log4j.PatternLayout +log4j.appender.oozieinstrumentation.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n + +log4j.appender.oozieaudit=org.apache.log4j.DailyRollingFileAppender +log4j.appender.oozieaudit.DatePattern='.'yyyy-MM-dd +log4j.appender.oozieaudit.File=${oozie.log.dir}/oozie-audit.log +log4j.appender.oozieaudit.Append=true +log4j.appender.oozieaudit.layout=org.apache.log4j.PatternLayout +log4j.appender.oozieaudit.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n + +log4j.appender.openjpa=org.apache.log4j.DailyRollingFileAppender +log4j.appender.openjpa.DatePattern='.'yyyy-MM-dd +log4j.appender.openjpa.File=${oozie.log.dir}/oozie-jpa.log +log4j.appender.openjpa.Append=true +log4j.appender.openjpa.layout=org.apache.log4j.PatternLayout +log4j.appender.openjpa.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n + +log4j.logger.openjpa=INFO, openjpa +log4j.logger.oozieops=INFO, oozieops +log4j.logger.oozieinstrumentation=ALL, oozieinstrumentation +log4j.logger.oozieaudit=ALL, oozieaudit +log4j.logger.org.apache.oozie=INFO, oozie, oozieError +log4j.logger.org.apache.hadoop=WARN, oozie +log4j.logger.org.mortbay=WARN, oozie +log4j.logger.org.hsqldb=WARN, oozie +log4j.logger.org.apache.hadoop.security.authentication.server=WARN, oozie + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/cdc18ecb/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/configuration/oozie-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/configuration/oozie-site.xml b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/configuration/oozie-site.xml new file mode 100644 index 0000000..f68369a --- /dev/null +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/configuration/oozie-site.xml @@ -0,0 +1,254 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration supports_final="true"> + <!-- + Refer to the oozie-default.xml file for the complete list of + Oozie configuration properties and their default values. + --> + + + + + + + + + + + + + + + + + + + + + + + + + + <property> + <name>oozie.authentication.simple.anonymous.allowed</name> + <value>true</value> + <description> + Indicates if anonymous requests are allowed. + This setting is meaningful only when using 'simple' authentication. + </description> + <on-ambari-upgrade add="false"/> + </property> + + <property> + <name>oozie.authentication.kerberos.name.rules</name> + <description>The mapping from kerberos principal names to local OS user names.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>oozie.authentication.type</name> + <value>simple</value> + <description> + Authentication used for Oozie HTTP endpoint, the supported values are: simple | kerberos | + #AUTHENTICATION_HANDLER_CLASSNAME#. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>oozie.base.url</name> + <value>http://localhost:11000/oozie</value> + <description>Base Oozie URL.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>oozie.credentials.credentialclasses</name> + <value>hcat=org.apache.oozie.action.hadoop.HCatCredentials,hive2=org.apache.oozie.action.hadoop.Hive2Credentials</value> + <description> + Credential Class to be used for HCat. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>oozie.service.HadoopAccessorService.hadoop.configurations</name> + <value>*={{hadoop_conf_dir}}</value> + <description> + Comma separated AUTHORITY=HADOOP_CONF_DIR, where AUTHORITY is the HOST:PORT of + the Hadoop service (JobTracker, HDFS). The wildcard '*' configuration is + used when there is no exact match for an authority. The HADOOP_CONF_DIR contains + the relevant Hadoop *-site.xml files. If the path is relative is looked within + the Oozie configuration directory; though the path can be absolute (i.e. to point + to Hadoop client conf/ directories in the local filesystem. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>oozie.service.HadoopAccessorService.kerberos.enabled</name> + <value>false</value> + <description> + Indicates if Oozie is configured to use Kerberos. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>oozie.service.URIHandlerService.uri.handlers</name> + <value>org.apache.oozie.dependency.FSURIHandler,org.apache.oozie.dependency.HCatURIHandler + </value> + <description> + Enlist the different uri handlers supported for data availability checks. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>oozie.services.ext</name> + <value> + org.apache.oozie.service.JMSAccessorService,org.apache.oozie.service.PartitionDependencyManagerService,org.apache.oozie.service.HCatAccessorService + </value> + <description> + To add/replace services defined in 'oozie.services' with custom implementations. + Class names must be separated by commas. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>oozie.db.schema.name</name> + <value>oozie</value> + <display-name>Database Name</display-name> + <description> + Oozie DataBase Name + </description> + <value-attributes> + <type>database</type> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>oozie.service.JPAService.jdbc.username</name> + <value>oozie</value> + <display-name>Database Username</display-name> + <description> + Database user name to use to connect to the database + </description> + <value-attributes> + <type>db_user</type> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property require-input="true"> + <name>oozie.service.JPAService.jdbc.password</name> + <value/> + <display-name>Database Password</display-name> + <property-type>PASSWORD</property-type> + <description> + DB user password. + + IMPORTANT: if password is emtpy leave a 1 space string, the service trims the value, + if empty Configuration assumes it is NULL. + </description> + <value-attributes> + <type>password</type> + <overridable>false</overridable> + <keystore>true</keystore> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>oozie.service.JPAService.jdbc.driver</name> + <value>org.apache.derby.jdbc.EmbeddedDriver</value> + <display-name>JDBC Driver Class</display-name> + <description> + JDBC driver class. + </description> + <value-attributes> + <overridable>false</overridable> + </value-attributes> + <depends-on> + <property> + <type>oozie-env</type> + <name>oozie_database</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>oozie.service.JPAService.jdbc.url</name> + <value>jdbc:derby:${oozie.data.dir}/${oozie.db.schema.name}-db;create=true</value> + <display-name>Database URL</display-name> + <description> + JDBC URL. + </description> + <value-attributes> + <overridable>false</overridable> + </value-attributes> + <depends-on> + <property> + <type>oozie-env</type> + <name>oozie_database</name> + </property> + <property> + <type>oozie-site</type> + <name>oozie.db.schema.name</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>oozie.service.AuthorizationService.security.enabled</name> + <value>true</value> + <description> + Specifies whether security (user name/admin role) is enabled or not. + If disabled any user can manage Oozie system and manage any job. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>oozie.authentication.simple.anonymous.allowed</name> + <value>true</value> + <description> + Indicates if anonymous requests are allowed when using 'simple' authentication. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>oozie.service.SparkConfigurationService.spark.configurations</name> + <value>*={{spark_conf_dir}}</value> + <description> + Comma separated AUTHORITY=SPARK_CONF_DIR, where AUTHORITY is the + HOST:PORT of the ResourceManager of a YARN cluster. The wildcard '*' + configuration is used when there is no exact match for an authority. + The SPARK_CONF_DIR contains the relevant spark-defaults.conf properties + file. If the path is relative is looked within the Oozie configuration + directory; though the path can be absolute. This is only used when the + Spark master is set to either "yarn-client" or "yarn-cluster". + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>oozie.action.retry.interval</name> + <value>30</value> + <description> + The interval between retries of an action in case of failure + </description> + <value-attributes> + <type>custom</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/cdc18ecb/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/kerberos.json b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/kerberos.json new file mode 100644 index 0000000..f1092f5 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/kerberos.json @@ -0,0 +1,70 @@ +{ + "services": [ + { + "name": "OOZIE", + "identities": [ + { + "name": "/spnego" + }, + { + "name": "/smokeuser" + } + ], + "auth_to_local_properties" : [ + "oozie-site/oozie.authentication.kerberos.name.rules" + ], + "configurations": [ + { + "oozie-site": { + "oozie.authentication.type": "kerberos", + "oozie.service.AuthorizationService.authorization.enabled": "true", + "oozie.service.HadoopAccessorService.kerberos.enabled": "true", + "local.realm": "${realm}", + "oozie.credentials.credentialclasses": "hcat=org.apache.oozie.action.hadoop.HCatCredentials,hive2=org.apache.oozie.action.hadoop.Hive2Credentials", + "oozie.zookeeper.secure" : "true" + } + } + ], + "components": [ + { + "name": "OOZIE_SERVER", + "identities": [ + { + "name": "/HDFS/NAMENODE/hdfs" + }, + { + "name": "oozie_server", + "principal": { + "value": "oozie/_HOST@${realm}", + "type" : "service", + "configuration": "oozie-site/oozie.service.HadoopAccessorService.kerberos.principal", + "local_username" : "${oozie-env/oozie_user}" + }, + "keytab": { + "file": "${keytab_dir}/oozie.service.keytab", + "owner": { + "name": "${oozie-env/oozie_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "oozie-site/oozie.service.HadoopAccessorService.keytab.file" + } + }, + { + "name": "/spnego", + "principal": { + "configuration": "oozie-site/oozie.authentication.kerberos.principal" + }, + "keytab": { + "configuration": "oozie-site/oozie.authentication.kerberos.keytab" + } + } + ] + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/ambari/blob/cdc18ecb/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/metainfo.xml b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/metainfo.xml new file mode 100644 index 0000000..d351cbe --- /dev/null +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/metainfo.xml @@ -0,0 +1,203 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>OOZIE</name> + <credential-store> + <supported>true</supported> + <enabled>true</enabled> + </credential-store> + <displayName>Oozie</displayName> + <comment>System for workflow coordination and execution of Apache Hadoop jobs. This also includes the installation of the optional Oozie Web Console which relies on and will install the <a target="_blank" href="http://www.sencha.com/legal/open-source-faq/">ExtJS</a> Library. + </comment> + <version>4.2.0.3.0</version> + <components> + <component> + <name>OOZIE_SERVER</name> + <displayName>Oozie Server</displayName> + <category>MASTER</category> + <cardinality>1+</cardinality> + <versionAdvertised>true</versionAdvertised> + <reassignAllowed>true</reassignAllowed> + <dependencies> + <dependency> + <name>HDFS/HDFS_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>YARN/YARN_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> + <commandScript> + <script>scripts/oozie_server.py</script> + <scriptType>PYTHON</scriptType> + <timeout>1800</timeout> + </commandScript> + <logs> + <log> + <logId>oozie_app</logId> + <primary>true</primary> + </log> + </logs> + </component> + + <component> + <name>OOZIE_CLIENT</name> + <displayName>Oozie Client</displayName> + <category>CLIENT</category> + <cardinality>1+</cardinality> + <versionAdvertised>true</versionAdvertised> + <dependencies> + <dependency> + <name>HDFS/HDFS_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> + <commandScript> + <script>scripts/oozie_client.py</script> + <scriptType>PYTHON</scriptType> + </commandScript> + <configFiles> + <configFile> + <type>xml</type> + <fileName>oozie-site.xml</fileName> + <dictionaryName>oozie-site</dictionaryName> + </configFile> + <configFile> + <type>env</type> + <fileName>oozie-env.sh</fileName> + <dictionaryName>oozie-env</dictionaryName> + </configFile> + <configFile> + <type>env</type> + <fileName>oozie-log4j.properties</fileName> + <dictionaryName>oozie-log4j</dictionaryName> + </configFile> + </configFiles> + </component> + </components> + + <osSpecifics> + <osSpecific> + <osFamily>any</osFamily> + <packages> + <package> + <name>zip</name> + </package> + <package> + <name>mysql-connector-java</name> + <skipUpgrade>true</skipUpgrade> + <condition>should_install_mysql_connector</condition> + </package> + <package> + <name>extjs</name> + </package> + </packages> + </osSpecific> + <osSpecific> + <osFamily>redhat5,amazon2015,redhat6,redhat7,suse11,suse12</osFamily> + <packages> + <package> + <name>oozie_${stack_version}</name> + </package> + <package> + <name>falcon_${stack_version}</name> + </package> + </packages> + </osSpecific> + <osSpecific> + <osFamily>debian7,ubuntu12,ubuntu14,ubuntu16</osFamily> + <packages> + <package> + <name>oozie-${stack_version}</name> + </package> + <package> + <name>falcon-${stack_version}</name> + </package> + <package> + <name>extjs</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <commandScript> + <script>scripts/service_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + + + <quickLinksConfigurations> + <quickLinksConfiguration> + <fileName>quicklinks.json</fileName> + <default>true</default> + </quickLinksConfiguration> + </quickLinksConfigurations> + + <requiredServices> + <service>YARN</service> + </requiredServices> + + <themes> + <theme> + <fileName>theme.json</fileName> + <default>true</default> + </theme> + </themes> + + <configuration-dependencies> + <config-type>oozie-site</config-type> + <config-type>oozie-env</config-type> + <config-type>oozie-log4j</config-type> + <config-type>yarn-site</config-type> + <config-type>hive-site</config-type> + <config-type>tez-site</config-type> + <config-type>mapred-site</config-type> + <config-type>core-site</config-type> + <config-type>application-properties</config-type> + </configuration-dependencies> + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/cdc18ecb/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/alerts/alert_check_oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/alerts/alert_check_oozie_server.py b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/alerts/alert_check_oozie_server.py new file mode 100644 index 0000000..0e9fe74 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/alerts/alert_check_oozie_server.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python + +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import os +import re + +from resource_management.core import global_lock +from resource_management.core.environment import Environment +from resource_management.core.resources import Execute +from resource_management.libraries.functions import format +from resource_management.libraries.functions import get_kinit_path +from resource_management.libraries.functions import get_klist_path +from ambari_commons.os_check import OSConst, OSCheck +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from urlparse import urlparse + +STACK_ROOT_PATTERN = "{{ stack_root }}" +RESULT_CODE_OK = 'OK' +RESULT_CODE_CRITICAL = 'CRITICAL' +RESULT_CODE_UNKNOWN = 'UNKNOWN' + +if OSCheck.is_windows_family(): + OOZIE_ENV_HTTPS_RE = r"set\s+OOZIE_HTTPS_PORT=(\d+)" +else: + OOZIE_ENV_HTTPS_RE = r"export\s+OOZIE_HTTPS_PORT=(\d+)" + +# The configured Kerberos executable search paths, if any +KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY = '{{kerberos-env/executable_search_paths}}' + +OOZIE_URL_KEY = '{{oozie-site/oozie.base.url}}' +SECURITY_ENABLED = '{{cluster-env/security_enabled}}' +OOZIE_USER = '{{oozie-env/oozie_user}}' +OOZIE_CONF_DIR = "{0}/current/oozie-server/conf".format(STACK_ROOT_PATTERN) +OOZIE_CONF_DIR_LEGACY = '/etc/oozie/conf' +OOZIE_HTTPS_PORT = '{{oozie-site/oozie.https.port}}' +OOZIE_ENV_CONTENT = '{{oozie-env/content}}' + +USER_KEYTAB_KEY = '{{oozie-site/oozie.service.HadoopAccessorService.keytab.file}}' +USER_PRINCIPAL_KEY = '{{oozie-site/oozie.service.HadoopAccessorService.kerberos.principal}}' +USER_KEY = '{{oozie-env/oozie_user}}' + +# default keytab location +USER_KEYTAB_SCRIPT_PARAM_KEY = 'default.oozie.keytab' +USER_KEYTAB_DEFAULT = '/etc/security/keytabs/oozie.headless.keytab' + +# default user principal +USER_PRINCIPAL_SCRIPT_PARAM_KEY = 'default.oozie.principal' +USER_PRINCIPAL_DEFAULT = '[email protected]' + +# default user +USER_DEFAULT = 'oozie' + +STACK_ROOT_KEY = '{{cluster-env/stack_root}}' +STACK_ROOT_DEFAULT = '/usr/hdp' + +class KerberosPropertiesNotFound(Exception): pass + +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def get_tokens(): + """ + Returns a tuple of tokens in the format {{site/property}} that will be used + to build the dictionary passed into execute + """ + return (OOZIE_URL_KEY,) + +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +def get_tokens(): + """ + Returns a tuple of tokens in the format {{site/property}} that will be used + to build the dictionary passed into execute + """ + return (OOZIE_URL_KEY, USER_PRINCIPAL_KEY, SECURITY_ENABLED, USER_KEYTAB_KEY, KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY, + USER_KEY, OOZIE_HTTPS_PORT, OOZIE_ENV_CONTENT, STACK_ROOT_KEY) + +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def get_check_command(oozie_url, host_name, configurations): + from resource_management.libraries.functions import reload_windows_env + reload_windows_env() + oozie_home = os.environ['OOZIE_HOME'] + oozie_cmd = os.path.join(oozie_home, 'bin', 'oozie.cmd') + command = format("cmd /c {oozie_cmd} admin -oozie {oozie_url} -status") + return (command, None, None) + +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +def get_check_command(oozie_url, host_name, configurations, parameters, only_kinit): + kerberos_env = None + + user = USER_DEFAULT + if USER_KEY in configurations: + user = configurations[USER_KEY] + + if is_security_enabled(configurations): + # defaults + user_keytab = USER_KEYTAB_DEFAULT + user_principal = USER_PRINCIPAL_DEFAULT + + # check script params + if USER_PRINCIPAL_SCRIPT_PARAM_KEY in parameters: + user_principal = parameters[USER_PRINCIPAL_SCRIPT_PARAM_KEY] + user_principal = user_principal.replace('_HOST', host_name.lower()) + if USER_KEYTAB_SCRIPT_PARAM_KEY in parameters: + user_keytab = parameters[USER_KEYTAB_SCRIPT_PARAM_KEY] + + # check configurations last as they should always take precedence + if USER_PRINCIPAL_KEY in configurations: + user_principal = configurations[USER_PRINCIPAL_KEY] + user_principal = user_principal.replace('_HOST', host_name.lower()) + if USER_KEYTAB_KEY in configurations: + user_keytab = configurations[USER_KEYTAB_KEY] + + # Create the kerberos credentials cache (ccache) file and set it in the environment to use + # when executing curl + env = Environment.get_instance() + ccache_file = "{0}{1}oozie_alert_cc_{2}".format(env.tmp_dir, os.sep, os.getpid()) + kerberos_env = {'KRB5CCNAME': ccache_file} + + # Get the configured Kerberos executable search paths, if any + kerberos_executable_search_paths = None + if KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY in configurations: + kerberos_executable_search_paths = configurations[KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY] + + klist_path_local = get_klist_path(kerberos_executable_search_paths) + kinit_path_local = get_kinit_path(kerberos_executable_search_paths) + kinit_part_command = format("{kinit_path_local} -l 5m20s -c {ccache_file} -kt {user_keytab} {user_principal}; ") + + # Determine if we need to kinit by testing to see if the relevant cache exists and has + # non-expired tickets. Tickets are marked to expire after 5 minutes to help reduce the number + # it kinits we do but recover quickly when keytabs are regenerated + + if only_kinit: + kinit_command = kinit_part_command + else: + kinit_command = "{0} -s {1} || ".format(klist_path_local, ccache_file) + kinit_part_command + + # prevent concurrent kinit + kinit_lock = global_lock.get_lock(global_lock.LOCK_TYPE_KERBEROS) + kinit_lock.acquire() + try: + Execute(kinit_command, environment=kerberos_env, user=user) + finally: + kinit_lock.release() + + # Configure stack root + stack_root = STACK_ROOT_DEFAULT + if STACK_ROOT_KEY in configurations: + stack_root = configurations[STACK_ROOT_KEY].lower() + + # oozie configuration directory using a symlink + oozie_config_directory = OOZIE_CONF_DIR.replace(STACK_ROOT_PATTERN, stack_root) + if not os.path.exists(oozie_config_directory): + oozie_config_directory = OOZIE_CONF_DIR_LEGACY + + command = "source {0}/oozie-env.sh ; oozie admin -oozie {1} -status".format( + oozie_config_directory, oozie_url) + + return (command, kerberos_env, user) + +def execute(configurations={}, parameters={}, host_name=None): + """ + Returns a tuple containing the result code and a pre-formatted result label + + Keyword arguments: + configurations (dictionary): a mapping of configuration key to value + parameters (dictionary): a mapping of script parameter key to value + host_name (string): the name of this host where the alert is running + """ + + if configurations is None: + return (RESULT_CODE_UNKNOWN, ['There were no configurations supplied to the script.']) + + if not OOZIE_URL_KEY in configurations: + return (RESULT_CODE_UNKNOWN, ['The Oozie URL is a required parameter.']) + + https_port = None + # try to get https port form oozie-env content + if OOZIE_ENV_CONTENT in configurations: + for line in configurations[OOZIE_ENV_CONTENT].splitlines(): + result = re.match(OOZIE_ENV_HTTPS_RE, line) + + if result is not None: + https_port = result.group(1) + # or from oozie-site.xml + if https_port is None and OOZIE_HTTPS_PORT in configurations: + https_port = configurations[OOZIE_HTTPS_PORT] + + oozie_url = configurations[OOZIE_URL_KEY] + + # construct proper url for https + if https_port is not None: + parsed_url = urlparse(oozie_url) + oozie_url = oozie_url.replace(parsed_url.scheme, "https") + if parsed_url.port is None: + oozie_url.replace(parsed_url.hostname, ":".join([parsed_url.hostname, str(https_port)])) + else: + oozie_url = oozie_url.replace(str(parsed_url.port), str(https_port)) + + # https will not work with localhost address, we need put fqdn + if https_port is None: + oozie_url = oozie_url.replace(urlparse(oozie_url).hostname, host_name) + + (code, msg) = get_check_result(oozie_url, host_name, configurations, parameters, False) + + # sometimes real lifetime for ticket is less than we have set(5m20s aS of now) + # so i've added this double check with rekinit command to be sure thaT it's not problem with ticket lifetime + if is_security_enabled(configurations) and code == RESULT_CODE_CRITICAL: + (code, msg) = get_check_result(oozie_url, host_name, configurations, parameters, True) + + return (code, msg) + + +def get_check_result(oozie_url, host_name, configurations, parameters, only_kinit): + try: + command, env, user = get_check_command(oozie_url, host_name, configurations, parameters, only_kinit) + # execute the command + Execute(command, environment=env, user=user) + + return (RESULT_CODE_OK, ["Successful connection to {0}".format(oozie_url)]) + except KerberosPropertiesNotFound, ex: + return (RESULT_CODE_UNKNOWN, [str(ex)]) + except Exception, ex: + return (RESULT_CODE_CRITICAL, [str(ex)]) + +def is_security_enabled(configurations): + security_enabled = False + if SECURITY_ENABLED in configurations: + security_enabled = str(configurations[SECURITY_ENABLED]).upper() == 'TRUE' + + return security_enabled http://git-wip-us.apache.org/repos/asf/ambari/blob/cdc18ecb/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/files/oozieSmoke2.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/files/oozieSmoke2.sh b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/files/oozieSmoke2.sh new file mode 100644 index 0000000..60716ae --- /dev/null +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/files/oozieSmoke2.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# + +export os_family=$1 +export oozie_lib_dir=$2 +export oozie_conf_dir=$3 +export oozie_bin_dir=$4 +export oozie_server_url=$5 +export oozie_examples_dir=$6 +export hadoop_conf_dir=$7 +export hadoop_bin_dir=$8 +export smoke_test_user=$9 +export job_name=${10} +export security_enabled=${11} +export smoke_user_keytab=${12} +export kinit_path_local=${13} +export smokeuser_principal=${14} + +function checkOozieJobStatus { + local job_id=$1 + local num_of_tries=$2 + #default num_of_tries to 10 if not present + num_of_tries=${num_of_tries:-10} + local i=0 + local rc=1 + local cmd="source ${oozie_conf_dir}/oozie-env.sh ; ${oozie_bin_dir}/oozie job -oozie ${OOZIE_SERVER} -info $job_id" + /var/lib/ambari-agent/ambari-sudo.sh su ${smoke_test_user} -s /bin/bash - -c "$cmd" + while [ $i -lt $num_of_tries ] ; do + cmd_output=`/var/lib/ambari-agent/ambari-sudo.sh su ${smoke_test_user} -s /bin/bash - -c "$cmd"` + (IFS='';echo $cmd_output) + act_status=$(IFS='';echo $cmd_output | grep ^Status | cut -d':' -f2 | sed 's| ||g') + echo "workflow_status=$act_status" + if [ "RUNNING" == "$act_status" ]; then + #increment the counter and get the status again after waiting for 15 secs + sleep 15 + (( i++ )) + elif [ "SUCCEEDED" == "$act_status" ]; then + rc=0; + break; + else + rc=1 + break; + fi + done + return $rc +} + +export OOZIE_EXIT_CODE=0 +export OOZIE_SERVER=$oozie_server_url + +cd $oozie_examples_dir + +if [[ $security_enabled == "True" ]]; then + kinitcmd="${kinit_path_local} -kt ${smoke_user_keytab} ${smokeuser_principal}; " +else + kinitcmd="" +fi + +cmd="${kinitcmd}source ${oozie_conf_dir}/oozie-env.sh ; ${oozie_bin_dir}/oozie -Doozie.auth.token.cache=false job -oozie $OOZIE_SERVER -config $oozie_examples_dir/examples/apps/${job_name}/job.properties -run" +echo $cmd +job_info=`/var/lib/ambari-agent/ambari-sudo.sh su ${smoke_test_user} -s /bin/bash - -c "$cmd" | grep "job:"` +job_id="`echo $job_info | cut -d':' -f2`" +checkOozieJobStatus "$job_id" 15 +OOZIE_EXIT_CODE="$?" +exit $OOZIE_EXIT_CODE http://git-wip-us.apache.org/repos/asf/ambari/blob/cdc18ecb/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/files/prepareOozieHdfsDirectories.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/files/prepareOozieHdfsDirectories.sh b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/files/prepareOozieHdfsDirectories.sh new file mode 100644 index 0000000..f2bee2d --- /dev/null +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/files/prepareOozieHdfsDirectories.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# + +export oozie_conf_dir=$1 +export oozie_examples_dir=$2 +export hadoop_conf_dir=$3 +export JOBTRACKER=$4 +export NAMENODE=$5 +export QUEUE=$6 +export JOB_NAME=$7 + +cd $oozie_examples_dir + +/var/lib/ambari-agent/ambari-sudo.sh tar -zxf oozie-examples.tar.gz +/var/lib/ambari-agent/ambari-sudo.sh chmod -R o+rx examples + +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|nameNode=hdfs://localhost:8020|nameNode=$NAMENODE|g" examples/apps/$JOB_NAME/job.properties +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|nameNode=hdfs://localhost:9000|nameNode=$NAMENODE|g" examples/apps/$JOB_NAME/job.properties +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|jobTracker=localhost:8021|jobTracker=$JOBTRACKER|g" examples/apps/$JOB_NAME/job.properties +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|jobTracker=localhost:9001|jobTracker=$JOBTRACKER|g" examples/apps/$JOB_NAME/job.properties +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|jobTracker=localhost:8032|jobTracker=$JOBTRACKER|g" examples/apps/$JOB_NAME/job.properties +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|queueName=default|queueName=$QUEUE|g" examples/apps/$JOB_NAME/job.properties +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|oozie.wf.application.path=hdfs://localhost:9000|oozie.wf.application.path=$NAMENODE|g" examples/apps/$JOB_NAME/job.properties http://git-wip-us.apache.org/repos/asf/ambari/blob/cdc18ecb/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/files/wrap_ooziedb.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/files/wrap_ooziedb.sh b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/files/wrap_ooziedb.sh new file mode 100644 index 0000000..36576b5 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/files/wrap_ooziedb.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +OUT=`cd /var/tmp/oozie && /usr/lib/oozie/bin/ooziedb.sh "$@" 2>&1` +EC=$? +echo $OUT +GRVAR=`echo ${OUT} | grep -o "java.lang.Exception: DB schema exists"` +if [ ${EC} -ne 0 ] && [ -n "$GRVAR" ] +then + exit 0 +else + exit $EC +fi http://git-wip-us.apache.org/repos/asf/ambari/blob/cdc18ecb/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/check_oozie_server_status.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/check_oozie_server_status.py b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/check_oozie_server_status.py new file mode 100644 index 0000000..7c69779 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/check_oozie_server_status.py @@ -0,0 +1,38 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from ambari_commons import OSConst + + +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def check_oozie_server_status(): + import status_params + from resource_management.libraries.functions.windows_service_utils import check_windows_service_status + + check_windows_service_status(status_params.oozie_server_win_service_name) + + +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +def check_oozie_server_status(): + import status_params + from resource_management.libraries.functions.check_process_status import check_process_status + + check_process_status(status_params.pid_file) + http://git-wip-us.apache.org/repos/asf/ambari/blob/cdc18ecb/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py new file mode 100644 index 0000000..def0545 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py @@ -0,0 +1,516 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +# Python Imports +import os +import re + +# Resource Management Imports +from resource_management.core.resources.service import ServiceConfig +from resource_management.core.resources.system import Directory, Execute, File +from resource_management.core.source import DownloadSource +from resource_management.core.source import InlineTemplate +from resource_management.core.source import Template +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.version import format_stack_version +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.oozie_prepare_war import prepare_war +from resource_management.libraries.functions.copy_tarball import get_current_version +from resource_management.libraries.resources.xml_config import XmlConfig +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions.security_commons import update_credential_provider_path +from resource_management.core.resources.packaging import Package +from resource_management.core.shell import as_user, as_sudo, call +from resource_management.core.exceptions import Fail + +from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster, setup_atlas_hook +from ambari_commons.constants import SERVICE, UPGRADE_TYPE_NON_ROLLING, UPGRADE_TYPE_ROLLING +from resource_management.libraries.functions.constants import Direction + +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from ambari_commons import OSConst +from ambari_commons.inet_utils import download_file + +from resource_management.core import Logger + +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def oozie(is_server=False): + import params + + from status_params import oozie_server_win_service_name + + XmlConfig("oozie-site.xml", + conf_dir=params.oozie_conf_dir, + configurations=params.config['configurations']['oozie-site'], + owner=params.oozie_user, + mode='f', + configuration_attributes=params.config['configuration_attributes']['oozie-site'] + ) + + File(os.path.join(params.oozie_conf_dir, "oozie-env.cmd"), + owner=params.oozie_user, + content=InlineTemplate(params.oozie_env_cmd_template) + ) + + Directory(params.oozie_tmp_dir, + owner=params.oozie_user, + create_parents = True, + ) + + if is_server: + # Manually overriding service logon user & password set by the installation package + ServiceConfig(oozie_server_win_service_name, + action="change_user", + username = params.oozie_user, + password = Script.get_password(params.oozie_user)) + + download_file(os.path.join(params.config['hostLevelParams']['jdk_location'], "sqljdbc4.jar"), + os.path.join(params.oozie_root, "extra_libs", "sqljdbc4.jar") + ) + webapps_sqljdbc_path = os.path.join(params.oozie_home, "oozie-server", "webapps", "oozie", "WEB-INF", "lib", "sqljdbc4.jar") + if os.path.isfile(webapps_sqljdbc_path): + download_file(os.path.join(params.config['hostLevelParams']['jdk_location'], "sqljdbc4.jar"), + webapps_sqljdbc_path + ) + download_file(os.path.join(params.config['hostLevelParams']['jdk_location'], "sqljdbc4.jar"), + os.path.join(params.oozie_home, "share", "lib", "oozie", "sqljdbc4.jar") + ) + download_file(os.path.join(params.config['hostLevelParams']['jdk_location'], "sqljdbc4.jar"), + os.path.join(params.oozie_home, "temp", "WEB-INF", "lib", "sqljdbc4.jar") + ) + +# TODO: see if see can remove this +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +def oozie(is_server=False): + import params + + if is_server: + params.HdfsResource(params.oozie_hdfs_user_dir, + type="directory", + action="create_on_execute", + owner=params.oozie_user, + mode=params.oozie_hdfs_user_mode + ) + params.HdfsResource(None, action="execute") + Directory(params.conf_dir, + create_parents = True, + owner = params.oozie_user, + group = params.user_group + ) + + params.oozie_site = update_credential_provider_path(params.oozie_site, + 'oozie-site', + os.path.join(params.conf_dir, 'oozie-site.jceks'), + params.oozie_user, + params.user_group + ) + + XmlConfig("oozie-site.xml", + conf_dir = params.conf_dir, + configurations = params.oozie_site, + configuration_attributes=params.config['configuration_attributes']['oozie-site'], + owner = params.oozie_user, + group = params.user_group, + mode = 0664 + ) + File(format("{conf_dir}/oozie-env.sh"), + owner=params.oozie_user, + content=InlineTemplate(params.oozie_env_sh_template), + group=params.user_group, + ) + + # On some OS this folder could be not exists, so we will create it before pushing there files + Directory(params.limits_conf_dir, + create_parents=True, + owner='root', + group='root' + ) + + File(os.path.join(params.limits_conf_dir, 'oozie.conf'), + owner='root', + group='root', + mode=0644, + content=Template("oozie.conf.j2") + ) + + if (params.log4j_props != None): + File(format("{params.conf_dir}/oozie-log4j.properties"), + mode=0644, + group=params.user_group, + owner=params.oozie_user, + content=InlineTemplate(params.log4j_props) + ) + elif (os.path.exists(format("{params.conf_dir}/oozie-log4j.properties"))): + File(format("{params.conf_dir}/oozie-log4j.properties"), + mode=0644, + group=params.user_group, + owner=params.oozie_user + ) + + if params.stack_version_formatted and check_stack_feature(StackFeature.OOZIE_ADMIN_USER, params.stack_version_formatted): + File(format("{params.conf_dir}/adminusers.txt"), + mode=0644, + group=params.user_group, + owner=params.oozie_user, + content=Template('adminusers.txt.j2', oozie_admin_users=params.oozie_admin_users) + ) + else: + File ( format("{params.conf_dir}/adminusers.txt"), + owner = params.oozie_user, + group = params.user_group + ) + + if params.jdbc_driver_name == "com.mysql.jdbc.Driver" or \ + params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \ + params.jdbc_driver_name == "org.postgresql.Driver" or \ + params.jdbc_driver_name == "oracle.jdbc.driver.OracleDriver": + File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), + content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")), + ) + pass + + oozie_ownership() + + if is_server: + oozie_server_specific() + +def oozie_ownership(): + import params + + File ( format("{conf_dir}/hadoop-config.xml"), + owner = params.oozie_user, + group = params.user_group + ) + + File ( format("{conf_dir}/oozie-default.xml"), + owner = params.oozie_user, + group = params.user_group + ) + + Directory ( format("{conf_dir}/action-conf"), + owner = params.oozie_user, + group = params.user_group + ) + + File ( format("{conf_dir}/action-conf/hive.xml"), + owner = params.oozie_user, + group = params.user_group + ) + +def oozie_server_specific(): + import params + + no_op_test = as_user(format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"), user=params.oozie_user) + + File(params.pid_file, + action="delete", + not_if=no_op_test + ) + + oozie_server_directories = [format("{oozie_home}/{oozie_tmp_dir}"), params.oozie_pid_dir, params.oozie_log_dir, params.oozie_tmp_dir, params.oozie_data_dir, params.oozie_lib_dir, params.oozie_webapps_dir, params.oozie_webapps_conf_dir, params.oozie_server_dir] + Directory( oozie_server_directories, + owner = params.oozie_user, + group = params.user_group, + mode = 0755, + create_parents = True, + cd_access="a", + ) + + Directory(params.oozie_libext_dir, + create_parents = True, + ) + + hashcode_file = format("{oozie_home}/.hashcode") + skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share") + + untar_sharelib = ('tar','-xvf',format('{oozie_home}/oozie-sharelib.tar.gz'),'-C',params.oozie_home) + + Execute( untar_sharelib, # time-expensive + not_if = format("{no_op_test} || {skip_recreate_sharelib}"), + sudo = True, + ) + + configure_cmds = [] + configure_cmds.append(('cp', params.ext_js_path, params.oozie_libext_dir)) + configure_cmds.append(('chown', format('{oozie_user}:{user_group}'), format('{oozie_libext_dir}/{ext_js_file}'))) + + Execute( configure_cmds, + not_if = no_op_test, + sudo = True, + ) + + Directory(params.oozie_webapps_conf_dir, + owner = params.oozie_user, + group = params.user_group, + recursive_ownership = True, + recursion_follow_links = True, + ) + + # download the database JAR + download_database_library_if_needed() + + #falcon el extension + if params.has_falcon_host: + Execute(format('{sudo} cp {falcon_home}/oozie/ext/falcon-oozie-el-extension-*.jar {oozie_libext_dir}'), + not_if = no_op_test) + + Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'), + not_if = no_op_test) + + if params.lzo_enabled and len(params.all_lzo_packages) > 0: + Package(params.all_lzo_packages, + retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, + retry_count=params.agent_stack_retry_count) + Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), + not_if = no_op_test, + ) + + prepare_war(params) + + File(hashcode_file, + mode = 0644, + ) + + if params.stack_version_formatted and check_stack_feature(StackFeature.OOZIE_CREATE_HIVE_TEZ_CONFIGS, params.stack_version_formatted): + # Create hive-site and tez-site configs for oozie + Directory(params.hive_conf_dir, + create_parents = True, + owner = params.oozie_user, + group = params.user_group + ) + if 'hive-site' in params.config['configurations']: + hive_site_config = update_credential_provider_path(params.config['configurations']['hive-site'], + 'hive-site', + os.path.join(params.hive_conf_dir, 'hive-site.jceks'), + params.oozie_user, + params.user_group + ) + XmlConfig("hive-site.xml", + conf_dir=params.hive_conf_dir, + configurations=hive_site_config, + configuration_attributes=params.config['configuration_attributes']['hive-site'], + owner=params.oozie_user, + group=params.user_group, + mode=0644 + ) + if 'tez-site' in params.config['configurations']: + XmlConfig( "tez-site.xml", + conf_dir = params.hive_conf_dir, + configurations = params.config['configurations']['tez-site'], + configuration_attributes=params.config['configuration_attributes']['tez-site'], + owner = params.oozie_user, + group = params.user_group, + mode = 0664 + ) + + # If Atlas is also installed, need to generate Atlas Hive hook (hive-atlas-application.properties file) in directory + # {stack_root}/{current_version}/atlas/hook/hive/ + # Because this is a .properties file instead of an xml file, it will not be read automatically by Oozie. + # However, should still save the file on this host so that can upload it to the Oozie Sharelib in DFS. + if has_atlas_in_cluster(): + atlas_hook_filepath = os.path.join(params.hive_conf_dir, params.atlas_hook_filename) + Logger.info("Has atlas in cluster, will save Atlas Hive hook into location %s" % str(atlas_hook_filepath)) + setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.oozie_user, params.user_group) + + Directory(params.oozie_server_dir, + owner = params.oozie_user, + group = params.user_group, + recursive_ownership = True, + ) + if params.security_enabled: + File(os.path.join(params.conf_dir, 'zkmigrator_jaas.conf'), + owner=params.oozie_user, + group=params.user_group, + content=Template("zkmigrator_jaas.conf.j2") + ) + +def __parse_sharelib_from_output(output): + """ + Return the parent directory of the first path from the output of the "oozie admin -shareliblist command $comp" + Output will match pattern like: + + Potential errors + [Available ShareLib] + hive + hdfs://server:8020/user/oozie/share/lib/lib_20160811235630/hive/file1.jar + hdfs://server:8020/user/oozie/share/lib/lib_20160811235630/hive/file2.jar + """ + if output is not None: + pattern = re.compile(r"\[Available ShareLib\]\n\S*?\n(.*share.*)", re.IGNORECASE) + m = pattern.search(output) + if m and len(m.groups()) == 1: + jar_path = m.group(1) + # Remove leading/trailing spaces and get the containing directory + sharelib_dir = os.path.dirname(jar_path.strip()) + return sharelib_dir + return None + +def copy_atlas_hive_hook_to_dfs_share_lib(upgrade_type=None, upgrade_direction=None): + """ + If the Atlas Hive Hook direcotry is present, Atlas is installed, and this is the first Oozie Server, + then copy the entire contents of that directory to the Oozie Sharelib in DFS, e.g., + /usr/$stack/$current_version/atlas/hook/hive/ -> hdfs:///user/oozie/share/lib/lib_$timetamp/hive + + :param upgrade_type: If in the middle of a stack upgrade, the type as UPGRADE_TYPE_ROLLING or UPGRADE_TYPE_NON_ROLLING + :param upgrade_direction: If in the middle of a stack upgrade, the direction as Direction.UPGRADE or Direction.DOWNGRADE. + """ + import params + + # Calculate the effective version since this code can also be called during EU/RU in the upgrade direction. + effective_version = params.stack_version_formatted if upgrade_type is None else format_stack_version(params.version) + if not check_stack_feature(StackFeature.ATLAS_HOOK_SUPPORT, effective_version): + return + + # Important that oozie_server_hostnames is sorted by name so that this only runs on a single Oozie server. + if not (len(params.oozie_server_hostnames) > 0 and params.hostname == params.oozie_server_hostnames[0]): + Logger.debug("Will not attempt to copy Atlas Hive hook to DFS since this is not the first Oozie Server " + "sorted by hostname.") + return + + if not has_atlas_in_cluster(): + Logger.debug("Will not attempt to copy Atlas Hve hook to DFS since Atlas is not installed on the cluster.") + return + + if upgrade_type is not None and upgrade_direction == Direction.DOWNGRADE: + Logger.debug("Will not attempt to copy Atlas Hve hook to DFS since in the middle of Rolling/Express upgrade " + "and performing a Downgrade.") + return + + current_version = get_current_version() + atlas_hive_hook_dir = format("{stack_root}/{current_version}/atlas/hook/hive/") + if not os.path.exists(atlas_hive_hook_dir): + Logger.error(format("ERROR. Atlas is installed in cluster but this Oozie server doesn't " + "contain directory {atlas_hive_hook_dir}")) + return + + atlas_hive_hook_impl_dir = os.path.join(atlas_hive_hook_dir, "atlas-hive-plugin-impl") + + num_files = len([name for name in os.listdir(atlas_hive_hook_impl_dir) if os.path.exists(os.path.join(atlas_hive_hook_impl_dir, name))]) + Logger.info("Found %d files/directories inside Atlas Hive hook impl directory %s"% (num_files, atlas_hive_hook_impl_dir)) + + # This can return over 100 files, so take the first 5 lines after "Available ShareLib" + # Use -oozie http(s):localhost:{oozie_server_admin_port}/oozie as oozie-env does not export OOZIE_URL + command = format(r'source {conf_dir}/oozie-env.sh ; oozie admin -oozie {oozie_base_url} -shareliblist hive | grep "\[Available ShareLib\]" -A 5') + Execute(command, + user=params.oozie_user, + tries=10, + try_sleep=5, + logoutput=True, + ) + + hive_sharelib_dir = __parse_sharelib_from_output(out) + + if hive_sharelib_dir is None: + raise Fail("Could not parse Hive sharelib from output.") + + Logger.info(format("Parsed Hive sharelib = {hive_sharelib_dir} and will attempt to copy/replace {num_files} files to it from {atlas_hive_hook_impl_dir}")) + + params.HdfsResource(hive_sharelib_dir, + type="directory", + action="create_on_execute", + source=atlas_hive_hook_impl_dir, + user=params.hdfs_user, + owner=params.oozie_user, + group=params.hdfs_user, + mode=0755, + recursive_chown=True, + recursive_chmod=True, + replace_existing_files=True + ) + + Logger.info("Copying Atlas Hive hook properties file to Oozie Sharelib in DFS.") + atlas_hook_filepath_source = os.path.join(params.hive_conf_dir, params.atlas_hook_filename) + atlas_hook_file_path_dest_in_dfs = os.path.join(hive_sharelib_dir, params.atlas_hook_filename) + params.HdfsResource(atlas_hook_file_path_dest_in_dfs, + type="file", + source=atlas_hook_filepath_source, + action="create_on_execute", + owner=params.oozie_user, + group=params.hdfs_user, + mode=0755, + replace_existing_files=True + ) + params.HdfsResource(None, action="execute") + + # Update the sharelib after making any changes + # Use -oozie http(s):localhost:{oozie_server_admin_port}/oozie as oozie-env does not export OOZIE_URL + Execute(format("source {conf_dir}/oozie-env.sh ; oozie admin -oozie {oozie_base_url} -sharelibupdate"), + user=params.oozie_user, + tries=5, + try_sleep=5, + logoutput=True, + ) + + +def download_database_library_if_needed(target_directory = None): + """ + Downloads the library to use when connecting to the Oozie database, if + necessary. The library will be downloaded to 'params.target' unless + otherwise specified. + :param target_directory: the location where the database library will be + downloaded to. + :return: + """ + import params + jdbc_drivers = ["com.mysql.jdbc.Driver", + "com.microsoft.sqlserver.jdbc.SQLServerDriver", + "oracle.jdbc.driver.OracleDriver","sap.jdbc4.sqlanywhere.IDriver"] + + # check to see if the JDBC driver name is in the list of ones that need to + # be downloaded + if params.jdbc_driver_name not in jdbc_drivers or not params.jdbc_driver_jar: + return + + if params.previous_jdbc_jar and os.path.isfile(params.previous_jdbc_jar): + File(params.previous_jdbc_jar, action='delete') + + # if the target directory is not specified + if target_directory is None: + target_jar_with_directory = params.target + else: + # create the full path using the supplied target directory and the JDBC JAR + target_jar_with_directory = target_directory + os.path.sep + params.jdbc_driver_jar + + if not os.path.exists(target_jar_with_directory): + File(params.downloaded_custom_connector, + content = DownloadSource(params.driver_curl_source)) + + if params.sqla_db_used: + untar_sqla_type2_driver = ('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir) + + Execute(untar_sqla_type2_driver, sudo = True) + + Execute(format("yes | {sudo} cp {jars_path_in_archive} {oozie_libext_dir}")) + + Directory(params.jdbc_libs_dir, + create_parents = True) + + Execute(format("yes | {sudo} cp {libs_path_in_archive} {jdbc_libs_dir}")) + + Execute(format("{sudo} chown -R {oozie_user}:{user_group} {oozie_libext_dir}/*")) + + else: + Execute(('cp', '--remove-destination', params.downloaded_custom_connector, target_jar_with_directory), + path=["/bin", "/usr/bin/"], + sudo = True) + + File(target_jar_with_directory, owner = params.oozie_user, + group = params.user_group) http://git-wip-us.apache.org/repos/asf/ambari/blob/cdc18ecb/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie_client.py b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie_client.py new file mode 100644 index 0000000..f98ecfd --- /dev/null +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie_client.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions import conf_select, stack_select +from resource_management.libraries.functions.constants import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.core.logger import Logger +from resource_management.core.exceptions import ClientComponentHasNoStatus + +from oozie import oozie +from oozie_service import oozie_service + + +class OozieClient(Script): + + def get_component_name(self): + return "oozie-client" + + def install(self, env): + self.install_packages(env) + self.configure(env) + + + def configure(self, env): + import params + env.set_params(params) + + oozie(is_server=False) + + def status(self, env): + raise ClientComponentHasNoStatus() + + + def pre_upgrade_restart(self, env, upgrade_type=None): + import params + env.set_params(params) + + # this function should not execute if the version can't be determined or + # the stack does not support rolling upgrade + if not (params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version)): + return + + Logger.info("Executing Oozie Client Stack Upgrade pre-restart") + conf_select.select(params.stack_name, "oozie", params.version) + stack_select.select("oozie-client", params.version) + + # We substitute some configs (oozie.authentication.kerberos.principal) before generation (see oozie.py and params.py). + # This function returns changed configs (it's used for config generation before config download) + def generate_configs_get_xml_file_content(self, filename, dictionary): + if dictionary == 'oozie-site': + import params + config = self.get_config() + return {'configurations': params.oozie_site, + 'configuration_attributes': config['configuration_attributes'][dictionary]} + else: + return super(OozieClient, self).generate_configs_get_xml_file_content(filename, dictionary) + +if __name__ == "__main__": + OozieClient().execute()
