http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/configuration/oozie-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/configuration/oozie-env.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/configuration/oozie-env.xml new file mode 100755 index 0000000..f4b161e --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/configuration/oozie-env.xml @@ -0,0 +1,189 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + <property> + <name>oozie_user</name> + <display-name>Oozie User</display-name> + <value>oozie</value> + <property-type>USER</property-type> + <description>Oozie User.</description> + </property> + <property> + <name>oozie_admin_users</name> + <value>{oozie_user}, oozie-admin</value> + <description>Oozie admin users.</description> + </property> + <property> + <name>oozie_database</name> + <display-name>Oozie Database</display-name> + <value>New Derby Database</value> + <description>Oozie Server Database.</description> + <value-attributes> + <overridable>false</overridable> + </value-attributes> + </property> + <property> + <name>oozie_derby_database</name> + <value>Derby</value> + <description>Oozie Derby Database</description> + </property> + <property> + <name>oozie_data_dir</name> + <display-name>Oozie Data Dir</display-name> + <value>/hadoop/oozie/data</value> + <description>Data directory in which the Oozie DB exists</description> + <value-attributes> + <type>directory</type> + <empty-value-valid>true</empty-value-valid> + <overridable>false</overridable> + </value-attributes> + </property> + <property> + <name>oozie_log_dir</name> + <display-name>Oozie Log Dir</display-name> + <value>/var/log/oozie</value> + <description>Directory for oozie logs</description> + <value-attributes> + <type>directory</type> + <editable-only-at-install>true</editable-only-at-install> + <overridable>false</overridable> + </value-attributes> + </property> + <property> + <name>oozie_pid_dir</name> + <display-name>Oozie PID Dir</display-name> + <value>/var/run/oozie</value> + <description>Directory in which the pid files for oozie reside.</description> + <value-attributes> + <type>directory</type> + <editable-only-at-install>true</editable-only-at-install> + <overridable>false</overridable> + </value-attributes> + </property> + <property> + <name>oozie_admin_port</name> + <display-name>Oozie Server Admin Port</display-name> + <value>11001</value> + <description>The admin port Oozie server runs.</description> + <value-attributes> + <overridable>false</overridable> + <type>int</type> + </value-attributes> + </property> + <property> + <name>oozie_initial_heapsize</name> + <value>1024</value> + <description>Oozie initial heap size.</description> + </property> + <property> + <name>oozie_heapsize</name> + <value>2048</value> + <description>Oozie heap size.</description> + </property> + <property> + <name>oozie_permsize</name> + <value>256</value> + <description>Oozie permanent generation size.</description> + </property> + + + <!-- oozie-env.sh --> + <property> + <name>content</name> + <description>This is the jinja template for oozie-env.sh file</description> + <value> +#!/bin/bash + +if [ -d "/usr/lib/bigtop-tomcat" ]; then + export OOZIE_CONFIG=${OOZIE_CONFIG:-{{conf_dir}}} + export CATALINA_BASE={{oozie_server_dir}} + export CATALINA_TMPDIR=${CATALINA_TMPDIR:-/var/tmp/oozie} + export OOZIE_CATALINA_HOME=/usr/lib/bigtop-tomcat +fi + +#Set JAVA HOME +export JAVA_HOME={{java_home}} + +export JRE_HOME=${JAVA_HOME} + +# Set Oozie specific environment variables here. + +# Settings for the Embedded Tomcat that runs Oozie +# Java System properties for Oozie should be specified in this variable +# This is needed so that Oozie does not run into OOM or GC Overhead limit +# exceeded exceptions. If the oozie server is handling large number of +# workflows/coordinator jobs, the memory settings may need to be revised + +{% if java_version < 8 %} +export CATALINA_OPTS="$CATALINA_OPTS -Xms{{oozie_initial_heapsize}} -Xmx{{oozie_heapsize}} -XX:MaxPermSize={{oozie_permsize}}" +{% else %} +export CATALINA_OPTS="$CATALINA_OPTS -Xms{{oozie_initial_heapsize}} -Xmx{{oozie_heapsize}} -XX:MaxMetaspaceSize={{oozie_permsize}}" +{% endif %} +# Oozie configuration file to load from Oozie configuration directory +# +# export OOZIE_CONFIG_FILE=oozie-site.xml + +# Oozie logs directory +# +export OOZIE_LOG={{oozie_log_dir}} + +# Oozie pid directory +# +export CATALINA_PID={{pid_file}} + +#Location of the data for oozie +export OOZIE_DATA={{oozie_data_dir}} + +# Oozie Log4J configuration file to load from Oozie configuration directory +# +# export OOZIE_LOG4J_FILE=oozie-log4j.properties + +# Reload interval of the Log4J configuration file, in seconds +# +# export OOZIE_LOG4J_RELOAD=10 + +# The port Oozie server runs +# +export OOZIE_HTTP_PORT={{oozie_server_port}} + +# The admin port Oozie server runs +# +export OOZIE_ADMIN_PORT={{oozie_server_admin_port}} + +# The host name Oozie server runs on +# +# export OOZIE_HTTP_HOSTNAME=`hostname -f` + +# The base URL for callback URLs to Oozie +# +# export OOZIE_BASE_URL="http://${OOZIE_HTTP_HOSTNAME}:${OOZIE_HTTP_PORT}/oozie" +export JAVA_LIBRARY_PATH={{hadoop_lib_home}}/native + +# At least 1 minute of retry time to account for server downtime during +# upgrade/downgrade +export OOZIE_CLIENT_OPTS="${OOZIE_CLIENT_OPTS} -Doozie.connection.retry.count=5 " + + </value> + </property> + +</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/configuration/oozie-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/configuration/oozie-log4j.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/configuration/oozie-log4j.xml new file mode 100755 index 0000000..2ca87c3 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/configuration/oozie-log4j.xml @@ -0,0 +1,146 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration supports_final="true"> + + <property> + <name>content</name> + <description>Custom log4j.properties</description> + <value> + # + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + # + + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. See accompanying LICENSE file. + # + + # If the Java System property 'oozie.log.dir' is not defined at Oozie start up time + # XLogService sets its value to '${oozie.home}/logs' + + # The appender that Oozie uses must be named 'oozie' (i.e. log4j.appender.oozie) + + # Using the RollingFileAppender with the OozieRollingPolicy will roll the log file every hour and retain up to MaxHistory number of + # log files. If FileNamePattern ends with ".gz" it will create gzip files. + log4j.appender.oozie=org.apache.log4j.rolling.RollingFileAppender + log4j.appender.oozie.RollingPolicy=org.apache.oozie.util.OozieRollingPolicy + log4j.appender.oozie.File=${oozie.log.dir}/oozie.log + log4j.appender.oozie.Append=true + log4j.appender.oozie.layout=org.apache.log4j.PatternLayout + log4j.appender.oozie.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - SERVER[${oozie.instance.id}] %m%n + # The FileNamePattern must end with "-%d{yyyy-MM-dd-HH}.gz" or "-%d{yyyy-MM-dd-HH}" and also start with the + # value of log4j.appender.oozie.File + log4j.appender.oozie.RollingPolicy.FileNamePattern=${log4j.appender.oozie.File}-%d{yyyy-MM-dd-HH} + # The MaxHistory controls how many log files will be retained (6480 hours / 24 hours per day = 270 days); -1 to disable + log4j.appender.oozie.RollingPolicy.MaxHistory=6480 + + + + log4j.appender.oozieError=org.apache.log4j.rolling.RollingFileAppender + log4j.appender.oozieError.RollingPolicy=org.apache.oozie.util.OozieRollingPolicy + log4j.appender.oozieError.File=${oozie.log.dir}/oozie-error.log + log4j.appender.oozieError.Append=true + log4j.appender.oozieError.layout=org.apache.log4j.PatternLayout + log4j.appender.oozieError.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - SERVER[${oozie.instance.id}] %m%n + # The FileNamePattern must end with "-%d{yyyy-MM-dd-HH}.gz" or "-%d{yyyy-MM-dd-HH}" and also start with the + # value of log4j.appender.oozieError.File + log4j.appender.oozieError.RollingPolicy.FileNamePattern=${log4j.appender.oozieError.File}-%d{yyyy-MM-dd-HH} + # The MaxHistory controls how many log files will be retained (6480 hours / 24 hours per day = 270 days); -1 to disable + log4j.appender.oozieError.RollingPolicy.MaxHistory=6480 + log4j.appender.oozieError.filter.1 = org.apache.log4j.varia.LevelMatchFilter + log4j.appender.oozieError.filter.1.levelToMatch = WARN + log4j.appender.oozieError.filter.2 = org.apache.log4j.varia.LevelMatchFilter + log4j.appender.oozieError.filter.2.levelToMatch = ERROR + log4j.appender.oozieError.filter.3 = org.apache.log4j.varia.LevelMatchFilter + log4j.appender.oozieError.filter.3.levelToMatch = FATAL + log4j.appender.oozieError.filter.4 = org.apache.log4j.varia.DenyAllFilter + + + + # Uncomment the below two lines to use the DailyRollingFileAppender instead + # The DatePattern must end with either "dd" or "HH" + #log4j.appender.oozie=org.apache.log4j.DailyRollingFileAppender + #log4j.appender.oozie.DatePattern='.'yyyy-MM-dd-HH + + log4j.appender.oozieops=org.apache.log4j.DailyRollingFileAppender + log4j.appender.oozieops.DatePattern='.'yyyy-MM-dd + log4j.appender.oozieops.File=${oozie.log.dir}/oozie-ops.log + log4j.appender.oozieops.Append=true + log4j.appender.oozieops.layout=org.apache.log4j.PatternLayout + log4j.appender.oozieops.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n + + log4j.appender.oozieinstrumentation=org.apache.log4j.DailyRollingFileAppender + log4j.appender.oozieinstrumentation.DatePattern='.'yyyy-MM-dd + log4j.appender.oozieinstrumentation.File=${oozie.log.dir}/oozie-instrumentation.log + log4j.appender.oozieinstrumentation.Append=true + log4j.appender.oozieinstrumentation.layout=org.apache.log4j.PatternLayout + log4j.appender.oozieinstrumentation.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n + + log4j.appender.oozieaudit=org.apache.log4j.rolling.RollingFileAppender + log4j.appender.oozieaudit.RollingPolicy=org.apache.oozie.util.OozieRollingPolicy + log4j.appender.oozieaudit.File=${oozie.log.dir}/oozie-audit.log + log4j.appender.oozieaudit.Append=true + log4j.appender.oozieaudit.layout=org.apache.log4j.PatternLayout + log4j.appender.oozieaudit.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n + log4j.appender.oozieaudit.RollingPolicy.FileNamePattern=${log4j.appender.oozieaudit.File}.%d{yyyy-MM-dd} + log4j.appender.oozieaudit.RollingPolicy.MaxHistory=6480 + + + log4j.appender.openjpa=org.apache.log4j.DailyRollingFileAppender + log4j.appender.openjpa.DatePattern='.'yyyy-MM-dd + log4j.appender.openjpa.File=${oozie.log.dir}/oozie-jpa.log + log4j.appender.openjpa.Append=true + log4j.appender.openjpa.layout=org.apache.log4j.PatternLayout + log4j.appender.openjpa.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n + + log4j.logger.openjpa=INFO, openjpa + log4j.logger.oozieops=INFO, oozieops + log4j.logger.oozieinstrumentation=ALL, oozieinstrumentation + log4j.logger.oozieaudit=ALL, oozieaudit + log4j.logger.org.apache.oozie=INFO, oozie, oozieError + log4j.logger.org.apache.hadoop=WARN, oozie + log4j.logger.org.mortbay=WARN, oozie + log4j.logger.org.hsqldb=WARN, oozie + log4j.logger.org.apache.hadoop.security.authentication.server=WARN, oozie + </value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/configuration/oozie-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/configuration/oozie-site.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/configuration/oozie-site.xml new file mode 100755 index 0000000..1cd7162 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/configuration/oozie-site.xml @@ -0,0 +1,382 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration supports_final="true"> + + <!-- + Refer to the oozie-default.xml file for the complete list of + Oozie configuration properties and their default values. + --> + <property> + <name>oozie.base.url</name> + <value>http://localhost:11000/oozie</value> + <description>Base Oozie URL.</description> + </property> + + <property> + <name>oozie.system.id</name> + <value>oozie-${user.name}</value> + <description> + The Oozie system ID. + </description> + </property> + + <property> + <name>oozie.systemmode</name> + <value>NORMAL</value> + <description> + System mode for Oozie at startup. + </description> + </property> + + <property> + <name>oozie.service.AuthorizationService.security.enabled</name> + <value>true</value> + <description> + Specifies whether security (user name/admin role) is enabled or not. + If disabled any user can manage Oozie system and manage any job. + </description> + </property> + + <property> + <name>oozie.service.PurgeService.older.than</name> + <value>30</value> + <description> + Jobs older than this value, in days, will be purged by the PurgeService. + </description> + </property> + + <property> + <name>oozie.service.PurgeService.purge.interval</name> + <value>3600</value> + <description> + Interval at which the purge service will run, in seconds. + </description> + </property> + + <property> + <name>oozie.service.CallableQueueService.queue.size</name> + <value>1000</value> + <description>Max callable queue size</description> + </property> + + <property> + <name>oozie.service.CallableQueueService.threads</name> + <value>10</value> + <description>Number of threads used for executing callables</description> + </property> + + <property> + <name>oozie.service.CallableQueueService.callable.concurrency</name> + <value>3</value> + <description> + Maximum concurrency for a given callable type. + Each command is a callable type (submit, start, run, signal, job, jobs, suspend,resume, etc). + Each action type is a callable type (Map-Reduce, Pig, SSH, FS, sub-workflow, etc). + All commands that use action executors (action-start, action-end, action-kill and action-check) use + the action type as the callable type. + </description> + </property> + + <property> + <name>oozie.service.coord.normal.default.timeout</name> + <value>120</value> + <description>Default timeout for a coordinator action input check (in minutes) for normal job. + -1 means infinite timeout + </description> + </property> + + <property> + <name>oozie.service.coord.check.maximum.frequency</name> + <value>false</value> + <description> + When true, Oozie will reject any coordinators with a frequency faster than 5 minutes. It is not recommended to disable + this check or submit coordinators with frequencies faster than 5 minutes: doing so can cause unintended behavior and + additional system stress. + </description> + </property> + + <property> + <name>oozie.db.schema.name</name> + <value>oozie</value> + <display-name>Database Name</display-name> + <description> + Oozie DataBase Name + </description> + <value-attributes> + <type>database</type> + <overridable>false</overridable> + </value-attributes> + </property> + + <property> + <name>oozie.authentication.type</name> + <value>simple</value> + <description> + Authentication used for Oozie HTTP endpoint, the supported values are: simple | kerberos | + #AUTHENTICATION_HANDLER_CLASSNAME#. + </description> + </property> + + <property> + <name>oozie.service.WorkflowAppService.system.libpath</name> + <value>/user/${user.name}/share/lib</value> + <description> + System library path to use for workflow applications. + This path is added to workflow application if their job properties sets + the property 'oozie.use.system.libpath' to true. + </description> + </property> + + <property> + <name>use.system.libpath.for.mapreduce.and.pig.jobs</name> + <value>false</value> + <description> + If set to true, submissions of MapReduce and Pig jobs will include + automatically the system library path, thus not requiring users to + specify where the Pig JAR files are. Instead, the ones from the system + library path are used. + </description> + </property> + <property> + <name>oozie.authentication.kerberos.name.rules</name> + <value> + RULE:[2:$1@$0]([jt]t@.*TODO-KERBEROS-DOMAIN)s/.*/TODO-MAPREDUSER/ + RULE:[2:$1@$0]([nd]n@.*TODO-KERBEROS-DOMAIN)s/.*/TODO-HDFSUSER/ + RULE:[2:$1@$0](hm@.*TODO-KERBEROS-DOMAIN)s/.*/TODO-HBASE-USER/ + RULE:[2:$1@$0](rs@.*TODO-KERBEROS-DOMAIN)s/.*/TODO-HBASE-USER/ + DEFAULT + </value> + <description>The mapping from kerberos principal names to local OS user names.</description> + </property> + <property> + <name>oozie.service.HadoopAccessorService.hadoop.configurations</name> + <value>*=/usr/iop/current/hadoop-client/conf</value> + <description> + Comma separated AUTHORITY=HADOOP_CONF_DIR, where AUTHORITY is the HOST:PORT of + the Hadoop service (JobTracker, HDFS). The wildcard '*' configuration is + used when there is no exact match for an authority. The HADOOP_CONF_DIR contains + the relevant Hadoop *-site.xml files. If the path is relative is looked within + the Oozie configuration directory; though the path can be absolute (i.e. to point + to Hadoop client conf/ directories in the local filesystem. + </description> + </property> + <property> + <name>oozie.service.ActionService.executor.ext.classes</name> + <value> + org.apache.oozie.action.email.EmailActionExecutor, + org.apache.oozie.action.hadoop.HiveActionExecutor, + org.apache.oozie.action.hadoop.ShellActionExecutor, + org.apache.oozie.action.hadoop.SqoopActionExecutor, + org.apache.oozie.action.hadoop.DistcpActionExecutor + </value> + <description> + List of ActionExecutors extension classes (separated by commas). Only action types with associated executors can + be used in workflows. This property is a convenience property to add extensions to the built in executors without + having to include all the built in ones. + </description> + </property> + + <property> + <name>oozie.service.SchemaService.wf.ext.schemas</name> + <value>shell-action-0.1.xsd,email-action-0.1.xsd,hive-action-0.2.xsd,sqoop-action-0.2.xsd,ssh-action-0.1.xsd,distcp-action-0.1.xsd,shell-action-0.2.xsd,oozie-sla-0.1.xsd,oozie-sla-0.2.xsd,hive-action-0.3.xsd</value> + <description> + Schemas for additional actions types. IMPORTANT: if there are no schemas leave a 1 space string, the service + trims the value, if empty Configuration assumes it is NULL. + </description> + </property> + <property> + <name>oozie.service.JPAService.create.db.schema</name> + <value>false</value> + <description> + Creates Oozie DB. + + If set to true, it creates the DB schema if it does not exist. If the DB schema exists is a NOP. + If set to false, it does not create the DB schema. If the DB schema does not exist it fails start up. + </description> + </property> + + <property> + <name>oozie.service.JPAService.jdbc.driver</name> + <value>org.apache.derby.jdbc.EmbeddedDriver</value> + <display-name>JDBC Driver Class</display-name> + <description> + JDBC driver class. + </description> + <value-attributes> + <overridable>false</overridable> + </value-attributes> + </property> + + <property> + <name>oozie.service.JPAService.jdbc.url</name> + <value>jdbc:derby:${oozie.data.dir}/${oozie.db.schema.name}-db;create=true</value> + <display-name>Database URL</display-name> + <description> + JDBC URL. + </description> + <value-attributes> + <overridable>false</overridable> + </value-attributes> + </property> + + <property> + <name>oozie.service.JPAService.jdbc.username</name> + <value>oozie</value> + <display-name>Database Username</display-name> + <description> + Database user name to use to connect to the database + </description> + <value-attributes> + <type>db_user</type> + <overridable>false</overridable> + </value-attributes> + </property> + + <property require-input = "true"> + <name>oozie.service.JPAService.jdbc.password</name> + <value></value> + <display-name>Database Password</display-name> + <property-type>PASSWORD</property-type> + <description> + DB user password. + + IMPORTANT: if password is empty leave a 1 space string, the service trims the value, + if empty Configuration assumes it is NULL. + </description> + <value-attributes> + <overridable>false</overridable> + </value-attributes> + </property> + + <property> + <name>oozie.service.JPAService.pool.max.active.conn</name> + <value>10</value> + <description> + Max number of connections. + </description> + </property> + + <property> + <name>oozie.services</name> + <value> + org.apache.oozie.service.SchedulerService, + org.apache.oozie.service.InstrumentationService, + org.apache.oozie.service.CallableQueueService, + org.apache.oozie.service.UUIDService, + org.apache.oozie.service.ELService, + org.apache.oozie.service.AuthorizationService, + org.apache.oozie.service.UserGroupInformationService, + org.apache.oozie.service.HadoopAccessorService, + org.apache.oozie.service.JobsConcurrencyService, + org.apache.oozie.service.URIHandlerService, + org.apache.oozie.service.MemoryLocksService, + org.apache.oozie.service.DagXLogInfoService, + org.apache.oozie.service.SchemaService, + org.apache.oozie.service.LiteWorkflowAppService, + org.apache.oozie.service.JPAService, + org.apache.oozie.service.StoreService, + org.apache.oozie.service.CoordinatorStoreService, + org.apache.oozie.service.SLAStoreService, + org.apache.oozie.service.DBLiteWorkflowStoreService, + org.apache.oozie.service.CallbackService, + org.apache.oozie.service.ShareLibService, + org.apache.oozie.service.ActionService, + org.apache.oozie.service.ActionCheckerService, + org.apache.oozie.service.RecoveryService, + org.apache.oozie.service.PurgeService, + org.apache.oozie.service.CoordinatorEngineService, + org.apache.oozie.service.BundleEngineService, + org.apache.oozie.service.DagEngineService, + org.apache.oozie.service.CoordMaterializeTriggerService, + org.apache.oozie.service.StatusTransitService, + org.apache.oozie.service.PauseTransitService, + org.apache.oozie.service.GroupsService, + org.apache.oozie.service.ProxyUserService, + org.apache.oozie.service.XLogStreamingService, + org.apache.oozie.service.JvmPauseMonitorService + </value> + <description> + All services to be created and managed by Oozie Services singleton. + Class names must be separated by commas. + </description> + </property> + <property> + <name>oozie.service.URIHandlerService.uri.handlers</name> + <value>org.apache.oozie.dependency.FSURIHandler,org.apache.oozie.dependency.HCatURIHandler</value> + <description> + Enlist the different uri handlers supported for data availability checks. + </description> + </property> + <property> + <name>oozie.services.ext</name> + <value>org.apache.oozie.service.JMSAccessorService,org.apache.oozie.service.PartitionDependencyManagerService,org.apache.oozie.service.HCatAccessorService + </value> + <description> + To add/replace services defined in 'oozie.services' with custom implementations. + Class names must be separated by commas. + </description> + </property> + <property> + <name>oozie.service.coord.push.check.requeue.interval</name> + <value>30000</value> + <description> + Command re-queue interval for push dependencies (in millisecond). + </description> + </property> + <property> + <name>oozie.credentials.credentialclasses</name> + <value>hcat=org.apache.oozie.action.hadoop.HCatCredentials</value> + <description> + Credential Class to be used for HCat. + </description> + </property> + + <property> + <name>oozie.service.SchemaService.wf.ext.schemas</name> + <value>shell-action-0.1.xsd,shell-action-0.2.xsd,shell-action-0.3.xsd,email-action-0.1.xsd,email-action-0.2.xsd,hive-action-0.2.xsd,hive-action-0.3.xsd,hive-action-0.4.xsd,hive-action-0.5.xsd,sqoop-action-0.2.xsd,sqoop-action-0.3.xsd,sqoop-action-0.4.xsd,ssh-action-0.1.xsd,ssh-action-0.2.xsd,distcp-action-0.1.xsd,distcp-action-0.2.xsd,oozie-sla-0.1.xsd,oozie-sla-0.2.xsd</value> + </property> + + <property> + <name>oozie.service.AuthorizationService.security.enabled</name> + <value>true</value> + <description> + Specifies whether security (user name/admin role) is enabled or not. + If disabled any user can manage Oozie system and manage any job. + </description> + </property> + + <property> + <name>oozie.service.HadoopAccessorService.kerberos.enabled</name> + <value>false</value> + <description> + Indicates if Oozie is configured to use Kerberos. + </description> + </property> + + <property> + <name>oozie.authentication.simple.anonymous.allowed</name> + <value>true</value> + <description> + Indicates if anonymous requests are allowed. + This setting is meaningful only when using 'simple' authentication. + </description> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/kerberos.json b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/kerberos.json new file mode 100755 index 0000000..dce65d0 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/kerberos.json @@ -0,0 +1,70 @@ +{ + "services": [ + { + "name": "OOZIE", + "identities": [ + { + "name": "/spnego" + }, + { + "name": "/smokeuser" + }, + { + "name": "/hdfs" + } + ], + "auth_to_local_properties" : [ + "oozie-site/oozie.authentication.kerberos.name.rules" + ], + "configurations": [ + { + "oozie-site": { + "oozie.authentication.type": "kerberos", + "oozie.service.AuthorizationService.authorization.enabled": "true", + "oozie.service.HadoopAccessorService.kerberos.enabled": "true", + "local.realm": "${realm}", + "oozie.credentials.credentialclasses": "hcat=org.apache.oozie.action.hadoop.HCatCredentials,hive2=org.apache.oozie.action.hadoop.Hive2Credentials", + "oozie.authentication.kerberos.name.rules": "" + } + } + ], + "components": [ + { + "name": "OOZIE_SERVER", + "identities": [ + { + "name": "oozie_server", + "principal": { + "value": "oozie/_HOST@${realm}", + "type" : "service", + "configuration": "oozie-site/oozie.service.HadoopAccessorService.kerberos.principal", + "local_username" : "${oozie-env/oozie_user}" + }, + "keytab": { + "file": "${keytab_dir}/oozie.service.keytab", + "owner": { + "name": "${oozie-env/oozie_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "oozie-site/oozie.service.HadoopAccessorService.keytab.file" + } + }, + { + "name": "/spnego", + "principal": { + "configuration": "oozie-site/oozie.authentication.kerberos.principal" + }, + "keytab": { + "configuration": "oozie-site/oozie.authentication.kerberos.keytab" + } + } + ] + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/metainfo.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/metainfo.xml new file mode 100755 index 0000000..2835c84 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/metainfo.xml @@ -0,0 +1,176 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>OOZIE</name> + <displayName>Oozie</displayName> + <comment>System for workflow coordination and execution of Apache Hadoop jobs. This also includes the installation of the optional Oozie Web Console which relies on and will install the <a target="_blank" href="http://www.sencha.com/legal/open-source-faq/">ExtJS</a> Library.</comment> + <version>4.2.0</version> + <components> + <component> + <name>OOZIE_SERVER</name> + <displayName>Oozie Server</displayName> + <category>MASTER</category> + <cardinality>1</cardinality> + <versionAdvertised>true</versionAdvertised> + <dependencies> + <dependency> + <name>HDFS/HDFS_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>YARN/YARN_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> + <commandScript> + <script>scripts/oozie_server.py</script> + <scriptType>PYTHON</scriptType> + <timeout>1200</timeout> + </commandScript> + </component> + + <component> + <name>OOZIE_CLIENT</name> + <displayName>Oozie Client</displayName> + <category>CLIENT</category> + <cardinality>1+</cardinality> + <versionAdvertised>true</versionAdvertised> + <dependencies> + <dependency> + <name>HDFS/HDFS_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> + <commandScript> + <script>scripts/oozie_client.py</script> + <scriptType>PYTHON</scriptType> + </commandScript> + <configFiles> + <configFile> + <type>xml</type> + <fileName>oozie-site.xml</fileName> + <dictionaryName>oozie-site</dictionaryName> + </configFile> + <configFile> + <type>env</type> + <fileName>oozie-env.sh</fileName> + <dictionaryName>oozie-env</dictionaryName> + </configFile> + <configFile> + <type>env</type> + <fileName>oozie-log4j.properties</fileName> + <dictionaryName>oozie-log4j</dictionaryName> + </configFile> + </configFiles> + </component> + </components> + + <osSpecifics> + <osSpecific> + <osFamily>any</osFamily> + <packages> + <package> + <name>zip</name> + </package> + <package> + <name>mysql-connector-java</name> + </package> + <package> + <name>extjs</name> + </package> + <package> + <name>oozie-client</name> + </package> + </packages> + </osSpecific> + <osSpecific> + <osFamily>redhat6,suse11</osFamily> + <packages> + <package> + <name>oozie</name> + </package> + <!--package> + <name>falcon</name> + </package--> + </packages> + </osSpecific> + <osSpecific> + <osFamily>ubuntu12</osFamily> + <packages> + <package> + <name>oozie</name> + </package> + <!--package> + <name>falcon</name> + </package--> + <package> + <name>extjs</name> + </package> + <package> + <name>libxml2-utils</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <commandScript> + <script>scripts/service_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + + <requiredServices> + <service>YARN</service> + </requiredServices> + + <configuration-dependencies> + <config-type>oozie-site</config-type> + <config-type>oozie-env</config-type> + <config-type>oozie-log4j</config-type> + <config-type>yarn-site</config-type> + <config-type>hive-site</config-type> + </configuration-dependencies> + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/alerts/alert_check_oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/alerts/alert_check_oozie_server.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/alerts/alert_check_oozie_server.py new file mode 100755 index 0000000..7c26775 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/alerts/alert_check_oozie_server.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python + +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import os +import re + +from resource_management.core import global_lock +from resource_management.core.environment import Environment +from resource_management.core.resources import Execute +from resource_management.libraries.functions import format +from resource_management.libraries.functions import get_kinit_path +from resource_management.libraries.functions import get_klist_path +from ambari_commons.os_check import OSConst, OSCheck +from urlparse import urlparse + +RESULT_CODE_OK = 'OK' +RESULT_CODE_CRITICAL = 'CRITICAL' +RESULT_CODE_UNKNOWN = 'UNKNOWN' + +OOZIE_ENV_HTTPS_RE = r"export\s+OOZIE_HTTPS_PORT=(\d+)" + +# The configured Kerberos executable search paths, if any +KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY = '{{kerberos-env/executable_search_paths}}' + +OOZIE_URL_KEY = '{{oozie-site/oozie.base.url}}' +SECURITY_ENABLED = '{{cluster-env/security_enabled}}' +OOZIE_USER = '{{oozie-env/oozie_user}}' +OOZIE_CONF_DIR = '/usr/iop/current/oozie-server/conf' +OOZIE_CONF_DIR_LEGACY = '/etc/oozie/conf' +OOZIE_HTTPS_PORT = '{{oozie-site/oozie.https.port}}' +OOZIE_ENV_CONTENT = '{{oozie-env/content}}' + +USER_KEYTAB_KEY = '{{oozie-site/oozie.service.HadoopAccessorService.keytab.file}}' +USER_PRINCIPAL_KEY = '{{oozie-site/oozie.service.HadoopAccessorService.kerberos.principal}}' +USER_KEY = '{{oozie-env/oozie_user}}' + +# default keytab location +USER_KEYTAB_SCRIPT_PARAM_KEY = 'default.oozie.keytab' +USER_KEYTAB_DEFAULT = '/etc/security/keytabs/oozie.headless.keytab' + +# default user principal +USER_PRINCIPAL_SCRIPT_PARAM_KEY = 'default.oozie.principal' +USER_PRINCIPAL_DEFAULT = 'oo...@example.com' + +# default user +USER_DEFAULT = 'oozie' + +class KerberosPropertiesNotFound(Exception): pass + +def get_tokens(): + """ + Returns a tuple of tokens in the format {{site/property}} that will be used + to build the dictionary passed into execute + """ + return (OOZIE_URL_KEY, USER_PRINCIPAL_KEY, SECURITY_ENABLED, USER_KEYTAB_KEY, KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY, + USER_KEY, OOZIE_HTTPS_PORT, OOZIE_ENV_CONTENT) + +def get_check_command(oozie_url, host_name, configurations, parameters, only_kinit): + kerberos_env = None + + user = USER_DEFAULT + if USER_KEY in configurations: + user = configurations[USER_KEY] + + if is_security_enabled(configurations): + # defaults + user_keytab = USER_KEYTAB_DEFAULT + user_principal = USER_PRINCIPAL_DEFAULT + + # check script params + if USER_PRINCIPAL_SCRIPT_PARAM_KEY in parameters: + user_principal = parameters[USER_PRINCIPAL_SCRIPT_PARAM_KEY] + user_principal = user_principal.replace('_HOST', host_name.lower()) + if USER_KEYTAB_SCRIPT_PARAM_KEY in parameters: + user_keytab = parameters[USER_KEYTAB_SCRIPT_PARAM_KEY] + + # check configurations last as they should always take precedence + if USER_PRINCIPAL_KEY in configurations: + user_principal = configurations[USER_PRINCIPAL_KEY] + user_principal = user_principal.replace('_HOST', host_name.lower()) + if USER_KEYTAB_KEY in configurations: + user_keytab = configurations[USER_KEYTAB_KEY] + + # Create the kerberos credentials cache (ccache) file and set it in the environment to use + # when executing curl + env = Environment.get_instance() + ccache_file = "{0}{1}oozie_alert_cc_{2}".format(env.tmp_dir, os.sep, os.getpid()) + kerberos_env = {'KRB5CCNAME': ccache_file} + + # Get the configured Kerberos executable search paths, if any + kerberos_executable_search_paths = None + if KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY in configurations: + kerberos_executable_search_paths = configurations[KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY] + + klist_path_local = get_klist_path(kerberos_executable_search_paths) + kinit_path_local = get_kinit_path(kerberos_executable_search_paths) + kinit_part_command = format("{kinit_path_local} -l 5m20s -c {ccache_file} -kt {user_keytab} {user_principal}; ") + + # Determine if we need to kinit by testing to see if the relevant cache exists and has + # non-expired tickets. Tickets are marked to expire after 5 minutes to help reduce the number + # it kinits we do but recover quickly when keytabs are regenerated + + if only_kinit: + kinit_command = kinit_part_command + else: + kinit_command = "{0} -s {1} || ".format(klist_path_local, ccache_file) + kinit_part_command + + # prevent concurrent kinit + kinit_lock = global_lock.get_lock(global_lock.LOCK_TYPE_KERBEROS) + kinit_lock.acquire() + try: + Execute(kinit_command, environment=kerberos_env, user=user) + finally: + kinit_lock.release() + + oozie_config_directory = OOZIE_CONF_DIR_LEGACY + if os.path.exists(OOZIE_CONF_DIR): + oozie_config_directory = OOZIE_CONF_DIR + + command = "source {0}/oozie-env.sh ; oozie admin -oozie {1} -status".format( + oozie_config_directory, oozie_url) + + return (command, kerberos_env, user) + +def execute(configurations={}, parameters={}, host_name=None): + """ + Returns a tuple containing the result code and a pre-formatted result label + + Keyword arguments: + configurations (dictionary): a mapping of configuration key to value + parameters (dictionary): a mapping of script parameter key to value + host_name (string): the name of this host where the alert is running + """ + + if configurations is None: + return (RESULT_CODE_UNKNOWN, ['There were no configurations supplied to the script.']) + + if not OOZIE_URL_KEY in configurations: + return (RESULT_CODE_UNKNOWN, ['The Oozie URL is a required parameter.']) + + https_port = None + # try to get https port form oozie-env content + if OOZIE_ENV_CONTENT in configurations: + for line in configurations[OOZIE_ENV_CONTENT].splitlines(): + result = re.match(OOZIE_ENV_HTTPS_RE, line) + + if result is not None: + https_port = result.group(1) + # or from oozie-site.xml + if https_port is None and OOZIE_HTTPS_PORT in configurations: + https_port = configurations[OOZIE_HTTPS_PORT] + + oozie_url = configurations[OOZIE_URL_KEY] + + # construct proper url for https + if https_port is not None: + parsed_url = urlparse(oozie_url) + oozie_url = oozie_url.replace(parsed_url.scheme, "https") + if parsed_url.port is None: + oozie_url.replace(parsed_url.hostname, ":".join([parsed_url.hostname, str(https_port)])) + else: + oozie_url = oozie_url.replace(str(parsed_url.port), str(https_port)) + + # https will not work with localhost address, we need put fqdn + if https_port is None: + oozie_url = oozie_url.replace(urlparse(oozie_url).hostname, host_name) + + (code, msg) = get_check_result(oozie_url, host_name, configurations, parameters, False) + + # sometimes real lifetime for ticket is less than we have set(5m20s aS of now) + # so i've added this double check with rekinit command to be sure thaT it's not problem with ticket lifetime + if is_security_enabled(configurations) and code == RESULT_CODE_CRITICAL: + (code, msg) = get_check_result(oozie_url, host_name, configurations, parameters, True) + + return (code, msg) + + +def get_check_result(oozie_url, host_name, configurations, parameters, only_kinit): + try: + command, env, user = get_check_command(oozie_url, host_name, configurations, parameters, only_kinit) + # execute the command + Execute(command, environment=env, user=user) + + return (RESULT_CODE_OK, ["Successful connection to {0}".format(oozie_url)]) + except KerberosPropertiesNotFound, ex: + return (RESULT_CODE_UNKNOWN, [str(ex)]) + except Exception, ex: + return (RESULT_CODE_CRITICAL, [str(ex)]) + +def is_security_enabled(configurations): + security_enabled = False + if SECURITY_ENABLED in configurations: + security_enabled = str(configurations[SECURITY_ENABLED]).upper() == 'TRUE' + + return security_enabled http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/files/oozieSmoke2.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/files/oozieSmoke2.sh b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/files/oozieSmoke2.sh new file mode 100755 index 0000000..e0d29b4 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/files/oozieSmoke2.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# + +export os_family=$1 +export oozie_lib_dir=$2 +export oozie_conf_dir=$3 +export oozie_bin_dir=$4 +export oozie_server_url=$5 +export oozie_examples_dir=$6 +export hadoop_conf_dir=$7 +export hadoop_bin_dir=$8 +export smoke_test_user=$9 +export security_enabled=${10} +export smoke_user_keytab=${11} +export kinit_path_local=${12} +export smokeuser_principal=${13} + +function getValueFromField { + xmllint $1 | grep "<name>$2</name>" -C 2 | grep '<value>' | cut -d ">" -f2 | cut -d "<" -f1 + return $? +} + +function checkOozieJobStatus { + local job_id=$1 + local num_of_tries=$2 + #default num_of_tries to 10 if not present + num_of_tries=${num_of_tries:-10} + local i=0 + local rc=1 + local cmd="source ${oozie_conf_dir}/oozie-env.sh ; ${oozie_bin_dir}/oozie job -oozie ${OOZIE_SERVER} -info $job_id" + /var/lib/ambari-agent/ambari-sudo.sh su ${smoke_test_user} -s /bin/bash - -c "$cmd" + while [ $i -lt $num_of_tries ] ; do + cmd_output=`/var/lib/ambari-agent/ambari-sudo.sh su ${smoke_test_user} -s /bin/bash - -c "$cmd"` + (IFS='';echo $cmd_output) + act_status=$(IFS='';echo $cmd_output | grep ^Status | cut -d':' -f2 | sed 's| ||g') + echo "workflow_status=$act_status" + if [ "RUNNING" == "$act_status" ]; then + #increment the counter and get the status again after waiting for 15 secs + sleep 15 + (( i++ )) + elif [ "SUCCEEDED" == "$act_status" ]; then + rc=0; + break; + else + rc=1 + break; + fi + done + return $rc +} + +export OOZIE_EXIT_CODE=0 +export OOZIE_SERVER=$oozie_server_url + +cd $oozie_examples_dir + +if [[ $security_enabled == "True" ]]; then + kinitcmd="${kinit_path_local} -kt ${smoke_user_keytab} ${smokeuser_principal}; " +else + kinitcmd="" +fi + +cmd="${kinitcmd}source ${oozie_conf_dir}/oozie-env.sh ; ${oozie_bin_dir}/oozie -Doozie.auth.token.cache=false job -oozie $OOZIE_SERVER -config $oozie_examples_dir/examples/apps/map-reduce/job.properties -run" +echo $cmd +job_info=`/var/lib/ambari-agent/ambari-sudo.sh su ${smoke_test_user} -s /bin/bash - -c "$cmd" | grep "job:"` +job_id="`echo $job_info | cut -d':' -f2`" +checkOozieJobStatus "$job_id" 15 +OOZIE_EXIT_CODE="$?" +exit $OOZIE_EXIT_CODE http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/files/prepareOozieHdfsDirectories.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/files/prepareOozieHdfsDirectories.sh b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/files/prepareOozieHdfsDirectories.sh new file mode 100755 index 0000000..79a1bfc --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/files/prepareOozieHdfsDirectories.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# + +export oozie_conf_dir=$1 +export oozie_examples_dir=$2 +export hadoop_conf_dir=$3 + +function getValueFromField { + xmllint $1 | grep "<name>$2</name>" -C 2 | grep '<value>' | cut -d ">" -f2 | cut -d "<" -f1 + return $? +} + +export JOBTRACKER=`getValueFromField ${hadoop_conf_dir}/yarn-site.xml yarn.resourcemanager.address` +export NAMENODE=`getValueFromField ${hadoop_conf_dir}/core-site.xml fs.defaultFS` + +cd $oozie_examples_dir + +/var/lib/ambari-agent/ambari-sudo.sh tar -zxf oozie-examples.tar.gz +/var/lib/ambari-agent/ambari-sudo.sh chmod -R o+rx examples + +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|nameNode=hdfs://localhost:8020|nameNode=$NAMENODE|g" examples/apps/map-reduce/job.properties +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|nameNode=hdfs://localhost:9000|nameNode=$NAMENODE|g" examples/apps/map-reduce/job.properties +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|jobTracker=localhost:8021|jobTracker=$JOBTRACKER|g" examples/apps/map-reduce/job.properties +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|jobTracker=localhost:9001|jobTracker=$JOBTRACKER|g" examples/apps/map-reduce/job.properties +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|jobTracker=localhost:8032|jobTracker=$JOBTRACKER|g" examples/apps/map-reduce/job.properties +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|oozie.wf.application.path=hdfs://localhost:9000|oozie.wf.application.path=$NAMENODE|g" examples/apps/map-reduce/job.properties http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/files/wrap_ooziedb.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/files/wrap_ooziedb.sh b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/files/wrap_ooziedb.sh new file mode 100755 index 0000000..bdd43ca --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/files/wrap_ooziedb.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +OUT=`cd /var/tmp/oozie && /usr/lib/oozie/bin/ooziedb.sh "$@" 2>&1` +EC=$? +echo $OUT +GRVAR=`echo ${OUT} | grep -o "java.lang.Exception: DB schema exists"` +if [ ${EC} -ne 0 ] && [ -n "$GRVAR" ] +then + exit 0 +else + exit $EC +fi http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/oozie.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/oozie.py new file mode 100755 index 0000000..6cd2b71 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/oozie.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +import hashlib +import os + +from resource_management.core.resources import Directory +from resource_management.core.resources import File +from resource_management.core.resources.system import Execute +from resource_management.core.source import DownloadSource +from resource_management.core.source import InlineTemplate +from resource_management.core.source import Template +from resource_management.libraries.functions import format +from resource_management.libraries.functions import compare_versions +from resource_management.libraries.resources.xml_config import XmlConfig +from resource_management.core.resources.packaging import Package +from resource_management.core.shell import as_user + + +# TODO: see if see can remove this +def oozie(is_server=False): + import params + + if is_server: + params.HdfsResource(params.oozie_hdfs_user_dir, + type="directory", + action="create_on_execute", + owner=params.oozie_user, + mode=params.oozie_hdfs_user_mode + ) + params.HdfsResource(None, action="execute") + + Directory(params.conf_dir, + create_parents = True, + owner = params.oozie_user, + group = params.user_group + ) + XmlConfig("oozie-site.xml", + conf_dir = params.conf_dir, + configurations = params.oozie_site, + configuration_attributes=params.config['configuration_attributes']['oozie-site'], + owner = params.oozie_user, + group = params.user_group, + mode = 0660 + ) + File(format("{conf_dir}/oozie-env.sh"), + owner=params.oozie_user, + content=InlineTemplate(params.oozie_env_sh_template), + group=params.user_group, + ) + + if (params.log4j_props != None): + File(format("{params.conf_dir}/oozie-log4j.properties"), + mode=0644, + group=params.user_group, + owner=params.oozie_user, + content=params.log4j_props + ) + elif (os.path.exists(format("{params.conf_dir}/oozie-log4j.properties"))): + File(format("{params.conf_dir}/oozie-log4j.properties"), + mode=0644, + group=params.user_group, + owner=params.oozie_user + ) + + File(format("{params.conf_dir}/adminusers.txt"), + mode=0644, + group=params.user_group, + owner=params.oozie_user, + content=Template('adminusers.txt.j2', oozie_user=params.oozie_user) + ) + + environment = { + "no_proxy": format("{ambari_server_hostname}") + } + + if params.jdbc_driver_name == "com.mysql.jdbc.Driver" or \ + params.jdbc_driver_name == "org.postgresql.Driver" or \ + params.jdbc_driver_name == "oracle.jdbc.driver.OracleDriver": + File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), + content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")), + ) + pass + + oozie_ownership() + + if is_server: + oozie_server_specific() + +def download_database_library_if_needed(): + """ + Downloads the library to use when connecting to the Oozie database, if + necessary. The library will be downloaded to 'params.target' unless + otherwise specified. + :param target_directory: the location where the database library will be + downloaded to. + :return: + """ + import params + # check to see if the JDBC driver name is in the list of ones that need to + # be downloaded + if params.jdbc_driver_name=="com.mysql.jdbc.Driver" or \ + params.jdbc_driver_name=="oracle.jdbc.driver.OracleDriver": + File(params.downloaded_custom_connector, + content = DownloadSource(params.driver_curl_source), + ) + Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.target), + #creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc + path=["/bin", "/usr/bin/"], + sudo = True) + + File ( params.target, + owner = params.oozie_user, + group = params.user_group + ) + + +def oozie_ownership(): + import params + + File ( format("{conf_dir}/hadoop-config.xml"), + owner = params.oozie_user, + group = params.user_group + ) + + File ( format("{conf_dir}/oozie-default.xml"), + owner = params.oozie_user, + group = params.user_group, + mode = 0644 + ) + + Directory ( format("{conf_dir}/action-conf"), + owner = params.oozie_user, + group = params.user_group + ) + + File ( format("{conf_dir}/action-conf/hive.xml"), + owner = params.oozie_user, + group = params.user_group + ) + +def oozie_server_specific(): + import params + + no_op_test = as_user(format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"), user=params.oozie_user) + File(params.pid_file, + action="delete", + not_if=no_op_test + ) + + oozie_server_directories = [format("{oozie_home}/{oozie_tmp_dir}"), params.oozie_pid_dir, params.oozie_log_dir, params.oozie_tmp_dir, params.oozie_data_dir, params.oozie_lib_dir, params.oozie_webapps_dir, params.oozie_webapps_conf_dir, params.oozie_server_dir] + Directory( oozie_server_directories, + owner = params.oozie_user, + group = params.user_group, + mode = 0755, + create_parents = True, + cd_access="a", + ) + + Directory(params.oozie_libext_dir, + create_parents=True, + ) + + hashcode_file = format("{oozie_home}/.hashcode") + hashcode = hashlib.md5(format('{oozie_home}/oozie-sharelib.tar.gz')).hexdigest() + skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share && [[ `cat {hashcode_file}` == '{hashcode}' ]]") + + untar_sharelib = ('tar','-xvf',format('{oozie_home}/oozie-sharelib.tar.gz'),'-C',params.oozie_home) + + Execute( untar_sharelib, # time-expensive + not_if = format("{no_op_test} || {skip_recreate_sharelib}"), + sudo = True, + ) + configure_cmds = [] + #configure_cmds.append(('tar','-xvf',format('{oozie_home}/oozie-sharelib.tar.gz'),'-C',params.oozie_home)) + #configure_cmds.append(('cp', params.ext_js_path, params.oozie_libext_dir)) + #configure_cmds.append(('chown', format('{oozie_user}:{user_group}'), format('{oozie_libext_dir}/{ext_js_file}'))) + configure_cmds.append(('chown', '-RL', format('{oozie_user}:{user_group}'), params.oozie_webapps_conf_dir)) + + + Execute( configure_cmds, + not_if = no_op_test, + sudo = True, + ) + + if params.jdbc_driver_name=="com.mysql.jdbc.Driver" or \ + params.jdbc_driver_name=="oracle.jdbc.driver.OracleDriver": + File(params.downloaded_custom_connector, + content = DownloadSource(params.driver_curl_source), + ) + + + Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.target), + #creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc + path=["/bin", "/usr/bin/"], + sudo = True) + + File ( params.target, + owner = params.oozie_user, + group = params.user_group + ) + + #falcon el extension + if params.has_falcon_host: + Execute(format('rm -rf {oozie_libext_dir}/falcon-oozie-el-extension.jar'),) + if params.security_enabled: + Execute(format('/usr/bin/kinit -kt /etc/security/keytabs/hdfs.headless.keytab {hdfs_principal_name}')) + Execute(format('hadoop fs -get /user/falcon/temp/falcon-oozie-el-extension.jar {oozie_libext_dir}'), + not_if = no_op_test, + ) + Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension.jar'), + not_if = no_op_test, + ) + #if params.lzo_enabled and len(params.lzo_packages_for_current_host) > 0: + # Package(params.lzo_packages_for_current_host) + # Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), + # not_if = no_op_test, + # ) + + prepare_war_cmd_file = format("{oozie_home}/.prepare_war_cmd") + prepare_war_cmd = format("cd {oozie_tmp_dir} && {oozie_setup_sh} prepare-war {oozie_secure}") + skip_prepare_war_cmd = format("test -f {prepare_war_cmd_file} && [[ `cat {prepare_war_cmd_file}` == '{prepare_war_cmd}' ]]") + + Execute(prepare_war_cmd, # time-expensive + user = params.oozie_user, + not_if = format("{no_op_test} || {skip_recreate_sharelib} && {skip_prepare_war_cmd}") + ) + File(hashcode_file, + content = hashcode, + mode = 0644, + ) + File(prepare_war_cmd_file, + content = prepare_war_cmd, + mode = 0644, + ) + + # Create hive-site and tez-site configs for oozie + Directory(params.hive_conf_dir, + create_parents = True, + owner = params.oozie_user, + group = params.user_group + ) + if 'hive-site' in params.config['configurations']: + XmlConfig("hive-site.xml", + conf_dir=params.hive_conf_dir, + configurations=params.config['configurations']['hive-site'], + configuration_attributes=params.config['configuration_attributes']['hive-site'], + owner=params.oozie_user, + group=params.user_group, + mode=0640 + ) + '''if 'tez-site' in params.config['configurations']: + XmlConfig( "tez-site.xml", + conf_dir = params.hive_conf_dir, + configurations = params.config['configurations']['tez-site'], + configuration_attributes=params.config['configuration_attributes']['tez-site'], + owner = params.oozie_user, + group = params.user_group, + mode = 0664 + )''' + Execute(('chown', '-R', format("{oozie_user}:{user_group}"), params.oozie_server_dir), + sudo=True + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/oozie_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/oozie_client.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/oozie_client.py new file mode 100755 index 0000000..b36d2ec --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/oozie_client.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management import * +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select + +from oozie import oozie +from oozie_service import oozie_service + + +class OozieClient(Script): + + def get_component_name(self): + return "oozie-client" + + def install(self, env): + self.install_packages(env) + self.configure(env) + + + def configure(self, env): + import params + env.set_params(params) + + oozie(is_server=False) + + def status(self, env): + raise ClientComponentHasNoStatus() + + + def pre_upgrade_restart(self, env, upgrade_type=None): + import params + env.set_params(params) + + # this function should not execute if the version can't be determined or + # is not at least IOP 4.0.0.0 + if not params.version or compare_versions(format_stack_version(params.version), '4.0.0.0') < 0: + return + + Logger.info("Executing Oozie Client Rolling Upgrade pre-restart") + conf_select.select(params.stack_name, "oozie", params.version) + stack_select.select("oozie-client", params.version) + #Execute(format("stack-select set oozie-client {version}")) + + # We substitute some configs (oozie.authentication.kerberos.principal) before generation (see oozie.py and params.py). + # This function returns changed configs (it's used for config generation before config download) + def generate_configs_get_xml_file_content(self, filename, dictionary): + if dictionary == 'oozie-site': + import params + config = self.get_config() + return {'configurations': params.oozie_site, + 'configuration_attributes': config['configuration_attributes'][dictionary]} + else: + return super(OozieClient, self).generate_configs_get_xml_file_content(filename, dictionary) + +if __name__ == "__main__": + OozieClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/oozie_server.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/oozie_server.py new file mode 100755 index 0000000..d6d6d37 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/oozie_server.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import oozie_server_upgrade + +from resource_management.core import Logger +from resource_management.core.resources.system import Execute +from resource_management.libraries.functions import format +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions.constants import Direction +from resource_management.libraries.script import Script +from resource_management.libraries.functions import check_process_status +from resource_management.libraries.functions import compare_versions +from resource_management.libraries.functions import format_stack_version +from resource_management.libraries.functions.security_commons import build_expectations +from resource_management.libraries.functions.security_commons import cached_kinit_executor +from resource_management.libraries.functions.security_commons import get_params_from_filesystem +from resource_management.libraries.functions.security_commons import validate_security_config_properties +from resource_management.libraries.functions.security_commons import FILE_TYPE_XML + +from oozie import oozie +from oozie_service import oozie_service + + +class OozieServer(Script): + + def get_component_name(self): + return "oozie-server" + + def install(self, env): + self.install_packages(env) + + + def configure(self, env, upgrade_type=None): + import params + + #TODO: needed? + if upgrade_type == "nonrolling" and params.upgrade_direction == Direction.UPGRADE and \ + params.version and compare_versions(format_stack_version(params.version), '4.1.0.0') >= 0: + conf_select.select(params.stack_name, "oozie", params.version) + # In order for the "/usr/hdp/current/oozie-<client/server>" point to the new version of + # oozie, we need to create the symlinks both for server and client. + # This is required as both need to be pointing to new installed oozie version. + + # Sets the symlink : eg: /usr/hdp/current/oozie-client -> /usr/hdp/2.3.x.y-<version>/oozie + stack_select.select("oozie-client", params.version) + # Sets the symlink : eg: /usr/hdp/current/oozie-server -> /usr/hdp/2.3.x.y-<version>/oozie + stack_select.select("oozie-server", params.version) + + env.set_params(params) + oozie(is_server=True) + + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + #TODO remove this when config command will be implemented + self.configure(env) + + # preparing the WAR file must run after configure since configure writes out + # oozie-env.sh which is needed to have the right environment directories setup! + if upgrade_type is not None: + oozie_server_upgrade.prepare_warfile(); + + oozie_service(action='start', upgrade_type=upgrade_type) + + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + oozie_service(action='stop', upgrade_type=upgrade_type) + + + def status(self, env): + import status_params + env.set_params(status_params) + check_process_status(status_params.pid_file) + + + def security_status(self, env): + + import status_params + env.set_params(status_params) + + if status_params.security_enabled: + expectations = { + "oozie-site": + build_expectations('oozie-site', + { + "oozie.authentication.type": "kerberos", + "oozie.service.AuthorizationService.security.enabled": "true", + "oozie.service.HadoopAccessorService.kerberos.enabled": "true" + }, + [ + "local.realm", + "oozie.authentication.kerberos.principal", + "oozie.authentication.kerberos.keytab", + "oozie.service.HadoopAccessorService.kerberos.principal", + "oozie.service.HadoopAccessorService.keytab.file" + ], + None) + } + + security_params = get_params_from_filesystem(status_params.conf_dir, + {'oozie-site.xml': FILE_TYPE_XML}) + result_issues = validate_security_config_properties(security_params, expectations) + if not result_issues: # If all validations passed successfully + try: + # Double check the dict before calling execute + if ('oozie-site' not in security_params + or 'oozie.authentication.kerberos.principal' not in security_params['oozie-site'] + or 'oozie.authentication.kerberos.keytab' not in security_params['oozie-site'] + or 'oozie.service.HadoopAccessorService.kerberos.principal' not in security_params['oozie-site'] + or 'oozie.service.HadoopAccessorService.keytab.file' not in security_params['oozie-site']): + self.put_structured_out({"securityState": "UNSECURED"}) + self.put_structured_out({"securityIssuesFound": "Keytab file or principal are not set property."}) + return + + cached_kinit_executor(status_params.kinit_path_local, + status_params.oozie_user, + security_params['oozie-site']['oozie.authentication.kerberos.keytab'], + security_params['oozie-site']['oozie.authentication.kerberos.principal'], + status_params.hostname, + status_params.tmp_dir) + cached_kinit_executor(status_params.kinit_path_local, + status_params.oozie_user, + security_params['oozie-site']['oozie.service.HadoopAccessorService.keytab.file'], + security_params['oozie-site']['oozie.service.HadoopAccessorService.kerberos.principal'], + status_params.hostname, + status_params.tmp_dir) + self.put_structured_out({"securityState": "SECURED_KERBEROS"}) + except Exception as e: + self.put_structured_out({"securityState": "ERROR"}) + self.put_structured_out({"securityStateErrorInfo": str(e)}) + else: + issues = [] + for cf in result_issues: + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) + self.put_structured_out({"securityState": "UNSECURED"}) + else: + self.put_structured_out({"securityState": "UNSECURED"}) + + + def pre_upgrade_restart(self, env, upgrade_type=None): + """ + Performs the tasks surrounding the Oozie startup when a rolling upgrade + is in progress. This includes backing up the configuration, updating + the database, preparing the WAR, and installing the sharelib in HDFS. + :param env: + :return: + """ + import params + env.set_params(params) + + # this function should not execute if the version can't be determined or + # is not at least IOP 4.0.0.0 + if not params.version or compare_versions(format_stack_version(params.version), '4.0.0.0') < 0: + return + + Logger.info("Executing Oozie Server Rolling Upgrade pre-restart") + + oozie_server_upgrade.backup_configuration() + + conf_select.select(params.stack_name, "oozie", params.version) + stack_select.select("oozie-server", params.version) + #Execute(format("stack-select set oozie-server {version}")) + + oozie_server_upgrade.restore_configuration() + #oozie_server_upgrade.prepare_libext_directory() + oozie_server_upgrade.upgrade_oozie() + + +if __name__ == "__main__": + OozieServer().execute()