http://git-wip-us.apache.org/repos/asf/bigtop/blob/a745f030/bigtop-packages/src/rpm/oozie/SPECS/oozie.spec ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/oozie/SPECS/oozie.spec b/bigtop-packages/src/rpm/oozie/SPECS/oozie.spec deleted file mode 100644 index 933481a..0000000 --- a/bigtop-packages/src/rpm/oozie/SPECS/oozie.spec +++ /dev/null @@ -1,232 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -%define usr_bin /usr/bin -%define lib_oozie /usr/lib/oozie -%define man_dir /usr/share/man -%define conf_oozie %{_sysconfdir}/%{name}/conf -%define conf_oozie_dist %{conf_oozie}.dist -%define tomcat_conf_oozie %{_sysconfdir}/%{name}/tomcat-conf -%define data_oozie /var/lib/oozie -%define lib_hadoop /usr/lib/hadoop - -%if %{!?suse_version:1}0 - %define doc_oozie %{_docdir}/oozie-%{oozie_version} - %define initd_dir %{_sysconfdir}/rc.d/init.d - %define alternatives_cmd alternatives -%else - -# Only tested on openSUSE 11.4. le'ts update it for previous release when confirmed -%if 0%{suse_version} > 1130 -%define suse_check \# Define an empty suse_check for compatibility with older sles -%endif - -# SLES is more strict anc check all symlinks point to valid path -# But we do point to a hadoop jar which is not there at build time -# (but would be at install time). -# Since our package build system does not handle dependencies, -# these symlink checks are deactivated -%define __os_install_post \ - %{suse_check} ; \ - /usr/lib/rpm/brp-compress ; \ - %{nil} - - %define doc_oozie %{_docdir}/oozie - %define initd_dir %{_sysconfdir}/rc.d - %define alternatives_cmd update-alternatives -%endif - -Name: oozie -Version: %{oozie_version} -Release: %{oozie_release} -Summary: Oozie is a system that runs workflows of Hadoop jobs. -URL: http://oozie.apache.org/ -Group: Development/Libraries -Buildroot: %{_topdir}/INSTALL/%{name}-%{version} -License: ASL 2.0 -Source0: %{name}-%{oozie_base_version}.tar.gz -Source1: do-component-build -Source2: install_oozie.sh -Source3: oozie.1 -Source4: oozie-env.sh -Source5: oozie.init -Source6: catalina.properties -Source7: context.xml -Source8: hive.xml -Source9: tomcat-deployment.sh -Source10: oozie-site.xml -Source11: bigtop.bom -#BIGTOP_PATCH_FILES -Requires(pre): /usr/sbin/groupadd, /usr/sbin/useradd -Requires(post): /sbin/chkconfig -Requires(preun): /sbin/chkconfig, /sbin/service -Requires: oozie-client = %{version}, hadoop-client, bigtop-tomcat -BuildArch: noarch - -%description - Oozie is a system that runs workflows of Hadoop jobs. - Oozie workflows are actions arranged in a control dependency DAG (Direct - Acyclic Graph). - - Oozie coordinator functionality allows to start workflows at regular - frequencies and when data becomes available in HDFS. - - An Oozie workflow may contain the following types of actions nodes: - map-reduce, map-reduce streaming, map-reduce pipes, pig, file-system, - sub-workflows, java, hive, sqoop and ssh (deprecated). - - Flow control operations within the workflow can be done using decision, - fork and join nodes. Cycles in workflows are not supported. - - Actions and decisions can be parameterized with job properties, actions - output (i.e. Hadoop counters) and HDFS file information (file exists, - file size, etc). Formal parameters are expressed in the workflow definition - as ${VARIABLE NAME} variables. - - A Workflow application is an HDFS directory that contains the workflow - definition (an XML file), all the necessary files to run all the actions: - JAR files for Map/Reduce jobs, shells for streaming Map/Reduce jobs, native - libraries, Pig scripts, and other resource files. - - Running workflow jobs is done via command line tools, a WebServices API - or a Java API. - - Monitoring the system and workflow jobs can be done via a web console, the - command line tools, the WebServices API and the Java API. - - Oozie is a transactional system and it has built in automatic and manual - retry capabilities. - - In case of workflow job failure, the workflow job can be rerun skipping - previously completed actions, the workflow application can be patched before - being rerun. - - -%package client -Version: %{version} -Release: %{release} -Summary: Client for Oozie Workflow Engine -URL: http://oozie.apache.org/ -Group: Development/Libraries -License: ASL 2.0 -BuildArch: noarch -Requires: bigtop-utils >= 0.7 - - -%description client - Oozie client is a command line client utility that allows remote - administration and monitoring of worflows. Using this client utility - you can submit worflows, start/suspend/resume/kill workflows and - find out their status at any instance. Apart from such operations, - you can also change the status of the entire system, get vesion - information. This client utility also allows you to validate - any worflows before they are deployed to the Oozie server. - - -%prep -%setup -n oozie-%{oozie_base_version} - -#BIGTOP_PATCH_COMMANDS - -%build - mkdir -p distro/downloads - env DO_MAVEN_DEPLOY="" FULL_VERSION=%{oozie_base_version} bash -x %{SOURCE1} - -%install -%__rm -rf $RPM_BUILD_ROOT - sh %{SOURCE2} --extra-dir=$RPM_SOURCE_DIR --build-dir=$PWD --server-dir=$RPM_BUILD_ROOT --client-dir=$RPM_BUILD_ROOT --docs-dir=$RPM_BUILD_ROOT%{doc_oozie} --initd-dir=$RPM_BUILD_ROOT%{initd_dir} --conf-dir=$RPM_BUILD_ROOT%{conf_oozie_dist} - -%__ln_s -f %{data_oozie}/ext-2.2 $RPM_BUILD_ROOT/%{lib_oozie}/webapps/oozie/ext-2.2 -%__rm -rf $RPM_BUILD_ROOT/%{lib_oozie}/webapps/oozie/docs -%__ln_s -f %{doc_oozie} $RPM_BUILD_ROOT/%{lib_oozie}/webapps/oozie/docs - -# Oozie server -%__rm -rf $RPM_BUILD_ROOT/%{lib_oozie}/lib/hadoop-*.jar -%__ln_s -f %{lib_hadoop}/client/hadoop-annotations.jar $RPM_BUILD_ROOT/%{lib_oozie}/lib/ -%__ln_s -f %{lib_hadoop}/client/hadoop-auth.jar $RPM_BUILD_ROOT/%{lib_oozie}/lib/ -%__ln_s -f %{lib_hadoop}/client/hadoop-common.jar $RPM_BUILD_ROOT/%{lib_oozie}/lib/ -%__ln_s -f %{lib_hadoop}/client/hadoop-hdfs.jar $RPM_BUILD_ROOT/%{lib_oozie}/lib/ -%__ln_s -f %{lib_hadoop}/client/hadoop-mapreduce-client-app.jar $RPM_BUILD_ROOT/%{lib_oozie}/lib/ -%__ln_s -f %{lib_hadoop}/client/hadoop-mapreduce-client-common.jar $RPM_BUILD_ROOT/%{lib_oozie}/lib/ -%__ln_s -f %{lib_hadoop}/client/hadoop-mapreduce-client-core.jar $RPM_BUILD_ROOT/%{lib_oozie}/lib/ -%__ln_s -f %{lib_hadoop}/client/hadoop-mapreduce-client-jobclient.jar $RPM_BUILD_ROOT/%{lib_oozie}/lib/ -%__ln_s -f %{lib_hadoop}/client/hadoop-mapreduce-client-shuffle.jar $RPM_BUILD_ROOT/%{lib_oozie}/lib/ -%__ln_s -f %{lib_hadoop}/client/hadoop-yarn-api.jar $RPM_BUILD_ROOT/%{lib_oozie}/lib/ -%__ln_s -f %{lib_hadoop}/client/hadoop-yarn-client.jar $RPM_BUILD_ROOT/%{lib_oozie}/lib/ -%__ln_s -f %{lib_hadoop}/client/hadoop-yarn-common.jar $RPM_BUILD_ROOT/%{lib_oozie}/lib/ -%__ln_s -f %{lib_hadoop}/client/hadoop-yarn-server-common.jar $RPM_BUILD_ROOT/%{lib_oozie}/lib/ - -%__install -d -m 0755 $RPM_BUILD_ROOT/usr/bin - -%__install -d -m 0755 %{buildroot}/%{_localstatedir}/log/oozie -%__install -d -m 0755 %{buildroot}/%{_localstatedir}/run/oozie - -%pre -getent group oozie >/dev/null || /usr/sbin/groupadd -r oozie >/dev/null -getent passwd oozie >/dev/null || /usr/sbin/useradd --comment "Oozie User" --shell /bin/false -M -r -g oozie --home %{data_oozie} oozie >/dev/null - -%post -%{alternatives_cmd} --install %{conf_oozie} %{name}-conf %{conf_oozie_dist} 30 -%{alternatives_cmd} --install %{tomcat_conf_oozie} %{name}-tomcat-conf %{tomcat_conf_oozie}.http 30 -%{alternatives_cmd} --install %{tomcat_conf_oozie} %{name}-tomcat-conf %{tomcat_conf_oozie}.https 20 - -/sbin/chkconfig --add oozie - -%preun -if [ "$1" = 0 ]; then - rm -r /etc/oozie/conf/tomcat-conf - /sbin/service oozie stop > /dev/null - /sbin/chkconfig --del oozie - %{alternatives_cmd} --remove %{name}-tomcat-conf %{tomcat_conf_oozie}.http || : - %{alternatives_cmd} --remove %{name}-tomcat-conf %{tomcat_conf_oozie}.https || : - %{alternatives_cmd} --remove %{name}-conf %{conf_oozie_dist} || : -fi - -%postun -if [ $1 -ge 1 ]; then - /sbin/service oozie condrestart > /dev/null -fi - -%files -%defattr(-,root,root) -%config(noreplace) %{conf_oozie_dist} -%config(noreplace) %{tomcat_conf_oozie}.* -%{usr_bin}/oozie-setup -%{lib_oozie}/bin/oozie-sys.sh -%{lib_oozie}/bin/oozie-env.sh -%{lib_oozie}/bin/oozied.sh -%{lib_oozie}/bin/ooziedb.sh -%{lib_oozie}/bin/oozie-setup.sh -%{lib_oozie}/webapps -%{lib_oozie}/libtools -%{lib_oozie}/lib -%{lib_oozie}/oozie-sharelib.tar.gz -%{lib_oozie}/libext -%{lib_oozie}/tomcat-deployment.sh -%{initd_dir}/oozie -%defattr(-, oozie, oozie) -%dir %{_sysconfdir}/%{name} -%dir %{_localstatedir}/log/oozie -%dir %{_localstatedir}/run/oozie -%attr(0755,oozie,oozie) %{data_oozie} - -%files client -%defattr(-,root,root) -%{usr_bin}/oozie -%dir %{lib_oozie} -%{lib_oozie}/bin/oozie -%{lib_oozie}/conf/oozie-client-env.sh -%{lib_oozie}/lib -%doc %{doc_oozie} -%{man_dir}/man1/oozie.1.*
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a745f030/bigtop-packages/src/rpm/oozie/SRPMS/.gitignore ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/oozie/SRPMS/.gitignore b/bigtop-packages/src/rpm/oozie/SRPMS/.gitignore deleted file mode 100644 index e69de29..0000000 http://git-wip-us.apache.org/repos/asf/bigtop/blob/a745f030/bigtop-tests/smoke-tests/oozie/build.gradle ---------------------------------------------------------------------- diff --git a/bigtop-tests/smoke-tests/oozie/build.gradle b/bigtop-tests/smoke-tests/oozie/build.gradle deleted file mode 100644 index 9d9f8f0..0000000 --- a/bigtop-tests/smoke-tests/oozie/build.gradle +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -def tests_to_include() { - return [ - "TestOozieSmoke.groovy" - ]; -} - -dependencies { - if (System.env.HADOOP_CONF_DIR) testRuntime files(System.env.HADOOP_CONF_DIR) -} - -sourceSets { - test { - groovy { - srcDirs = [ - "${BIGTOP_HOME}/bigtop-tests/test-artifacts/oozie/src/main/groovy/org/apache/bigtop/itest/ooziesmoke" - ] - exclude { - FileTreeElement elem -> (doExclude(elem.getName())) - } - } - } -} - -test.doFirst { - checkEnv(["HADOOP_CONF_DIR", "OOZIE_URL", "OOZIE_TAR_HOME"]) - // OOZIE_TAR_HOME should point to the directory where oozie-examples.tar.gz is located - // Usually, it could be found in $HADOOP_HOME/share/doc/oozie - systemProperty "org.apache.bigtop.itest.oozie_tar_home", System.getenv('OOZIE_TAR_HOME') -} http://git-wip-us.apache.org/repos/asf/bigtop/blob/a745f030/bigtop-tests/test-artifacts/fatjar/pom.xml ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/fatjar/pom.xml b/bigtop-tests/test-artifacts/fatjar/pom.xml index 961747e..28f5370 100644 --- a/bigtop-tests/test-artifacts/fatjar/pom.xml +++ b/bigtop-tests/test-artifacts/fatjar/pom.xml @@ -75,12 +75,6 @@ </dependency> <dependency> <groupId>org.apache.bigtop.itest</groupId> - <artifactId>oozie-smoke</artifactId> - <version>${project.version}</version> - <optional>true</optional> - </dependency> - <dependency> - <groupId>org.apache.bigtop.itest</groupId> <artifactId>solr-smoke</artifactId> <version>${project.version}</version> <optional>true</optional> http://git-wip-us.apache.org/repos/asf/bigtop/blob/a745f030/bigtop-tests/test-artifacts/oozie/pom.xml ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/oozie/pom.xml b/bigtop-tests/test-artifacts/oozie/pom.xml deleted file mode 100644 index 22fc597..0000000 --- a/bigtop-tests/test-artifacts/oozie/pom.xml +++ /dev/null @@ -1,39 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> - - <parent> - <groupId>org.apache.bigtop.itest</groupId> - <artifactId>bigtop-smokes</artifactId> - <version>1.3.0</version> - <relativePath>../pom.xml</relativePath> - </parent> - - <groupId>org.apache.bigtop.itest</groupId> - <artifactId>oozie-smoke</artifactId> - <version>1.3.0</version> - <name>ooziesmoke</name> - - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - </dependency> - </dependencies> -</project> http://git-wip-us.apache.org/repos/asf/bigtop/blob/a745f030/bigtop-tests/test-artifacts/oozie/src/main/groovy/org/apache/bigtop/itest/ooziesmoke/TestOozieSmoke.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/oozie/src/main/groovy/org/apache/bigtop/itest/ooziesmoke/TestOozieSmoke.groovy b/bigtop-tests/test-artifacts/oozie/src/main/groovy/org/apache/bigtop/itest/ooziesmoke/TestOozieSmoke.groovy deleted file mode 100644 index 657dd65..0000000 --- a/bigtop-tests/test-artifacts/oozie/src/main/groovy/org/apache/bigtop/itest/ooziesmoke/TestOozieSmoke.groovy +++ /dev/null @@ -1,141 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.bigtop.itest.ooziesmoke - -import org.junit.Test - -import static junit.framework.Assert.assertNotNull -import static org.junit.Assert.assertTrue -import org.apache.bigtop.itest.shell.Shell -import org.junit.AfterClass -import org.junit.BeforeClass -import static junit.framework.Assert.assertEquals -import org.apache.hadoop.conf.Configuration - -class TestOozieSmoke { - private static final int WAIT_TIMEOUT = 60000; - private static Shell sh = new Shell("/bin/bash -s"); - private static String tmp_dir = "oozie.${(new Date().getTime())}"; - - private static String oozie_url; - private static String resourcemanager; - private static String namenode; - private static String oozie_tar_home; - - @BeforeClass - static void setUp() { - Configuration conf = new Configuration(); - conf.addResource('yarn-site.xml'); - - oozie_url = System.getProperty("org.apache.bigtop.itest.oozie_url", "http://localhost:11000/oozie";); - resourcemanager = conf.get("yarn.resourcemanager.address") - resourcemanager = System.getProperty("org.apache.bigtop.itest.resourcemanager", resourcemanager); - namenode = conf.get('fs.defaultFS') ? conf.get('fs.defaultFS') : conf.get('fs.default.name') - namenode = System.getProperty("org.apache.bigtop.itest.namenode", namenode); - assertNotNull("resourcemanager hostname isn't set", resourcemanager) - assertNotNull("namenode hostname isn't set", namenode) - - oozie_tar_home = System.getProperty("org.apache.bigtop.itest.oozie_tar_home", - (new File("/usr/share/doc/packages/oozie/")).exists() ? - "/usr/share/doc/packages/oozie/" : - "/usr/share/doc/oozie*/"); - - sh.exec("mkdir /tmp/${tmp_dir}", - "cd /tmp/${tmp_dir}", - "tar xzf ${oozie_tar_home}/oozie-examples.tar.gz", - "hadoop fs -mkdir ${tmp_dir}", - "hadoop fs -put examples ${tmp_dir}"); - assertEquals("Failed to put examples onto HDFS", - 0, sh.ret); - } - - @AfterClass - static void tearDown() { - sh.exec("rm -rf /tmp/${tmp_dir}", - "hadoop fs -rmr ${tmp_dir}"); - } - - void testOozieExamplesCommon(String testname) { - sh.exec("oozie job -oozie ${oozie_url} -run -DjobTracker=${resourcemanager} -DnameNode=${namenode} " + - "-DexamplesRoot=${tmp_dir}/examples -config /tmp/${tmp_dir}/examples/apps/${testname}/job.properties"); - assertEquals("Oozie job submition ${testname} failed", - 0, sh.ret); - - String jobId = sh.out[0].replaceAll(/job: /, ""); - while (sh.exec("oozie job -oozie ${oozie_url} -info ${jobId}").out.join(' ') =~ /Status\s*:\s*RUNNING/) { - sleep(WAIT_TIMEOUT); - } - assertTrue("Oozie job ${testname} returned ${sh.out.join(' ')} instead of SUCCEEDED", - (sh.out.join(' ') =~ /Status\s*:\s*SUCCEEDED/).find()); - } - - @Test(timeout = 300000L) - public void testNoOp() { - testOozieExamplesCommon("no-op"); - } - - @Test(timeout = 300000L) - public void testJavaMain() { - testOozieExamplesCommon("java-main"); - } - - @Test(timeout = 300000L) - public void testMapReduce() { - testOozieExamplesCommon("map-reduce"); - } - - @Test(timeout = 300000L) - public void testCustomMain() { - testOozieExamplesCommon("custom-main"); - } - - @Test(timeout = 300000L) - public void testHadoopEl() { - testOozieExamplesCommon("hadoop-el"); - } - - @Test(timeout = 300000L) - public void testStreaming() { - testOozieExamplesCommon("streaming"); - } - - @Test(timeout = 300000L) - public void testPig() { - testOozieExamplesCommon("pig"); - } - - @Test(timeout = 300000L) - public void testHive() { - testOozieExamplesCommon("hive"); - } - - @Test(timeout = 300000L) - public void testSubwf() { - testOozieExamplesCommon("subwf"); - } - - @Test(timeout = 300000L) - public void testSsh() { - // testOozieExamplesCommon("ssh"); - } - - @Test(timeout = 300000L) - public void testDemo() { - // testOozieExamplesCommon("demo"); - } -}
