Repository: oozie
Updated Branches:
  refs/heads/master a16de8980 -> 00275b71a


OOZIE-2273 MiniOozie does not work outside of Oozie (rkanter)


Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/00275b71
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/00275b71
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/00275b71

Branch: refs/heads/master
Commit: 00275b71a626a5035b9802309aadf7df378c5e59
Parents: a16de89
Author: Robert Kanter <rkan...@cloudera.com>
Authored: Tue Sep 20 15:16:29 2016 -0700
Committer: Robert Kanter <rkan...@cloudera.com>
Committed: Tue Sep 20 15:16:29 2016 -0700

----------------------------------------------------------------------
 .../command/coord/TestCoordRerunXCommand.java   |  2 +-
 ...TestHAPartitionDependencyManagerService.java |  1 +
 .../org/apache/oozie/test/XHCatTestCase.java    |  1 +
 .../java/org/apache/oozie/test/XTestCase.java   | 82 +++++++++++++++-----
 .../oozie/example/TestLocalOozieExample.java    |  1 -
 .../org/apache/oozie/test/WorkflowTest.java     |  1 -
 release-log.txt                                 |  1 +
 .../action/hadoop/TestPigActionExecutor.java    |  1 -
 .../apache/oozie/action/hadoop/TestPigMain.java |  1 -
 9 files changed, 67 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/oozie/blob/00275b71/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java
----------------------------------------------------------------------
diff --git 
a/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java 
b/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java
index cdfc9d6..891e6ec 100644
--- 
a/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java
+++ 
b/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java
@@ -569,7 +569,7 @@ public class TestCoordRerunXCommand extends XDataTestCase {
      * @throws Exception
      */
     public void testCoordRerunCleanupForHCat() throws Exception {
-
+        super.setupHCatalogServer();
         services = super.setupServicesForHCatalog();
         services.init();
 

http://git-wip-us.apache.org/repos/asf/oozie/blob/00275b71/core/src/test/java/org/apache/oozie/service/TestHAPartitionDependencyManagerService.java
----------------------------------------------------------------------
diff --git 
a/core/src/test/java/org/apache/oozie/service/TestHAPartitionDependencyManagerService.java
 
b/core/src/test/java/org/apache/oozie/service/TestHAPartitionDependencyManagerService.java
index d681d42..3e1df07 100644
--- 
a/core/src/test/java/org/apache/oozie/service/TestHAPartitionDependencyManagerService.java
+++ 
b/core/src/test/java/org/apache/oozie/service/TestHAPartitionDependencyManagerService.java
@@ -47,6 +47,7 @@ public class TestHAPartitionDependencyManagerService extends 
ZKXTestCase {
     protected void setUp() throws Exception {
         super.setUp();
         services = super.setupServicesForHCatalog(Services.get());
+        super.setupHCatalogServer();
         // disable recovery service
         services.getConf().setInt(RecoveryService.CONF_SERVICE_INTERVAL, 
1000000);
         // disable regular cache purge

http://git-wip-us.apache.org/repos/asf/oozie/blob/00275b71/core/src/test/java/org/apache/oozie/test/XHCatTestCase.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/test/XHCatTestCase.java 
b/core/src/test/java/org/apache/oozie/test/XHCatTestCase.java
index 85ee1f2..2adbee7 100644
--- a/core/src/test/java/org/apache/oozie/test/XHCatTestCase.java
+++ b/core/src/test/java/org/apache/oozie/test/XHCatTestCase.java
@@ -40,6 +40,7 @@ public abstract class XHCatTestCase extends XFsTestCase {
     @Override
     protected void setUp() throws Exception {
         super.setUp();
+        super.setupHCatalogServer();
         hcatServer = super.getHCatalogServer();
     }
 

http://git-wip-us.apache.org/repos/asf/oozie/blob/00275b71/core/src/test/java/org/apache/oozie/test/XTestCase.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/test/XTestCase.java 
b/core/src/test/java/org/apache/oozie/test/XTestCase.java
index 3dd99d7..70f77be 100644
--- a/core/src/test/java/org/apache/oozie/test/XTestCase.java
+++ b/core/src/test/java/org/apache/oozie/test/XTestCase.java
@@ -130,12 +130,12 @@ public abstract class XTestCase extends TestCase {
                 OOZIE_SRC_DIR = new File(OOZIE_SRC_DIR, "core");
             }
             if (!OOZIE_SRC_DIR.exists()) {
-                System.err.println();
-                System.err.println("Could not determine project root 
directory");
-                System.err.println();
-                System.exit(-1);
+                // We're probably being run from outside of Oozie (e.g. 
MiniOozie), so just use a dummy location here.
+                // Anything that uses this location should have a fallback 
anyway.
+                OOZIE_SRC_DIR = new File(".");
+            } else {
+                OOZIE_SRC_DIR = OOZIE_SRC_DIR.getParentFile();
             }
-            OOZIE_SRC_DIR = OOZIE_SRC_DIR.getParentFile();
 
             String testPropsFile = System.getProperty(OOZIE_TEST_PROPERTIES, 
"test.properties");
             File file = new File(testPropsFile).isAbsolute()
@@ -343,7 +343,7 @@ public abstract class XTestCase extends TestCase {
             else {
                 // If we still can't find it, then exit
                 System.err.println();
-                System.err.println(XLog.format("Custom configuration file for 
testing does no exist [{0}]",
+                System.err.println(XLog.format("Custom configuration file for 
testing does not exist [{0}]",
                                                source.getAbsolutePath()));
                 System.err.println();
                 System.exit(-1);
@@ -369,8 +369,26 @@ public abstract class XTestCase extends TestCase {
         File actionConfDir = new File(testCaseConfDir, "action-conf");
         actionConfDir.mkdir();
         source = new File(OOZIE_SRC_DIR, 
"core/src/test/resources/hadoop-config.xml");
+        InputStream hadoopConfigResourceStream = null;
+        if (!source.exists()) {
+            // If we can't find it, try using the class loader (useful if 
we're using XTestCase from outside core)
+            URL sourceURL = 
getClass().getClassLoader().getResource("hadoop-config.xml");
+            if (sourceURL != null) {
+                hadoopConfigResourceStream = sourceURL.openStream();
+            }
+            else {
+                // If we still can't find it, then exit
+                System.err.println();
+                System.err.println(XLog.format("hadoop-config.xml 
configuration file for testing does not exist [{0}]",
+                        source.getAbsolutePath()));
+                System.err.println();
+                System.exit(-1);
+            }
+        } else {
+            hadoopConfigResourceStream = new FileInputStream(source);
+        }
         target = new File(hadoopConfDir, "hadoop-site.xml");
-        IOUtils.copyStream(new FileInputStream(source), new 
FileOutputStream(target));
+        IOUtils.copyStream(hadoopConfigResourceStream, new 
FileOutputStream(target));
 
         if (System.getProperty("oozielocal.log") == null) {
             setSystemProperty("oozielocal.log", "/tmp/oozielocal.log");
@@ -400,10 +418,14 @@ public abstract class XTestCase extends TestCase {
             os.close();
         }
 
-        if (System.getProperty("oozie.test.metastore.server", 
"true").equals("true")) {
+        if (System.getProperty("oozie.test.metastore.server", 
"false").equals("true")) {
             setupHCatalogServer();
         }
 
+        if (System.getProperty("oozie.test.hive.server.2", 
"false").equals("true")) {
+            setupHiveServer2();
+        }
+
         // Cleanup any leftover database data to make sure we start each test 
with an empty database
         if (cleanUpDBTables) {
             cleanUpDBTables();
@@ -415,10 +437,8 @@ public abstract class XTestCase extends TestCase {
      */
     @Override
     protected void tearDown() throws Exception {
-        if (hiveserver2 != null && hiveserver2.isStarted()) {
-            hiveserver2.stop();
-            hiveserver2 = null;
-        }
+        tearDownHiveServer2();
+        tearDownHCatalogServer();
         resetSystemProperties();
         sysProps = null;
         testCaseDir = null;
@@ -881,6 +901,7 @@ public abstract class XTestCase extends TestCase {
     private static MiniMRCluster mrCluster = null;
     private static MiniHCatServer hcatServer = null;
     private static MiniHS2 hiveserver2 = null;
+    private static HiveConf hs2Config = null;
 
     private void setUpEmbeddedHadoop(String testCaseDir) throws Exception {
         if (dfsCluster == null && mrCluster == null) {
@@ -989,7 +1010,7 @@ public abstract class XTestCase extends TestCase {
       return conf;
     }
 
-    private void setupHCatalogServer() throws Exception {
+    protected void setupHCatalogServer() throws Exception {
         if (hcatServer == null) {
             hcatServer = new MiniHCatServer(RUNMODE.SERVER, createJobConf());
             hcatServer.start();
@@ -997,17 +1018,38 @@ public abstract class XTestCase extends TestCase {
         }
     }
 
+    protected void tearDownHCatalogServer() throws Exception {
+        // TODO: This doesn't properly shutdown the metastore.  For now, just 
keep the current one running once it's been started
+//        if (hcatServer != null) {
+//            hcatServer.shutdown();
+//            hcatServer = null;
+//            log.info("Metastore server shutdown");
+//        }
+    }
+
     protected void setupHiveServer2() throws Exception {
         if (hiveserver2 == null) {
             setSystemProperty("test.tmp.dir", getTestCaseDir());
-            // Make HS2 use our Mini cluster by copying all configs to 
HiveConf; also had to hack MiniHS2
-            HiveConf hconf = new HiveConf();
-            Configuration jobConf = createJobConf();
-            for (Map.Entry<String, String> pair: jobConf) {
-                hconf.set(pair.getKey(), pair.getValue());
+            // We cache the HS2 config because it's expensive to build
+            if (hs2Config == null) {
+                // Make HS2 use our Mini cluster by copying all configs to 
HiveConf; also had to hack MiniHS2
+                hs2Config = new HiveConf();
+                Configuration jobConf = createJobConf();
+                for (Map.Entry<String, String> pair : jobConf) {
+                    hs2Config.set(pair.getKey(), pair.getValue());
+                }
             }
-            hiveserver2 = new MiniHS2(hconf, dfsCluster.getFileSystem());
+            hiveserver2 = new MiniHS2(hs2Config, dfsCluster.getFileSystem());
             hiveserver2.start(new HashMap<String, String>());
+            log.info("Hive Server 2 started at " + hiveserver2.getJdbcURL());
+        }
+    }
+
+    protected void tearDownHiveServer2() {
+        if (hiveserver2 != null && hiveserver2.isStarted()) {
+            hiveserver2.stop();
+            hiveserver2 = null;
+            log.info("Hive Server 2 shutdown");
         }
     }
 
@@ -1036,6 +1078,8 @@ public abstract class XTestCase extends TestCase {
         catch (Exception ex) {
             System.out.println(ex);
         }
+        // This is tied to the MiniCluster because it inherits configs from 
there
+        hs2Config = null;
     }
 
     private static void shutdownMiniCluster2() {

http://git-wip-us.apache.org/repos/asf/oozie/blob/00275b71/examples/src/test/java/org/apache/oozie/example/TestLocalOozieExample.java
----------------------------------------------------------------------
diff --git 
a/examples/src/test/java/org/apache/oozie/example/TestLocalOozieExample.java 
b/examples/src/test/java/org/apache/oozie/example/TestLocalOozieExample.java
index 2c819b7..9b7bfe4 100644
--- a/examples/src/test/java/org/apache/oozie/example/TestLocalOozieExample.java
+++ b/examples/src/test/java/org/apache/oozie/example/TestLocalOozieExample.java
@@ -34,7 +34,6 @@ public class TestLocalOozieExample extends XFsTestCase {
 
     @Override
     protected void setUp() throws Exception {
-        System.setProperty("oozie.test.metastore.server", "false");
         super.setUp();
         oozieLocalLog = System.getProperty("oozielocal.log");
         System.setProperty("oozielocal.log", 
getTestCaseDir()+"/oozielocal.log");

http://git-wip-us.apache.org/repos/asf/oozie/blob/00275b71/minitest/src/test/java/org/apache/oozie/test/WorkflowTest.java
----------------------------------------------------------------------
diff --git a/minitest/src/test/java/org/apache/oozie/test/WorkflowTest.java 
b/minitest/src/test/java/org/apache/oozie/test/WorkflowTest.java
index 64027d9..2845f0a 100644
--- a/minitest/src/test/java/org/apache/oozie/test/WorkflowTest.java
+++ b/minitest/src/test/java/org/apache/oozie/test/WorkflowTest.java
@@ -42,7 +42,6 @@ public class WorkflowTest extends MiniOozieTestCase {
 
     @Override
     protected void setUp() throws Exception {
-        System.setProperty("oozie.test.metastore.server", "false");
         System.setProperty(XLogService.LOG4J_FILE, "oozie-log4j.properties");
         super.setUp();
     }

http://git-wip-us.apache.org/repos/asf/oozie/blob/00275b71/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 0dd42d0..71d9dfe 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
 -- Oozie 4.3.0 release (trunk - unreleased)
 
+OOZIE-2273 MiniOozie does not work outside of Oozie (rkanter)
 OOZIE-2667 Optimize queries for DB export (gezapeti via rkanter)
 OOZIE-1793 Improve find bugs reporting for Oozie (rkanter)
 OOZIE-2572 SLA DURATION miss not shown when job is running for longer than 
expected time (satishsaley via puru)

http://git-wip-us.apache.org/repos/asf/oozie/blob/00275b71/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java
----------------------------------------------------------------------
diff --git 
a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java
 
b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java
index 09d3da3..25092ce 100644
--- 
a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java
+++ 
b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java
@@ -76,7 +76,6 @@ public class TestPigActionExecutor extends 
ActionExecutorTestCase {
 
     @Override
     protected void setUp() throws Exception {
-        System.setProperty("oozie.test.metastore.server", "false");
         super.setUp();
         PigTestCase.resetPigStats();
     }

http://git-wip-us.apache.org/repos/asf/oozie/blob/00275b71/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java
----------------------------------------------------------------------
diff --git 
a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java 
b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java
index 1ea612d..081d319 100644
--- a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java
+++ b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java
@@ -40,7 +40,6 @@ public class TestPigMain extends PigTestCase {
 
     @Override
     protected void setUp() throws Exception {
-        System.setProperty("oozie.test.metastore.server", "false");
         super.setUp();
         SECURITY_MANAGER = System.getSecurityManager();
         resetPigStats();

Reply via email to