Repository: oozie Updated Branches: refs/heads/master 800ea758e -> 0601fc228
OOZIE-3038 Make all Oozie JUnit tests pass on dist_test (andras.piros via gezapeti) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/0601fc22 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/0601fc22 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/0601fc22 Branch: refs/heads/master Commit: 0601fc228ce2d5a9610efd7644e5adab611f8a64 Parents: 800ea75 Author: Gezapeti Cseh <gezap...@gmail.com> Authored: Mon Aug 28 17:23:15 2017 +0200 Committer: Gezapeti Cseh <gezap...@gmail.com> Committed: Mon Aug 28 17:23:18 2017 +0200 ---------------------------------------------------------------------- .../oozie/action/ssh/TestSshActionExecutor.java | 30 +++++++ .../apache/oozie/servlet/TestAdminServlet.java | 5 +- .../oozie/servlet/TestV1AdminServlet.java | 5 +- .../java/org/apache/oozie/test/XTestCase.java | 28 ++++-- .../apache/oozie/util/TestGraphGenerator.java | 95 ++++++-------------- release-log.txt | 1 + 6 files changed, 87 insertions(+), 77 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/0601fc22/core/src/test/java/org/apache/oozie/action/ssh/TestSshActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/ssh/TestSshActionExecutor.java b/core/src/test/java/org/apache/oozie/action/ssh/TestSshActionExecutor.java index 16dd29a..00daa77 100644 --- a/core/src/test/java/org/apache/oozie/action/ssh/TestSshActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/ssh/TestSshActionExecutor.java @@ -159,6 +159,10 @@ public class TestSshActionExecutor extends XFsTestCase { @Override protected void setUp() throws Exception { + if (!isSshPresent()) { + return; + } + super.setUp(); services = new Services(); services.init(); @@ -170,6 +174,19 @@ public class TestSshActionExecutor extends XFsTestCase { fs.delete(path, true); } + private boolean isSshPresent() { + int exitValue; + try { + final Process process = Runtime.getRuntime().exec("ssh -V"); + process.waitFor(); + exitValue = process.exitValue(); + } catch (final IOException | InterruptedException e) { + exitValue = 1; + } + + return exitValue == 0; + } + protected String getActionXMLSchema() { return "uri:oozie-workflow:0.1"; } @@ -647,7 +664,20 @@ public class TestSshActionExecutor extends XFsTestCase { @Override protected void tearDown() throws Exception { + if (!isSshPresent()) { + return; + } + services.destroy(); super.tearDown(); } + + @Override + protected void runTest() throws Throwable { + if (!isSshPresent()) { + return; + } + + super.runTest(); + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/0601fc22/core/src/test/java/org/apache/oozie/servlet/TestAdminServlet.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/servlet/TestAdminServlet.java b/core/src/test/java/org/apache/oozie/servlet/TestAdminServlet.java index 595cb26..0d7b362 100644 --- a/core/src/test/java/org/apache/oozie/servlet/TestAdminServlet.java +++ b/core/src/test/java/org/apache/oozie/servlet/TestAdminServlet.java @@ -71,7 +71,10 @@ public class TestAdminServlet extends DagServletTestCase { assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); assertTrue(conn.getHeaderField("content-type").startsWith(RestConstants.JSON_CONTENT_TYPE)); JSONObject json = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream())); - assertTrue(json.containsKey(Shell.WINDOWS ? "USERNAME" : "USER")); + assertTrue("USERNAME, USER or HOME property not found in json", + json.containsKey(Shell.WINDOWS ? "USERNAME" : "USER") || + json.containsKey("HOME") + ); return null; } }); http://git-wip-us.apache.org/repos/asf/oozie/blob/0601fc22/core/src/test/java/org/apache/oozie/servlet/TestV1AdminServlet.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/servlet/TestV1AdminServlet.java b/core/src/test/java/org/apache/oozie/servlet/TestV1AdminServlet.java index 1e66537..0113751 100644 --- a/core/src/test/java/org/apache/oozie/servlet/TestV1AdminServlet.java +++ b/core/src/test/java/org/apache/oozie/servlet/TestV1AdminServlet.java @@ -75,7 +75,10 @@ public class TestV1AdminServlet extends DagServletTestCase { assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); assertTrue(conn.getHeaderField("content-type").startsWith(RestConstants.JSON_CONTENT_TYPE)); JSONObject json = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream())); - assertTrue(json.containsKey(Shell.WINDOWS ? "USERNAME" : "USER")); + assertTrue("USERNAME, USER or HOME property not found in json", + json.containsKey(Shell.WINDOWS ? "USERNAME" : "USER") || + json.containsKey("HOME") + ); return null; } }); http://git-wip-us.apache.org/repos/asf/oozie/blob/0601fc22/core/src/test/java/org/apache/oozie/test/XTestCase.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/test/XTestCase.java b/core/src/test/java/org/apache/oozie/test/XTestCase.java index 50e2452..584aa12 100644 --- a/core/src/test/java/org/apache/oozie/test/XTestCase.java +++ b/core/src/test/java/org/apache/oozie/test/XTestCase.java @@ -336,10 +336,11 @@ public abstract class XTestCase extends TestCase { testCaseConfDir = createTestCaseSubDir("conf"); // load test Oozie site - String oozieTestDB = System.getProperty("oozie.test.db", "hsqldb"); - String defaultOozieSize = - new File(OOZIE_SRC_DIR, "core/src/test/resources/" + oozieTestDB + "-oozie-site.xml").getAbsolutePath(); - String customOozieSite = System.getProperty("oozie.test.config.file", defaultOozieSize); + final String oozieTestDB = System.getProperty("oozie.test.db", "hsqldb"); + final String oozieSiteFileName = oozieTestDB + "-oozie-site.xml"; + final String defaultOozieSite = + new File(OOZIE_SRC_DIR, "core/src/test/resources/" + oozieSiteFileName).getAbsolutePath(); + final String customOozieSite = System.getProperty("oozie.test.config.file", defaultOozieSite); File source = new File(customOozieSite); if(!source.isAbsolute()) { source = new File(OOZIE_SRC_DIR, customOozieSite); @@ -347,11 +348,28 @@ public abstract class XTestCase extends TestCase { source = source.getAbsoluteFile(); InputStream oozieSiteSourceStream = null; if (source.exists()) { + log.info("Reading Oozie test resource from file. [source.name={0}]", source.getName()); oozieSiteSourceStream = new FileInputStream(source); } else { // If we can't find it, try using the class loader (useful if we're using XTestCase from outside core) - URL sourceURL = getClass().getClassLoader().getResource(oozieTestDB + "-oozie-site.xml"); + log.info("Oozie test resource file doesn't exist. [source.name={0}]", source.getName()); + final String testResourceName; + if (customOozieSite.lastIndexOf(Path.SEPARATOR) > -1) { + final String customOozieSiteFileName = customOozieSite.substring(customOozieSite.lastIndexOf(Path.SEPARATOR) + 1); + if (customOozieSiteFileName.equals(oozieSiteFileName)) { + testResourceName = oozieSiteFileName; + } + else { + testResourceName = customOozieSiteFileName; + } + } + else { + testResourceName = oozieSiteFileName; + } + log.info("Reading Oozie test resource from classpath. [testResourceName={0};source.name={1}]", + testResourceName, source.getName()); + final URL sourceURL = getClass().getClassLoader().getResource(testResourceName); if (sourceURL != null) { oozieSiteSourceStream = sourceURL.openStream(); } http://git-wip-us.apache.org/repos/asf/oozie/blob/0601fc22/core/src/test/java/org/apache/oozie/util/TestGraphGenerator.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/util/TestGraphGenerator.java b/core/src/test/java/org/apache/oozie/util/TestGraphGenerator.java index a92d468..002e925 100644 --- a/core/src/test/java/org/apache/oozie/util/TestGraphGenerator.java +++ b/core/src/test/java/org/apache/oozie/util/TestGraphGenerator.java @@ -27,9 +27,6 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; -import java.nio.MappedByteBuffer; -import java.nio.channels.FileChannel; -import java.nio.charset.Charset; public class TestGraphGenerator extends XTestCase { @@ -37,18 +34,18 @@ public class TestGraphGenerator extends XTestCase { try { new GraphGenerator(null, null); } - catch (IllegalArgumentException iae) { + catch (final IllegalArgumentException iae) { Assert.assertTrue("Construction with illegal args failed as expected: " + iae.getMessage(), true); } try { new GraphGenerator("<workflow></workflow>", null); } - catch (IllegalArgumentException iae) { + catch (final IllegalArgumentException iae) { Assert.assertTrue("Construction with illegal args failed as expected: " + iae.getMessage(), true); } Assert.assertNotNull(new GraphGenerator("<workflow></workflow>", new WorkflowJobBean())); Assert.assertNotNull(new GraphGenerator(null, new WorkflowJobBean())); - WorkflowJobBean jsonWFJob = new WorkflowJobBean(); + final WorkflowJobBean jsonWFJob = new WorkflowJobBean(); jsonWFJob.setAppName("My Test App"); jsonWFJob.setId("My Test ID"); Assert.assertNotNull(new GraphGenerator("<workflow></workflow>", jsonWFJob)); @@ -57,94 +54,52 @@ public class TestGraphGenerator extends XTestCase { } public void testWrite() { - WorkflowJobBean jsonWFJob = new WorkflowJobBean(); + final WorkflowJobBean jsonWFJob = new WorkflowJobBean(); jsonWFJob.setAppName("My Test App"); jsonWFJob.setId("My Test ID"); - String png1 = "src/test/resources/tmp1.png"; - String png2 = "src/test/resources/tmp2.png"; - try { - GraphGenerator g = new GraphGenerator(readFile("src/test/resources/graphWF.xml"), jsonWFJob); - g.write(new FileOutputStream(new File(png1))); - } - catch (Exception e) { - Assert.fail("Write PNG failed for graphWF.xml: " + e.getMessage()); - } + generateAndAssertPng(jsonWFJob, "graphWF.xml", false); - File f1 = new File(png1); - try { - // Check if a valid file was written - Assert.assertNotNull(ImageIO.read(f1)); - } - catch (IOException io) { - Assert.fail("Not a valid PNG: " + io.getMessage()); - } + generateAndAssertPng(jsonWFJob, "graphWF.xml", true); try { - GraphGenerator g = new GraphGenerator(readFile("src/test/resources/graphWF.xml"), jsonWFJob, true); - g.write(new FileOutputStream(new File(png2))); + final String content = IOUtils.getResourceAsString("invalidGraphWF.xml", -1); + final GraphGenerator g = new GraphGenerator(content, jsonWFJob, true); + g.write(new org.apache.hadoop.io.IOUtils.NullOutputStream()); } - catch (Exception e) { - Assert.fail("Write PNG failed for graphWF.xml: " + e.getMessage()); - } - - File f2 = new File(png2); - try { - // Check if a valid file was written - Assert.assertNotNull(ImageIO.read(f2)); - } - catch (IOException io) { - Assert.fail("Not a valid PNG: " + io.getMessage()); + catch (final Exception e) { + Assert.fail("Write PNG failed for invalidGraphWF.xml: " + e.getMessage()); } + } - Assert.assertTrue(f1.length() < f2.length()); - f1.delete(); - f2.delete(); - + private void generateAndAssertPng(final WorkflowJobBean workflowJob, final String path, final boolean showKill) { try { - GraphGenerator g = new GraphGenerator(readFile("src/test/resources/invalidGraphWF.xml"), jsonWFJob, true); - g.write(new FileOutputStream(new File("src/test/resources/invalid.png"))); + final File outputPng = File.createTempFile("graph-output", path); + final String content = IOUtils.getResourceAsString(path, -1); + final GraphGenerator g = new GraphGenerator(content, workflowJob); + g.write(new FileOutputStream(outputPng)); + Assert.assertNotNull("PNG read error", ImageIO.read(new FileInputStream(outputPng))); } - catch (Exception e) { - Assert.fail("Write PNG failed for invalidGraphWF.xml: " + e.getMessage()); + catch (final Exception e) { + Assert.fail(String.format("Read or write PNG without kill failed for %s: %s", path, e.getMessage())); } - new File("src/test/resources/invalid.png").delete(); } public void testJobDAGLimit_more() throws IOException { - WorkflowJobBean jsonWFJob = new WorkflowJobBean(); + final WorkflowJobBean jsonWFJob = new WorkflowJobBean(); jsonWFJob.setAppName("My Test App"); jsonWFJob.setId("My Test ID"); - String txt = "src/test/resources/tmp1.txt"; try { - GraphGenerator g = new GraphGenerator(readFile("src/test/resources/graphWF_26_actions.xml"), jsonWFJob); - g.write(new FileOutputStream(new File(txt))); + final String content = IOUtils.getResourceAsString("graphWF_26_actions.xml", -1); + final GraphGenerator g = new GraphGenerator(content, jsonWFJob); + g.write(new FileOutputStream(File.createTempFile("graph-output", "over-limit"))); Assert.fail("This should not get executed"); } - catch (Exception e) { + catch (final Exception e) { Assert.assertTrue(e.getMessage().startsWith( "Can't display the graph. Number of actions are more than display limit")); } - - File f1 = new File(txt); - f1.delete(); - - } - - private static String readFile(String path) throws IOException { - File f = new File(path); - System.out.println("Reading input file " + f.getAbsolutePath()); - FileInputStream stream = new FileInputStream(f); - try { - FileChannel fc = stream.getChannel(); - MappedByteBuffer bb = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size()); - /* Instead of using default, pass in a decoder. */ - return Charset.defaultCharset().decode(bb).toString(); - } - finally { - stream.close(); - } } } http://git-wip-us.apache.org/repos/asf/oozie/blob/0601fc22/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 1db2874..c2b3683 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.0.0 release (trunk - unreleased) +OOZIE-3038 Make all Oozie JUnit tests pass on dist_test (andras.piros via gezapeti) OOZIE-2940 Possible NullPointerException in WorkflowActionBean (dionusos via gezapeti) OOZIE-3036 Spark 2.2.0 support: tell Spark not to get any delegation tokens (andras.piros via gezapeti) OOZIE-3028 Oozie Pig Action fails with no python dependencies (dbist13 via rohini)
