Repository: oozie Updated Branches: refs/heads/master 6cdc4d5d8 -> 88aa654d6
OOZIE-3236 Fix flaky test TestHiveActionExecutor#testHiveAction (pbacsko via gezapeti) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/88aa654d Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/88aa654d Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/88aa654d Branch: refs/heads/master Commit: 88aa654d66e8954a7e250f19513678da705177c3 Parents: 6cdc4d5 Author: Gezapeti Cseh <gezap...@apache.org> Authored: Wed May 16 13:46:42 2018 +0200 Committer: Gezapeti Cseh <gezap...@apache.org> Committed: Wed May 16 13:46:42 2018 +0200 ---------------------------------------------------------------------- .../java/org/apache/oozie/test/XTestCase.java | 18 ++++++++++++++--- release-log.txt | 1 + .../action/hadoop/TestHiveActionExecutor.java | 21 +------------------- 3 files changed, 17 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/88aa654d/core/src/test/java/org/apache/oozie/test/XTestCase.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/test/XTestCase.java b/core/src/test/java/org/apache/oozie/test/XTestCase.java index a8be36b..436cfc2 100644 --- a/core/src/test/java/org/apache/oozie/test/XTestCase.java +++ b/core/src/test/java/org/apache/oozie/test/XTestCase.java @@ -125,6 +125,7 @@ import org.apache.openjpa.persistence.RollbackException; public abstract class XTestCase extends TestCase { private static EnumSet<YarnApplicationState> YARN_TERMINAL_STATES = EnumSet.of(YarnApplicationState.FAILED, YarnApplicationState.KILLED, YarnApplicationState.FINISHED); + private static final int DEFAULT_YARN_TIMEOUT = 60_000; private Map<String, String> sysProps; private String testCaseDir; private String testCaseConfDir; @@ -1313,8 +1314,8 @@ public abstract class XTestCase extends TestCase { return services; } - protected YarnApplicationState waitUntilYarnAppState(String externalId, final EnumSet<YarnApplicationState> acceptedStates) - throws HadoopAccessorException, IOException, YarnException { + protected YarnApplicationState waitUntilYarnAppState(String externalId, final EnumSet<YarnApplicationState> acceptedStates, + int timeoutMs) throws HadoopAccessorException, IOException, YarnException { final ApplicationId appId = ConverterUtils.toApplicationId(externalId); final MutableObject<YarnApplicationState> finalState = new MutableObject<YarnApplicationState>(); @@ -1322,7 +1323,7 @@ public abstract class XTestCase extends TestCase { final YarnClient yarnClient = Services.get().get(HadoopAccessorService.class).createYarnClient(getTestUser(), conf); try { - waitFor(60 * 1000, new Predicate() { + waitFor(timeoutMs, new Predicate() { @Override public boolean evaluate() throws Exception { YarnApplicationState state = yarnClient.getApplicationReport(appId).getYarnApplicationState(); @@ -1341,12 +1342,23 @@ public abstract class XTestCase extends TestCase { return finalState.getValue(); } + protected YarnApplicationState waitUntilYarnAppState(String externalId, final EnumSet<YarnApplicationState> acceptedStates) + throws HadoopAccessorException, IOException, YarnException { + return waitUntilYarnAppState(externalId, acceptedStates, DEFAULT_YARN_TIMEOUT); + } + protected void waitUntilYarnAppDoneAndAssertSuccess(String externalId) throws HadoopAccessorException, IOException, YarnException { YarnApplicationState state = waitUntilYarnAppState(externalId, YARN_TERMINAL_STATES); assertEquals("YARN App state for app " + externalId, YarnApplicationState.FINISHED, state); } + protected void waitUntilYarnAppDoneAndAssertSuccess(String externalId, int timeout) + throws HadoopAccessorException, IOException, YarnException { + YarnApplicationState state = waitUntilYarnAppState(externalId, YARN_TERMINAL_STATES, timeout); + assertEquals("YARN App state for app " + externalId, YarnApplicationState.FINISHED, state); + } + protected void waitUntilYarnAppKilledAndAssertSuccess(String externalId) throws HadoopAccessorException, IOException, YarnException { YarnApplicationState state = waitUntilYarnAppState(externalId, YARN_TERMINAL_STATES); http://git-wip-us.apache.org/repos/asf/oozie/blob/88aa654d/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 222c255..6dbe922 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.1.0 release (trunk - unreleased) +OOZIE-3236 Fix flaky test TestHiveActionExecutor#testHiveAction (pbacsko via gezapeti) OOZIE-3235 Upgrade ActiveMQ to 5.15.3 (matijhs via andras.piros) OOZIE-3217 Enable definition of admin users using oozie-site.xml (orova via andras.piros) OOZIE-3219 Cannot compile with hadoop 3.1.0 (dbist13, andras.piros) http://git-wip-us.apache.org/repos/asf/oozie/blob/88aa654d/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java ---------------------------------------------------------------------- diff --git a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java index 0e99fac..82f93d8 100644 --- a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java +++ b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java @@ -18,9 +18,6 @@ package org.apache.oozie.action.hadoop; -import java.io.FileInputStream; -import java.io.InputStream; -import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; import java.text.MessageFormat; @@ -36,8 +33,6 @@ import org.apache.oozie.WorkflowJobBean; import org.apache.oozie.client.WorkflowAction; import org.apache.oozie.service.ConfigurationService; import org.apache.oozie.service.WorkflowAppService; -import org.apache.oozie.util.ClassUtils; -import org.apache.oozie.util.IOUtils; import org.apache.oozie.util.XConfiguration; import org.jdom.Namespace; @@ -55,7 +50,6 @@ public class TestHiveActionExecutor extends ActionExecutorTestCase { private static final String OUTPUT_DIRNAME = "output"; private static final String DATA_FILENAME = "data.txt"; - @SuppressWarnings("unchecked") public void testSetupMethods() throws Exception { HiveActionExecutor ae = new HiveActionExecutor(); assertEquals(Arrays.asList(HiveMain.class), ae.getLauncherClasses()); @@ -155,7 +149,7 @@ public class TestHiveActionExecutor extends ActionExecutorTestCase { Context context = createContext(getActionScriptXml()); Namespace ns = Namespace.getNamespace("uri:oozie:hive-action:0.2"); final String launcherId = submitAction(context, ns); - waitUntilYarnAppDoneAndAssertSuccess(launcherId); + waitUntilYarnAppDoneAndAssertSuccess(launcherId, 180 * 1000); Configuration conf = new XConfiguration(); conf.set("user.name", getTestUser()); Map<String, String> actionData = LauncherHelper.getActionData(getFileSystem(), context.getActionDir(), @@ -218,19 +212,6 @@ public class TestHiveActionExecutor extends ActionExecutorTestCase { return jobId; } - private String copyJar(String targetFile, Class<?> anyContainedClass) - throws Exception { - String file = ClassUtils.findContainingJar(anyContainedClass); - System.out.println("[copy-jar] class: " + anyContainedClass - + ", local jar ==> " + file); - Path targetPath = new Path(getAppPath(), targetFile); - FileSystem fs = getFileSystem(); - InputStream is = new FileInputStream(file); - OutputStream os = fs.create(new Path(getAppPath(), targetPath)); - IOUtils.copyStream(is, os); - return targetPath.toString(); - } - private Context createContext(String actionXml) throws Exception { HiveActionExecutor ae = new HiveActionExecutor();
