Repository: oozie Updated Branches: refs/heads/master a3bcad6c0 -> 5f6b9159d
OOZIE-2847 Oozie Ha timing issue (dionusos via asasvari) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/5f6b9159 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/5f6b9159 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/5f6b9159 Branch: refs/heads/master Commit: 5f6b9159dda6a1045bcbe7c88d17c6e6af46bc51 Parents: a3bcad6 Author: Attila Sasvari <asasv...@cloudera.com> Authored: Tue Feb 6 13:19:46 2018 +0100 Committer: Attila Sasvari <asasv...@cloudera.com> Committed: Tue Feb 6 13:19:46 2018 +0100 ---------------------------------------------------------------------- release-log.txt | 2 ++ .../apache/oozie/action/hadoop/LauncherAM.java | 12 ++++--- .../oozie/action/hadoop/TestLauncherAM.java | 37 ++++++++++++++++---- 3 files changed, 40 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/5f6b9159/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index ccea120..4029944 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,7 @@ -- Oozie 5.0.0 release (trunk - unreleased) +OOZIE-2847 Oozie Ha timing issue (dionusos via asasvari) +OOZIE-1717 amend Add indexes to speed up db queries (asasvari) OOZIE-3168 Remove -secure option from DG_QuickStart.twiki and from oozie-setup.sh (kmarton via gezapeti) OOZIE-1717 Add indexes to speed up db queries (asasvari) OOZIE-3157 Setup truststore so that it also works in HTTP only mode (kmarton via asasvari) http://git-wip-us.apache.org/repos/asf/oozie/blob/5f6b9159/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java index 9cacd2e..cab1452 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java @@ -467,18 +467,20 @@ public class LauncherAM { private void setRecoveryId() throws LauncherException { try { ApplicationId applicationId = containerId.getApplicationAttemptId().getApplicationId(); - String applicationIdStr = applicationId.toString(); + final String applicationIdStr = applicationId.toString(); - String recoveryId = Preconditions.checkNotNull(launcherConf.get(OOZIE_ACTION_RECOVERY_ID), + final String recoveryId = Preconditions.checkNotNull(launcherConf.get(OOZIE_ACTION_RECOVERY_ID), "RecoveryID should not be null"); - Path path = new Path(actionDir, recoveryId); + final Path path = new Path(actionDir, recoveryId); if (!hdfsOperations.fileExists(path, launcherConf)) { hdfsOperations.writeStringToFile(path, launcherConf, applicationIdStr); } else { - String id = hdfsOperations.readFileContents(path, launcherConf); + final String id = hdfsOperations.readFileContents(path, launcherConf); - if (!applicationIdStr.equals(id)) { + if (id == null || id.isEmpty()) { + hdfsOperations.writeStringToFile(path, launcherConf, applicationIdStr); + } else if (!applicationIdStr.equals(id)) { throw new LauncherException(MessageFormat.format( "YARN Id mismatch, action file [{0}] declares Id [{1}] current Id [{2}]", path, id, applicationIdStr)); http://git-wip-us.apache.org/repos/asf/oozie/blob/5f6b9159/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java index 96d3e1d..533d5b2 100644 --- a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java +++ b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java @@ -17,6 +17,9 @@ */ package org.apache.oozie.action.hadoop; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import static org.apache.oozie.action.hadoop.LauncherAM.ACTION_DATA_EXTERNAL_CHILD_IDS; import static org.apache.oozie.action.hadoop.LauncherAM.ACTION_DATA_NEW_ID; import static org.apache.oozie.action.hadoop.LauncherAM.ACTION_DATA_OUTPUT_PROPS; @@ -45,36 +48,32 @@ import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyInt; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.times; +import org.mockito.Mockito; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyZeroInteractions; import java.io.File; import java.io.IOException; import java.io.StringReader; -import java.security.PrivilegedExceptionAction; import java.text.MessageFormat; import java.util.Map; import java.util.Properties; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.oozie.action.hadoop.LauncherAM.LauncherSecurityManager; import org.apache.oozie.action.hadoop.LauncherAM.OozieActionResult; -import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; import org.mockito.Mock; -import org.mockito.invocation.InvocationOnMock; +import static org.mockito.Mockito.when; import org.mockito.runners.MockitoJUnitRunner; -import org.mockito.stubbing.Answer; @RunWith(MockitoJUnitRunner.class) public class TestLauncherAM { @@ -91,6 +90,8 @@ public class TestLauncherAM { private static final String EXIT_CODE_0 = "0"; private static final String DUMMY_XML = "<dummy>dummyXml</dummy>"; + public static final String ACTION_DIR = "/tmp/"; + @Rule public ExpectedException thrown = ExpectedException.none(); @@ -618,4 +619,28 @@ public class TestLauncherAM { return this; } } + + @Test + public void testRecoveryWritesJobId() throws IOException, InterruptedException, LauncherException, + NoSuchFieldException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { + //create empty file on the following path: ACTION_DIR/RECOVERY_ID + final Path path = new Path(ACTION_DIR, "1"); + when(hdfsOperationsMock.readFileContents(any(Path.class), eq(launcherJobConfig))).thenReturn(EMPTY_STRING); + when(hdfsOperationsMock.fileExists(any(Path.class), eq(launcherJobConfig))).thenReturn(true); + + //run launchermapper with the same ACTION_DIR/RECOVERY_ID + final Field f = launcherAM.getClass().getDeclaredField("actionDir"); + f.setAccessible(true); + f.set(launcherAM, path); + final Method m = launcherAM.getClass().getDeclaredMethod("setRecoveryId"); + m.setAccessible(true); + m.invoke(launcherAM); + + //check empty file, launcherMapper should have written RECOVERY_ID into it + Mockito.verify(hdfsOperationsMock).writeStringToFile( + eq(new Path(path, "1")), + eq(launcherJobConfig), + eq("application_1479473450392_0001")); + + } }
