Repository: oozie Updated Branches: refs/heads/master 75312da08 -> 792e46716
OOZIE-2385 org.apache.oozie.TestCoordinatorEngineStreamLog.testCoordLogStreaming is flakey (rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/792e4671 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/792e4671 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/792e4671 Branch: refs/heads/master Commit: 792e467162100bfb07ee377db3444f7105fb17ff Parents: 75312da Author: Robert Kanter <[email protected]> Authored: Thu Oct 8 15:03:13 2015 -0700 Committer: Robert Kanter <[email protected]> Committed: Thu Oct 8 15:03:13 2015 -0700 ---------------------------------------------------------------------- .../oozie/TestCoordinatorEngineStreamLog.java | 137 ++++--------------- release-log.txt | 1 + 2 files changed, 30 insertions(+), 108 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/792e4671/core/src/test/java/org/apache/oozie/TestCoordinatorEngineStreamLog.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/TestCoordinatorEngineStreamLog.java b/core/src/test/java/org/apache/oozie/TestCoordinatorEngineStreamLog.java index 6d8e48a..3eb1016 100644 --- a/core/src/test/java/org/apache/oozie/TestCoordinatorEngineStreamLog.java +++ b/core/src/test/java/org/apache/oozie/TestCoordinatorEngineStreamLog.java @@ -18,14 +18,9 @@ package org.apache.oozie; -import java.io.File; -import java.io.FileWriter; import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; import java.io.StringWriter; import java.io.Writer; -import java.net.URI; import java.util.Collections; import java.util.Comparator; import java.util.Date; @@ -33,23 +28,20 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.oozie.client.CoordinatorAction; -import org.apache.oozie.client.OozieClient; +import org.apache.oozie.client.CoordinatorJob; +import org.apache.oozie.client.Job; import org.apache.oozie.client.rest.RestConstants; +import org.apache.oozie.executor.jpa.CoordActionQueryExecutor; +import org.apache.oozie.executor.jpa.CoordJobQueryExecutor; import org.apache.oozie.service.DagXLogInfoService; import org.apache.oozie.service.Services; import org.apache.oozie.service.XLogStreamingService; -import org.apache.oozie.test.XFsTestCase; +import org.apache.oozie.test.XDataTestCase; import org.apache.oozie.util.DateUtils; import org.apache.oozie.util.XLogFilter; -import org.apache.oozie.util.IOUtils; -import org.apache.oozie.util.XLogUserFilterParam; -import org.apache.oozie.util.XConfiguration; -public class TestCoordinatorEngineStreamLog extends XFsTestCase { +public class TestCoordinatorEngineStreamLog extends XDataTestCase { private Services services; @Override @@ -65,23 +57,6 @@ public class TestCoordinatorEngineStreamLog extends XFsTestCase { super.tearDown(); } - private void writeToFile(String appXml, String appPath) throws Exception { - File wf = new File(new URI(appPath).getPath()); - PrintWriter out = null; - try { - out = new PrintWriter(new FileWriter(wf)); - out.println(appXml); - } - catch (IOException iex) { - throw iex; - } - finally { - if (out != null) { - out.close(); - } - } - } - static class DummyXLogStreamingService extends XLogStreamingService { XLogFilter filter; Date startTime; @@ -101,12 +76,14 @@ public class TestCoordinatorEngineStreamLog extends XFsTestCase { } public void testCoordLogStreaming() throws Exception { + services.setService(DummyXLogStreamingService.class); + new DagXLogInfoService().init(services); CoordinatorEngine ce = createCoordinatorEngine(); - final String jobId = runJobsImpl(ce, 6); + final String jobId = createJobs(6); CoordinatorJobBean cjb = ce.getCoordJob(jobId); Date createdDate = cjb.getCreatedTime(); - Date endDate = new Date(); + Date endDate = cjb.getEndTime(); assertTrue(endDate.after(createdDate)); List<CoordinatorAction> list = cjb.getActions(); @@ -115,16 +92,14 @@ public class TestCoordinatorEngineStreamLog extends XFsTestCase { return a.getId().compareTo(b.getId()); } }); - - endDate = new Date(); - - Thread.sleep(2000); + // Test 1.to test if fields are injected ce.streamLog(jobId, new StringWriter(), new HashMap<String, String[]>()); DummyXLogStreamingService service = (DummyXLogStreamingService) services.get(XLogStreamingService.class); XLogFilter filter = service.filter; assertEquals(filter.getFilterParams().get(DagXLogInfoService.JOB), jobId); - assertTrue(endDate.before(service.endTime)); + assertEquals(cjb.getCreatedTime(), service.startTime); + assertEquals(cjb.getLastModifiedTime(), service.endTime); // Test2 @@ -234,76 +209,22 @@ public class TestCoordinatorEngineStreamLog extends XFsTestCase { } } - private String runJobsImpl(final CoordinatorEngine ce, int count) throws Exception { - try { - Services.get().getConf().setBoolean("oozie.service.coord.check.maximum.frequency", false); - services.setService(DummyXLogStreamingService.class); - // need to re-define the parameters that are cleared upon the service - // reset: - new DagXLogInfoService().init(services); - - Configuration conf = new XConfiguration(); - - final String appPath = getTestCaseFileUri("coordinator.xml"); - final long now = System.currentTimeMillis(); - final String start = DateUtils.formatDateOozieTZ(new Date(now)); - long e = now + 1000 * 60 * count; - final String end = DateUtils.formatDateOozieTZ(new Date(e)); - - String wfXml = IOUtils.getResourceAsString("wf-no-op.xml", -1); - writeToFile(wfXml, getFsTestCaseDir(), "workflow.xml"); - - String appXml = "<coordinator-app name=\"NAME\" frequency=\"${coord:minutes(1)}\" start=\"" + start - + "\" end=\"" + end + "\" timezone=\"UTC\" " + "xmlns=\"uri:oozie:coordinator:0.1\"> " + "<controls> " - + " <timeout>1</timeout> " + " <concurrency>1</concurrency> " + " <execution>LIFO</execution> " - + "</controls> " + "<action> " + " <workflow> " + " <app-path>" + getFsTestCaseDir() - + "/workflow.xml</app-path>" - + " <configuration> <property> <name>inputA</name> <value>valueA</value> </property> " - + " <property> <name>inputB</name> <value>valueB</value> " + " </property></configuration> " - + "</workflow>" + "</action> " + "</coordinator-app>"; - writeToFile(appXml, appPath); - conf.set(OozieClient.COORDINATOR_APP_PATH, appPath); - conf.set(OozieClient.USER_NAME, getTestUser()); - - final String jobId = ce.submitJob(conf, true); - waitFor(1000 * 60 * count, new Predicate() { - @Override - public boolean evaluate() throws Exception { - try { - List<CoordinatorAction> actions = ce.getCoordJob(jobId).getActions(); - if (actions.size() < 1) { - return false; - } - for (CoordinatorAction action : actions) { - CoordinatorAction.Status actionStatus = action.getStatus(); - if (actionStatus != CoordinatorAction.Status.SUCCEEDED) { - return false; - } - } - return true; - } - catch (Exception ex) { - ex.printStackTrace(); - return false; - } - } - }); - // Assert all the actions are succeeded (useful for waitFor() timeout - // case): - final List<CoordinatorAction> actions = ce.getCoordJob(jobId).getActions(); - for (CoordinatorAction action : actions) { - assertEquals(CoordinatorAction.Status.SUCCEEDED, action.getStatus()); - } - return jobId; - } finally { - Services.get().getConf().setBoolean("oozie.service.coord.check.maximum.frequency", true); + private String createJobs(int numActions) throws Exception { + long time = System.currentTimeMillis(); + CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.SUCCEEDED, false, true); + job.setCreatedTime(new Date(time)); + time += 1000 * 60; + for (int i = 1; i <= numActions; i++) { + CoordinatorActionBean action = createCoordAction(job.getId(), i, CoordinatorAction.Status.SUCCEEDED, + "coord-action-get.xml", 0, new Date(time)); + action.setCreatedTime(new Date(time)); + time += 1000 * 60; + action.setLastModifiedTime(new Date(time)); + time += 1000 * 60; + addRecordToCoordActionTable(action, null); } - } - - private void writeToFile(String content, Path appPath, String fileName) throws IOException { - FileSystem fs = getFileSystem(); - Writer writer = new OutputStreamWriter(fs.create(new Path(appPath, fileName), true)); - writer.write(content); - writer.close(); + job.setEndTime(new Date(time)); + CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQueryExecutor.CoordJobQuery.UPDATE_COORD_JOB, job); + return job.getId(); } } http://git-wip-us.apache.org/repos/asf/oozie/blob/792e4671/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 30578b6..257d08b 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2385 org.apache.oozie.TestCoordinatorEngineStreamLog.testCoordLogStreaming is flakey (rkanter) OOZIE-2382 org.apache.oozie.action.hadoop.TestPigMain.testPig_withNullExternalID is flakey (rkanter) OOZIE-2379 org.apache.oozie.command.coord.TestCoordPushDependencyCheckXCommand.testRequeueOnException is flakey (rkanter) OOZIE-2378 org.apache.oozie.client.TestOozieCLI.testAdminInstrumentation is flakey (rkanter)
