Repository: tez Updated Branches: refs/heads/master d3fd82832 -> 1468457e6
TEZ-3413. ConcurrentModificationException in HistoryEventTimelineConversion for AppLaunchedEvent. (Jonathan Eagles via jlowe) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/1468457e Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/1468457e Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/1468457e Branch: refs/heads/master Commit: 1468457e671dfcf2780cf17d2655e086aef120c9 Parents: d3fd828 Author: Jason Lowe <[email protected]> Authored: Wed Aug 17 15:08:34 2016 +0000 Committer: Jason Lowe <[email protected]> Committed: Wed Aug 17 15:08:34 2016 +0000 ---------------------------------------------------------------------- CHANGES.txt | 3 ++ .../apache/tez/dag/history/utils/DAGUtils.java | 3 +- .../ats/TestHistoryEventTimelineConversion.java | 34 ++++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/1468457e/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 62c69b0..a8e7080 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3413. ConcurrentModificationException in HistoryEventTimelineConversion for AppLaunchedEvent. TEZ-3352. MRInputHelpers getStringProperty() should not fail if property value is null. TEZ-3409. Log dagId along with other information when submitting a dag. TEZ-3384. Fix TestATSV15HistoryLoggingService::testDAGGroupingGroupingEnabled unit test. @@ -97,6 +98,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3413. ConcurrentModificationException in HistoryEventTimelineConversion for AppLaunchedEvent. TEZ-3352. MRInputHelpers getStringProperty() should not fail if property value is null. TEZ-3409. Log dagId along with other information when submitting a dag. TEZ-3384. Fix TestATSV15HistoryLoggingService::testDAGGroupingGroupingEnabled unit test. @@ -579,6 +581,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3413. ConcurrentModificationException in HistoryEventTimelineConversion for AppLaunchedEvent. TEZ-3286. Allow clients to set processor reserved memory per vertex (instead of per container). TEZ-3223. Support a NullHistoryLogger to disable history logging if needed. TEZ-3293. Fetch failures can cause a shuffle hang waiting for memory merge that never starts. http://git-wip-us.apache.org/repos/asf/tez/blob/1468457e/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java index f1ac0ab..d8d2407 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java @@ -475,7 +475,8 @@ public class DAGUtils { } public static Map<String, String> convertConfigurationToATSMap(Configuration conf) { - Iterator<Entry<String, String>> iter = conf.iterator(); + // Copy configuration to avoid CME since iterator is not thread safe until HADOOP-13500 + Iterator<Entry<String, String>> iter = new Configuration(conf).iterator(); Map<String, String> atsConf = new TreeMap<String, String>(); while (iter.hasNext()) { Entry<String, String> entry = iter.next(); http://git-wip-us.apache.org/repos/asf/tez/blob/1468457e/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java index dabd6a5..62fb335 100644 --- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java +++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; import java.util.Random; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -232,6 +233,39 @@ public class TestHistoryEventTimelineConversion { } + @Test(timeout = 5000) + public void testConvertAppLaunchedEventConcurrentModificationException() + throws InterruptedException { + long launchTime = random.nextLong(); + long submitTime = random.nextLong(); + final Configuration conf = new Configuration(false); + conf.set("foo", "bar"); + conf.set("applicationId", "1234"); + final AtomicBoolean shutdown = new AtomicBoolean(false); + + Thread confChanger = new Thread() { + @Override + public void run() { + int i = 1; + while (!shutdown.get()) { + // trigger an actual change to test concurrency with Configuration#iterator + conf.set("test" + i++, "test"); + } + } + }; + + confChanger.start(); + try { + MockVersionInfo mockVersionInfo = new MockVersionInfo(); + AppLaunchedEvent event = new AppLaunchedEvent(applicationId, launchTime, + submitTime, user, conf, mockVersionInfo); + HistoryEventTimelineConversion.convertToTimelineEntity(event); + } finally { + shutdown.set(true); + confChanger.join(); + } + } + @SuppressWarnings("unchecked") @Test(timeout = 5000) public void testConvertAppLaunchedEvent() {
