Repository: tez
Updated Branches:
refs/heads/branch-0.8 119c47621 -> 90519d9f7
TEZ-3413. ConcurrentModificationException in HistoryEventTimelineConversion for
AppLaunchedEvent. (Jonathan Eagles via jlowe)
(cherry picked from commit 1468457e671dfcf2780cf17d2655e086aef120c9)
Conflicts:
CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/90519d9f
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/90519d9f
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/90519d9f
Branch: refs/heads/branch-0.8
Commit: 90519d9f7b8e3358279daaec0e05fd878d131d23
Parents: 119c476
Author: Jason Lowe <[email protected]>
Authored: Wed Aug 17 15:15:31 2016 +0000
Committer: Jason Lowe <[email protected]>
Committed: Wed Aug 17 15:15:31 2016 +0000
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../apache/tez/dag/history/utils/DAGUtils.java | 3 +-
.../ats/TestHistoryEventTimelineConversion.java | 34 ++++++++++++++++++++
3 files changed, 38 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/90519d9f/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index b78f48d..87adaa2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3413. ConcurrentModificationException in HistoryEventTimelineConversion
for AppLaunchedEvent.
TEZ-3352. MRInputHelpers getStringProperty() should not fail if property
value is null.
TEZ-3409. Log dagId along with other information when submitting a dag.
TEZ-3384. Fix
TestATSV15HistoryLoggingService::testDAGGroupingGroupingEnabled unit test.
@@ -499,6 +500,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3413. ConcurrentModificationException in HistoryEventTimelineConversion
for AppLaunchedEvent.
TEZ-3286. Allow clients to set processor reserved memory per vertex (instead
of per container).
TEZ-3223. Support a NullHistoryLogger to disable history logging if needed.
TEZ-3293. Fetch failures can cause a shuffle hang waiting for memory merge
that never starts.
http://git-wip-us.apache.org/repos/asf/tez/blob/90519d9f/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
----------------------------------------------------------------------
diff --git
a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
index f1ac0ab..d8d2407 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
@@ -475,7 +475,8 @@ public class DAGUtils {
}
public static Map<String, String> convertConfigurationToATSMap(Configuration
conf) {
- Iterator<Entry<String, String>> iter = conf.iterator();
+ // Copy configuration to avoid CME since iterator is not thread safe until
HADOOP-13500
+ Iterator<Entry<String, String>> iter = new Configuration(conf).iterator();
Map<String, String> atsConf = new TreeMap<String, String>();
while (iter.hasNext()) {
Entry<String, String> entry = iter.next();
http://git-wip-us.apache.org/repos/asf/tez/blob/90519d9f/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
----------------------------------------------------------------------
diff --git
a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
index dabd6a5..62fb335 100644
---
a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
+++
b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -232,6 +233,39 @@ public class TestHistoryEventTimelineConversion {
}
+ @Test(timeout = 5000)
+ public void testConvertAppLaunchedEventConcurrentModificationException()
+ throws InterruptedException {
+ long launchTime = random.nextLong();
+ long submitTime = random.nextLong();
+ final Configuration conf = new Configuration(false);
+ conf.set("foo", "bar");
+ conf.set("applicationId", "1234");
+ final AtomicBoolean shutdown = new AtomicBoolean(false);
+
+ Thread confChanger = new Thread() {
+ @Override
+ public void run() {
+ int i = 1;
+ while (!shutdown.get()) {
+ // trigger an actual change to test concurrency with
Configuration#iterator
+ conf.set("test" + i++, "test");
+ }
+ }
+ };
+
+ confChanger.start();
+ try {
+ MockVersionInfo mockVersionInfo = new MockVersionInfo();
+ AppLaunchedEvent event = new AppLaunchedEvent(applicationId, launchTime,
+ submitTime, user, conf, mockVersionInfo);
+ HistoryEventTimelineConversion.convertToTimelineEntity(event);
+ } finally {
+ shutdown.set(true);
+ confChanger.join();
+ }
+ }
+
@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testConvertAppLaunchedEvent() {
