Repository: tez
Updated Branches:
refs/heads/branch-0.5 0f6c1086f -> 3e5991a5c
TEZ-2203. Intern strings in tez counters (bikas)
(cherry picked from commit 6e15b2f6dac8471d266b8f313038f3767fedab04)
Dropping TestMockDAGAppMaster.java since the test is ignored and mainly
for manual memory testing.
Conflicts:
CHANGES.txt
tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java
(cherry picked from commit edc2720980712dfcafcc944482f1e116b1d50030)
Conflicts:
CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/3e5991a5
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/3e5991a5
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/3e5991a5
Branch: refs/heads/branch-0.5
Commit: 3e5991a5cba5e81764ee5778e408fd4226cc0806
Parents: 0f6c108
Author: Bikas Saha <[email protected]>
Authored: Fri Mar 20 12:27:02 2015 -0700
Committer: Bikas Saha <[email protected]>
Committed: Mon Sep 21 14:33:51 2015 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../tez/common/counters/AbstractCounterGroup.java | 7 ++++---
.../tez/common/counters/FileSystemCounterGroup.java | 9 ++++++---
.../tez/common/counters/FrameworkCounterGroup.java | 3 +--
.../org/apache/tez/common/counters/GenericCounter.java | 12 ++++++------
5 files changed, 18 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/3e5991a5/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index f9e4734..0646dfc 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
TEZ-2552. CRC errors can cause job to run for very long time in large jobs.
ALL CHANGES:
+ TEZ-2203. Intern strings in tez counters
TEZ-2834. Make Tez preemption resilient to incorrect free resource reported
by YARN
TEZ-2812. Preemption sometimes does not respect heartbeats between
preemptions
http://git-wip-us.apache.org/repos/asf/tez/blob/3e5991a5/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounterGroup.java
----------------------------------------------------------------------
diff --git
a/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounterGroup.java
b/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounterGroup.java
index 7e791d7..a4b153f 100644
---
a/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounterGroup.java
+++
b/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounterGroup.java
@@ -28,6 +28,7 @@ import java.util.concurrent.ConcurrentSkipListMap;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.util.StringInterner;
import com.google.common.collect.Iterators;
@@ -49,8 +50,8 @@ public abstract class AbstractCounterGroup<T extends
TezCounter>
public AbstractCounterGroup(String name, String displayName,
Limits limits) {
- this.name = name;
- this.displayName = displayName;
+ this.name = StringInterner.weakIntern(name);
+ this.displayName = StringInterner.weakIntern(displayName);
this.limits = limits;
}
@@ -160,7 +161,7 @@ public abstract class AbstractCounterGroup<T extends
TezCounter>
@Override
public synchronized void readFields(DataInput in) throws IOException {
- displayName = Text.readString(in);
+ displayName = StringInterner.weakIntern(Text.readString(in));
counters.clear();
int size = WritableUtils.readVInt(in);
for (int i = 0; i < size; i++) {
http://git-wip-us.apache.org/repos/asf/tez/blob/3e5991a5/tez-api/src/main/java/org/apache/tez/common/counters/FileSystemCounterGroup.java
----------------------------------------------------------------------
diff --git
a/tez-api/src/main/java/org/apache/tez/common/counters/FileSystemCounterGroup.java
b/tez-api/src/main/java/org/apache/tez/common/counters/FileSystemCounterGroup.java
index 88d47ca..5024154 100644
---
a/tez-api/src/main/java/org/apache/tez/common/counters/FileSystemCounterGroup.java
+++
b/tez-api/src/main/java/org/apache/tez/common/counters/FileSystemCounterGroup.java
@@ -29,13 +29,16 @@ import java.util.Locale;
import java.util.Map;
import com.google.common.base.Joiner;
+
import static com.google.common.base.Preconditions.*;
+
import com.google.common.collect.AbstractIterator;
import com.google.common.collect.Iterators;
import com.google.common.collect.Maps;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.util.StringInterner;
/**
* An abstract class to provide common implementation of the filesystem
@@ -54,7 +57,7 @@ public abstract class FileSystemCounterGroup<C extends
TezCounter>
// Just a few local casts probably worth not having to carry it around.
private final Map<String, Object[]> map =
new ConcurrentSkipListMap<String, Object[]>();
- private String displayName = "File System Counters";
+ private String displayName = StringInterner.weakIntern("File System
Counters");
private static final Joiner NAME_JOINER = Joiner.on('_');
@@ -65,7 +68,7 @@ public abstract class FileSystemCounterGroup<C extends
TezCounter>
private long value;
public FSCounter(String scheme, FileSystemCounter ref) {
- this.scheme = scheme;
+ this.scheme = scheme; // this is interned in the checkScheme() method
via a map
key = ref;
}
@@ -122,7 +125,7 @@ public abstract class FileSystemCounterGroup<C extends
TezCounter>
@Override
public void setDisplayName(String displayName) {
- this.displayName = displayName;
+ this.displayName = StringInterner.weakIntern(displayName);
}
@Override
http://git-wip-us.apache.org/repos/asf/tez/blob/3e5991a5/tez-api/src/main/java/org/apache/tez/common/counters/FrameworkCounterGroup.java
----------------------------------------------------------------------
diff --git
a/tez-api/src/main/java/org/apache/tez/common/counters/FrameworkCounterGroup.java
b/tez-api/src/main/java/org/apache/tez/common/counters/FrameworkCounterGroup.java
index 66b6e33..3a4aa97 100644
---
a/tez-api/src/main/java/org/apache/tez/common/counters/FrameworkCounterGroup.java
+++
b/tez-api/src/main/java/org/apache/tez/common/counters/FrameworkCounterGroup.java
@@ -26,7 +26,6 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Iterator;
-import com.google.common.base.Joiner;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.WritableUtils;
@@ -60,7 +59,7 @@ public abstract class FrameworkCounterGroup<T extends Enum<T>,
public FrameworkCounter(T ref, String groupName) {
key = ref;
- this.groupName = groupName;
+ this.groupName = groupName; // this is interned in the fmap/i2s of
CounterGroupFactory
}
@Override
http://git-wip-us.apache.org/repos/asf/tez/blob/3e5991a5/tez-api/src/main/java/org/apache/tez/common/counters/GenericCounter.java
----------------------------------------------------------------------
diff --git
a/tez-api/src/main/java/org/apache/tez/common/counters/GenericCounter.java
b/tez-api/src/main/java/org/apache/tez/common/counters/GenericCounter.java
index 5477606..4bb4c76 100644
--- a/tez-api/src/main/java/org/apache/tez/common/counters/GenericCounter.java
+++ b/tez-api/src/main/java/org/apache/tez/common/counters/GenericCounter.java
@@ -25,6 +25,7 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.util.StringInterner;
/**
* A generic counter implementation
@@ -41,13 +42,12 @@ public class GenericCounter extends AbstractCounter {
}
public GenericCounter(String name, String displayName) {
- this.name = name;
- this.displayName = displayName;
+ this(name, displayName, 0);
}
public GenericCounter(String name, String displayName, long value) {
- this.name = name;
- this.displayName = displayName;
+ this.name = StringInterner.weakIntern(name);
+ this.displayName = StringInterner.weakIntern(displayName);
this.value = value;
}
@@ -58,8 +58,8 @@ public class GenericCounter extends AbstractCounter {
@Override
public synchronized void readFields(DataInput in) throws IOException {
- name = Text.readString(in);
- displayName = in.readBoolean() ? Text.readString(in) : name;
+ name = StringInterner.weakIntern(Text.readString(in));
+ displayName = in.readBoolean() ?
StringInterner.weakIntern(Text.readString(in)) : name;
value = WritableUtils.readVLong(in);
}
