Author: ddas
Date: Tue Sep 9 00:02:32 2008
New Revision: 693385
URL: http://svn.apache.org/viewvc?rev=693385&view=rev
Log:
HADOOP-3970. Provides a way to recover counters written to JobHistory.
Contributed by Amar Kamat.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/core/org/apache/hadoop/util/StringUtils.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Counters.java
Modified: hadoop/core/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=693385&r1=693384&r2=693385&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue Sep 9 00:02:32 2008
@@ -138,6 +138,9 @@
HADOOP-3581. Prevents memory intensive user tasks from taking down
nodes. (Vinod K V via ddas)
+ HADOOP-3970. Provides a way to recover counters written to JobHistory.
+ (Amar Kamat via ddas)
+
IMPROVEMENTS
HADOOP-3908. Fuse-dfs: better error message if llibhdfs.so doesn't exist.
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/util/StringUtils.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/util/StringUtils.java?rev=693385&r1=693384&r2=693385&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/util/StringUtils.java
(original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/util/StringUtils.java Tue Sep
9 00:02:32 2008
@@ -327,16 +327,11 @@
}
ArrayList<String> strList = new ArrayList<String>();
StringBuilder split = new StringBuilder();
- int numPreEscapes = 0;
- for (int i=0; i<str.length(); i++) {
- char curChar = str.charAt(i);
- if (numPreEscapes==0 && curChar == separator) { // separator
- strList.add(split.toString());
- split.setLength(0); // clear the split
- } else {
- split.append(curChar);
- numPreEscapes = (curChar == escapeChar)?(++numPreEscapes)%2:0;
- }
+ int index = 0;
+ while ((index = findNext(str, separator, escapeChar, index, split)) >= 0) {
+ ++index; // move over the separator for next search
+ strList.add(split.toString());
+ split.setLength(0); // reset the buffer
}
strList.add(split.toString());
// remove trailing empty split(s)
@@ -348,6 +343,33 @@
}
/**
+ * Finds the first occurrence of the separator character ignoring the escaped
+ * separators starting from the index. Note the substring between the index
+ * and the position of the separator is passed.
+ * @param str the source string
+ * @param separator the character to find
+ * @param escapeChar character used to escape
+ * @param start from where to search
+ * @param split used to pass back the extracted string
+ */
+ public static int findNext(String str, char separator, char escapeChar,
+ int start, StringBuilder split) {
+ int numPreEscapes = 0;
+ for (int i = start; i < str.length(); i++) {
+ char curChar = str.charAt(i);
+ if (numPreEscapes == 0 && curChar == separator) { // separator
+ return i;
+ } else {
+ split.append(curChar);
+ numPreEscapes = (curChar == escapeChar)
+ ? (++numPreEscapes) % 2
+ : 0;
+ }
+ }
+ return -1;
+ }
+
+ /**
* Escape commas in the string using the default escape char
* @param str a string
* @return an escaped string
@@ -367,13 +389,31 @@
*/
public static String escapeString(
String str, char escapeChar, char charToEscape) {
+ return escapeString(str, escapeChar, new char[] {charToEscape});
+ }
+
+ // check if the character array has the character
+ private static boolean hasChar(char[] chars, char character) {
+ for (char target : chars) {
+ if (character == target) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * @param charsToEscape array of characters to be escaped
+ */
+ public static String escapeString(String str, char escapeChar,
+ char[] charsToEscape) {
if (str == null) {
return null;
}
StringBuilder result = new StringBuilder();
for (int i=0; i<str.length(); i++) {
char curChar = str.charAt(i);
- if (curChar == escapeChar || curChar == charToEscape) {
+ if (curChar == escapeChar || hasChar(charsToEscape, curChar)) {
// special char
result.append(escapeChar);
}
@@ -402,6 +442,14 @@
*/
public static String unEscapeString(
String str, char escapeChar, char charToEscape) {
+ return unEscapeString(str, escapeChar, new char[] {charToEscape});
+ }
+
+ /**
+ * @param charsToEscape array of characters to unescape
+ */
+ public static String unEscapeString(String str, char escapeChar,
+ char[] charsToEscape) {
if (str == null) {
return null;
}
@@ -410,7 +458,7 @@
for (int i=0; i<str.length(); i++) {
char curChar = str.charAt(i);
if (hasPreEscape) {
- if (curChar != escapeChar && curChar != charToEscape) {
+ if (curChar != escapeChar && !hasChar(charsToEscape, curChar)) {
// no special char
throw new IllegalArgumentException("Illegal escaped string " + str +
" unescaped " + escapeChar + " at " + (i-1));
@@ -419,9 +467,9 @@
result.append(curChar);
hasPreEscape = false;
} else {
- if (curChar == charToEscape) {
+ if (hasChar(charsToEscape, curChar)) {
throw new IllegalArgumentException("Illegal escaped string " + str +
- " unescaped " + charToEscape + " at " + i);
+ " unescaped " + curChar + " at " + i);
} else if (curChar == escapeChar) {
hasPreEscape = true;
} else {
@@ -431,7 +479,7 @@
}
if (hasPreEscape ) {
throw new IllegalArgumentException("Illegal escaped string " + str +
- ", not expecting " + charToEscape + " in the end." );
+ ", not expecting " + escapeChar + " in the end." );
}
return result.toString();
}
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Counters.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Counters.java?rev=693385&r1=693384&r2=693385&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Counters.java
(original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Counters.java Tue Sep
9 00:02:32 2008
@@ -21,6 +21,7 @@
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.text.ParseException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -29,11 +30,14 @@
import java.util.Map;
import java.util.MissingResourceException;
import java.util.ResourceBundle;
+import java.util.Set;
import org.apache.commons.logging.*;
+import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.util.StringUtils;
/**
* A set of named counters.
@@ -47,6 +51,15 @@
*/
public class Counters implements Writable, Iterable<Counters.Group> {
private static final Log LOG = LogFactory.getLog(Counters.class);
+ private static final char GROUP_OPEN = '{';
+ private static final char GROUP_CLOSE = '}';
+ private static final char COUNTER_OPEN = '[';
+ private static final char COUNTER_CLOSE = ']';
+ private static final char UNIT_OPEN = '(';
+ private static final char UNIT_CLOSE = ')';
+ private static char[] charsToEscape = {GROUP_OPEN, GROUP_CLOSE,
+ COUNTER_OPEN, COUNTER_CLOSE,
+ UNIT_OPEN, UNIT_CLOSE};
//private static Log log = LogFactory.getLog("Counters.class");
@@ -112,6 +125,41 @@
}
/**
+ * Set the display name of the counter.
+ */
+ public synchronized void setDisplayName(String displayName) {
+ this.displayName = displayName;
+ }
+
+ /**
+ * Returns the compact stringified version of the counter in the format
+ * [(actual-name)(display-name)(value)]
+ */
+ public synchronized String makeEscapedCompactString() {
+ StringBuffer buf = new StringBuffer();
+ buf.append(COUNTER_OPEN);
+
+ // Add the counter name
+ buf.append(UNIT_OPEN);
+ buf.append(escape(getName()));
+ buf.append(UNIT_CLOSE);
+
+ // Add the display name
+ buf.append(UNIT_OPEN);
+ buf.append(escape(getDisplayName()));
+ buf.append(UNIT_CLOSE);
+
+ // Add the value
+ buf.append(UNIT_OPEN);
+ buf.append(this.value);
+ buf.append(UNIT_CLOSE);
+
+ buf.append(COUNTER_CLOSE);
+
+ return buf.toString();
+ }
+
+ /**
* What is the current value of this counter?
* @return the current value
*/
@@ -181,6 +229,48 @@
}
/**
+ * Set the display name
+ */
+ public void setDisplayName(String displayName) {
+ this.displayName = displayName;
+ }
+
+ /**
+ * Returns the compact stringified version of the group in the format
+ * {(actual-name)(display-name)(value)[][][]} where [] are compact strings
for the
+ * counters within.
+ */
+ public String makeEscapedCompactString() {
+ StringBuffer buf = new StringBuffer();
+ buf.append(GROUP_OPEN); // group start
+
+ // Add the group name
+ buf.append(UNIT_OPEN);
+ buf.append(escape(getName()));
+ buf.append(UNIT_CLOSE);
+
+ // Add the display name
+ buf.append(UNIT_OPEN);
+ buf.append(escape(getDisplayName()));
+ buf.append(UNIT_CLOSE);
+
+ // write the value
+ for(Counter counter: subcounters.values()) {
+ buf.append(counter.makeEscapedCompactString());
+ }
+
+ buf.append(GROUP_CLOSE); // group end
+ return buf.toString();
+ }
+
+ /**
+ * Returns the names of the counters within
+ */
+ public synchronized Set<String> getCounterNames() {
+ return subcounters.keySet();
+ }
+
+ /**
* Returns the value of the specified counter, or 0 if the counter does
* not exist.
*/
@@ -499,6 +589,125 @@
return buffer.toString();
}
+ /**
+ * Represent the counter in a textual format that can be converted back to
+ * its object form
+ * @return the string in the following format
+ *
{(groupname)(group-displayname)[(countername)(displayname)(value)][][]}{}{}
+ */
+ public synchronized String makeEscapedCompactString() {
+ StringBuffer buffer = new StringBuffer();
+ for(Group group: this){
+ buffer.append(group.makeEscapedCompactString());
+ }
+ return buffer.toString();
+ }
+
+ // Extracts a block (data enclosed within delimeters) ignoring escape
+ // sequences. Throws ParseException if an incomplete block is found else
+ // returns null.
+ private static String getBlock(String str, char open, char close,
+ IntWritable index) throws ParseException {
+ StringBuilder split = new StringBuilder();
+ int next = StringUtils.findNext(str, open, StringUtils.ESCAPE_CHAR,
+ index.get(), split);
+ split.setLength(0); // clear the buffer
+ if (next >= 0) {
+ ++next; // move over '('
+
+ next = StringUtils.findNext(str, close, StringUtils.ESCAPE_CHAR,
+ next, split);
+ if (next >= 0) {
+ ++next; // move over ')'
+ index.set(next);
+ return split.toString(); // found a block
+ } else {
+ throw new ParseException("Unexpected end of block", next);
+ }
+ }
+ return null; // found nothing
+ }
+
+ /**
+ * Convert a stringified counter representation into a counter object. Note
+ * that the counter can be recovered if its stringified using
+ * [EMAIL PROTECTED] #makeEscapedCompactString()}.
+ * @return a Counter
+ */
+ public static Counters fromEscapedCompactString(String compactString)
+ throws ParseException {
+ Counters counters = new Counters();
+ IntWritable index = new IntWritable(0);
+
+ // Get the group to work on
+ String groupString =
+ getBlock(compactString, GROUP_OPEN, GROUP_CLOSE, index);
+
+ while (groupString != null) {
+ IntWritable groupIndex = new IntWritable(0);
+
+ // Get the actual name
+ String groupName =
+ getBlock(groupString, UNIT_OPEN, UNIT_CLOSE, groupIndex);
+ groupName = unescape(groupName);
+
+ // Get the display name
+ String groupDisplayName =
+ getBlock(groupString, UNIT_OPEN, UNIT_CLOSE, groupIndex);
+ groupDisplayName = unescape(groupDisplayName);
+
+ // Get the counters
+ Group group = counters.getGroup(groupName);
+ group.setDisplayName(groupDisplayName);
+
+ String counterString =
+ getBlock(groupString, COUNTER_OPEN, COUNTER_CLOSE, groupIndex);
+
+ while (counterString != null) {
+ IntWritable counterIndex = new IntWritable(0);
+
+ // Get the actual name
+ String counterName =
+ getBlock(counterString, UNIT_OPEN, UNIT_CLOSE, counterIndex);
+ counterName = unescape(counterName);
+
+ // Get the display name
+ String counterDisplayName =
+ getBlock(counterString, UNIT_OPEN, UNIT_CLOSE, counterIndex);
+ counterDisplayName = unescape(counterDisplayName);
+
+ // Get the value
+ long value =
+ Long.parseLong(getBlock(counterString, UNIT_OPEN, UNIT_CLOSE,
+ counterIndex));
+
+ // Add the counter
+ Counter counter = group.getCounterForName(counterName);
+ counter.setDisplayName(counterDisplayName);
+ counter.increment(value);
+
+ // Get the next counter
+ counterString =
+ getBlock(groupString, COUNTER_OPEN, COUNTER_CLOSE, groupIndex);
+ }
+
+ groupString = getBlock(compactString, GROUP_OPEN, GROUP_CLOSE, index);
+ }
+ return counters;
+ }
+
+ // Escapes all the delimiters for counters i.e {,[,(,),],}
+ private static String escape(String string) {
+ return StringUtils.escapeString(string, StringUtils.ESCAPE_CHAR,
+ charsToEscape);
+ }
+
+ // Unescapes all the delimiters for counters i.e {,[,(,),],}
+ private static String unescape(String string) {
+ return StringUtils.unEscapeString(string, StringUtils.ESCAPE_CHAR,
+ charsToEscape);
+ }
+
public static class Application {
//special counters which are written by the application and are
//used by the framework.
