Author: omalley
Date: Fri Mar 4 04:28:40 2011
New Revision: 1077555
URL: http://svn.apache.org/viewvc?rev=1077555&view=rev
Log:
commit fac50cf14ffe968281ba41fecdf4d946d3f942fa
Author: Arun C Murthy <[email protected]>
Date: Mon Jul 19 10:51:07 2010 -0700
MAPREDUCE-323. Fixed underscores in user and job names, also fixed some
debug code. Contributed by Dick King.
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobTracker.java
hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/jobhistory.jsp
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java?rev=1077555&r1=1077554&r2=1077555&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java
(original)
+++
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java
Fri Mar 4 04:28:40 2011
@@ -98,6 +98,8 @@ public class JobHistory {
static final String DONE_DIRECTORY_FORMAT_DIRNAME
= "version-" + DONE_DIRECTORY_FORMAT_VERSION;
+ static final String UNDERSCORE_ESCAPE = "%5F";
+
public static final Log LOG = LogFactory.getLog(JobHistory.class);
private static final char DELIMITER = ' ';
static final char LINE_DELIMITER_CHAR = '.';
@@ -135,7 +137,7 @@ public class JobHistory {
static final String CONF_FILE_NAME_SUFFIX = "_conf.xml";
// XXXXX debug mode -- set this to false for production
- private static final boolean DEBUG_MODE = true;
+ private static final boolean DEBUG_MODE = false;
private static final int SERIAL_NUMBER_DIRECTORY_DIGITS = 6;
private static final int SERIAL_NUMBER_LOW_DIGITS = DEBUG_MODE ? 1 : 3;
@@ -334,7 +336,7 @@ public class JobHistory {
private static String serialNumberDirectoryComponent(JobID id) {
return String.format(SERIAL_NUMBER_FORMAT,
- new Integer(jobSerialNumber(id)))
+ Integer.valueOf(jobSerialNumber(id)))
.substring(0, SERIAL_NUMBER_DIRECTORY_DIGITS);
}
@@ -533,10 +535,19 @@ public class JobHistory {
//permission
if (!DONEDIR_FS.exists(DONE)) {
LOG.info("Creating DONE folder at "+ DONE);
- if (! DONEDIR_FS.mkdirs(DONE,
+ if (!DONEDIR_FS.mkdirs(DONE,
new FsPermission(HISTORY_DIR_PERMISSION))) {
throw new IOException("Mkdirs failed to create " + DONE.toString());
}
+
+ Path versionSubdir = new Path(DONE, DONE_DIRECTORY_FORMAT_DIRNAME);
+
+ if (!DONEDIR_FS.exists(versionSubdir)) {
+ if (!DONEDIR_FS.mkdirs(versionSubdir,
+ new FsPermission(HISTORY_DIR_PERMISSION))) {
+ throw new IOException("Mkdirs failed to create " + versionSubdir);
+ }
+ }
}
fileManager.start();
}
@@ -981,6 +992,44 @@ public class JobHistory {
String[] jobDetails = logFileName.split("_");
return jobDetails[4];
}
+
+
+ // This code will be inefficient if the subject contains dozens of
underscores
+ static String escapeUnderscores(String escapee) {
+ return replaceStringInstances(escapee, "_", UNDERSCORE_ESCAPE);
+ }
+
+ static String nonOccursString(String logFileName) {
+ int adHocIndex = 0;
+
+ String unfoundString = "q" + adHocIndex;
+
+ while (logFileName.contains(unfoundString)) {
+ unfoundString = "q" + ++adHocIndex;
+ }
+
+ return unfoundString + "q";
+ }
+
+ // I tolerate this code because I expect a low number of
+ // occurrences in a relatively short string
+ static String replaceStringInstances
+ (String logFileName, String old, String replacement) {
+ int index = logFileName.indexOf(old);
+
+ while (index > 0) {
+ logFileName = (logFileName.substring(0, index)
+ + replacement
+ + replaceStringInstances
+ (logFileName.substring(index + old.length()),
+ old, replacement));
+
+ index = logFileName.indexOf(old);
+ }
+
+ return logFileName;
+ }
+
/**
* Helper class for logging or reading back events related to job start,
finish or failure.
@@ -1079,6 +1128,15 @@ public class JobHistory {
*/
public static String encodeJobHistoryFileName(String logFileName)
throws IOException {
+ String replacementUnderscoreEscape = null;
+
+ if (logFileName.contains(UNDERSCORE_ESCAPE)) {
+ replacementUnderscoreEscape = nonOccursString(logFileName);
+
+ logFileName = replaceStringInstances
+ (logFileName, UNDERSCORE_ESCAPE, replacementUnderscoreEscape);
+ }
+
String encodedFileName = null;
try {
encodedFileName = URLEncoder.encode(logFileName, "UTF-8");
@@ -1088,6 +1146,12 @@ public class JobHistory {
ioe.setStackTrace(uee.getStackTrace());
throw ioe;
}
+
+ if (replacementUnderscoreEscape != null) {
+ encodedFileName = replaceStringInstances
+ (encodedFileName, replacementUnderscoreEscape, UNDERSCORE_ESCAPE);
+ }
+
return encodedFileName;
}
@@ -1173,8 +1237,8 @@ public class JobHistory {
return
id.toString() + "_"
+ submitTime + "_"
- + getUserName(jobConf) + "_"
- + trimJobName(getJobName(jobConf));
+ + escapeUnderscores(getUserName(jobConf)) + "_"
+ + escapeUnderscores(trimJobName(getJobName(jobConf)));
}
/**
@@ -1226,9 +1290,13 @@ public class JobHistory {
}
// Make the pattern matching the job's history file
- final Pattern historyFilePattern =
- Pattern.compile(id.toString() + "_" + DIGITS + "_" + user + "_"
- + escapeRegexChars(jobName) + "+");
+
+ final String regexp
+ = id.toString() + "_" + DIGITS + "_" + user + "_"
+ + escapeRegexChars(jobName) + "+";
+
+ final Pattern historyFilePattern = Pattern.compile(regexp);
+
// a path filter that matches 4 parts of the filenames namely
// - jt-hostname
// - job-id
@@ -1236,14 +1304,16 @@ public class JobHistory {
// - jobname
PathFilter filter = new PathFilter() {
public boolean accept(Path path) {
- String fileName = path.getName();
+ String unescapedFileName = path.getName();
+ String fileName = null;
try {
- fileName = decodeJobHistoryFileName(fileName);
+ fileName = decodeJobHistoryFileName(unescapedFileName);
} catch (IOException ioe) {
LOG.info("Error while decoding history file " + fileName + "."
+ " Ignoring file.", ioe);
return false;
}
+
return historyFilePattern.matcher(fileName).find();
}
};
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobTracker.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobTracker.java?rev=1077555&r1=1077554&r2=1077555&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobTracker.java
(original)
+++
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobTracker.java
Fri Mar 4 04:28:40 2011
@@ -2257,11 +2257,11 @@ public class JobTracker implements MRCon
infoServer.setAttribute
("serialNumberDirectoryDigits",
- new Integer(JobHistory.serialNumberDirectoryDigits()));
+ Integer.valueOf(JobHistory.serialNumberDirectoryDigits()));
infoServer.setAttribute
("serialNumberTotalDigits",
- new Integer(JobHistory.serialNumberTotalDigits()));
+ Integer.valueOf(JobHistory.serialNumberTotalDigits()));
return new Path(historyLogDir).getFileSystem(conf);
}
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/jobhistory.jsp
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/jobhistory.jsp?rev=1077555&r1=1077554&r2=1077555&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/jobhistory.jsp
(original)
+++
hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/jobhistory.jsp
Fri Mar 4 04:28:40 2011
@@ -64,13 +64,11 @@ window.location.href = url;
final int FILENAME_JOBNAME_PART = FILENAME_JOBID_END + 2;
- // XXXXXXXX debug code -- should start with 20
- final int[] SCAN_SIZES = { 3, 5, 20, 50, 200 };
+ final int[] SCAN_SIZES = { 20, 50, 200 };
final int FILES_PER_SCAN = 1000;
- // XXXXX debug -- this should be 100.
- final int DEFAULT_PAGE_SIZE = 10;
+ final int DEFAULT_PAGE_SIZE = 100;
final String DEFAULT_DATE_GLOB_COMPONENT = "*/*/*";
@@ -86,47 +84,54 @@ window.location.href = url;
final String parts[] = dateSplit[0].split(":");
- final String user = (parts.length >= 1)
- ? parts[0].toLowerCase()
- : "";
+ final String rawUser = (parts.length >= 1)
+ ? parts[0].toLowerCase()
+ : "";
+
+ final String userInFname
+ = escapeUnderscores(JobHistory.JobInfo.encodeJobHistoryFileName(
+ HtmlQuoting.unquoteHtmlChars(rawUser))).toLowerCase();
final int currentScanSizeIndex
= (request.getParameter("scansize") == null)
? 0 : Integer.parseInt(request.getParameter("scansize"));
- // DEBUG we temporarily allow a "date" with a leading digit of 4 or 5,
- // and a "month" with a leading digit of 2, because for testing we will
- // use hours resp. minutes for months resp. days.
final String SEARCH_PARSE_REGEX
- = "([0-2]?[0-9])/([0-5]?[0-9])/((?:2[0-9])[0-9][0-9])";
+ = "([0-1]?[0-9])/([0-3]?[0-9])/((?:2[0-9])[0-9][0-9])";
final Pattern dateSearchParse = Pattern.compile(SEARCH_PARSE_REGEX);
- final String jobname = (parts.length >= 2)
- ? parts[1].toLowerCase()
- : "";
+ final String rawJobname = (parts.length >= 2)
+ ? parts[1].toLowerCase()
+ : "";
+
+ final String jobnameKeywordInFname
+ = escapeUnderscores(JobHistory.JobInfo.encodeJobHistoryFileName(
+ HtmlQuoting.unquoteHtmlChars(rawJobname))).toLowerCase();
+
PathFilter jobLogFileFilter = new PathFilter() {
- // unquote params before encoding for search
- final String uqUser = JobHistory.JobInfo.encodeJobHistoryFileName(
- HtmlQuoting.unquoteHtmlChars(user));
- final String uqJobname = JobHistory.JobInfo.encodeJobHistoryFileName(
- HtmlQuoting.unquoteHtmlChars(jobname));
private boolean matchUser(String fileName) {
// return true if
// - user is not specified
// - user matches
- return "".equals(uqUser) ||
uqUser.equals(fileName.split("_")[FILENAME_USER_PART]);
+ return "".equals(userInFname)
+ || userInFname.equals(fileName.split("_")[FILENAME_USER_PART]
+ .toLowerCase());
}
private boolean matchJobName(String fileName) {
// return true if
// - jobname is not specified
// - jobname contains the keyword
- return "".equals(uqJobname) ||
fileName.split("_")[FILENAME_JOBNAME_PART].toLowerCase().contains(uqJobname);
+ return "".equals(jobnameKeywordInFname)
+ || fileName.split("_")[FILENAME_JOBNAME_PART].toLowerCase()
+ .contains(jobnameKeywordInFname);
}
public boolean accept(Path path) {
- return !(path.getName().endsWith(".xml")) && matchUser(path.getName())
&& matchJobName(path.getName());
+ String name = path.getName();
+
+ return !(name.endsWith(".xml")) && matchUser(name) &&
matchJobName(name);
}
};
@@ -290,8 +295,8 @@ window.location.href = url;
+ "</h2>");
}
- out.println("<!-- user : " + user +
- ", jobname : " + jobname + "-->");
+ out.println("<!-- user : " + rawUser +
+ ", jobname : " + rawJobname + "-->");
if (null == jobFiles || jobFiles.length == 0) {
out.println("No files found!");
return ;
@@ -364,13 +369,13 @@ window.location.href = url;
+ (sizeIsExact
? ""
: ", <b>" + numHistoryFiles + "</b> gotten"));
- if (!"".equals(user)) {
+ if (!"".equals(rawUser)) {
// show the user if present
- out.println(" for user <b>" + user + "</b>");
+ out.println(" for user <b>" + rawUser + "</b>");
}
- if (!"".equals(jobname)) {
+ if (!"".equals(rawJobname)) {
out.println(" with jobname having the keyword <b>" +
- jobname + "</b> in it.");
+ rawJobname + "</b> in it.");
// show the jobname keyword if present
}
if (!DEFAULT_DATE_GLOB_COMPONENT.equals(dateComponent)) {
@@ -430,9 +435,14 @@ window.location.href = url;
Set<String> displayedJobs = new HashSet<String>();
for (int i = start - 1; i < start + length - 1; ++i) {
Path jobFile = jobFiles[i];
+
+ String fname = jobFile.getName();
+ String marker = JobHistory.nonOccursString(fname);
+ String reescapedFname = JobHistory.replaceStringInstances(fname,
+ JobHistory.UNDERSCORE_ESCAPE, marker);
String decodedJobFileName =
- JobHistory.JobInfo.decodeJobHistoryFileName(jobFile.getName());
+ JobHistory.JobInfo.decodeJobHistoryFileName(reescapedFname);
String[] jobDetails = decodedJobFileName.split("_");
String trackerStartTime = jobDetails[1];
@@ -440,8 +450,10 @@ window.location.href = url;
+ "_" + jobDetails[JOB_ID_START + 1]
+ "_" + jobDetails[JOB_ID_START + 2]);
String submitTimestamp = jobDetails[FILENAME_SUBMIT_TIMESTAMP_PART];
- String userName = jobDetails[FILENAME_USER_PART];
- String jobName = jobDetails[FILENAME_JOBNAME_PART];
+ String userName =
JobHistory.replaceStringInstances(jobDetails[FILENAME_USER_PART],
+ marker, JobHistory.UNDERSCORE_ESCAPE);
+ String jobName =
JobHistory.replaceStringInstances(jobDetails[FILENAME_JOBNAME_PART],
+ marker, JobHistory.UNDERSCORE_ESCAPE);
// Check if the job is already displayed. There can be multiple job
// history files for jobs that have restarted
@@ -452,8 +464,11 @@ window.location.href = url;
}
// Encode the logfile name again to cancel the decoding done by the
browser
- String encodedJobFileName =
+ String preEncodedJobFileName =
JobHistory.JobInfo.encodeJobHistoryFileName(jobFile.getName());
+
+ String encodedJobFileName =
+ JobHistory.replaceStringInstances(preEncodedJobFileName, "%5F",
"%255F");
%>
<center>
<%
@@ -478,11 +493,37 @@ window.location.href = url;
out.print("<td>" + new Date(Long.parseLong(timestamp)) + "</td>");
out.print("<td>" + "<a href=\"jobdetailshistory.jsp?logFile="
+ logFile.toString() + "\">" + jobId + "</a></td>");
- out.print("<td>" + HtmlQuoting.quoteHtmlChars(jobName) + "</td>");
- out.print("<td>" + HtmlQuoting.quoteHtmlChars(user) + "</td>");
+ out.print("<td>"
+ + HtmlQuoting.quoteHtmlChars(unescapeUnderscores(jobName))
+ + "</td>");
+ out.print("<td>"
+ + HtmlQuoting.quoteHtmlChars(unescapeUnderscores(user))
+ + "</td>");
out.print("</tr>");
}
+ private String escapeUnderscores(String rawString) {
+ return convertStrings(rawString, "_", "%5F");
+ }
+
+ private String unescapeUnderscores(String rawString) {
+ return convertStrings(rawString, "%5F", "_");
+ }
+
+ // inefficient if there are a lot of underscores
+ private String convertStrings(String escapedString, String from, String
to) {
+ int firstEscape = escapedString.indexOf(from);
+
+ if (firstEscape < 0) {
+ return escapedString;
+ }
+
+ return escapedString.substring(0, firstEscape)
+ + to
+ + unescapeUnderscores(escapedString.substring
+ (firstEscape + from.length()));
+ }
+
private void printNavigationTool(int pageno, int size, int max,
String searchPlusScan, JspWriter out)
throws IOException {
