Author: sradia
Date: Fri Jun 1 01:04:40 2012
New Revision: 1344955
URL: http://svn.apache.org/viewvc?rev=1344955&view=rev
Log:
HADOOP-8411 TestStorageDirecotyFailure, TestTaskLogsTruncater, TestWebHdfsUrl
and TestSecurityUtil fail on Windows (Ivan Mitic via Sanjay Radia)
Modified:
hadoop/common/branches/branch-1-win/CHANGES.txt
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskLogsTruncater.java
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageDirectoryFailure.java
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/security/TestSecurityUtil.java
Modified: hadoop/common/branches/branch-1-win/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.txt?rev=1344955&r1=1344954&r2=1344955&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1-win/CHANGES.txt Fri Jun 1 01:04:40 2012
@@ -20,6 +20,9 @@ branch-hadoop-1-win - unreleased
HADOOP-8235 Support file permissions and ownership on Windows for
RawLocalFileSystem (Chuan Liu via sanjay)
+ HADOOP-8411 TestStorageDirecotyFailure, TestTaskLogsTruncater,
TestWebHdfsUrl and TestSecurityUtil
+ fail on Windows (Ivan Mitic via Sanjay Radia)
+
Release 1.1.0 - unreleased
NEW FEATURES
Modified:
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskLogsTruncater.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskLogsTruncater.java?rev=1344955&r1=1344954&r2=1344955&view=diff
==============================================================================
---
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskLogsTruncater.java
(original)
+++
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskLogsTruncater.java
Fri Jun 1 01:04:40 2012
@@ -46,6 +46,7 @@ import org.apache.hadoop.mapred.TaskLog.
import org.apache.hadoop.mapred.TaskLog.LogFileDetail;
import org.apache.hadoop.mapreduce.server.tasktracker.JVMInfo;
import org.apache.hadoop.mapreduce.server.tasktracker.userlogs.UserLogManager;
+import org.apache.hadoop.util.Shell;
/**
* The class for truncating the user logs.
@@ -284,7 +285,7 @@ public class TaskLogsTruncater {
// ////// End of closing the file streams ////////////
// ////// Commit the changes from tmp file to the logFile ////////////
- if (!tmpFile.renameTo(logFile)) {
+ if (!renameAtomicWithOverride(tmpFile, logFile)) {
// If the tmpFile cannot be renamed revert back
// updatedTaskLogFileDetails to maintain the consistency of the
// original log file
@@ -305,6 +306,43 @@ public class TaskLogsTruncater {
}
/**
+ * Renames a source file into a target file. Overrides the target file
+ * if it already exists.
+ */
+ private boolean renameAtomicWithOverride(File source, File target) {
+ if (Shell.WINDOWS && target.exists()) {
+ // Rename into the existing file fails on Windows, hence we have to
+ // rename using a temp file to provide some level of atomic behavior
+ // (and revert changes back if something fails)
+ File tmpTargetFile = new File(target.getPath() + ".tmp");
+ if (target.renameTo(tmpTargetFile)) {
+ if (source.renameTo(target)) {
+ // Rename succeeded, try to delete the target backup
+ if (!tmpTargetFile.delete()) {
+ LOG.warn("Cannot delete tmpTargetFile "
+ + tmpTargetFile.getAbsolutePath());
+ tmpTargetFile.deleteOnExit();
+ }
+ // Return true anyways as we successfully renamed source
+ // into target
+ return true;
+ } else {
+ // Revert back the original
+ if (!tmpTargetFile.renameTo(target)) {
+ LOG.error("Cannot revert back the original log file "
+ + target.getAbsolutePath());
+ }
+ return false;
+ }
+ } else {
+ return false;
+ }
+ } else {
+ return source.renameTo(target);
+ }
+ }
+
+ /**
* @param lInfo
* @param taskLogFileDetails
* @param updatedTaskLogFileDetails
Modified:
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageDirectoryFailure.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageDirectoryFailure.java?rev=1344955&r1=1344954&r2=1344955&view=diff
==============================================================================
---
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageDirectoryFailure.java
(original)
+++
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageDirectoryFailure.java
Fri Jun 1 01:04:40 2012
@@ -44,6 +44,7 @@ import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
+import org.apache.hadoop.util.Shell;
/**
* Test that the NN stays up as long as it has a valid storage directory and
@@ -125,6 +126,17 @@ public class TestStorageDirectoryFailure
assertTrue(Arrays.equals(buff, readFile(name, buff.length)));
}
+ /** Fully delete the storage directory */
+ private void FullyDeleteStorageDir(String name) throws IOException {
+ if (Shell.WINDOWS) {
+ // Files that are currently in use cannot be deleted on Windows, hence
+ // we are closing the log files explicitly.
+ cluster.getNameNode().getFSImage().getEditLog().close();
+ cluster.getNameNode().getFSImage().unlockAll();
+ }
+ FileUtil.fullyDelete(new File(name));
+ }
+
@Test
/** Remove storage dirs and checkpoint to trigger detection */
public void testCheckpointAfterFailingFirstNamedir() throws IOException {
@@ -133,7 +145,7 @@ public class TestStorageDirectoryFailure
checkFileCreation("file0");
// Remove the 1st storage dir
- FileUtil.fullyDelete(new File(nameDirs.get(0)));
+ FullyDeleteStorageDir(nameDirs.get(0));
secondaryNN.doCheckpoint();
assertEquals(1, numRemovedDirs());
assertEquals(nameDirs.get(0), getRemovedDirs().get(0).getRoot().getPath());
@@ -141,7 +153,7 @@ public class TestStorageDirectoryFailure
checkFileCreation("file1");
// Remove the 2nd
- FileUtil.fullyDelete(new File(nameDirs.get(1)));
+ FullyDeleteStorageDir(nameDirs.get(1));
secondaryNN.doCheckpoint();
assertEquals(2, numRemovedDirs());
assertEquals(nameDirs.get(1), getRemovedDirs().get(1).getRoot().getPath());
@@ -156,7 +168,7 @@ public class TestStorageDirectoryFailure
// After the checkpoint, we should be dead. Verify fatalExit was
// called and that eg a checkpoint fails.
- FileUtil.fullyDelete(new File(nameDirs.get(2)));
+ FullyDeleteStorageDir(nameDirs.get(2));
try {
secondaryNN.doCheckpoint();
fail("There's no storage to retrieve an image from");
@@ -183,7 +195,7 @@ public class TestStorageDirectoryFailure
checkFileCreation("file0");
- FileUtil.fullyDelete(new File(nameDirs.get(0)));
+ FullyDeleteStorageDir(nameDirs.get(0));
secondaryNN.doCheckpoint();
assertEquals(1, numRemovedDirs());
assertEquals(nameDirs.get(0), getRemovedDirs().get(0).getRoot().getPath());
Modified:
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java?rev=1344955&r1=1344954&r2=1344955&view=diff
==============================================================================
---
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
(original)
+++
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
Fri Jun 1 01:04:40 2012
@@ -71,14 +71,14 @@ public class TestWebHdfsUrl {
fsPath, new TokenArgumentParam(tokenString));
Assert.assertEquals(
generateUrlQueryPrefix(PutOpParam.Op.RENEWDELEGATIONTOKEN,
- ugi.getUserName())
+ ugi.getShortUserName())
+ "&token=" + tokenString, renewTokenUrl.getQuery());
Token<DelegationTokenIdentifier> delegationToken = new
Token<DelegationTokenIdentifier>(
token);
delegationToken.setKind(WebHdfsFileSystem.TOKEN_KIND);
Assert.assertEquals(
generateUrlQueryPrefix(PutOpParam.Op.CANCELDELEGATIONTOKEN,
- ugi.getUserName())
+ ugi.getShortUserName())
+ "&token="
+ tokenString
+ "&"
Modified:
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/security/TestSecurityUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/security/TestSecurityUtil.java?rev=1344955&r1=1344954&r2=1344955&view=diff
==============================================================================
---
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/security/TestSecurityUtil.java
(original)
+++
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/security/TestSecurityUtil.java
Fri Jun 1 01:04:40 2012
@@ -95,7 +95,9 @@ public class TestSecurityUtil {
@Test
public void testLocalHostNameForNullOrWild() throws Exception {
- String local = SecurityUtil.getLocalHostName();
+ // Get local host name and convert to lower case to be consistent with
+ // SecurityUtil.replacePattern
+ String local = SecurityUtil.getLocalHostName().toLowerCase();
assertEquals("hdfs/" + local + "@REALM", SecurityUtil.getServerPrincipal(
"hdfs/_HOST@REALM", (String) null));
assertEquals("hdfs/" + local + "@REALM", SecurityUtil.getServerPrincipal(
