Author: acmurthy
Date: Wed Nov 28 15:17:21 2012
New Revision: 1414750
URL: http://svn.apache.org/viewvc?rev=1414750&view=rev
Log:
Rebased on trunk.
Added:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathAccessDeniedException.java
- copied unchanged from r1414746,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathAccessDeniedException.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathExistsException.java
- copied unchanged from r1414746,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathExistsException.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIOException.java
- copied unchanged from r1414746,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIOException.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIsDirectoryException.java
- copied unchanged from r1414746,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIsDirectoryException.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIsNotDirectoryException.java
- copied unchanged from r1414746,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIsNotDirectoryException.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIsNotEmptyDirectoryException.java
- copied unchanged from r1414746,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIsNotEmptyDirectoryException.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathNotFoundException.java
- copied unchanged from r1414746,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathNotFoundException.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathOperationException.java
- copied unchanged from r1414746,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathOperationException.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathPermissionException.java
- copied unchanged from r1414746,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathPermissionException.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDelegationTokenRenewer.java
- copied unchanged from r1414746,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDelegationTokenRenewer.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java
- copied unchanged from r1414746,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java
Removed:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathExceptions.java
Modified:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/CHANGES.txt
(contents, props changed)
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/docs/
(props changed)
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/
(props changed)
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Mkdir.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Test.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touchz.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/core/
(props changed)
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathExceptions.java
Modified:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
---
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/CHANGES.txt
Wed Nov 28 15:17:21 2012
@@ -132,7 +132,14 @@ Trunk (Unreleased)
HADOOP-9004. Allow security unit tests to use external KDC. (Stephen Chu
via suresh)
- HADOOP-6616. Improve documentation for rack awareness. (Adam Faris via
jghoman)
+ HADOOP-6616. Improve documentation for rack awareness. (Adam Faris via
+ jghoman)
+
+ HADOOP-9075. FileContext#FSLinkResolver should be made static.
+ (Arpit Agarwal via suresh)
+
+ HADOOP-9093. Move all the Exception in PathExceptions to o.a.h.fs package.
+ (suresh)
BUG FIXES
@@ -373,6 +380,9 @@ Release 2.0.3-alpha - Unreleased
HADOOP-9035. Generalize setup of LoginContext (daryn via bobby)
+ HADOOP-9042. Add a test for umask in FileSystemContractBaseTest.
+ (Colin McCabe via eli)
+
OPTIMIZATIONS
HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
@@ -437,6 +447,12 @@ Release 2.0.3-alpha - Unreleased
HADOOP-6607. Add different variants of non caching HTTP headers. (tucu)
+ HADOOP-9049. DelegationTokenRenewer needs to be Singleton and FileSystems
+ should register/deregister to/from. (Karthik Kambatla via tomwhite)
+
+ HADOOP-9064. Augment DelegationTokenRenewer API to cancel the tokens on
+ calls to removeRenewAction. (kkambatl via tucu)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
@@ -1139,6 +1155,15 @@ Release 0.23.6 - UNRELEASED
BUG FIXES
+ HADOOP-9072. Hadoop-Common-0.23-Build Fails to build in Jenkins
+ (Robert Parker via tgraves)
+
+ HADOOP-8992. Enhance unit-test coverage of class HarFileSystem (Ivan A.
+ Veselovsky via bobby)
+
+ HADOOP-9038. unit-tests for AllocatorPerContext.PathIterator (Ivan A.
+ Veselovsky via bobby)
+
Release 0.23.5 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1411704-1414746
Modified:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
---
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
(original)
+++
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
Wed Nov 28 15:17:21 2012
@@ -291,5 +291,13 @@
<Field name="previousSnapshot" />
<Bug pattern="IS2_INCONSISTENT_SYNC" />
</Match>
-
+ <!--
+ The method uses a generic type T that extends two other types
+ T1 and T2. Findbugs complains of a cast from T1 to T2.
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.fs.DelegationTokenRenewer" />
+ <Method name="removeRenewAction" />
+ <Bug pattern="BC_UNCONFIRMED_CAST" />
+ </Match>
</FindBugsFilter>
Propchange:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1411704-1414746
Propchange:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1411704-1414746
Modified:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
---
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java
(original)
+++
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java
Wed Nov 28 15:17:21 2012
@@ -24,6 +24,8 @@ import java.util.concurrent.DelayQueue;
import java.util.concurrent.Delayed;
import java.util.concurrent.TimeUnit;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
@@ -33,8 +35,11 @@ import org.apache.hadoop.util.Time;
* A daemon thread that waits for the next file system to renew.
*/
@InterfaceAudience.Private
-public class DelegationTokenRenewer<T extends FileSystem &
DelegationTokenRenewer.Renewable>
+public class DelegationTokenRenewer
extends Thread {
+ private static final Log LOG = LogFactory
+ .getLog(DelegationTokenRenewer.class);
+
/** The renewable interface used by the renewer. */
public interface Renewable {
/** @return the renew token. */
@@ -93,7 +98,7 @@ public class DelegationTokenRenewer<T ex
* @param newTime the new time
*/
private void updateRenewalTime() {
- renewalTime = RENEW_CYCLE + Time.now();
+ renewalTime = renewCycle + Time.now();
}
/**
@@ -134,34 +139,82 @@ public class DelegationTokenRenewer<T ex
}
/** Wait for 95% of a day between renewals */
- private static final int RENEW_CYCLE = 24 * 60 * 60 * 950;
+ private static final int RENEW_CYCLE = 24 * 60 * 60 * 950;
- private DelayQueue<RenewAction<T>> queue = new DelayQueue<RenewAction<T>>();
+ @InterfaceAudience.Private
+ protected static int renewCycle = RENEW_CYCLE;
- public DelegationTokenRenewer(final Class<T> clazz) {
+ /** Queue to maintain the RenewActions to be processed by the {@link #run()}
*/
+ private volatile DelayQueue<RenewAction<?>> queue = new
DelayQueue<RenewAction<?>>();
+
+ /**
+ * Create the singleton instance. However, the thread can be started lazily
in
+ * {@link #addRenewAction(FileSystem)}
+ */
+ private static DelegationTokenRenewer INSTANCE = null;
+
+ private DelegationTokenRenewer(final Class<? extends FileSystem> clazz) {
super(clazz.getSimpleName() + "-" +
DelegationTokenRenewer.class.getSimpleName());
setDaemon(true);
}
+ public static synchronized DelegationTokenRenewer getInstance() {
+ if (INSTANCE == null) {
+ INSTANCE = new DelegationTokenRenewer(FileSystem.class);
+ }
+ return INSTANCE;
+ }
+
/** Add a renew action to the queue. */
- public void addRenewAction(final T fs) {
+ public synchronized <T extends FileSystem & Renewable> void
addRenewAction(final T fs) {
queue.add(new RenewAction<T>(fs));
+ if (!isAlive()) {
+ start();
+ }
+ }
+
+ /**
+ * Remove the associated renew action from the queue
+ *
+ * @throws IOException
+ */
+ public synchronized <T extends FileSystem & Renewable> void
removeRenewAction(
+ final T fs) throws IOException {
+ for (RenewAction<?> action : queue) {
+ if (action.weakFs.get() == fs) {
+ try {
+ fs.getRenewToken().cancel(fs.getConf());
+ } catch (InterruptedException ie) {
+ LOG.error("Interrupted while canceling token for " + fs.getUri()
+ + "filesystem");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(ie.getStackTrace());
+ }
+ }
+ queue.remove(action);
+ return;
+ }
+ }
}
+ @SuppressWarnings("static-access")
@Override
public void run() {
for(;;) {
- RenewAction<T> action = null;
+ RenewAction<?> action = null;
try {
- action = queue.take();
- if (action.renew()) {
- action.updateRenewalTime();
- queue.add(action);
+ synchronized (this) {
+ action = queue.take();
+ if (action.renew()) {
+ action.updateRenewalTime();
+ queue.add(action);
+ }
}
} catch (InterruptedException ie) {
return;
} catch (Exception ie) {
- T.LOG.warn("Failed to renew token, action=" + action, ie);
+ action.weakFs.get().LOG.warn("Failed to renew token, action=" + action,
+ ie);
}
}
}
Modified:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
---
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
(original)
+++
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
Wed Nov 28 15:17:21 2012
@@ -1119,7 +1119,7 @@ public final class FileContext {
* @param target The symlink's absolute target
* @return Fully qualified version of the target.
*/
- private Path qualifySymlinkTarget(final AbstractFileSystem pathFS,
+ private static Path qualifySymlinkTarget(final AbstractFileSystem pathFS,
Path pathWithLink, Path target) {
// NB: makeQualified uses the target's scheme and authority, if
// specified, and the scheme and authority of pathFS, if not.
@@ -2321,7 +2321,7 @@ public final class FileContext {
* Class used to perform an operation on and resolve symlinks in a
* path. The operation may potentially span multiple file systems.
*/
- protected abstract class FSLinkResolver<T> {
+ protected static abstract class FSLinkResolver<T> {
// The maximum number of symbolic link components in a path
private static final int MAX_PATH_LINKS = 32;
Modified:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
---
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
(original)
+++
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
Wed Nov 28 15:17:21 2012
@@ -584,13 +584,6 @@ public class HarFileSystem extends Filte
public String getName() {
return name;
}
-
- public List<String> getChildren() {
- return children;
- }
- public String getFileName() {
- return name;
- }
public String getPartName() {
return partName;
}
@@ -662,15 +655,6 @@ public class HarFileSystem extends Filte
hstatus.getStartIndex(), hstatus.getLength(), bufferSize);
}
- /*
- * create throws an exception in Har filesystem.
- * The archive once created cannot be changed.
- */
- public FSDataOutputStream create(Path f, int bufferSize)
- throws IOException {
- throw new IOException("Har: Create not allowed");
- }
-
@Override
public FSDataOutputStream create(Path f,
FsPermission permission,
@@ -1106,4 +1090,11 @@ public class HarFileSystem extends Filte
}
}
}
+
+ /*
+ * testing purposes only:
+ */
+ HarMetaData getMetadata() {
+ return metadata;
+ }
}
Modified:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
---
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
(original)
+++
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
Wed Nov 28 15:17:21 2012
@@ -481,12 +481,15 @@ public class LocalDirAllocator {
@Override
public Path next() {
- Path result = next;
+ final Path result = next;
try {
advance();
} catch (IOException ie) {
throw new RuntimeException("Can't check existance of " + next, ie);
}
+ if (result == null) {
+ throw new NoSuchElementException();
+ }
return result;
}
Modified:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
---
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
(original)
+++
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
Wed Nov 28 15:17:21 2012
@@ -34,7 +34,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.shell.PathExceptions.PathNotFoundException;
+import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.util.StringUtils;
/**
Modified:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
---
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
(original)
+++
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
Wed Nov 28 15:17:21 2012
@@ -27,12 +27,12 @@ import org.apache.hadoop.fs.FSDataOutput
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FilterFileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.shell.PathExceptions.PathExistsException;
-import org.apache.hadoop.fs.shell.PathExceptions.PathIOException;
-import org.apache.hadoop.fs.shell.PathExceptions.PathIsDirectoryException;
-import org.apache.hadoop.fs.shell.PathExceptions.PathIsNotDirectoryException;
-import org.apache.hadoop.fs.shell.PathExceptions.PathNotFoundException;
-import org.apache.hadoop.fs.shell.PathExceptions.PathOperationException;
+import org.apache.hadoop.fs.PathExistsException;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.PathIsDirectoryException;
+import org.apache.hadoop.fs.PathIsNotDirectoryException;
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.fs.PathOperationException;
import org.apache.hadoop.io.IOUtils;
/**
Modified:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
---
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
(original)
+++
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
Wed Nov 28 15:17:21 2012
@@ -27,7 +27,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.shell.PathExceptions.PathIsDirectoryException;
+import org.apache.hadoop.fs.PathIsDirectoryException;
import org.apache.hadoop.io.IOUtils;
/** Various commands for copy files */
Modified:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
---
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java
(original)
+++
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java
Wed Nov 28 15:17:21 2012
@@ -24,11 +24,11 @@ import java.util.LinkedList;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.PathIsDirectoryException;
+import org.apache.hadoop.fs.PathIsNotDirectoryException;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
import org.apache.hadoop.fs.Trash;
-import org.apache.hadoop.fs.shell.PathExceptions.PathIOException;
-import org.apache.hadoop.fs.shell.PathExceptions.PathIsDirectoryException;
-import org.apache.hadoop.fs.shell.PathExceptions.PathIsNotDirectoryException;
-import
org.apache.hadoop.fs.shell.PathExceptions.PathIsNotEmptyDirectoryException;
/**
* Classes that delete paths
Modified:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
---
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
(original)
+++
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
Wed Nov 28 15:17:21 2012
@@ -38,7 +38,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.shell.PathExceptions.PathIsDirectoryException;
+import org.apache.hadoop.fs.PathIsDirectoryException;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.IOUtils;
Modified:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Mkdir.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Mkdir.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
---
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Mkdir.java
(original)
+++
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Mkdir.java
Wed Nov 28 15:17:21 2012
@@ -24,10 +24,10 @@ import java.util.LinkedList;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.shell.PathExceptions.PathExistsException;
-import org.apache.hadoop.fs.shell.PathExceptions.PathIOException;
-import org.apache.hadoop.fs.shell.PathExceptions.PathIsNotDirectoryException;
-import org.apache.hadoop.fs.shell.PathExceptions.PathNotFoundException;
+import org.apache.hadoop.fs.PathExistsException;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.PathIsNotDirectoryException;
+import org.apache.hadoop.fs.PathNotFoundException;
/**
* Create the given dir
Modified:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
---
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java
(original)
+++
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java
Wed Nov 28 15:17:21 2012
@@ -23,8 +23,8 @@ import java.util.LinkedList;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.fs.shell.CopyCommands.CopyFromLocal;
-import org.apache.hadoop.fs.shell.PathExceptions.PathIOException;
/** Various commands for moving files */
@InterfaceAudience.Private
Modified:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
---
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java
(original)
+++
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java
Wed Nov 28 15:17:21 2012
@@ -32,10 +32,10 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.shell.PathExceptions.PathIOException;
-import org.apache.hadoop.fs.shell.PathExceptions.PathIsDirectoryException;
-import org.apache.hadoop.fs.shell.PathExceptions.PathIsNotDirectoryException;
-import org.apache.hadoop.fs.shell.PathExceptions.PathNotFoundException;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.PathIsDirectoryException;
+import org.apache.hadoop.fs.PathIsNotDirectoryException;
+import org.apache.hadoop.fs.PathNotFoundException;
/**
* Encapsulates a Path (path), its FileStatus (stat), and its FileSystem (fs).
Modified:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
---
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java
(original)
+++
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java
Wed Nov 28 15:17:21 2012
@@ -25,7 +25,7 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.BlockLocation;
-import org.apache.hadoop.fs.shell.PathExceptions.PathIOException;
+import org.apache.hadoop.fs.PathIOException;
/**
* Modifies the replication factor
Modified:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
---
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java
(original)
+++
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java
Wed Nov 28 15:17:21 2012
@@ -25,7 +25,7 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.shell.PathExceptions.PathIsDirectoryException;
+import org.apache.hadoop.fs.PathIsDirectoryException;
import org.apache.hadoop.io.IOUtils;
/**
Modified:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Test.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Test.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
---
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Test.java
(original)
+++
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Test.java
Wed Nov 28 15:17:21 2012
@@ -23,7 +23,6 @@ import java.util.LinkedList;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.shell.PathExceptions.PathNotFoundException;
/**
* Perform shell-like file tests
Modified:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touchz.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touchz.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
---
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touchz.java
(original)
+++
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touchz.java
Wed Nov 28 15:17:21 2012
@@ -23,9 +23,9 @@ import java.util.LinkedList;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.shell.PathExceptions.PathIOException;
-import org.apache.hadoop.fs.shell.PathExceptions.PathIsDirectoryException;
-import org.apache.hadoop.fs.shell.PathExceptions.PathNotFoundException;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.PathIsDirectoryException;
+import org.apache.hadoop.fs.PathNotFoundException;
/**
* Unix touch like commands
Propchange:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1411704-1414746
Modified:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
---
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
(original)
+++
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
Wed Nov 28 15:17:21 2012
@@ -23,11 +23,13 @@ import java.io.IOException;
import junit.framework.TestCase;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
/**
* <p>
@@ -43,7 +45,7 @@ import org.apache.hadoop.fs.Path;
* </p>
*/
public abstract class FileSystemContractBaseTest extends TestCase {
-
+ protected final static String TEST_UMASK = "062";
protected FileSystem fs;
protected byte[] data = new byte[getBlockSize() * 2]; // two blocks of data
{
@@ -151,7 +153,26 @@ public abstract class FileSystemContract
assertFalse(fs.exists(testDeepSubDir));
}
-
+
+ public void testMkdirsWithUmask() throws Exception {
+ if (fs.getScheme().equals("s3") || fs.getScheme().equals("s3n")) {
+ // skip permission tests for S3FileSystem until HDFS-1333 is fixed.
+ return;
+ }
+ Configuration conf = fs.getConf();
+ String oldUmask =
conf.get(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY);
+ try {
+ conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, TEST_UMASK);
+ final Path dir = new Path("/test/newDir");
+ assertTrue(fs.mkdirs(dir, new FsPermission((short)0777)));
+ FileStatus status = fs.getFileStatus(dir);
+ assertTrue(status.isDirectory());
+ assertEquals((short)0715, status.getPermission().toShort());
+ } finally {
+ conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, oldUmask);
+ }
+ }
+
public void testGetFileStatusThrowsExceptionForNonExistentFile()
throws Exception {
try {
Modified:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
---
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
(original)
+++
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
Wed Nov 28 15:17:21 2012
@@ -22,6 +22,8 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.Shell;
@@ -32,6 +34,7 @@ import org.junit.runners.Parameterized.P
import org.junit.Test;
import static org.junit.Assert.*;
+import static org.junit.Assume.*;
/** This test LocalDirAllocator works correctly;
* Every test case uses different buffer dirs to
@@ -318,7 +321,7 @@ public class TestLocalDirAllocator {
*/
@Test
public void testNoSideEffects() throws IOException {
- if (isWindows) return;
+ assumeTrue(!isWindows);
String dir = buildBufferDir(ROOT, 0);
try {
conf.set(CONTEXT, dir);
@@ -339,8 +342,7 @@ public class TestLocalDirAllocator {
*/
@Test
public void testGetLocalPathToRead() throws IOException {
- if (isWindows)
- return;
+ assumeTrue(!isWindows);
String dir = buildBufferDir(ROOT, 0);
try {
conf.set(CONTEXT, dir);
@@ -354,7 +356,60 @@ public class TestLocalDirAllocator {
Shell.execCommand(new String[] { "chmod", "u+w", BUFFER_DIR_ROOT });
rmBufferDirs();
}
+ }
+
+ /**
+ * Test that {@link LocalDirAllocator#getAllLocalPathsToRead(String,
Configuration)}
+ * returns correct filenames and "file" schema.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testGetAllLocalPathsToRead() throws IOException {
+ assumeTrue(!isWindows);
+
+ String dir0 = buildBufferDir(ROOT, 0);
+ String dir1 = buildBufferDir(ROOT, 1);
+ try {
+ conf.set(CONTEXT, dir0 + "," + dir1);
+ assertTrue(localFs.mkdirs(new Path(dir0)));
+ assertTrue(localFs.mkdirs(new Path(dir1)));
+
+ localFs.create(new Path(dir0 + Path.SEPARATOR + FILENAME));
+ localFs.create(new Path(dir1 + Path.SEPARATOR + FILENAME));
+
+ // check both the paths are returned as paths to read:
+ final Iterable<Path> pathIterable =
dirAllocator.getAllLocalPathsToRead(FILENAME, conf);
+ int count = 0;
+ for (final Path p: pathIterable) {
+ count++;
+ assertEquals(FILENAME, p.getName());
+ assertEquals("file", p.getFileSystem(conf).getUri().getScheme());
+ }
+ assertEquals(2, count);
+ // test #next() while no element to iterate any more:
+ try {
+ Path p = pathIterable.iterator().next();
+ assertFalse("NoSuchElementException must be thrown, but returned ["+p
+ +"] instead.", true); // exception expected
+ } catch (NoSuchElementException nsee) {
+ // okay
+ }
+
+ // test modification not allowed:
+ final Iterable<Path> pathIterable2 =
dirAllocator.getAllLocalPathsToRead(FILENAME, conf);
+ final Iterator<Path> it = pathIterable2.iterator();
+ try {
+ it.remove();
+ assertFalse(true); // exception expected
+ } catch (UnsupportedOperationException uoe) {
+ // okay
+ }
+ } finally {
+ Shell.execCommand(new String[] { "chmod", "u+w", BUFFER_DIR_ROOT });
+ rmBufferDirs();
+ }
}
@Test
Modified:
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathExceptions.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathExceptions.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
---
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathExceptions.java
(original)
+++
hadoop/common/branches/MR-2454/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathExceptions.java
Wed Nov 28 15:17:21 2012
@@ -23,7 +23,7 @@ import static org.junit.Assert.assertEqu
import java.io.IOException;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.shell.PathExceptions.PathIOException;
+import org.apache.hadoop.fs.PathIOException;
import org.junit.Test;
public class TestPathExceptions {
