Author: suresh
Date: Thu Dec 13 04:31:33 2012
New Revision: 1421075
URL: http://svn.apache.org/viewvc?rev=1421075&view=rev
Log:
Merging trunk to branch-trunk-win
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.txt
(contents, props changed)
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/docs/
(props changed)
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/
(props changed)
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsMappingWithFallback.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/core/
(props changed)
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1421075&r1=1421074&r2=1421075&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Dec 13 04:31:33 2012
@@ -298,6 +298,9 @@ Trunk (Unreleased)
HADOOP-9131. Turn off TestLocalFileSystem#testListStatusWithColons on
Windows. (Chris Nauroth via suresh)
+ HADOOP-8957 AbstractFileSystem#IsValidName should be overridden for
+ embedded file systems like ViewFs (Chris Nauroth via Sanjay Radia)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
@@ -485,6 +488,9 @@ Release 2.0.3-alpha - Unreleased
HADOOP-9113. o.a.h.fs.TestDelegationTokenRenewer is failing intermittently.
(Karthik Kambatla via eli)
+ HADOOP-9135. JniBasedUnixGroupsMappingWithFallback should log at debug
+ rather than info during fallback. (Colin Patrick McCabe via todd)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1420493-1421074
Propchange:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1420493-1421074
Propchange:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1420493-1421074
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java?rev=1421075&r1=1421074&r2=1421075&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
Thu Dec 13 04:31:33 2012
@@ -85,14 +85,20 @@ public abstract class AbstractFileSystem
}
/**
- * Prohibits names which contain a ".", "..", ":" or "/"
+ * Returns true if the specified string is considered valid in the path part
+ * of a URI by this file system. The default implementation enforces the
rules
+ * of HDFS, but subclasses may override this method to implement specific
+ * validation rules for specific file systems.
+ *
+ * @param src String source filename to check, path part of the URI
+ * @return boolean true if the specified string is considered valid
*/
- private static boolean isValidName(String src) {
- // Check for ".." "." ":" "/"
+ public boolean isValidName(String src) {
+ // Prohibit ".." "." and anything containing ":"
StringTokenizer tokens = new StringTokenizer(src, Path.SEPARATOR);
while(tokens.hasMoreTokens()) {
String element = tokens.nextToken();
- if (element.equals("target/generated-sources") ||
+ if (element.equals("..") ||
element.equals(".") ||
(element.indexOf(":") >= 0)) {
return false;
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java?rev=1421075&r1=1421074&r2=1421075&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
Thu Dec 13 04:31:33 2012
@@ -278,4 +278,9 @@ public abstract class FilterFs extends A
public List<Token<?>> getDelegationTokens(String renewer) throws IOException
{
return myFs.getDelegationTokens(renewer);
}
+
+ @Override
+ public boolean isValidName(String src) {
+ return myFs.isValidName(src);
+ }
}
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java?rev=1421075&r1=1421074&r2=1421075&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
Thu Dec 13 04:31:33 2012
@@ -157,6 +157,14 @@ public class RawLocalFs extends Delegate
}
}
+ @Override
+ public boolean isValidName(String src) {
+ // Different local file systems have different validation rules. Skip
+ // validation here and just let the OS handle it. This is consistent with
+ // RawLocalFileSystem.
+ return true;
+ }
+
@Override
public Path getLinkTarget(Path f) throws IOException {
/* We should never get here. Valid local links are resolved transparently
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java?rev=1421075&r1=1421074&r2=1421075&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
Thu Dec 13 04:31:33 2012
@@ -83,7 +83,12 @@ class ChRootedFs extends AbstractFileSys
return new Path((chRootPathPart.isRoot() ? "" : chRootPathPartString)
+ path.toUri().getPath());
}
-
+
+ @Override
+ public boolean isValidName(String src) {
+ return myFs.isValidName(fullPath(new Path(src)).toUri().toString());
+ }
+
public ChRootedFs(final AbstractFileSystem fs, final Path theRoot)
throws URISyntaxException {
super(fs.getUri(), fs.getUri().getScheme(),
@@ -103,7 +108,7 @@ class ChRootedFs extends AbstractFileSys
// scheme:/// and scheme://authority/
myUri = new URI(myFs.getUri().toString() +
(myFs.getUri().getAuthority() == null ? "" : Path.SEPARATOR) +
- chRootPathPart.toString().substring(1));
+ chRootPathPart.toUri().getPath().substring(1));
super.checkPath(theRoot);
}
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java?rev=1421075&r1=1421074&r2=1421075&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
Thu Dec 13 04:31:33 2012
@@ -62,6 +62,9 @@ import org.apache.hadoop.util.Time;
@InterfaceAudience.Public
@InterfaceStability.Evolving /*Evolving for a release,to be changed to Stable
*/
public class ViewFileSystem extends FileSystem {
+
+ private static final Path ROOT_PATH = new Path(Path.SEPARATOR);
+
static AccessControlException readOnlyMountTable(final String operation,
final String p) {
return new AccessControlException(
@@ -97,23 +100,6 @@ public class ViewFileSystem extends File
Path homeDir = null;
/**
- * Prohibits names which contain a ".", "..", ":" or "/"
- */
- private static boolean isValidName(final String src) {
- // Check for ".." "." ":" "/"
- final StringTokenizer tokens = new StringTokenizer(src, Path.SEPARATOR);
- while(tokens.hasMoreTokens()) {
- String element = tokens.nextToken();
- if (element.equals("..") ||
- element.equals(".") ||
- (element.indexOf(":") >= 0)) {
- return false;
- }
- }
- return true;
- }
-
- /**
* Make the path Absolute and get the path-part of a pathname.
* Checks that URI matches this file system
* and that the path-part is a valid name.
@@ -124,10 +110,6 @@ public class ViewFileSystem extends File
private String getUriPath(final Path p) {
checkPath(p);
String s = makeAbsolute(p).toUri().getPath();
- if (!isValidName(s)) {
- throw new InvalidPathException("Path part " + s + " from URI" + p
- + " is not a valid filename.");
- }
return s;
}
@@ -672,7 +654,7 @@ public class ViewFileSystem extends File
PERMISSION_RRR, ugi.getUserName(), ugi.getGroupNames()[0],
new Path(theInternalDir.fullPath).makeQualified(
- myUri, null));
+ myUri, ROOT_PATH));
}
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java?rev=1421075&r1=1421074&r2=1421075&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
Thu Dec 13 04:31:33 2012
@@ -597,6 +597,12 @@ public class ViewFs extends AbstractFile
return result;
}
+ @Override
+ public boolean isValidName(String src) {
+ // Prefix validated at mount time and rest of path validated by mount
target.
+ return true;
+ }
+
/*
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsMappingWithFallback.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsMappingWithFallback.java?rev=1421075&r1=1421074&r2=1421075&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsMappingWithFallback.java
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsMappingWithFallback.java
Thu Dec 13 04:31:33 2012
@@ -37,7 +37,7 @@ public class JniBasedUnixGroupsMappingWi
if (NativeCodeLoader.isNativeCodeLoaded()) {
this.impl = new JniBasedUnixGroupsMapping();
} else {
- LOG.info("Falling back to shell based");
+ LOG.debug("Falling back to shell based");
this.impl = new ShellBasedUnixGroupsMapping();
}
if (LOG.isDebugEnabled()){
Propchange:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1420493-1421074
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java?rev=1421075&r1=1421074&r2=1421075&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java
Thu Dec 13 04:31:33 2012
@@ -25,6 +25,7 @@ import java.util.EnumSet;
import static org.apache.hadoop.fs.FileContextTestHelper.*;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.AbstractFileSystem;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileContextTestHelper;
@@ -36,6 +37,7 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
public class TestChRootedFs {
FileContext fc; // The ChRoootedFs
@@ -314,4 +316,21 @@ public class TestChRootedFs {
fc.getDefaultFileSystem().resolvePath(new Path("/nonExisting"));
}
+ @Test
+ public void testIsValidNameValidInBaseFs() throws Exception {
+ AbstractFileSystem baseFs = Mockito.spy(fc.getDefaultFileSystem());
+ ChRootedFs chRootedFs = new ChRootedFs(baseFs, new Path("/chroot"));
+ Mockito.doReturn(true).when(baseFs).isValidName(Mockito.anyString());
+ Assert.assertTrue(chRootedFs.isValidName("/test"));
+ Mockito.verify(baseFs).isValidName("/chroot/test");
+ }
+
+ @Test
+ public void testIsValidNameInvalidInBaseFs() throws Exception {
+ AbstractFileSystem baseFs = Mockito.spy(fc.getDefaultFileSystem());
+ ChRootedFs chRootedFs = new ChRootedFs(baseFs, new Path("/chroot"));
+ Mockito.doReturn(false).when(baseFs).isValidName(Mockito.anyString());
+ Assert.assertFalse(chRootedFs.isValidName("/test"));
+ Mockito.verify(baseFs).isValidName("/chroot/test");
+ }
}
