Author: szetszwo Date: Mon Apr 9 23:25:17 2012 New Revision: 1311518 URL: http://svn.apache.org/viewvc?rev=1311518&view=rev Log: Merge r1310141 through r1311517 from trunk.
Added: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/NotInMountpointException.java - copied unchanged from r1311517, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/NotInMountpointException.java Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed) hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/docs/ (props changed) hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/ (props changed) hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/core/ (props changed) hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestShellCommandFencer.java Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java?rev=1311518&r1=1311517&r2=1311518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java Mon Apr 9 23:25:17 2012 @@ -331,7 +331,14 @@ public class AuthenticationFilter implem HttpServletResponse httpResponse = (HttpServletResponse) response; try { boolean newToken = false; - AuthenticationToken token = getToken(httpRequest); + AuthenticationToken token; + try { + token = getToken(httpRequest); + } + catch (AuthenticationException ex) { + LOG.warn("AuthenticationToken ignored: " + ex.getMessage()); + token = null; + } if (token == null) { if (LOG.isDebugEnabled()) { LOG.debug("Request [{}] triggering authentication", getRequestURL(httpRequest)); @@ -371,6 +378,9 @@ public class AuthenticationFilter implem } filterChain.doFilter(httpRequest, httpResponse); } + else { + throw new AuthenticationException("Missing AuthenticationToken"); + } } catch (AuthenticationException ex) { if (!httpResponse.isCommitted()) { Cookie cookie = createCookie(""); Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java?rev=1311518&r1=1311517&r2=1311518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java Mon Apr 9 23:25:17 2012 @@ -23,10 +23,11 @@ import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.lang.reflect.Method; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** @@ -38,6 +39,8 @@ import org.apache.hadoop.classification. @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) @InterfaceStability.Evolving public class KerberosName { + private static final Logger LOG = LoggerFactory.getLogger(KerberosName.class); + /** The first component of the name */ private final String serviceName; /** The second component of the name. It may be null. */ @@ -81,6 +84,7 @@ public class KerberosName { try { defaultRealm = KerberosUtil.getDefaultRealm(); } catch (Exception ke) { + LOG.warn("Kerberos krb5 configuration not found, setting default realm to empty"); defaultRealm=""; } } Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java?rev=1311518&r1=1311517&r2=1311518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java Mon Apr 9 23:25:17 2012 @@ -349,7 +349,7 @@ public class TestAuthenticationFilter ex } } - private void _testDoFilterAuthentication(boolean withDomainPath) throws Exception { + private void _testDoFilterAuthentication(boolean withDomainPath, boolean invalidToken) throws Exception { AuthenticationFilter filter = new AuthenticationFilter(); try { FilterConfig config = Mockito.mock(FilterConfig.class); @@ -380,6 +380,12 @@ public class TestAuthenticationFilter ex Mockito.when(request.getRequestURL()).thenReturn(new StringBuffer("http://foo:8080/bar")); Mockito.when(request.getQueryString()).thenReturn("authenticated=true"); + if (invalidToken) { + Mockito.when(request.getCookies()).thenReturn( + new Cookie[] { new Cookie(AuthenticatedURL.AUTH_COOKIE, "foo")} + ); + } + HttpServletResponse response = Mockito.mock(HttpServletResponse.class); FilterChain chain = Mockito.mock(FilterChain.class); @@ -437,11 +443,15 @@ public class TestAuthenticationFilter ex } public void testDoFilterAuthentication() throws Exception { - _testDoFilterAuthentication(false); + _testDoFilterAuthentication(false, false); + } + + public void testDoFilterAuthenticationWithInvalidToken() throws Exception { + _testDoFilterAuthentication(false, true); } public void testDoFilterAuthenticationWithDomainPath() throws Exception { - _testDoFilterAuthentication(true); + _testDoFilterAuthentication(true, false); } public void testDoFilterAuthenticated() throws Exception { Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1311518&r1=1311517&r2=1311518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/CHANGES.txt Mon Apr 9 23:25:17 2012 @@ -252,6 +252,12 @@ Release 2.0.0 - UNRELEASED HADOOP-8007. Use substitution tokens for fencing argument (todd) + HADOOP-8077. HA: fencing method should be able to be configured on + a per-NN or per-NS basis (todd) + + HADOOP-8086. KerberosName silently sets defaultRealm to "" if the + Kerberos config is not found, it should log a WARN (tucu) + OPTIMIZATIONS BUG FIXES @@ -324,6 +330,14 @@ Release 2.0.0 - UNRELEASED HADOOP-8243. Security support broken in CLI (manual) failover controller (todd) + HADOOP-8251. Fix SecurityUtil.fetchServiceTicket after HADOOP-6941 (todd) + + HADOOP-8249. invalid hadoop-auth cookies should trigger authentication + if info is avail before returning HTTP 401 (tucu) + + HADOOP-8261. Har file system doesn't deal with FS URIs with a host but no + port. (atm) + BREAKDOWN OF HADOOP-7454 SUBTASKS HADOOP-7455. HA: Introduce HA Service Protocol Interface. (suresh) @@ -400,6 +414,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8180. Remove hsqldb since its not needed from pom.xml (Ravi Prakash via tgraves) + HADOOP-8014. ViewFileSystem does not correctly implement getDefaultBlockSize, + getDefaultReplication, getContentSummary (John George via bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Propchange: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/CHANGES.txt ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1310141-1311517 Propchange: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/docs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1310141-1311517 Propchange: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1310141-1311517 Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java?rev=1311518&r1=1311517&r2=1311518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java Mon Apr 9 23:25:17 2012 @@ -579,7 +579,8 @@ public abstract class FileSystem extends * * The FileSystem will simply return an elt containing 'localhost'. * - * @param p path of file to get locations for + * @param p path is used to identify an FS since an FS could have + * another FS that it could be delegating the call to * @param start offset into the given file * @param len length for which to get locations for */ @@ -602,10 +603,21 @@ public abstract class FileSystem extends return new FsServerDefaults(getDefaultBlockSize(), conf.getInt("io.bytes.per.checksum", 512), 64 * 1024, - getDefaultReplication(), + getDefaultReplication(), conf.getInt("io.file.buffer.size", 4096)); } - + + /** + * Return a set of server default configuration values + * @param p path is used to identify an FS since an FS could have + * another FS that it could be delegating the call to + * @return server default configuration values + * @throws IOException + */ + public FsServerDefaults getServerDefaults(Path p) throws IOException { + return getServerDefaults(); + } + /** * Return the fully-qualified path of path f resolving the path * through any symlinks or mount point @@ -653,8 +665,8 @@ public abstract class FileSystem extends throws IOException { return create(f, overwrite, getConf().getInt("io.file.buffer.size", 4096), - getDefaultReplication(), - getDefaultBlockSize()); + getDefaultReplication(f), + getDefaultBlockSize(f)); } /** @@ -668,8 +680,8 @@ public abstract class FileSystem extends throws IOException { return create(f, true, getConf().getInt("io.file.buffer.size", 4096), - getDefaultReplication(), - getDefaultBlockSize(), progress); + getDefaultReplication(f), + getDefaultBlockSize(f), progress); } /** @@ -683,7 +695,7 @@ public abstract class FileSystem extends return create(f, true, getConf().getInt("io.file.buffer.size", 4096), replication, - getDefaultBlockSize()); + getDefaultBlockSize(f)); } /** @@ -699,7 +711,7 @@ public abstract class FileSystem extends return create(f, true, getConf().getInt("io.file.buffer.size", 4096), replication, - getDefaultBlockSize(), progress); + getDefaultBlockSize(f), progress); } @@ -715,8 +727,8 @@ public abstract class FileSystem extends int bufferSize ) throws IOException { return create(f, overwrite, bufferSize, - getDefaultReplication(), - getDefaultBlockSize()); + getDefaultReplication(f), + getDefaultBlockSize(f)); } /** @@ -733,8 +745,8 @@ public abstract class FileSystem extends Progressable progress ) throws IOException { return create(f, overwrite, bufferSize, - getDefaultReplication(), - getDefaultBlockSize(), progress); + getDefaultReplication(f), + getDefaultBlockSize(f), progress); } @@ -1916,12 +1928,32 @@ public abstract class FileSystem extends return getConf().getLong("fs.local.block.size", 32 * 1024 * 1024); } + /** Return the number of bytes that large input files should be optimally + * be split into to minimize i/o time. The given path will be used to + * locate the actual filesystem. The full path does not have to exist. + * @param f path of file + * @return the default block size for the path's filesystem + */ + public long getDefaultBlockSize(Path f) { + return getDefaultBlockSize(); + } + /** * Get the default replication. */ public short getDefaultReplication() { return 1; } /** + * Get the default replication for a path. The given path will be used to + * locate the actual filesystem. The full path does not have to exist. + * @param path of the file + * @return default replication for the path's filesystem + */ + public short getDefaultReplication(Path path) { + return getDefaultReplication(); + } + + /** * Return a file status object that represents the path. * @param f The path we want information from * @return a FileStatus object Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java?rev=1311518&r1=1311517&r2=1311518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java Mon Apr 9 23:25:17 2012 @@ -28,6 +28,7 @@ import org.apache.hadoop.classification. import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.Progressable; @@ -267,6 +268,7 @@ public class FilterFileSystem extends Fi return fs.mkdirs(f, permission); } + /** * The src file is on the local disk. Add it to FS at * the given dst name. @@ -336,19 +338,42 @@ public class FilterFileSystem extends Fi return fs.getUsed(); } - /** Return the number of bytes that large input files should be optimally - * be split into to minimize i/o time. */ + @Override public long getDefaultBlockSize() { return fs.getDefaultBlockSize(); } - /** - * Get the default replication. - */ + @Override public short getDefaultReplication() { return fs.getDefaultReplication(); } + @Override + public FsServerDefaults getServerDefaults() throws IOException { + return fs.getServerDefaults(); + } + + // path variants delegate to underlying filesystem + @Override + public ContentSummary getContentSummary(Path f) throws IOException { + return fs.getContentSummary(f); + } + + @Override + public long getDefaultBlockSize(Path f) { + return fs.getDefaultBlockSize(f); + } + + @Override + public short getDefaultReplication(Path f) { + return fs.getDefaultReplication(f); + } + + @Override + public FsServerDefaults getServerDefaults(Path f) throws IOException { + return fs.getServerDefaults(f); + } + /** * Get file status. */ @@ -441,4 +466,4 @@ public class FilterFileSystem extends Fi Credentials credentials) throws IOException { return fs.getDelegationTokens(renewer, credentials); } -} \ No newline at end of file +} Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java?rev=1311518&r1=1311517&r2=1311518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java Mon Apr 9 23:25:17 2012 @@ -202,7 +202,8 @@ public class HarFileSystem extends Filte final String underLyingHost = i == host.length()? null: host.substring(i); int underLyingPort = rawURI.getPort(); String auth = (underLyingHost == null && underLyingPort == -1)? - null:(underLyingHost+":"+underLyingPort); + null:(underLyingHost+ + (underLyingPort == -1 ? "" : ":"+underLyingPort)); URI tmp = null; if (rawURI.getQuery() != null) { // query component not allowed Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java?rev=1311518&r1=1311517&r2=1311518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java Mon Apr 9 23:25:17 2012 @@ -23,6 +23,7 @@ import org.apache.hadoop.classification. import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileChecksum; @@ -209,11 +210,6 @@ class ChRootedFileSystem extends FilterF } @Override - public FsServerDefaults getServerDefaults() throws IOException { - return super.getServerDefaults(); - } - - @Override public FileStatus[] listStatus(final Path f) throws IOException { return super.listStatus(fullPath(f)); @@ -273,4 +269,42 @@ class ChRootedFileSystem extends FilterF public Path resolvePath(final Path p) throws IOException { return super.resolvePath(fullPath(p)); } + + @Override + public ContentSummary getContentSummary(Path f) throws IOException { + return super.getContentSummary(fullPath(f)); + } + + + private static Path rootPath = new Path(Path.SEPARATOR); + + @Override + public long getDefaultBlockSize() { + return getDefaultBlockSize(fullPath(rootPath)); + } + + @Override + public long getDefaultBlockSize(Path f) { + return super.getDefaultBlockSize(fullPath(f)); + } + + @Override + public short getDefaultReplication() { + return getDefaultReplication(fullPath(rootPath)); + } + + @Override + public short getDefaultReplication(Path f) { + return super.getDefaultReplication(fullPath(f)); + } + + @Override + public FsServerDefaults getServerDefaults() throws IOException { + return getServerDefaults(fullPath(rootPath)); + } + + @Override + public FsServerDefaults getServerDefaults(Path f) throws IOException { + return super.getServerDefaults(fullPath(f)); + } } Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java?rev=1311518&r1=1311517&r2=1311518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java Mon Apr 9 23:25:17 2012 @@ -34,6 +34,7 @@ import org.apache.hadoop.classification. import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileAlreadyExistsException; @@ -41,6 +42,7 @@ import org.apache.hadoop.fs.FileChecksum import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FsConstants; +import org.apache.hadoop.fs.FsServerDefaults; import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; @@ -471,6 +473,57 @@ public class ViewFileSystem extends File } @Override + public long getDefaultBlockSize() { + throw new NotInMountpointException("getDefaultBlockSize"); + } + + @Override + public short getDefaultReplication() { + throw new NotInMountpointException("getDefaultReplication"); + } + + @Override + public FsServerDefaults getServerDefaults() throws IOException { + throw new NotInMountpointException("getServerDefaults"); + } + + @Override + public long getDefaultBlockSize(Path f) { + try { + InodeTree.ResolveResult<FileSystem> res = + fsState.resolve(getUriPath(f), true); + return res.targetFileSystem.getDefaultBlockSize(res.remainingPath); + } catch (FileNotFoundException e) { + throw new NotInMountpointException(f, "getDefaultBlockSize"); + } + } + + @Override + public short getDefaultReplication(Path f) { + try { + InodeTree.ResolveResult<FileSystem> res = + fsState.resolve(getUriPath(f), true); + return res.targetFileSystem.getDefaultReplication(res.remainingPath); + } catch (FileNotFoundException e) { + throw new NotInMountpointException(f, "getDefaultReplication"); + } + } + + @Override + public FsServerDefaults getServerDefaults(Path f) throws IOException { + InodeTree.ResolveResult<FileSystem> res = + fsState.resolve(getUriPath(f), true); + return res.targetFileSystem.getServerDefaults(res.remainingPath); + } + + @Override + public ContentSummary getContentSummary(Path f) throws IOException { + InodeTree.ResolveResult<FileSystem> res = + fsState.resolve(getUriPath(f), true); + return res.targetFileSystem.getContentSummary(res.remainingPath); + } + + @Override public void setWriteChecksum(final boolean writeChecksum) { List<InodeTree.MountPoint<FileSystem>> mountPoints = fsState.getMountPoints(); @@ -742,5 +795,20 @@ public class ViewFileSystem extends File public void setVerifyChecksum(boolean verifyChecksum) { // Noop for viewfs } + + @Override + public FsServerDefaults getServerDefaults(Path f) throws IOException { + throw new NotInMountpointException(f, "getServerDefaults"); + } + + @Override + public long getDefaultBlockSize(Path f) { + throw new NotInMountpointException(f, "getDefaultBlockSize"); + } + + @Override + public short getDefaultReplication(Path f) { + throw new NotInMountpointException(f, "getDefaultReplication"); + } } } Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java?rev=1311518&r1=1311517&r2=1311518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java Mon Apr 9 23:25:17 2012 @@ -53,9 +53,6 @@ import com.google.common.collect.Lists; @InterfaceAudience.Private @InterfaceStability.Evolving public class NodeFencer { - public static final String CONF_METHODS_KEY = - "dfs.ha.fencing.methods"; - private static final String CLASS_RE = "([a-zA-Z0-9\\.\\$]+)"; private static final Pattern CLASS_WITH_ARGUMENT = Pattern.compile(CLASS_RE + "\\((.+?)\\)"); @@ -76,18 +73,18 @@ public class NodeFencer { private final List<FenceMethodWithArg> methods; - public NodeFencer(Configuration conf) + NodeFencer(Configuration conf, String spec) throws BadFencingConfigurationException { - this.methods = parseMethods(conf); + this.methods = parseMethods(conf, spec); } - public static NodeFencer create(Configuration conf) + public static NodeFencer create(Configuration conf, String confKey) throws BadFencingConfigurationException { - String confStr = conf.get(CONF_METHODS_KEY); + String confStr = conf.get(confKey); if (confStr == null) { return null; } - return new NodeFencer(conf); + return new NodeFencer(conf, confStr); } public boolean fence(HAServiceTarget fromSvc) { @@ -115,10 +112,10 @@ public class NodeFencer { return false; } - private static List<FenceMethodWithArg> parseMethods(Configuration conf) + private static List<FenceMethodWithArg> parseMethods(Configuration conf, + String spec) throws BadFencingConfigurationException { - String confStr = conf.get(CONF_METHODS_KEY); - String[] lines = confStr.split("\\s*\n\\s*"); + String[] lines = spec.split("\\s*\n\\s*"); List<FenceMethodWithArg> methods = Lists.newArrayList(); for (String line : lines) { Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java?rev=1311518&r1=1311517&r2=1311518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java Mon Apr 9 23:25:17 2012 @@ -171,7 +171,7 @@ public class SecurityUtil { } else { principalClass = Class.forName("sun.security.krb5.PrincipalName"); credentialsClass = Class.forName("sun.security.krb5.Credentials"); - krb5utilClass = Class.forName("sun.security.jgss.krb5"); + krb5utilClass = Class.forName("sun.security.jgss.krb5.Krb5Util"); } @SuppressWarnings("rawtypes") Constructor principalConstructor = principalClass.getConstructor(String.class, Propchange: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/core/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1310141-1311517 Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java?rev=1311518&r1=1311517&r2=1311518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java Mon Apr 9 23:25:17 2012 @@ -36,6 +36,7 @@ public final class FileSystemTestHelper System.getProperty("test.build.data", "target/test/data") + "/test"; private static final int DEFAULT_BLOCK_SIZE = 1024; private static final int DEFAULT_NUM_BLOCKS = 2; + private static final short DEFAULT_NUM_REPL = 1; private static String absTestRootDir = null; /** Hidden constructor */ @@ -99,9 +100,9 @@ public final class FileSystemTestHelper * Create files with numBlocks blocks each with block size blockSize. */ public static long createFile(FileSystem fSys, Path path, int numBlocks, - int blockSize, boolean createParent) throws IOException { + int blockSize, short numRepl, boolean createParent) throws IOException { FSDataOutputStream out = - fSys.create(path, false, 4096, fSys.getDefaultReplication(), blockSize ); + fSys.create(path, false, 4096, numRepl, blockSize ); byte[] data = getFileData(numBlocks, blockSize); out.write(data, 0, data.length); @@ -109,13 +110,19 @@ public final class FileSystemTestHelper return data.length; } + + public static long createFile(FileSystem fSys, Path path, int numBlocks, + int blockSize, boolean createParent) throws IOException { + return createFile(fSys, path, numBlocks, blockSize, fSys.getDefaultReplication(), true); + } + public static long createFile(FileSystem fSys, Path path, int numBlocks, int blockSize) throws IOException { return createFile(fSys, path, numBlocks, blockSize, true); - } + } public static long createFile(FileSystem fSys, Path path) throws IOException { - return createFile(fSys, path, DEFAULT_NUM_BLOCKS, DEFAULT_BLOCK_SIZE, true); + return createFile(fSys, path, DEFAULT_NUM_BLOCKS, DEFAULT_BLOCK_SIZE, DEFAULT_NUM_REPL, true); } public static long createFile(FileSystem fSys, String name) throws IOException { Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java?rev=1311518&r1=1311517&r2=1311518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java Mon Apr 9 23:25:17 2012 @@ -23,6 +23,7 @@ import java.net.URI; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.fs.FsConstants; @@ -170,7 +171,15 @@ public class TestChRootedFileSystem { Assert.assertTrue(fSys.isDirectory(FileSystemTestHelper.getTestRootPath(fSys,"/newDir/dirFooBar"))); Assert.assertTrue(fSysTarget.isDirectory(new Path(chrootedTo,"newDir/dirFooBar"))); } - + + @Test + public void testGetContentSummary() throws IOException { + // GetContentSummary of a dir + fSys.mkdirs(new Path("/newDir/dirFoo")); + ContentSummary cs = fSys.getContentSummary(new Path("/newDir/dirFoo")); + Assert.assertEquals(-1L, cs.getQuota()); + Assert.assertEquals(-1L, cs.getSpaceQuota()); + } /** * We would have liked renames across file system to fail but Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java?rev=1311518&r1=1311517&r2=1311518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java Mon Apr 9 23:25:17 2012 @@ -132,8 +132,7 @@ public class TestNodeFencer { throws BadFencingConfigurationException { System.err.println("Testing configuration:\n" + confStr); Configuration conf = new Configuration(); - conf.set(NodeFencer.CONF_METHODS_KEY, confStr); - return new NodeFencer(conf); + return new NodeFencer(conf, confStr); } /** Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestShellCommandFencer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestShellCommandFencer.java?rev=1311518&r1=1311517&r2=1311518&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestShellCommandFencer.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestShellCommandFencer.java Mon Apr 9 23:25:17 2012 @@ -71,8 +71,7 @@ public class TestShellCommandFencer { public void testCheckNoArgs() { try { Configuration conf = new Configuration(); - conf.set(NodeFencer.CONF_METHODS_KEY, "shell"); - new NodeFencer(conf); + new NodeFencer(conf, "shell"); fail("Didn't throw when passing no args to shell"); } catch (BadFencingConfigurationException confe) { assertTrue( @@ -85,8 +84,7 @@ public class TestShellCommandFencer { public void testCheckParensNoArgs() { try { Configuration conf = new Configuration(); - conf.set(NodeFencer.CONF_METHODS_KEY, "shell()"); - new NodeFencer(conf); + new NodeFencer(conf, "shell()"); fail("Didn't throw when passing no args to shell"); } catch (BadFencingConfigurationException confe) { assertTrue(