Author: arp
Date: Tue Sep 3 23:20:45 2013
New Revision: 1519885
URL: http://svn.apache.org/r1519885
Log:
Merging r1518852 through r1519883 from trunk to branch HDFS-2832
Added:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/security/
- copied from r1519883,
hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/security/
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/security/AccessPrivilege.java
- copied unchanged from r1519883,
hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/security/AccessPrivilege.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/security/NfsExports.java
- copied unchanged from r1519883,
hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/security/NfsExports.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/security/
- copied from r1519883,
hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/security/
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/security/TestNfsExports.java
- copied unchanged from r1519883,
hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/security/TestNfsExports.java
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt
(contents, props changed)
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/docs/
(props changed)
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/
(props changed)
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Stat.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/CompositeService.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/core/
(props changed)
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestStat.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountResponse.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Constant.java
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt
Tue Sep 3 23:20:45 2013
@@ -357,6 +357,9 @@ Release 2.3.0 - UNRELEASED
HADOOP-9877. Fix listing of snapshot directories in globStatus.
(Binglin Chang via Andrew Wang)
+ HADOOP-9909. org.apache.hadoop.fs.Stat should permit other LANG.
+ (Shinichi Yamashita via Andrew Wang)
+
Release 2.1.1-beta - UNRELEASED
INCOMPATIBLE CHANGES
@@ -396,6 +399,9 @@ Release 2.1.1-beta - UNRELEASED
HADOOP-9906. Move HAZKUtil to o.a.h.util.ZKUtil and make inner-classes
public (Karthik Kambatla via Sandy Ryza)
+ HADOOP-9918. Add addIfService to CompositeService (Karthik Kambatla via
+ Sandy Ryza)
+
OPTIMIZATIONS
BUG FIXES
@@ -435,6 +441,9 @@ Release 2.1.1-beta - UNRELEASED
HADOOP-9894. Race condition in Shell leads to logged error stream handling
exceptions (Arpit Agarwal)
+ HADOOP-9774. RawLocalFileSystem.listStatus() return absolute paths when
+ input path is relative on Windows. (Shanyu Zhao via ivanmi)
+
Release 2.1.0-beta - 2013-08-22
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1518852-1519883
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1518852-1519883
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1518852-1519883
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
Tue Sep 3 23:20:45 2013
@@ -182,6 +182,18 @@ public class Path implements Comparable
/** Construct a Path from components. */
public Path(String scheme, String authority, String path) {
checkPathArg( path );
+
+ // add a slash in front of paths with Windows drive letters
+ if (hasWindowsDrive(path) && path.charAt(0) != '/') {
+ path = "/" + path;
+ }
+
+ // add "./" in front of Linux relative paths so that a path containing
+ // a colon e.q. "a:b" will not be interpreted as scheme "a".
+ if (!WINDOWS && path.charAt(0) != '/') {
+ path = "./" + path;
+ }
+
initialize(scheme, authority, path, null);
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
Tue Sep 3 23:20:45 2013
@@ -393,7 +393,7 @@ public class RawLocalFileSystem extends
new DeprecatedRawLocalFileStatus(localf, getDefaultBlockSize(f),
this)};
}
- File[] names = localf.listFiles();
+ String[] names = localf.list();
if (names == null) {
return null;
}
@@ -401,7 +401,9 @@ public class RawLocalFileSystem extends
int j = 0;
for (int i = 0; i < names.length; i++) {
try {
- results[j] = getFileStatus(new Path(names[i].getAbsolutePath()));
+ // Assemble the path using the Path 3 arg constructor to make sure
+ // paths with colon are properly resolved on Linux
+ results[j] = getFileStatus(new Path(f, new Path(null, null,
names[i])));
j++;
} catch (FileNotFoundException e) {
// ignore the files not found since the dir list may have have changed
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Stat.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Stat.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Stat.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Stat.java
Tue Sep 3 23:20:45 2013
@@ -20,6 +20,8 @@ package org.apache.hadoop.fs;
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
import java.util.NoSuchElementException;
import java.util.StringTokenizer;
@@ -62,6 +64,10 @@ public class Stat extends Shell {
this.path = new Path(qualified.toUri().getPath());
this.blockSize = blockSize;
this.dereference = deref;
+ // LANG = C setting
+ Map<String, String> env = new HashMap<String, String>();
+ env.put("LANG", "C");
+ setEnvironment(env);
}
public FileStatus getFileStatus() throws IOException {
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/CompositeService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/CompositeService.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/CompositeService.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/CompositeService.java
Tue Sep 3 23:20:45 2013
@@ -64,6 +64,11 @@ public class CompositeService extends Ab
}
}
+ /**
+ * Add the passed {@link Service} to the list of services managed by this
+ * {@link CompositeService}
+ * @param service the {@link Service} to be added
+ */
protected void addService(Service service) {
if (LOG.isDebugEnabled()) {
LOG.debug("Adding service " + service.getName());
@@ -73,6 +78,21 @@ public class CompositeService extends Ab
}
}
+ /**
+ * If the passed object is an instance of {@link Service},
+ * add it to the list of services managed by this {@link CompositeService}
+ * @param object
+ * @return true if a service is added, false otherwise.
+ */
+ protected boolean addIfService(Object object) {
+ if (object instanceof Service) {
+ addService((Service) object);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
protected synchronized boolean removeService(Service service) {
synchronized (serviceList) {
return serviceList.add(service);
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
Tue Sep 3 23:20:45 2013
@@ -540,6 +540,13 @@ abstract public class Shell {
protected abstract void parseExecResult(BufferedReader lines)
throws IOException;
+ /**
+ * Get the environment variable
+ */
+ public String getEnvironment(String env) {
+ return environment.get(env);
+ }
+
/** get the current sub-process executing the given command
* @return process executing the command
*/
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1518852-1519883
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
Tue Sep 3 23:20:45 2013
@@ -280,6 +280,21 @@ public class TestLocalFileSystem {
stats[0].getPath().toUri().getPath());
}
+ @Test
+ public void testListStatusReturnConsistentPathOnWindows() throws IOException
{
+ assumeTrue(Shell.WINDOWS);
+ String dirNoDriveSpec = TEST_ROOT_DIR;
+ if (dirNoDriveSpec.charAt(1) == ':')
+ dirNoDriveSpec = dirNoDriveSpec.substring(2);
+
+ File file = new File(dirNoDriveSpec, "foo");
+ file.mkdirs();
+ FileStatus[] stats = fileSys.listStatus(new Path(dirNoDriveSpec));
+ assertEquals("Unexpected number of stats", 1, stats.length);
+ assertEquals("Bad path from stat", new
Path(file.getPath()).toUri().getPath(),
+ stats[0].getPath().toUri().getPath());
+ }
+
@Test(timeout = 10000)
public void testReportChecksumFailure() throws IOException {
base.mkdirs();
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java
Tue Sep 3 23:20:45 2013
@@ -158,7 +158,43 @@ public class TestPath extends TestCase {
assertEquals(new Path("c:/foo"), new Path("d:/bar", "c:/foo"));
}
}
-
+
+ @Test (timeout = 30000)
+ public void testPathThreeArgContructor() {
+ assertEquals(new Path("foo"), new Path(null, null, "foo"));
+ assertEquals(new Path("scheme:///foo"), new Path("scheme", null, "/foo"));
+ assertEquals(
+ new Path("scheme://authority/foo"),
+ new Path("scheme", "authority", "/foo"));
+
+ if (Path.WINDOWS) {
+ assertEquals(new Path("c:/foo/bar"), new Path(null, null, "c:/foo/bar"));
+ assertEquals(new Path("c:/foo/bar"), new Path(null, null,
"/c:/foo/bar"));
+ } else {
+ assertEquals(new Path("./a:b"), new Path(null, null, "a:b"));
+ }
+
+ // Resolution tests
+ if (Path.WINDOWS) {
+ assertEquals(
+ new Path("c:/foo/bar"),
+ new Path("/fou", new Path(null, null, "c:/foo/bar")));
+ assertEquals(
+ new Path("c:/foo/bar"),
+ new Path("/fou", new Path(null, null, "/c:/foo/bar")));
+ assertEquals(
+ new Path("/foo/bar"),
+ new Path("/foo", new Path(null, null, "bar")));
+ } else {
+ assertEquals(
+ new Path("/foo/bar/a:b"),
+ new Path("/foo/bar", new Path(null, null, "a:b")));
+ assertEquals(
+ new Path("/a:b"),
+ new Path("/foo/bar", new Path(null, null, "/a:b")));
+ }
+ }
+
@Test (timeout = 30000)
public void testEquals() {
assertFalse(new Path("/").equals(new Path("/foo")));
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestStat.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestStat.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestStat.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestStat.java
Tue Sep 3 23:20:45 2013
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.fs;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -119,4 +120,9 @@ public class TestStat {
// expected
}
}
+
+ @Test(timeout=10000)
+ public void testStatEnvironment() throws Exception {
+ assertEquals(stat.getEnvironment("LANG"), "C");
+ }
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountResponse.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountResponse.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountResponse.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountResponse.java
Tue Sep 3 23:20:45 2013
@@ -19,6 +19,7 @@ package org.apache.hadoop.mount;
import java.util.List;
+import org.apache.hadoop.nfs.security.NfsExports;
import org.apache.hadoop.oncrpc.RpcAcceptedReply;
import org.apache.hadoop.oncrpc.XDR;
import org.apache.hadoop.oncrpc.RpcAuthInfo.AuthFlavor;
@@ -59,15 +60,28 @@ public class MountResponse {
xdr.writeBoolean(false); // Value follows no
return xdr;
}
-
+
/** Response for RPC call {@link MountInterface.MNTPROC#EXPORT} */
- public static XDR writeExportList(XDR xdr, int xid, List<String> exports) {
+ public static XDR writeExportList(XDR xdr, int xid, List<String> exports,
+ List<NfsExports> hostMatcher) {
+ assert (exports.size() == hostMatcher.size());
+
RpcAcceptedReply.voidReply(xdr, xid);
- for (String export : exports) {
+ for (int i = 0; i < exports.size(); i++) {
xdr.writeBoolean(true); // Value follows - yes
- xdr.writeString(export);
- xdr.writeInt(0);
+ xdr.writeString(exports.get(i));
+
+ // List host groups
+ String[] hostGroups = hostMatcher.get(i).getHostGroupList();
+ if (hostGroups.length > 0) {
+ for (int j = 0; j < hostGroups.length; j++) {
+ xdr.writeBoolean(true); // Value follows - yes
+ xdr.writeVariableOpaque(hostGroups[j].getBytes());
+ }
+ }
+ xdr.writeBoolean(false); // Value follows - no more group
}
+
xdr.writeBoolean(false); // Value follows - no
return xdr;
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Constant.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Constant.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Constant.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Constant.java
Tue Sep 3 23:20:45 2013
@@ -192,13 +192,13 @@ public class Nfs3Constant {
public static final String EXPORTS_ALLOWED_HOSTS_SEPARATOR = ";";
/** Allowed hosts for nfs exports */
- public static final String EXPORTS_ALLOWED_HOSTS_KEY =
"hdfs.nfs.exports.allowed.hosts";
+ public static final String EXPORTS_ALLOWED_HOSTS_KEY =
"dfs.nfs.exports.allowed.hosts";
public static final String EXPORTS_ALLOWED_HOSTS_KEY_DEFAULT = "* rw";
/** Size for nfs exports cache */
- public static final String EXPORTS_CACHE_SIZE_KEY =
"hdfs.nfs.exports.cache.size";
+ public static final String EXPORTS_CACHE_SIZE_KEY =
"dfs.nfs.exports.cache.size";
public static final int EXPORTS_CACHE_SIZE_DEFAULT = 512;
/** Expiration time for nfs exports cache entry */
- public static final String EXPORTS_CACHE_EXPIRYTIME_MILLIS_KEY =
"hdfs.nfs.exports.cache.expirytime.millis";
+ public static final String EXPORTS_CACHE_EXPIRYTIME_MILLIS_KEY =
"dfs.nfs.exports.cache.expirytime.millis";
public static final long EXPORTS_CACHE_EXPIRYTIME_MILLIS_DEFAULT = 15 * 60 *
1000; // 15 min
public static final String FILE_DUMP_DIR_KEY = "dfs.nfs3.dump.dir";
