Author: cnauroth
Date: Sat Jan 18 06:50:19 2014
New Revision: 1559324
URL: http://svn.apache.org/r1559324
Log:
Merge trunk to HDFS-4685.
Added:
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
- copied unchanged from r1559323,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
- copied unchanged from r1559323,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
Modified:
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt
(contents, props changed)
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/bin/hadoop
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/docs/
(props changed)
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/
(props changed)
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/core/
(props changed)
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-minikdc/src/main/java/org/apache/hadoop/minikdc/MiniKdc.java
Modified:
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1559324&r1=1559323&r2=1559324&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt
Sat Jan 18 06:50:19 2014
@@ -98,7 +98,8 @@ Trunk (Unreleased)
HADOOP-8844. Add a plaintext fs -text test-case.
(Akira AJISAKA via harsh)
- HADOOP-9432 Add support for markdown .md files in site documentation
(stevel)
+ HADOOP-9432 Add support for markdown .md files in site documentation
+ (stevel)
HADOOP-9186. test-patch.sh should report build failure to JIRA.
(Binglin Chang via Colin Patrick McCabe)
@@ -110,6 +111,8 @@ Trunk (Unreleased)
HADOOP-10201. Add listing to KeyProvider API. (Larry McCay via omalley)
+ HADOOP-10177. Create CLI tools for managing keys. (Larry McCay via omalley)
+
BUG FIXES
HADOOP-9451. Fault single-layer config if node group topology is enabled.
@@ -117,8 +120,8 @@ Trunk (Unreleased)
HADOOP-8419. Fixed GzipCode NPE reset for IBM JDK. (Yu Li via eyang)
- HADOOP-8177. MBeans shouldn't try to register when it fails to create
MBeanName.
- (Devaraj K via umamahesh)
+ HADOOP-8177. MBeans shouldn't try to register when it fails to create
+ MBeanName. (Devaraj K via umamahesh)
HADOOP-8018. Hudson auto test for HDFS has started throwing javadoc
(Jon Eagles via bobby)
@@ -430,6 +433,9 @@ Release 2.4.0 - UNRELEASED
HADOOP-10173. Remove UGI from DIGEST-MD5 SASL server creation (daryn via
kihwal)
+ HADOOP-10228. FsPermission#fromShort() should cache FsAction.values().
+ (Haohui Mai via cnauroth)
+
BUG FIXES
HADOOP-9964. Fix deadlocks in TestHttpServer by synchronize
@@ -511,6 +517,19 @@ Release 2.4.0 - UNRELEASED
HADOOP-10214. Fix multithreaded correctness warnings in
ActiveStandbyElector
(Liang Xie via kasha)
+ HADOOP-10223. MiniKdc#main() should close the FileReader it creates.
+ (Ted Yu via tucu)
+
+ HADOOP-10236. Fix typo in o.a.h.ipc.Client#checkResponse. (Akira Ajisaka
+ via suresh)
+
+ HADOOP-10146. Workaround JDK7 Process fd close bug (daryn)
+
+ HADOOP-10125. no need to process RPC request if the client connection
+ has been dropped (Ming Ma via brandonli)
+
+ HADOOP-10235. Hadoop tarball has 2 versions of stax-api JARs. (tucu)
+
Release 2.3.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -599,6 +618,13 @@ Release 2.3.0 - UNRELEASED
HADOOP-10178. Configuration deprecation always emit "deprecated" warnings
when a new key is used. (Shanyu Zhao via cnauroth)
+ HADOOP-10234. "hadoop.cmd jar" does not propagate exit code. (cnauroth)
+
+ HADOOP-10240. Windows build instructions incorrectly state requirement of
+ protoc 2.4.1 instead of 2.5.0. (cnauroth)
+
+ HADOOP-10112. har file listing doesn't work with wild card. (brandonli)
+
Release 2.2.0 - 2013-10-13
INCOMPATIBLE CHANGES
@@ -1326,6 +1352,18 @@ Release 2.1.0-beta - 2013-08-22
HADOOP-9701. mvn site ambiguous links in hadoop-common. (kkambatl via tucu)
+Release 2.0.6-alpha - 08/22/2013
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
Release 2.0.5-alpha - 06/06/2013
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1557294-1559323
Modified:
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/bin/hadoop
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/bin/hadoop?rev=1559324&r1=1559323&r2=1559324&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/bin/hadoop
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/bin/hadoop
Sat Jan 18 06:50:19 2014
@@ -104,6 +104,8 @@ case $COMMAND in
CLASS=org.apache.hadoop.util.VersionInfo
elif [ "$COMMAND" = "jar" ] ; then
CLASS=org.apache.hadoop.util.RunJar
+ elif [ "$COMMAND" = "key" ] ; then
+ CLASS=org.apache.hadoop.crypto.key.KeyShell
elif [ "$COMMAND" = "checknative" ] ; then
CLASS=org.apache.hadoop.util.NativeLibraryChecker
elif [ "$COMMAND" = "distcp" ] ; then
Modified:
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd?rev=1559324&r1=1559323&r2=1559324&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
Sat Jan 18 06:50:19 2014
@@ -143,7 +143,7 @@ call :updatepath %HADOOP_BIN_PATH%
call %JAVA% %JAVA_HEAP_MAX% %HADOOP_OPTS% -classpath %CLASSPATH% %CLASS%
%hadoop-command-arguments%
- goto :eof
+ exit /b %ERRORLEVEL%
:fs
set CLASS=org.apache.hadoop.fs.FsShell
Propchange:
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1557294-1559323
Propchange:
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1557294-1559323
Modified:
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java?rev=1559324&r1=1559323&r2=1559324&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
Sat Jan 18 06:50:19 2014
@@ -77,7 +77,7 @@ public class JavaKeyStoreProvider extend
private JavaKeyStoreProvider(URI uri, Configuration conf) throws IOException
{
this.uri = uri;
path = unnestUri(uri);
- fs = FileSystem.get(conf);
+ fs = path.getFileSystem(conf);
// Get the password from the user's environment
String pw = System.getenv(KEYSTORE_PASSWORD_NAME);
if (pw == null) {
Modified:
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java?rev=1559324&r1=1559323&r2=1559324&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
Sat Jan 18 06:50:19 2014
@@ -245,6 +245,17 @@ public abstract class KeyProvider {
}
/**
+ * Indicates whether this provider represents a store
+ * that is intended for transient use - such as the UserProvider
+ * is. These providers are generally used to provide access to
+ * keying material rather than for long term storage.
+ * @return true if transient, false otherwise
+ */
+ public boolean isTransient() {
+ return false;
+ }
+
+ /**
* Get the key material for a specific version of the key. This method is
used
* when decrypting data.
* @param versionName the name of a specific version of the key
Modified:
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java?rev=1559324&r1=1559323&r2=1559324&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java
Sat Jan 18 06:50:19 2014
@@ -50,6 +50,11 @@ public class UserProvider extends KeyPro
}
@Override
+ public boolean isTransient() {
+ return true;
+ }
+
+ @Override
public KeyVersion getKeyVersion(String versionName) {
byte[] bytes = credentials.getSecretKey(new Text(versionName));
if (bytes == null) {
Modified:
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java?rev=1559324&r1=1559323&r2=1559324&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
Sat Jan 18 06:50:19 2014
@@ -136,8 +136,7 @@ public class FsPermission implements Wri
}
public void fromShort(short n) {
- FsAction[] v = FsAction.values();
-
+ FsAction[] v = FSACTION_VALUES;
set(v[(n >>> 6) & 7], v[(n >>> 3) & 7], v[n & 7], (((n >>> 9) & 1) == 1),
(((n >>> 10) & 1) == 1) );
}
@@ -233,6 +232,8 @@ public class FsPermission implements Wri
public static final int DEFAULT_UMASK =
CommonConfigurationKeys.FS_PERMISSIONS_UMASK_DEFAULT;
+ private static final FsAction[] FSACTION_VALUES = FsAction.values();
+
/**
* Get the user file creation mask (umask)
*
Modified:
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java?rev=1559324&r1=1559323&r2=1559324&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
Sat Jan 18 06:50:19 2014
@@ -286,7 +286,7 @@ public class Client {
if (!Arrays.equals(id, RpcConstants.DUMMY_CLIENT_ID)) {
if (!Arrays.equals(id, clientId)) {
throw new IOException("Client IDs not matched: local ID="
- + StringUtils.byteToHexString(clientId) + ", ID in reponse="
+ + StringUtils.byteToHexString(clientId) + ", ID in response="
+
StringUtils.byteToHexString(header.getClientId().toByteArray()));
}
}
Modified:
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1559324&r1=1559323&r2=1559324&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
Sat Jan 18 06:50:19 2014
@@ -2021,6 +2021,10 @@ public abstract class Server {
if (LOG.isDebugEnabled()) {
LOG.debug(Thread.currentThread().getName() + ": " + call + " for
RpcKind " + call.rpcKind);
}
+ if (!call.connection.channel.isOpen()) {
+ LOG.info(Thread.currentThread().getName() + ": skipped " + call);
+ continue;
+ }
String errorClass = null;
String error = null;
RpcStatusProto returnStatus = RpcStatusProto.SUCCESS;
Modified:
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java?rev=1559324&r1=1559323&r2=1559324&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
Sat Jan 18 06:50:19 2014
@@ -21,6 +21,7 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.io.InputStream;
import java.util.Arrays;
import java.util.Map;
import java.util.Timer;
@@ -511,7 +512,17 @@ abstract public class Shell {
}
// close the input stream
try {
- inReader.close();
+ // JDK 7 tries to automatically drain the input streams for us
+ // when the process exits, but since close is not synchronized,
+ // it creates a race if we close the stream first and the same
+ // fd is recycled. the stream draining thread will attempt to
+ // drain that fd!! it may block, OOM, or cause bizarre behavior
+ // see: https://bugs.openjdk.java.net/browse/JDK-8024521
+ // issue is fixed in build 7u60
+ InputStream stdout = process.getInputStream();
+ synchronized (stdout) {
+ inReader.close();
+ }
} catch (IOException ioe) {
LOG.warn("Error while closing the input stream", ioe);
}
@@ -524,7 +535,10 @@ abstract public class Shell {
LOG.warn("Interrupted while joining errThread");
}
try {
- errReader.close();
+ InputStream stderr = process.getErrorStream();
+ synchronized (stderr) {
+ errReader.close();
+ }
} catch (IOException ioe) {
LOG.warn("Error while closing the error stream", ioe);
}
Propchange:
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1557294-1559323
Modified:
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-minikdc/src/main/java/org/apache/hadoop/minikdc/MiniKdc.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-minikdc/src/main/java/org/apache/hadoop/minikdc/MiniKdc.java?rev=1559324&r1=1559323&r2=1559324&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-minikdc/src/main/java/org/apache/hadoop/minikdc/MiniKdc.java
(original)
+++
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-minikdc/src/main/java/org/apache/hadoop/minikdc/MiniKdc.java
Sat Jan 18 06:50:19 2014
@@ -125,7 +125,15 @@ public class MiniKdc {
+ file.getAbsolutePath());
}
Properties userConf = new Properties();
- userConf.load(new FileReader(file));
+ FileReader r = null;
+ try {
+ r = new FileReader(file);
+ userConf.load(r);
+ } finally {
+ if (r != null) {
+ r.close();
+ }
+ }
for (Map.Entry entry : userConf.entrySet()) {
conf.put(entry.getKey(), entry.getValue());
}
