Author: szetszwo
Date: Tue Sep 3 19:01:19 2013
New Revision: 1519796
URL: http://svn.apache.org/r1519796
Log:
Merge r1517887 through r1518850 from trunk.
Added:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ZKUtil.java
- copied unchanged from r1518850,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ZKUtil.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestZKUtil.java
- copied unchanged from r1518850,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestZKUtil.java
Removed:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAZKUtil.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestHAZKUtil.java
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt
(contents, props changed)
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/docs/
(props changed)
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/
(props changed)
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/core/
(props changed)
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElector.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElectorRealZK.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-minikdc/src/main/java/org/apache/hadoop/minikdc/MiniKdc.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-minikdc/src/test/java/org/apache/hadoop/minikdc/TestMiniKdc.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Constant.java
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt
Tue Sep 3 19:01:19 2013
@@ -327,6 +327,9 @@ Release 2.3.0 - UNRELEASED
HADOOP-9487 Deprecation warnings in Configuration should go to their
own log or otherwise be suppressible (Chu Tong via stevel)
+ HADOOP-9889. Refresh the Krb5 configuration when creating a new kdc in
+ Hadoop-MiniKDC (Wei Yan via Sandy Ryza)
+
OPTIMIZATIONS
HADOOP-9748. Reduce blocking on UGI.ensureInitialized (daryn)
@@ -362,6 +365,9 @@ Release 2.1.1-beta - UNRELEASED
IMPROVEMENTS
+ HADOOP-9910. proxy server start and stop documentation wrong
+ (Andre Kelpe via harsh)
+
HADOOP-9446. Support Kerberos SPNEGO for IBM JDK. (Yu Gao via llu)
HADOOP-9787. ShutdownHelper util to shutdown threads and threadpools.
@@ -387,6 +393,9 @@ Release 2.1.1-beta - UNRELEASED
HADOOP-9886. Turn warning message in RetryInvocationHandler to debug
(arpit)
+ HADOOP-9906. Move HAZKUtil to o.a.h.util.ZKUtil and make inner-classes
+ public (Karthik Kambatla via Sandy Ryza)
+
OPTIMIZATIONS
BUG FIXES
@@ -423,6 +432,9 @@ Release 2.1.1-beta - UNRELEASED
HADOOP-9887. globStatus does not correctly handle paths starting with a
drive
spec on Windows. (Chuan Liu via cnauroth)
+ HADOOP-9894. Race condition in Shell leads to logged error stream handling
+ exceptions (Arpit Agarwal)
+
Release 2.1.0-beta - 2013-08-22
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1517029-1518850
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1513717-1518850
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1517029-1518850
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
Tue Sep 3 19:01:19 2013
@@ -31,7 +31,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.ha.HAZKUtil.ZKAuthInfo;
+import org.apache.hadoop.util.ZKUtil.ZKAuthInfo;
import org.apache.hadoop.util.StringUtils;
import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.KeeperException;
@@ -47,7 +47,6 @@ import org.apache.zookeeper.KeeperExcept
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
/**
*
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
Tue Sep 3 19:01:19 2013
@@ -36,7 +36,8 @@ import org.apache.hadoop.ha.ActiveStandb
import org.apache.hadoop.ha.ActiveStandbyElector.ActiveStandbyElectorCallback;
import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo;
import org.apache.hadoop.ha.HAServiceProtocol.RequestSource;
-import org.apache.hadoop.ha.HAZKUtil.ZKAuthInfo;
+import org.apache.hadoop.util.ZKUtil;
+import org.apache.hadoop.util.ZKUtil.ZKAuthInfo;
import org.apache.hadoop.ha.HealthMonitor.State;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.security.AccessControlException;
@@ -313,18 +314,18 @@ public abstract class ZKFailoverControll
ZK_SESSION_TIMEOUT_DEFAULT);
// Parse ACLs from configuration.
String zkAclConf = conf.get(ZK_ACL_KEY, ZK_ACL_DEFAULT);
- zkAclConf = HAZKUtil.resolveConfIndirection(zkAclConf);
- List<ACL> zkAcls = HAZKUtil.parseACLs(zkAclConf);
+ zkAclConf = ZKUtil.resolveConfIndirection(zkAclConf);
+ List<ACL> zkAcls = ZKUtil.parseACLs(zkAclConf);
if (zkAcls.isEmpty()) {
zkAcls = Ids.CREATOR_ALL_ACL;
}
// Parse authentication from configuration.
String zkAuthConf = conf.get(ZK_AUTH_KEY);
- zkAuthConf = HAZKUtil.resolveConfIndirection(zkAuthConf);
+ zkAuthConf = ZKUtil.resolveConfIndirection(zkAuthConf);
List<ZKAuthInfo> zkAuths;
if (zkAuthConf != null) {
- zkAuths = HAZKUtil.parseAuth(zkAuthConf);
+ zkAuths = ZKUtil.parseAuth(zkAuthConf);
} else {
zkAuths = Collections.emptyList();
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
Tue Sep 3 19:01:19 2013
@@ -515,8 +515,13 @@ abstract public class Shell {
} catch (IOException ioe) {
LOG.warn("Error while closing the input stream", ioe);
}
- if (!completed.get()) {
- errThread.interrupt();
+ try {
+ if (!completed.get()) {
+ errThread.interrupt();
+ errThread.join();
+ }
+ } catch (InterruptedException ie) {
+ LOG.warn("Interrupted while joining errThread");
}
try {
errReader.close();
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
Tue Sep 3 19:01:19 2013
@@ -518,7 +518,7 @@ $ $HADOOP_YARN_HOME/sbin/yarn-daemon.sh
are used with load balancing it should be run on each of them:
----
-$ $HADOOP_YARN_HOME/bin/yarn start proxyserver --config $HADOOP_CONF_DIR
+$ $HADOOP_YARN_HOME/sbin/yarn-daemon.sh start proxyserver --config
$HADOOP_CONF_DIR
----
Start the MapReduce JobHistory Server with the following command, run on the
@@ -560,7 +560,7 @@ $ $HADOOP_YARN_HOME/sbin/yarn-daemon.sh
balancing it should be run on each of them:
----
-$ $HADOOP_YARN_HOME/bin/yarn stop proxyserver --config $HADOOP_CONF_DIR
+$ $HADOOP_YARN_HOME/sbin/yarn-daemon.sh stop proxyserver --config
$HADOOP_CONF_DIR
----
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1513717-1518850
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElector.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElector.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElector.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElector.java
Tue Sep 3 19:01:19 2013
@@ -41,7 +41,7 @@ import org.mockito.Mockito;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.ha.ActiveStandbyElector.ActiveStandbyElectorCallback;
import org.apache.hadoop.ha.ActiveStandbyElector.ActiveNotFoundException;
-import org.apache.hadoop.ha.HAZKUtil.ZKAuthInfo;
+import org.apache.hadoop.util.ZKUtil.ZKAuthInfo;
import org.apache.hadoop.test.GenericTestUtils;
public class TestActiveStandbyElector {
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElectorRealZK.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElectorRealZK.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElectorRealZK.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElectorRealZK.java
Tue Sep 3 19:01:19 2013
@@ -28,7 +28,7 @@ import java.util.UUID;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.ha.ActiveStandbyElector.ActiveStandbyElectorCallback;
import org.apache.hadoop.ha.ActiveStandbyElector.State;
-import org.apache.hadoop.ha.HAZKUtil.ZKAuthInfo;
+import org.apache.hadoop.util.ZKUtil.ZKAuthInfo;
import org.apache.hadoop.util.Shell;
import org.apache.log4j.Level;
import org.apache.zookeeper.ZooDefs.Ids;
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-minikdc/src/main/java/org/apache/hadoop/minikdc/MiniKdc.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-minikdc/src/main/java/org/apache/hadoop/minikdc/MiniKdc.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-minikdc/src/main/java/org/apache/hadoop/minikdc/MiniKdc.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-minikdc/src/main/java/org/apache/hadoop/minikdc/MiniKdc.java
Tue Sep 3 19:01:19 2013
@@ -59,6 +59,7 @@ import java.io.FileReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.StringReader;
+import java.lang.reflect.Method;
import java.net.InetAddress;
import java.net.ServerSocket;
import java.text.MessageFormat;
@@ -432,6 +433,17 @@ public class MiniKdc {
System.setProperty("sun.security.krb5.debug", conf.getProperty(DEBUG,
"false"));
+
+ // refresh the config
+ Class<?> classRef;
+ if (System.getProperty("java.vendor").contains("IBM")) {
+ classRef = Class.forName("com.ibm.security.krb5.internal.Config");
+ } else {
+ classRef = Class.forName("sun.security.krb5.Config");
+ }
+ Method refreshMethod = classRef.getMethod("refresh", new Class[0]);
+ refreshMethod.invoke(classRef, new Object[0]);
+
LOG.info("MiniKdc listening at port: {}", getPort());
LOG.info("MiniKdc setting JVM krb5.conf to: {}",
krb5conf.getAbsolutePath());
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-minikdc/src/test/java/org/apache/hadoop/minikdc/TestMiniKdc.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-minikdc/src/test/java/org/apache/hadoop/minikdc/TestMiniKdc.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-minikdc/src/test/java/org/apache/hadoop/minikdc/TestMiniKdc.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-minikdc/src/test/java/org/apache/hadoop/minikdc/TestMiniKdc.java
Tue Sep 3 19:01:19 2013
@@ -30,7 +30,11 @@ import javax.security.auth.login.Configu
import javax.security.auth.login.LoginContext;
import java.io.File;
import java.security.Principal;
-import java.util.*;
+import java.util.Set;
+import java.util.Map;
+import java.util.HashSet;
+import java.util.HashMap;
+import java.util.Arrays;
public class TestMiniKdc extends KerberosSecurityTestcase {
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Constant.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Constant.java?rev=1519796&r1=1519795&r2=1519796&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Constant.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Constant.java
Tue Sep 3 19:01:19 2013
@@ -201,4 +201,8 @@ public class Nfs3Constant {
public static final String EXPORTS_CACHE_EXPIRYTIME_MILLIS_KEY =
"hdfs.nfs.exports.cache.expirytime.millis";
public static final long EXPORTS_CACHE_EXPIRYTIME_MILLIS_DEFAULT = 15 * 60 *
1000; // 15 min
+ public static final String FILE_DUMP_DIR_KEY = "dfs.nfs3.dump.dir";
+ public static final String FILE_DUMP_DIR_DEFAULT = "/tmp/.hdfs-nfs";
+ public static final String ENABLE_FILE_DUMP_KEY = "dfs.nfs3.enableDump";
+ public static final boolean ENABLE_FILE_DUMP_DEFAULT = true;
}
