hadoop git commit: HADOOP-14970. MiniHadoopClusterManager doesn't respect lack of format option. Contributed by Erik Krogen
Repository: hadoop Updated Branches: refs/heads/trunk 226bedc02 -> 1a407bc99 HADOOP-14970. MiniHadoopClusterManager doesn't respect lack of format option. Contributed by Erik Krogen Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1a407bc9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1a407bc9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1a407bc9 Branch: refs/heads/trunk Commit: 1a407bc9906306801690bc75ff0f0456f8f265fd Parents: 226bedc Author: Erik Krogen Authored: Thu Apr 12 23:27:51 2018 -0700 Committer: Konstantin V Shvachko Committed: Thu Apr 12 23:27:51 2018 -0700 -- .../java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a407bc9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java index b89cdc0..d29dd34 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java @@ -156,6 +156,7 @@ public class MiniHadoopClusterManager { if (!noDFS) { dfs = new MiniDFSCluster.Builder(conf).nameNodePort(nnPort) .nameNodeHttpPort(nnHttpPort).numDataNodes(numDataNodes) + .format(dfsOpts == StartupOption.FORMAT) .startupOption(dfsOpts).build(); LOG.info("Started MiniDFSCluster -- namenode on port " + dfs.getNameNodePort()); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: HADOOP-14970. MiniHadoopClusterManager doesn't respect lack of format option. Contributed by Erik Krogen
Repository: hadoop Updated Branches: refs/heads/branch-3.0 64105868e -> bce57a139 HADOOP-14970. MiniHadoopClusterManager doesn't respect lack of format option. Contributed by Erik Krogen (cherry picked from commit 1a407bc9906306801690bc75ff0f0456f8f265fd) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bce57a13 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bce57a13 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bce57a13 Branch: refs/heads/branch-3.0 Commit: bce57a139b4423ad2e85563212bb3106fc084f35 Parents: 6410586 Author: Erik Krogen Authored: Thu Apr 12 23:27:51 2018 -0700 Committer: Konstantin V Shvachko Committed: Thu Apr 12 23:35:58 2018 -0700 -- .../java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bce57a13/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java index 324f0ca..4d3ca48 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java @@ -156,6 +156,7 @@ public class MiniHadoopClusterManager { if (!noDFS) { dfs = new MiniDFSCluster.Builder(conf).nameNodePort(nnPort) .nameNodeHttpPort(nnHttpPort).numDataNodes(numDataNodes) + .format(dfsOpts == StartupOption.FORMAT) .startupOption(dfsOpts).build(); LOG.info("Started MiniDFSCluster -- namenode on port " + dfs.getNameNodePort()); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: HADOOP-14970. MiniHadoopClusterManager doesn't respect lack of format option. Contributed by Erik Krogen
Repository: hadoop Updated Branches: refs/heads/branch-3.1 84531ad9b -> e343be46f HADOOP-14970. MiniHadoopClusterManager doesn't respect lack of format option. Contributed by Erik Krogen (cherry picked from commit 1a407bc9906306801690bc75ff0f0456f8f265fd) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e343be46 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e343be46 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e343be46 Branch: refs/heads/branch-3.1 Commit: e343be46f8aa0cabe25d85e24c1e1ec8e6ebb061 Parents: 84531ad Author: Erik Krogen Authored: Thu Apr 12 23:27:51 2018 -0700 Committer: Konstantin V Shvachko Committed: Thu Apr 12 23:31:18 2018 -0700 -- .../java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e343be46/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java index b89cdc0..d29dd34 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java @@ -156,6 +156,7 @@ public class MiniHadoopClusterManager { if (!noDFS) { dfs = new MiniDFSCluster.Builder(conf).nameNodePort(nnPort) .nameNodeHttpPort(nnHttpPort).numDataNodes(numDataNodes) + .format(dfsOpts == StartupOption.FORMAT) .startupOption(dfsOpts).build(); LOG.info("Started MiniDFSCluster -- namenode on port " + dfs.getNameNodePort()); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: HADOOP-14970. MiniHadoopClusterManager doesn't respect lack of format option. Contributed by Erik Krogen
Repository: hadoop Updated Branches: refs/heads/branch-2 7393020cb -> 31d061e4e HADOOP-14970. MiniHadoopClusterManager doesn't respect lack of format option. Contributed by Erik Krogen (cherry picked from commit 1a407bc9906306801690bc75ff0f0456f8f265fd) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/31d061e4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/31d061e4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/31d061e4 Branch: refs/heads/branch-2 Commit: 31d061e4e844147501518f64ec41cf8867cc70bd Parents: 7393020 Author: Erik Krogen Authored: Thu Apr 12 23:27:51 2018 -0700 Committer: Konstantin V Shvachko Committed: Thu Apr 12 23:39:47 2018 -0700 -- .../java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/31d061e4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java index 96d0027..becc768 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java @@ -156,6 +156,7 @@ public class MiniHadoopClusterManager { if (!noDFS) { dfs = new MiniDFSCluster.Builder(conf).nameNodePort(nnPort) .nameNodeHttpPort(nnHttpPort).numDataNodes(numDataNodes) + .format(dfsOpts == StartupOption.FORMAT) .startupOption(dfsOpts).build(); LOG.info("Started MiniDFSCluster -- namenode on port " + dfs.getNameNodePort()); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: HADOOP-14970. MiniHadoopClusterManager doesn't respect lack of format option. Contributed by Erik Krogen
Repository: hadoop Updated Branches: refs/heads/branch-2.8 1548205a8 -> c4ea1c8bb HADOOP-14970. MiniHadoopClusterManager doesn't respect lack of format option. Contributed by Erik Krogen (cherry picked from commit 1a407bc9906306801690bc75ff0f0456f8f265fd) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c4ea1c8b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c4ea1c8b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c4ea1c8b Branch: refs/heads/branch-2.8 Commit: c4ea1c8bb73a91be6233a855c62ea1a8119dbecb Parents: 1548205 Author: Erik Krogen Authored: Thu Apr 12 23:27:51 2018 -0700 Committer: Konstantin V Shvachko Committed: Thu Apr 12 23:52:25 2018 -0700 -- .../org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java| 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4ea1c8b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java index 2e8ba5e..4055b06 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java @@ -152,7 +152,9 @@ public class MiniHadoopClusterManager { URISyntaxException { if (!noDFS) { dfs = new MiniDFSCluster.Builder(conf).nameNodePort(nnPort) - .numDataNodes(numDataNodes).startupOption(dfsOpts).build(); + .numDataNodes(numDataNodes) + .format(dfsOpts == StartupOption.FORMAT) + .startupOption(dfsOpts).build(); LOG.info("Started MiniDFSCluster -- namenode on port " + dfs.getNameNodePort()); } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: HADOOP-14970. MiniHadoopClusterManager doesn't respect lack of format option. Contributed by Erik Krogen
Repository: hadoop Updated Branches: refs/heads/branch-2.9 48023bda1 -> 7b0573108 HADOOP-14970. MiniHadoopClusterManager doesn't respect lack of format option. Contributed by Erik Krogen (cherry picked from commit 1a407bc9906306801690bc75ff0f0456f8f265fd) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7b057310 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7b057310 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7b057310 Branch: refs/heads/branch-2.9 Commit: 7b05731080b73c820c5ecdb3f85eaa0233377ffd Parents: 48023bd Author: Erik Krogen Authored: Thu Apr 12 23:27:51 2018 -0700 Committer: Konstantin V Shvachko Committed: Thu Apr 12 23:43:16 2018 -0700 -- .../java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b057310/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java index 96d0027..becc768 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java @@ -156,6 +156,7 @@ public class MiniHadoopClusterManager { if (!noDFS) { dfs = new MiniDFSCluster.Builder(conf).nameNodePort(nnPort) .nameNodeHttpPort(nnHttpPort).numDataNodes(numDataNodes) + .format(dfsOpts == StartupOption.FORMAT) .startupOption(dfsOpts).build(); LOG.info("Started MiniDFSCluster -- namenode on port " + dfs.getNameNodePort()); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: HADOOP-14970. MiniHadoopClusterManager doesn't respect lack of format option. Contributed by Erik Krogen
Repository: hadoop Updated Branches: refs/heads/branch-2.7 b4d24d822 -> f7830f399 HADOOP-14970. MiniHadoopClusterManager doesn't respect lack of format option. Contributed by Erik Krogen (cherry picked from commit 1a407bc9906306801690bc75ff0f0456f8f265fd) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f7830f39 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f7830f39 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f7830f39 Branch: refs/heads/branch-2.7 Commit: f7830f399a73a88b20deeb881ef7b55af5e5adad Parents: b4d24d8 Author: Erik Krogen Authored: Thu Apr 12 23:27:51 2018 -0700 Committer: Konstantin V Shvachko Committed: Thu Apr 12 23:54:03 2018 -0700 -- .../org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java| 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7830f39/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java index 2e8ba5e..4055b06 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java @@ -152,7 +152,9 @@ public class MiniHadoopClusterManager { URISyntaxException { if (!noDFS) { dfs = new MiniDFSCluster.Builder(conf).nameNodePort(nnPort) - .numDataNodes(numDataNodes).startupOption(dfsOpts).build(); + .numDataNodes(numDataNodes) + .format(dfsOpts == StartupOption.FORMAT) + .startupOption(dfsOpts).build(); LOG.info("Started MiniDFSCluster -- namenode on port " + dfs.getNameNodePort()); } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: HDFS-13423. Ozone: Clean-up of ozone related change from hadoop-hdfs-project. Contributed by Nanda Kumar.
Repository: hadoop Updated Branches: refs/heads/HDFS-7240 c36a850af -> 584c573a5 HDFS-13423. Ozone: Clean-up of ozone related change from hadoop-hdfs-project. Contributed by Nanda Kumar. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/584c573a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/584c573a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/584c573a Branch: refs/heads/HDFS-7240 Commit: 584c573a5604d49522c4b7766fc52f4d3eb92496 Parents: c36a850 Author: Mukul Kumar Singh Authored: Fri Apr 13 14:13:06 2018 +0530 Committer: Mukul Kumar Singh Committed: Fri Apr 13 14:13:06 2018 +0530 -- .../java/org/apache/hadoop/hdds/HddsUtils.java | 46 .../hadoop/ozone/HddsDatanodeService.java | 3 +- .../hdfs/server/common/HdfsServerConstants.java | 3 +- .../hadoop/hdfs/server/common/StorageInfo.java | 4 -- .../hadoop/hdfs/server/datanode/DataNode.java | 14 +- .../web/RestCsrfPreventionFilterHandler.java| 2 +- .../hadoop-hdfs/src/main/proto/HdfsServer.proto | 8 .../namenode/TestFavoredNodesEndToEnd.java | 6 +-- .../hadoop/ozone/MiniOzoneClassicCluster.java | 19 +--- .../hadoop/ozone/MiniOzoneTestHelper.java | 22 -- .../apache/hadoop/ozone/RatisTestHelper.java| 2 +- .../hadoop/ozone/TestMiniOzoneCluster.java | 8 +--- .../apache/hadoop/ozone/TestOzoneHelper.java| 3 +- .../TestStorageContainerManagerHelper.java | 2 +- .../TestCloseContainerHandler.java | 2 +- .../ozoneimpl/TestOzoneContainerRatis.java | 3 +- .../container/ozoneimpl/TestRatisManager.java | 3 +- .../ksm/TestKeySpaceManagerRestInterface.java | 3 +- .../hadoop/ozone/ozShell/TestOzoneShell.java| 3 +- .../org/apache/hadoop/ozone/scm/TestSCMCli.java | 3 +- .../apache/hadoop/ozone/scm/TestSCMMetrics.java | 3 +- .../ozone/web/TestDistributedOzoneVolumes.java | 4 +- .../hadoop/ozone/web/TestLocalOzoneVolumes.java | 4 +- .../hadoop/ozone/web/TestOzoneWebAccess.java| 3 +- .../hadoop/ozone/web/client/TestBuckets.java| 3 +- .../hadoop/ozone/web/client/TestKeys.java | 4 +- .../ozone/web/client/TestOzoneClient.java | 3 +- .../hadoop/ozone/web/client/TestVolume.java | 3 +- .../web/netty/ObjectStoreRestHttpServer.java| 45 +-- 29 files changed, 109 insertions(+), 122 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/584c573a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java -- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java index f00f503..a0b5c47 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java @@ -22,18 +22,26 @@ import com.google.common.base.Optional; import com.google.common.base.Strings; import com.google.common.net.HostAndPort; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.net.DNS; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.net.InetSocketAddress; +import java.net.UnknownHostException; import java.nio.file.Paths; import java.util.Collection; import java.util.HashSet; +import static org.apache.hadoop.hdfs.DFSConfigKeys +.DFS_DATANODE_DNS_INTERFACE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys +.DFS_DATANODE_DNS_NAMESERVER_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED_DEFAULT; @@ -269,4 +277,42 @@ public class HddsUtils { } return dataNodeIDPath; } + + /** + * Returns the hostname for this datanode. If the hostname is not + * explicitly configured in the given config, then it is determined + * via the DNS class. + * + * @param conf Configuration + * + * @return the hostname (NB: may not be a FQDN) + * @throws UnknownHostException if the dfs.datanode.dns.interface + *option is used and the hostname can not be determined + */ + public static String getHostName(Configuration conf) + throws UnknownHostException { +String name = conf.get(DFS_DATANODE_HOST_NAME_KEY); +if (name == null) { + String dnsInterface = conf.get( + CommonConfigurationK
hadoop git commit: HDFS-13418. NetworkTopology should be configurable when enable DFSNetworkTopology. Contributed by Tao Jie.
Repository: hadoop Updated Branches: refs/heads/trunk 1a407bc99 -> 0725953ef HDFS-13418. NetworkTopology should be configurable when enable DFSNetworkTopology. Contributed by Tao Jie. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0725953e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0725953e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0725953e Branch: refs/heads/trunk Commit: 0725953efec89b35b7586b846abb01f7c5963b37 Parents: 1a407bc Author: Yiqun Lin Authored: Fri Apr 13 17:55:45 2018 +0800 Committer: Yiqun Lin Committed: Fri Apr 13 17:55:45 2018 +0800 -- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 7 +++ .../hadoop/hdfs/net/DFSNetworkTopology.java | 10 +++- .../src/main/resources/hdfs-default.xml | 14 ++ .../blockmanagement/TestDatanodeManager.java| 52 4 files changed, 81 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0725953e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index b4b9d97..b4dab4d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; +import org.apache.hadoop.hdfs.net.DFSNetworkTopology; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRackFaultTolerant; @@ -1177,6 +1178,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.use.dfs.network.topology"; public static final boolean DFS_USE_DFS_NETWORK_TOPOLOGY_DEFAULT = true; + public static final String DFS_NET_TOPOLOGY_IMPL_KEY = + "dfs.net.topology.impl"; + + public static final Class DFS_NET_TOPOLOGY_IMPL_DEFAULT = + DFSNetworkTopology.class; + // dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry @Deprecated public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY http://git-wip-us.apache.org/repos/asf/hadoop/blob/0725953e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java index e74cdec..7889ef4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java @@ -22,11 +22,13 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.net.Node; import org.apache.hadoop.net.NodeBase; +import org.apache.hadoop.util.ReflectionUtils; import java.util.ArrayList; import java.util.Collection; @@ -44,8 +46,12 @@ public class DFSNetworkTopology extends NetworkTopology { private static final Random RANDOM = new Random(); public static DFSNetworkTopology getInstance(Configuration conf) { -DFSNetworkTopology nt = new DFSNetworkTopology(); -return (DFSNetworkTopology)nt.init(DFSTopologyNodeImpl.FACTORY); + +DFSNetworkTopology nt = ReflectionUtils.newInstance(conf.getClass( +DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_KEY, +DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_DEFAULT, +DFSNetworkTopology.class), conf); +return (DFSNetworkTopology) nt.init(DFSTopologyNodeImpl.FACTORY); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/0725953e/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src
hadoop git commit: HDFS-13418. NetworkTopology should be configurable when enable DFSNetworkTopology. Contributed by Tao Jie.
Repository: hadoop Updated Branches: refs/heads/branch-3.1 e343be46f -> 16763d822 HDFS-13418. NetworkTopology should be configurable when enable DFSNetworkTopology. Contributed by Tao Jie. (cherry picked from commit 0725953efec89b35b7586b846abb01f7c5963b37) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/16763d82 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/16763d82 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/16763d82 Branch: refs/heads/branch-3.1 Commit: 16763d82297efd422ff7d62e94347c4f0097e9a8 Parents: e343be4 Author: Yiqun Lin Authored: Fri Apr 13 17:55:45 2018 +0800 Committer: Yiqun Lin Committed: Fri Apr 13 17:57:15 2018 +0800 -- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 7 +++ .../hadoop/hdfs/net/DFSNetworkTopology.java | 10 +++- .../src/main/resources/hdfs-default.xml | 14 ++ .../blockmanagement/TestDatanodeManager.java| 52 4 files changed, 81 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/16763d82/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index b4b9d97..b4dab4d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; +import org.apache.hadoop.hdfs.net.DFSNetworkTopology; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRackFaultTolerant; @@ -1177,6 +1178,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.use.dfs.network.topology"; public static final boolean DFS_USE_DFS_NETWORK_TOPOLOGY_DEFAULT = true; + public static final String DFS_NET_TOPOLOGY_IMPL_KEY = + "dfs.net.topology.impl"; + + public static final Class DFS_NET_TOPOLOGY_IMPL_DEFAULT = + DFSNetworkTopology.class; + // dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry @Deprecated public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY http://git-wip-us.apache.org/repos/asf/hadoop/blob/16763d82/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java index e74cdec..7889ef4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java @@ -22,11 +22,13 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.net.Node; import org.apache.hadoop.net.NodeBase; +import org.apache.hadoop.util.ReflectionUtils; import java.util.ArrayList; import java.util.Collection; @@ -44,8 +46,12 @@ public class DFSNetworkTopology extends NetworkTopology { private static final Random RANDOM = new Random(); public static DFSNetworkTopology getInstance(Configuration conf) { -DFSNetworkTopology nt = new DFSNetworkTopology(); -return (DFSNetworkTopology)nt.init(DFSTopologyNodeImpl.FACTORY); + +DFSNetworkTopology nt = ReflectionUtils.newInstance(conf.getClass( +DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_KEY, +DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_DEFAULT, +DFSNetworkTopology.class), conf); +return (DFSNetworkTopology) nt.init(DFSTopologyNodeImpl.FACTORY); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/16763d82/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml -- diff --git a/hadoop-hdfs-project/hadoop
hadoop git commit: HDFS-13418. NetworkTopology should be configurable when enable DFSNetworkTopology. Contributed by Tao Jie.
Repository: hadoop Updated Branches: refs/heads/branch-2 31d061e4e -> a7721082d HDFS-13418. NetworkTopology should be configurable when enable DFSNetworkTopology. Contributed by Tao Jie. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a7721082 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a7721082 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a7721082 Branch: refs/heads/branch-2 Commit: a7721082dc3d608dc8a3ac7e1ffa0de21781f00b Parents: 31d061e Author: Yiqun Lin Authored: Fri Apr 13 17:59:35 2018 +0800 Committer: Yiqun Lin Committed: Fri Apr 13 17:59:35 2018 +0800 -- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 7 +++ .../hadoop/hdfs/net/DFSNetworkTopology.java | 9 +++- .../src/main/resources/hdfs-default.xml | 14 ++ .../blockmanagement/TestDatanodeManager.java| 52 4 files changed, 80 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7721082/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 42ce05b..c9fef06 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; +import org.apache.hadoop.hdfs.net.DFSNetworkTopology; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.RamDiskReplicaLruTracker; @@ -1003,6 +1004,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.use.dfs.network.topology"; public static final boolean DFS_USE_DFS_NETWORK_TOPOLOGY_DEFAULT = true; + public static final String DFS_NET_TOPOLOGY_IMPL_KEY = + "dfs.net.topology.impl"; + + public static final Class DFS_NET_TOPOLOGY_IMPL_DEFAULT = + DFSNetworkTopology.class; + // dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry @Deprecated public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7721082/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java index e74cdec..f3074d5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java @@ -22,11 +22,13 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.net.Node; import org.apache.hadoop.net.NodeBase; +import org.apache.hadoop.util.ReflectionUtils; import java.util.ArrayList; import java.util.Collection; @@ -44,8 +46,11 @@ public class DFSNetworkTopology extends NetworkTopology { private static final Random RANDOM = new Random(); public static DFSNetworkTopology getInstance(Configuration conf) { -DFSNetworkTopology nt = new DFSNetworkTopology(); -return (DFSNetworkTopology)nt.init(DFSTopologyNodeImpl.FACTORY); +DFSNetworkTopology nt = ReflectionUtils.newInstance(conf.getClass( +DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_KEY, +DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_DEFAULT, +DFSNetworkTopology.class), conf); +return (DFSNetworkTopology) nt.init(DFSTopologyNodeImpl.FACTORY); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7721082/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/
hadoop git commit: HDFS-13438. Fix javadoc in FsVolumeList#removeVolume. Contributed by Shashikant Banerjee.
Repository: hadoop Updated Branches: refs/heads/trunk 0725953ef -> fa8b88ab2 HDFS-13438. Fix javadoc in FsVolumeList#removeVolume. Contributed by Shashikant Banerjee. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fa8b88ab Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fa8b88ab Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fa8b88ab Branch: refs/heads/trunk Commit: fa8b88ab2b272b29cf116a5de038d78fc4357b9d Parents: 0725953 Author: Bharat Viswanadham Authored: Fri Apr 13 08:56:02 2018 -0700 Committer: Bharat Viswanadham Committed: Fri Apr 13 08:56:02 2018 -0700 -- .../hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa8b88ab/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java index 75baf84..8f52ea7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java @@ -335,7 +335,7 @@ class FsVolumeList { /** * Dynamically remove volume in the list. - * @param volume the volume to be removed. + * @param storageLocation {@link StorageLocation} of the volume to be removed. * @param clearFailure set true to remove failure info for this volume. */ void removeVolume(StorageLocation storageLocation, boolean clearFailure) { - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: HDFS-13430. Fix TestEncryptionZonesWithKMS failure due to HADOOP-14445.
Repository: hadoop Updated Branches: refs/heads/trunk fa8b88ab2 -> 650359371 HDFS-13430. Fix TestEncryptionZonesWithKMS failure due to HADOOP-14445. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/65035937 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/65035937 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/65035937 Branch: refs/heads/trunk Commit: 650359371175fba416331e73aa03d2a96ccb90e5 Parents: fa8b88a Author: Xiao Chen Authored: Fri Apr 13 09:04:52 2018 -0700 Committer: Xiao Chen Committed: Fri Apr 13 09:05:17 2018 -0700 -- .../src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java | 3 +++ 1 file changed, 3 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/65035937/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java index 6f9ef29..51c6c4e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java @@ -110,6 +110,7 @@ import org.junit.Test; import org.junit.rules.Timeout; import org.mockito.Mockito; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_KEY; import static org.junit.Assert.assertNotNull; import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.anyLong; @@ -192,6 +193,8 @@ public class TestEncryptionZones { // Lower the batch size for testing conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES, 2); +// disable kms client copy legacy token logic because it's irrelevant. +conf.setBoolean(KMS_CLIENT_COPY_LEGACY_TOKEN_KEY, false); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); cluster.waitActive(); Logger.getLogger(EncryptionZoneManager.class).setLevel(Level.TRACE); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: HDFS-13438. Fix javadoc in FsVolumeList#removeVolume. Contributed by Shashikant Banerjee.
Repository: hadoop Updated Branches: refs/heads/branch-3.1 16763d822 -> 8df8cad01 HDFS-13438. Fix javadoc in FsVolumeList#removeVolume. Contributed by Shashikant Banerjee. (cherry picked from commit fa8b88ab2b272b29cf116a5de038d78fc4357b9d) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8df8cad0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8df8cad0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8df8cad0 Branch: refs/heads/branch-3.1 Commit: 8df8cad019836bd81005392667558a3fc88459be Parents: 16763d8 Author: Bharat Viswanadham Authored: Fri Apr 13 08:56:02 2018 -0700 Committer: Bharat Viswanadham Committed: Fri Apr 13 09:05:24 2018 -0700 -- .../hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8df8cad0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java index 75baf84..8f52ea7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java @@ -335,7 +335,7 @@ class FsVolumeList { /** * Dynamically remove volume in the list. - * @param volume the volume to be removed. + * @param storageLocation {@link StorageLocation} of the volume to be removed. * @param clearFailure set true to remove failure info for this volume. */ void removeVolume(StorageLocation storageLocation, boolean clearFailure) { - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: HDFS-13330. ShortCircuitCache#fetchOrCreate never retries. Contributed by Gabor Bota.
Repository: hadoop Updated Branches: refs/heads/trunk 650359371 -> e66e287ef HDFS-13330. ShortCircuitCache#fetchOrCreate never retries. Contributed by Gabor Bota. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e66e287e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e66e287e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e66e287e Branch: refs/heads/trunk Commit: e66e287efe2b43e710137a628f03c7df3ebdf498 Parents: 6503593 Author: Wei-Chiu Chuang Authored: Fri Apr 13 09:17:34 2018 -0700 Committer: Wei-Chiu Chuang Committed: Fri Apr 13 09:17:34 2018 -0700 -- .../hdfs/shortcircuit/ShortCircuitCache.java| 11 ++--- .../shortcircuit/TestShortCircuitCache.java | 26 2 files changed, 33 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e66e287e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java index b26652b..c2f0350 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java @@ -664,6 +664,7 @@ public class ShortCircuitCache implements Closeable { unref(replica); } + static final int FETCH_OR_CREATE_RETRY_TIMES = 3; /** * Fetch or create a replica. * @@ -678,11 +679,11 @@ public class ShortCircuitCache implements Closeable { */ public ShortCircuitReplicaInfo fetchOrCreate(ExtendedBlockId key, ShortCircuitReplicaCreator creator) { -Waitable newWaitable = null; +Waitable newWaitable; lock.lock(); try { ShortCircuitReplicaInfo info = null; - do { + for (int i = 0; i < FETCH_OR_CREATE_RETRY_TIMES; i++){ if (closed) { LOG.trace("{}: can't fethchOrCreate {} because the cache is closed.", this, key); @@ -692,11 +693,12 @@ public class ShortCircuitCache implements Closeable { if (waitable != null) { try { info = fetch(key, waitable); +break; } catch (RetriableException e) { LOG.debug("{}: retrying {}", this, e.getMessage()); } } - } while (false); + } if (info != null) return info; // We need to load the replica ourselves. newWaitable = new Waitable<>(lock.newCondition()); @@ -717,7 +719,8 @@ public class ShortCircuitCache implements Closeable { * * @throws RetriableException If the caller needs to retry. */ - private ShortCircuitReplicaInfo fetch(ExtendedBlockId key, + @VisibleForTesting // ONLY for testing + protected ShortCircuitReplicaInfo fetch(ExtendedBlockId key, Waitable waitable) throws RetriableException { // Another thread is already in the process of loading this // ShortCircuitReplica. So we simply wait for it to complete. http://git-wip-us.apache.org/repos/asf/hadoop/blob/e66e287e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java index 7ba0edc..5da6a25 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java @@ -65,6 +65,7 @@ import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.ShortCircuitReplica import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.ShmId; import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.Slot; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.ipc.RetriableException; import org.apache.hadoop.net.unix.DomainSocket; import org.apache.hadoop.net.unix.TemporarySocketDirectory; import org.apache.hadoop.security.token.SecretManager.InvalidToken; @@ -793,4 +794,29 @@ public class TestShortCircuitCache { cluster.shutdown(); sockDir.close(); } + + @Test + public void testFetchOrCreateRetries() throws Exception { +try(ShortCircuitCache cache = Mockito +.spy(new ShortCircuitCache(10, 1000
hadoop git commit: Revert "YARN-7810. Update TestDockerContainerRuntime to test with current user credential."
Repository: hadoop Updated Branches: refs/heads/branch-2 a7721082d -> cc2a2a8e0 Revert "YARN-7810. Update TestDockerContainerRuntime to test with current user credential." This reverts commit 724bffdb89cd62cc90a1f49c7c5e40998dc1cc0f. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cc2a2a8e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cc2a2a8e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cc2a2a8e Branch: refs/heads/branch-2 Commit: cc2a2a8e063ecdb23216830be8418736af79fe7b Parents: a772108 Author: Wei-Chiu Chuang Authored: Fri Apr 13 10:13:59 2018 -0700 Committer: Wei-Chiu Chuang Committed: Fri Apr 13 10:13:59 2018 -0700 -- .../runtime/TestDockerContainerRuntime.java | 178 +-- 1 file changed, 82 insertions(+), 96 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc2a2a8e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index ab38ea2..aef94a7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -81,8 +81,7 @@ public class TestDockerContainerRuntime { private HashMap env; private String image; private String uidGidPair; - private String runAsUser = System.getProperty("user.name"); - private String[] groups = {}; + private String runAsUser; private String user; private String appId; private String containerIdStr = containerId; @@ -131,37 +130,8 @@ public class TestDockerContainerRuntime { when(context.getEnvironment()).thenReturn(env); when(container.getUser()).thenReturn(submittingUser); -// Get the running user's uid and gid for remap -String uid = ""; -String gid = ""; -Shell.ShellCommandExecutor shexec1 = new Shell.ShellCommandExecutor( -new String[]{"id", "-u", runAsUser}); -Shell.ShellCommandExecutor shexec2 = new Shell.ShellCommandExecutor( -new String[]{"id", "-g", runAsUser}); -Shell.ShellCommandExecutor shexec3 = new Shell.ShellCommandExecutor( -new String[]{"id", "-G", runAsUser}); -try { - shexec1.execute(); - // get rid of newline at the end - uid = shexec1.getOutput().replaceAll("\n$", ""); -} catch (Exception e) { - LOG.info("Could not run id -u command: " + e); -} -try { - shexec2.execute(); - // get rid of newline at the end - gid = shexec2.getOutput().replaceAll("\n$", ""); -} catch (Exception e) { - LOG.info("Could not run id -g command: " + e); -} -try { - shexec3.execute(); - groups = shexec3.getOutput().replace("\n", " ").split(" "); -} catch (Exception e) { - LOG.info("Could not run id -G command: " + e); -} -uidGidPair = uid + ":" + gid; - +uidGidPair = ""; +runAsUser = "run_as_user"; user = "user"; appId = "app_id"; containerIdStr = containerId; @@ -331,7 +301,7 @@ public class TestDockerContainerRuntime { List dockerCommands = Files.readAllLines(Paths.get (dockerCommandFile), Charset.forName("UTF-8")); -int expected = 14; +int expected = 13; int counter = 0; Assert.assertEquals(expected, dockerCommands.size()); Assert.assertEquals("[docker-command-execution]", @@ -341,8 +311,6 @@ public class TestDockerContainerRuntime { Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++)); Assert.assertEquals(" detach=true", dockerCommands.get(counter++)); Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); -Assert.assertEquals(" group-add=" + String.join(",", groups), -dockerCommands.get(counter++)); Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++)); Assert .assertEquals(" image=busybox:latest", dockerCommands.get(counter++)); @@ -358,7 +326,7 @@ public class TestDockerContain
[02/14] hadoop git commit: HADOOP-15062. TestCryptoStreamsWithOpensslAesCtrCryptoCodec fails on Debian 9. Contributed by Miklos Szegedi.
HADOOP-15062. TestCryptoStreamsWithOpensslAesCtrCryptoCodec fails on Debian 9. Contributed by Miklos Szegedi. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9014f98b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9014f98b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9014f98b Branch: refs/heads/branch-3.1 Commit: 9014f98b603bbb680d92a54b414e4c38e329e892 Parents: ffc39ec Author: Yufei Gu Authored: Tue Mar 20 15:19:18 2018 -0700 Committer: Arpit Agarwal Committed: Fri Apr 13 10:12:01 2018 -0700 -- .../org/apache/hadoop/crypto/OpensslCipher.c| 33 1 file changed, 33 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9014f98b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c -- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c index c7984a3..abff7ea 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c @@ -27,8 +27,12 @@ #ifdef UNIX static EVP_CIPHER_CTX * (*dlsym_EVP_CIPHER_CTX_new)(void); static void (*dlsym_EVP_CIPHER_CTX_free)(EVP_CIPHER_CTX *); +#if OPENSSL_API_COMPAT < 0x1010L && OPENSSL_VERSION_NUMBER >= 0x1010L +static int (*dlsym_EVP_CIPHER_CTX_reset)(EVP_CIPHER_CTX *); +#else static int (*dlsym_EVP_CIPHER_CTX_cleanup)(EVP_CIPHER_CTX *); static void (*dlsym_EVP_CIPHER_CTX_init)(EVP_CIPHER_CTX *); +#endif static int (*dlsym_EVP_CIPHER_CTX_set_padding)(EVP_CIPHER_CTX *, int); static int (*dlsym_EVP_CIPHER_CTX_test_flags)(const EVP_CIPHER_CTX *, int); static int (*dlsym_EVP_CIPHER_CTX_block_size)(const EVP_CIPHER_CTX *); @@ -123,10 +127,16 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs "EVP_CIPHER_CTX_new"); LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_free, env, openssl, \ "EVP_CIPHER_CTX_free"); +#if OPENSSL_API_COMPAT < 0x1010L && OPENSSL_VERSION_NUMBER >= 0x1010L + LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_reset, env, openssl, \ + "EVP_CIPHER_CTX_reset"); +#else LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_cleanup, env, openssl, \ "EVP_CIPHER_CTX_cleanup"); LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_init, env, openssl, \ "EVP_CIPHER_CTX_init"); +#endif + LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_set_padding, env, openssl, \ "EVP_CIPHER_CTX_set_padding"); LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_test_flags, env, openssl, \ @@ -271,7 +281,11 @@ JNIEXPORT jlong JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_init (*env)->ReleaseByteArrayElements(env, key, jKey, 0); (*env)->ReleaseByteArrayElements(env, iv, jIv, 0); if (rc == 0) { +#if OPENSSL_API_COMPAT < 0x1010L && OPENSSL_VERSION_NUMBER >= 0x1010L +dlsym_EVP_CIPHER_CTX_reset(context); +#else dlsym_EVP_CIPHER_CTX_cleanup(context); +#endif THROW(env, "java/lang/InternalError", "Error in EVP_CipherInit_ex."); return (jlong)0; } @@ -334,7 +348,11 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_update int output_len = 0; if (!dlsym_EVP_CipherUpdate(context, output_bytes, &output_len, \ input_bytes, input_len)) { +#if OPENSSL_API_COMPAT < 0x1010L && OPENSSL_VERSION_NUMBER >= 0x1010L +dlsym_EVP_CIPHER_CTX_reset(context); +#else dlsym_EVP_CIPHER_CTX_cleanup(context); +#endif THROW(env, "java/lang/InternalError", "Error in EVP_CipherUpdate."); return 0; } @@ -376,7 +394,11 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_doFinal int output_len = 0; if (!dlsym_EVP_CipherFinal_ex(context, output_bytes, &output_len)) { +#if OPENSSL_API_COMPAT < 0x1010L && OPENSSL_VERSION_NUMBER >= 0x1010L +dlsym_EVP_CIPHER_CTX_reset(context); +#else dlsym_EVP_CIPHER_CTX_cleanup(context); +#endif THROW(env, "java/lang/InternalError", "Error in EVP_CipherFinal_ex."); return 0; } @@ -396,6 +418,16 @@ JNIEXPORT jstring JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_getLibrary (JNIEnv *env, jclass clazz) { #ifdef UNIX +#if OPENSSL_API_COMPAT < 0x1010L && OPENSSL_VERSION_NUMBER >= 0x1010L + if (dlsym_EVP_CIPHER_CTX_reset) { +Dl_info dl_info; +if(dladdr( +dlsym_EVP_CIPHER_CTX_reset, +&dl_info)) { + return (*env)->NewStringUTF(env, dl_info.dli_fname); +} +
[07/14] hadoop git commit: HDFS-11043. TestWebHdfsTimeouts fails. Contributed by Xiaoyu Yao and Chao Sun.
HDFS-11043. TestWebHdfsTimeouts fails. Contributed by Xiaoyu Yao and Chao Sun. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/867135af Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/867135af Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/867135af Branch: refs/heads/branch-3.1 Commit: 867135af28af5065db0f8f73707efa3c7672da73 Parents: 56acbe4 Author: Xiaoyu Yao Authored: Wed Mar 21 13:53:35 2018 -0700 Committer: Arpit Agarwal Committed: Fri Apr 13 10:14:47 2018 -0700 -- .../java/org/apache/hadoop/hdfs/web/TestWebHdfsTimeouts.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/867135af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTimeouts.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTimeouts.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTimeouts.java index 67c39e1..4743821 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTimeouts.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTimeouts.java @@ -66,10 +66,10 @@ public class TestWebHdfsTimeouts { private static final Log LOG = LogFactory.getLog(TestWebHdfsTimeouts.class); - private static final int CLIENTS_TO_CONSUME_BACKLOG = 100; + private static final int CLIENTS_TO_CONSUME_BACKLOG = 129; private static final int CONNECTION_BACKLOG = 1; - private static final int SHORT_SOCKET_TIMEOUT = 5; - private static final int TEST_TIMEOUT = 1; + private static final int SHORT_SOCKET_TIMEOUT = 200; + private static final int TEST_TIMEOUT = 10; private List clients; private WebHdfsFileSystem fs; - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[06/14] hadoop git commit: HDFS-13315. Add a test for the issue reported in HDFS-11481 which is fixed by HDFS-10997. Contributed by Yongjun Zhang.
HDFS-13315. Add a test for the issue reported in HDFS-11481 which is fixed by HDFS-10997. Contributed by Yongjun Zhang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/56acbe47 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/56acbe47 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/56acbe47 Branch: refs/heads/branch-3.1 Commit: 56acbe476b6659ee4e3f53fdb5888906625950c8 Parents: 2b46bd3 Author: Yongjun Zhang Authored: Tue Mar 20 23:00:39 2018 -0700 Committer: Arpit Agarwal Committed: Fri Apr 13 10:14:24 2018 -0700 -- .../hdfs/tools/snapshot/SnapshotDiff.java | 1 + .../org/apache/hadoop/hdfs/DFSTestUtil.java | 37 ++ .../apache/hadoop/hdfs/TestEncryptionZones.java | 41 .../snapshot/TestSnapshotDiffReport.java| 25 +--- 4 files changed, 80 insertions(+), 24 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/56acbe47/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java index 3838ca1..1237099 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java @@ -106,6 +106,7 @@ public class SnapshotDiff extends Configured implements Tool { } catch (IOException e) { String[] content = e.getLocalizedMessage().split("\n"); System.err.println("snapshotDiff: " + content[0]); + e.printStackTrace(System.err); return 1; } return 0; http://git-wip-us.apache.org/repos/asf/hadoop/blob/56acbe47/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index 1411a7f..4f9f260 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -121,6 +121,9 @@ import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState; import org.apache.hadoop.hdfs.protocol.ReplicatedBlockStats; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType; import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; @@ -2354,4 +2357,38 @@ public class DFSTestUtil { } return closedFiles; } + + /** + * Check the correctness of the snapshotDiff report. + * Make sure all items in the passed entries are in the snapshotDiff + * report. + */ + public static void verifySnapshotDiffReport(DistributedFileSystem fs, + Path dir, String from, String to, + DiffReportEntry... entries) throws IOException { +SnapshotDiffReport report = fs.getSnapshotDiffReport(dir, from, to); +// reverse the order of from and to +SnapshotDiffReport inverseReport = fs +.getSnapshotDiffReport(dir, to, from); +LOG.info(report.toString()); +LOG.info(inverseReport.toString() + "\n"); + +assertEquals(entries.length, report.getDiffList().size()); +assertEquals(entries.length, inverseReport.getDiffList().size()); + +for (DiffReportEntry entry : entries) { + if (entry.getType() == DiffType.MODIFY) { +assertTrue(report.getDiffList().contains(entry)); +assertTrue(inverseReport.getDiffList().contains(entry)); + } else if (entry.getType() == DiffType.DELETE) { +assertTrue(report.getDiffList().contains(entry)); +assertTrue(inverseReport.getDiffList().contains( +new DiffReportEntry(DiffType.CREATE, entry.getSourcePath(; + } else if (entry.getType() == DiffType.CREATE) { +assertTrue(report.getDiffList().contains(entry)); +assertTrue(inverseReport.getDiffList().contains( +new DiffReportEntry(DiffType.DELETE, entry.getSourcePath(; + } +} + } } http://git-wip-us.apache.org/rep
[01/14] hadoop git commit: HADOOP-14667. Flexible Visual Studio support. Contributed by Allen Wittenauer
Repository: hadoop Updated Branches: refs/heads/branch-3.1 8df8cad01 -> 994c7d66e HADOOP-14667. Flexible Visual Studio support. Contributed by Allen Wittenauer Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ffc39ec3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ffc39ec3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ffc39ec3 Branch: refs/heads/branch-3.1 Commit: ffc39ec3ab7939adfc4464372960251d32eb6435 Parents: 8df8cad Author: Chris Douglas Authored: Mon Mar 19 16:05:55 2018 -0700 Committer: Arpit Agarwal Committed: Fri Apr 13 10:11:35 2018 -0700 -- BUILDING.txt| 30 dev-support/bin/win-vs-upgrade.cmd | 39 dev-support/win-paths-eg.cmd| 49 hadoop-common-project/hadoop-common/pom.xml | 28 +++ .../src/main/native/native.vcxproj | 2 + .../hadoop-hdfs-native-client/pom.xml | 5 +- 6 files changed, 128 insertions(+), 25 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffc39ec3/BUILDING.txt -- diff --git a/BUILDING.txt b/BUILDING.txt index 6c266e5..6d752d4 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -348,7 +348,7 @@ Requirements: * Maven 3.0 or later * ProtocolBuffer 2.5.0 * CMake 3.1 or newer -* Windows SDK 7.1 or Visual Studio 2010 Professional +* Visual Studio 2010 Professional or Higher * Windows SDK 8.1 (if building CPU rate control for the container executor) * zlib headers (if building native code bindings for zlib) * Internet connection for first build (to fetch all Maven and Hadoop dependencies) @@ -359,18 +359,15 @@ Requirements: Unix command-line tools are also included with the Windows Git package which can be downloaded from http://git-scm.com/downloads -If using Visual Studio, it must be Visual Studio 2010 Professional (not 2012). +If using Visual Studio, it must be Professional level or higher. Do not use Visual Studio Express. It does not support compiling for 64-bit, -which is problematic if running a 64-bit system. The Windows SDK 7.1 is free to -download here: - -http://www.microsoft.com/en-us/download/details.aspx?id=8279 +which is problematic if running a 64-bit system. The Windows SDK 8.1 is available to download at: http://msdn.microsoft.com/en-us/windows/bg162891.aspx -Cygwin is neither required nor supported. +Cygwin is not required. -- Building: @@ -378,21 +375,12 @@ Building: Keep the source code tree in a short path to avoid running into problems related to Windows maximum path length limitation (for example, C:\hdc). -Run builds from a Windows SDK Command Prompt. (Start, All Programs, -Microsoft Windows SDK v7.1, Windows SDK 7.1 Command Prompt). - -JAVA_HOME must be set, and the path must not contain spaces. If the full path -would contain spaces, then use the Windows short path instead. - -You must set the Platform environment variable to either x64 or Win32 depending -on whether you're running a 64-bit or 32-bit system. Note that this is -case-sensitive. It must be "Platform", not "PLATFORM" or "platform". -Environment variables on Windows are usually case-insensitive, but Maven treats -them as case-sensitive. Failure to set this environment variable correctly will -cause msbuild to fail while building the native code in hadoop-common. +There is one support command file located in dev-support called win-paths-eg.cmd. +It should be copied somewhere convenient and modified to fit your needs. -set Platform=x64 (when building on a 64-bit system) -set Platform=Win32 (when building on a 32-bit system) +win-paths-eg.cmd sets up the environment for use. You will need to modify this +file. It will put all of the required components in the command path, +configure the bit-ness of the build, and set several optional components. Several tests require that the user must have the Create Symbolic Links privilege. http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffc39ec3/dev-support/bin/win-vs-upgrade.cmd -- diff --git a/dev-support/bin/win-vs-upgrade.cmd b/dev-support/bin/win-vs-upgrade.cmd new file mode 100644 index 000..d8c9d73 --- /dev/null +++ b/dev-support/bin/win-vs-upgrade.cmd @@ -0,0 +1,39 @@ +@ECHO OFF +@REM Licensed to the Apache Software Foundation (ASF) under one or more +@REM contributor license agreements. See the NOTICE file distributed with +@REM this work for additional information regarding copyright ownership. +@REM The ASF licenses this file to You under the Apache License, Version 2.0 +@REM (the "L
[04/14] hadoop git commit: HADOOP-15331. Fix a race condition causing parsing error of java.io.BufferedInputStream in class org.apache.hadoop.conf.Configuration. Contributed by Miklos Szegedi.
HADOOP-15331. Fix a race condition causing parsing error of java.io.BufferedInputStream in class org.apache.hadoop.conf.Configuration. Contributed by Miklos Szegedi. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6891da1e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6891da1e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6891da1e Branch: refs/heads/branch-3.1 Commit: 6891da1e750118f00aca58a2426d085055d8efc5 Parents: 922a0db Author: Yufei Gu Authored: Thu Mar 22 11:04:37 2018 -0700 Committer: Arpit Agarwal Committed: Fri Apr 13 10:13:34 2018 -0700 -- .../org/apache/hadoop/conf/Configuration.java | 5 +++- .../apache/hadoop/conf/TestConfiguration.java | 31 2 files changed, 35 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6891da1e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index 0b2196b..a69e4c4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -816,8 +816,11 @@ public class Configuration implements Iterable>, */ @SuppressWarnings("unchecked") public Configuration(Configuration other) { -this.resources = (ArrayList) other.resources.clone(); synchronized(other) { + // Make sure we clone a finalized state + // Resources like input streams can be processed only once + other.getProps(); + this.resources = (ArrayList) other.resources.clone(); if (other.properties != null) { this.properties = (Properties)other.properties.clone(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6891da1e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java -- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java index 265e007..c8e5f17 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.conf; +import java.io.BufferedInputStream; import java.io.BufferedWriter; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -2433,4 +2434,34 @@ public class TestConfiguration { System.setOut(output); } } + + /** + * Test race conditions between clone() and getProps(). + * Test for race conditions in the way Hadoop handles the Configuration + * class. The scenario is the following. Let's assume that there are two + * threads sharing the same Configuration class. One adds some resources + * to the configuration, while the other one clones it. Resources are + * loaded lazily in a deferred call to loadResources(). If the cloning + * happens after adding the resources but before parsing them, some temporary + * resources like input stream pointers are cloned. Eventually both copies + * will load the same input stream resources. + * One parses the input stream XML and closes it updating it's own copy of + * the resource. The other one has another pointer to the same input stream. + * When it tries to load it, it will crash with a stream closed exception. + */ + @Test + public void testResourceRace() { +InputStream is = +new BufferedInputStream(new ByteArrayInputStream( +"".getBytes())); +Configuration config = new Configuration(); +// Thread 1 +config.addResource(is); +// Thread 2 +Configuration confClone = new Configuration(conf); +// Thread 2 +confClone.get("firstParse"); +// Thread 1 +config.get("secondParse"); + } } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[13/14] hadoop git commit: YARN-8048. Support auto-spawning of admin configured services during bootstrap of RM (Rohith Sharma K S via wangda)
YARN-8048. Support auto-spawning of admin configured services during bootstrap of RM (Rohith Sharma K S via wangda) Change-Id: I2d8d61ccad55e1118009294d7e17822df3cd0fd5 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7031a853 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7031a853 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7031a853 Branch: refs/heads/branch-3.1 Commit: 7031a853f4fee3bd097ab5813006a1953c06b6cf Parents: d98c3ca Author: Wangda Tan Authored: Fri Apr 6 21:24:58 2018 -0700 Committer: Arpit Agarwal Committed: Fri Apr 13 10:18:48 2018 -0700 -- .../hadoop/yarn/conf/YarnConfiguration.java | 4 + .../hadoop-yarn-services-api/pom.xml| 5 + .../client/SystemServiceManagerImpl.java| 381 +++ .../service/client/TestSystemServiceImpl.java | 180 + .../users/sync/user1/example-app1.yarnfile | 16 + .../users/sync/user1/example-app2.yarnfile | 16 + .../users/sync/user1/example-app3.json | 16 + .../users/sync/user2/example-app1.yarnfile | 16 + .../users/sync/user2/example-app2.yarnfile | 16 + .../yarn/service/conf/YarnServiceConf.java | 2 + .../yarn/service/TestSystemServiceManager.java | 156 .../server/service/SystemServiceManager.java| 25 ++ .../yarn/server/service/package-info.java | 27 ++ .../server/resourcemanager/ResourceManager.java | 30 +- 14 files changed, 889 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7031a853/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 9898cba..58c288b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -341,6 +341,10 @@ public class YarnConfiguration extends Configuration { public static final String YARN_API_SERVICES_ENABLE = "yarn." + "webapp.api-service.enable"; + @Private + public static final String DEFAULT_YARN_API_SYSTEM_SERVICES_CLASS = + "org.apache.hadoop.yarn.service.client.SystemServiceManagerImpl"; + public static final String RM_RESOURCE_TRACKER_ADDRESS = RM_PREFIX + "resource-tracker.address"; public static final int DEFAULT_RM_RESOURCE_TRACKER_PORT = 8031; http://git-wip-us.apache.org/repos/asf/hadoop/blob/7031a853/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml index 0dfa92d..d45da09 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml @@ -71,6 +71,7 @@ **/*.json +**/*.yarnfile @@ -96,6 +97,10 @@ org.apache.hadoop + hadoop-yarn-server-common + + + org.apache.hadoop hadoop-common http://git-wip-us.apache.org/repos/asf/hadoop/blob/7031a853/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java new file mode 100644 index 000..225f8bd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java @@ -0,0 +1,381 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * T
[03/14] hadoop git commit: HADOOP-15330. Remove jdk1.7 profile from hadoop-annotations module
HADOOP-15330. Remove jdk1.7 profile from hadoop-annotations module Signed-off-by: Akira Ajisaka Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/922a0db1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/922a0db1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/922a0db1 Branch: refs/heads/branch-3.1 Commit: 922a0db13c16632baae2d7906082e625445bfe2a Parents: 9014f98 Author: fang zhenyi Authored: Wed Mar 21 14:01:26 2018 +0900 Committer: Arpit Agarwal Committed: Fri Apr 13 10:12:27 2018 -0700 -- hadoop-common-project/hadoop-annotations/pom.xml | 15 --- 1 file changed, 15 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/922a0db1/hadoop-common-project/hadoop-annotations/pom.xml -- diff --git a/hadoop-common-project/hadoop-annotations/pom.xml b/hadoop-common-project/hadoop-annotations/pom.xml index af9f183..c11d9ba 100644 --- a/hadoop-common-project/hadoop-annotations/pom.xml +++ b/hadoop-common-project/hadoop-annotations/pom.xml @@ -39,21 +39,6 @@ - jdk1.7 - -1.7 - - - - jdk.tools - jdk.tools - 1.7 - system - ${java.home}/../lib/tools.jar - - - - jdk1.8 1.8 - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[14/14] hadoop git commit: YARN-8091. Revisit checkUserAccessToQueue RM REST API. (wangda)
YARN-8091. Revisit checkUserAccessToQueue RM REST API. (wangda) Change-Id: I5fab3fe229c34e967487b7327c7b3c8ddf7cb795 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/994c7d66 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/994c7d66 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/994c7d66 Branch: refs/heads/branch-3.1 Commit: 994c7d66e02ef27db234bece49526c7a4c08f3b2 Parents: 7031a85 Author: Wangda Tan Authored: Mon Apr 2 15:22:05 2018 -0700 Committer: Arpit Agarwal Committed: Fri Apr 13 10:19:10 2018 -0700 -- .../webapp/RMWebServiceProtocol.java| 3 +- .../resourcemanager/webapp/RMWebServices.java | 40 ++-- .../webapp/dao/RMQueueAclInfo.java | 65 .../webapp/TestRMWebServices.java | 52 +--- .../webapp/DefaultRequestInterceptorREST.java | 5 +- .../webapp/FederationInterceptorREST.java | 3 +- .../server/router/webapp/RouterWebServices.java | 3 +- .../webapp/MockRESTRequestInterceptor.java | 5 +- .../PassThroughRESTRequestInterceptor.java | 3 +- 9 files changed, 127 insertions(+), 52 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/994c7d66/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java index 423c4e1..85ea07d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java @@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsEntryList; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.RMQueueAclInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDeleteRequestInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationSubmissionRequestInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateRequestInfo; @@ -673,7 +674,7 @@ public interface RMWebServiceProtocol { * @throws AuthorizationException if the user is not authorized to invoke this *method. */ - Response checkUserAccessToQueue(String queue, String username, + RMQueueAclInfo checkUserAccessToQueue(String queue, String username, String queueAclType, HttpServletRequest hsr) throws AuthorizationException; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/994c7d66/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index c40e8be..d30764d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -173,6 +173,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsEntr import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsEntryList; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
[10/14] hadoop git commit: YARN-1151. Ability to configure auxiliary services from HDFS-based JAR files. (Xuan Gong via wangda)
YARN-1151. Ability to configure auxiliary services from HDFS-based JAR files. (Xuan Gong via wangda) Change-Id: Ied37ff11e507fc86847753ba79486652c8fadfe9 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c9623714 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c9623714 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c9623714 Branch: refs/heads/branch-3.1 Commit: c962371430b8957500397fe71e944c953f46ed7d Parents: 077eda6 Author: Wangda Tan Authored: Fri Apr 6 21:25:57 2018 -0700 Committer: Arpit Agarwal Committed: Fri Apr 13 10:17:47 2018 -0700 -- .../hadoop/yarn/conf/YarnConfiguration.java | 3 + .../containermanager/AuxServices.java | 160 +- .../containermanager/ContainerManagerImpl.java | 3 +- .../containermanager/TestAuxServices.java | 167 +-- 4 files changed, 313 insertions(+), 20 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9623714/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index a2cc9b7..9898cba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2086,6 +2086,9 @@ public class YarnConfiguration extends Configuration { public static final String NM_AUX_SERVICES_CLASSPATH = NM_AUX_SERVICES + ".%s.classpath"; + public static final String NM_AUX_SERVICE_REMOTE_CLASSPATH = + NM_AUX_SERVICES + ".%s.remote-classpath"; + public static final String NM_AUX_SERVICES_SYSTEM_CLASSES = NM_AUX_SERVICES + ".%s.system-classes"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9623714/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java index 57cca50..c8b7a76 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager; +import java.io.IOException; +import java.net.URI; import java.nio.ByteBuffer; import java.util.Collection; import java.util.Collections; @@ -29,45 +31,70 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.service.Service; import org.apache.hadoop.service.ServiceStateChangeListener; import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext; import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext; import org.apache.hadoop.yarn.server.api.AuxiliaryLocalPathHandler; import org.apache.hadoop.yarn.server.api.AuxiliaryService; import org.apache.hadoop.yarn.server.api.ContainerInitializationContext; import org.apache.hadoop.yarn.server.ap
[11/14] hadoop git commit: YARN-8028. Support authorizeUserAccessToQueue in RMWebServices. Contributed by Wangda Tan.
YARN-8028. Support authorizeUserAccessToQueue in RMWebServices. Contributed by Wangda Tan. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/21717db6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/21717db6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/21717db6 Branch: refs/heads/branch-3.1 Commit: 21717db6a02b4e619033bfab8bd82a0f52c892c4 Parents: c962371 Author: Sunil G Authored: Sun Mar 18 11:00:30 2018 +0530 Committer: Arpit Agarwal Committed: Fri Apr 13 10:18:12 2018 -0700 -- .../security/QueueACLsManager.java | 1 - .../resourcemanager/webapp/RMWSConsts.java | 8 ++ .../webapp/RMWebServiceProtocol.java| 18 + .../resourcemanager/webapp/RMWebServices.java | 55 - .../webapp/TestRMWebServices.java | 84 .../webapp/DefaultRequestInterceptorREST.java | 9 +++ .../webapp/FederationInterceptorREST.java | 6 ++ .../server/router/webapp/RouterWebServices.java | 17 .../webapp/MockRESTRequestInterceptor.java | 6 ++ .../PassThroughRESTRequestInterceptor.java | 8 ++ 10 files changed, 208 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/21717db6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java index 530cb25..4c22a55 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java @@ -114,7 +114,6 @@ public class QueueACLsManager { // version is added for the moving the application case. The check has // extra logging to distinguish between the queue not existing in the // application move request case and the real access denied case. - if (scheduler instanceof CapacityScheduler) { CSQueue queue = ((CapacityScheduler) scheduler).getQueue(targetQueue); if (queue == null) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/21717db6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java index 5a945da..29ae81b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java @@ -174,6 +174,12 @@ public final class RMWSConsts { public static final String GET_CONTAINER = "/apps/{appid}/appattempts/{appattemptid}/containers/{containerid}"; + /** + * Path for {code checkUserAccessToQueue#} + */ + public static final String CHECK_USER_ACCESS_TO_QUEUE = + "/queues/{queue}/access"; + // QueryParams for RMWebServiceProtocol public static final String TIME = "time"; @@ -183,6 +189,7 @@ public final class RMWSConsts { public static final String FINAL_STATUS = "finalStatus"; public static final String USER = "user"; public static final String QUEUE = "queue"; + public static final String QUEUES = "queues"; public static final String LIMIT = "limit"; public static final String STARTED_TIME_BEGIN = "startedTimeBegin"; public static final String STARTED_TIME_END = "startedTimeEnd"; @@ -209,6 +216,7 @@ public final class RMWSConsts { public static final String GET_LABELS = "get-labels"; public static final String DESELECTS = "deSelects"; public sta
[12/14] hadoop git commit: YARN-8040. [UI2] New YARN UI webapp does not respect current pathname for REST api. Contributed by Sunil G.
YARN-8040. [UI2] New YARN UI webapp does not respect current pathname for REST api. Contributed by Sunil G. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d98c3ca5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d98c3ca5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d98c3ca5 Branch: refs/heads/branch-3.1 Commit: d98c3ca5fd8795eb10fe3e68215934f083ab1135 Parents: 21717db Author: Sunil G Authored: Sun Mar 18 10:44:33 2018 +0530 Committer: Arpit Agarwal Committed: Fri Apr 13 10:18:39 2018 -0700 -- .../hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js | 7 ++- 1 file changed, 6 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d98c3ca5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js index d047ed1..83df971 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js @@ -40,7 +40,7 @@ function getTimeLineV1URL(rmhost) { function updateConfigs(application) { var hostname = window.location.hostname; - var rmhost = hostname + (window.location.port ? ':' + window.location.port: ''); + var rmhost = hostname + (window.location.port ? ':' + window.location.port: '') + skipTrailingSlash(window.location.pathname); if(!ENV.hosts.rmWebAddress) { ENV.hosts.rmWebAddress = rmhost; @@ -130,3 +130,8 @@ export default { before: 'env', initialize }; + +const skipTrailingSlash = function(path) { + path = path.replace('ui2/', ''); + return path.replace(/\/$/, ''); +}; - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[05/14] hadoop git commit: HADOOP-15332. Fix typos in hadoop-aws markdown docs. Contributed by Gabor Bota.
HADOOP-15332. Fix typos in hadoop-aws markdown docs. Contributed by Gabor Bota. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2b46bd3f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2b46bd3f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2b46bd3f Branch: refs/heads/branch-3.1 Commit: 2b46bd3f4548fefe292e9fedffe2fdf7f2f12be7 Parents: 6891da1 Author: Sean Mackrory Authored: Tue Mar 20 21:11:51 2018 -0700 Committer: Arpit Agarwal Committed: Fri Apr 13 10:13:56 2018 -0700 -- .../tools/hadoop-aws/committer_architecture.md | 76 ++-- .../markdown/tools/hadoop-aws/committers.md | 18 ++--- .../markdown/tools/hadoop-aws/encryption.md | 8 +-- .../src/site/markdown/tools/hadoop-aws/index.md | 34 - .../site/markdown/tools/hadoop-aws/s3guard.md | 4 +- .../site/markdown/tools/hadoop-aws/testing.md | 20 +++--- .../tools/hadoop-aws/troubleshooting_s3a.md | 2 +- 7 files changed, 81 insertions(+), 81 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b46bd3f/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committer_architecture.md -- diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committer_architecture.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committer_architecture.md index b974ea8..e4ba75d 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committer_architecture.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committer_architecture.md @@ -28,7 +28,7 @@ The standard commit algorithms (the `FileOutputCommitter` and its v1 and v2 algo rely on directory rename being an `O(1)` atomic operation: callers output their work to temporary directories in the destination filesystem, then rename these directories to the final destination as way of committing work. -This is the perfect solution for commiting work against any filesystem with +This is the perfect solution for committing work against any filesystem with consistent listing operations and where the `FileSystem.rename()` command is an atomic `O(1)` operation. @@ -60,7 +60,7 @@ delayed completion of multi-part PUT operations That is: tasks write all data as multipart uploads, *but delay the final commit action until until the final, single job commit action.* Only that data committed in the job commit action will be made visible; work from speculative -and failed tasks will not be instiantiated. As there is no rename, there is no +and failed tasks will not be instantiated. As there is no rename, there is no delay while data is copied from a temporary directory to the final directory. The duration of the commit will be the time needed to determine which commit operations to construct, and to execute them. @@ -109,7 +109,7 @@ This is traditionally implemented via a `FileSystem.rename()` call. It is useful to differentiate between a *task-side commit*: an operation performed in the task process after its work, and a *driver-side task commit*, in which - the Job driver perfoms the commit operation. Any task-side commit work will + the Job driver performs the commit operation. Any task-side commit work will be performed across the cluster, and may take place off the critical part for job execution. However, unless the commit protocol requires all tasks to await a signal from the job driver, task-side commits cannot instantiate their output @@ -241,7 +241,7 @@ def commitTask(fs, jobAttemptPath, taskAttemptPath, dest): fs.rename(taskAttemptPath, taskCommittedPath) ``` -On a genuine fileystem this is an `O(1)` directory rename. +On a genuine filesystem this is an `O(1)` directory rename. On an object store with a mimiced rename, it is `O(data)` for the copy, along with overhead for listing and deleting all files (For S3, that's @@ -257,13 +257,13 @@ def abortTask(fs, jobAttemptPath, taskAttemptPath, dest): fs.delete(taskAttemptPath, recursive=True) ``` -On a genuine fileystem this is an `O(1)` operation. On an object store, +On a genuine filesystem this is an `O(1)` operation. On an object store, proportional to the time to list and delete files, usually in batches. ### Job Commit -Merge all files/directories in all task commited paths into final destination path. +Merge all files/directories in all task committed paths into final destination path. Optionally; create 0-byte `_SUCCESS` file in destination path. ```python @@ -420,9 +420,9 @@ by renaming the files. A a key difference is that the v1 algorithm commits a source directory to via a directory rename, which is traditionally an `O(1)` operation. -In constrast, the v2 algorithm lists al
[08/14] hadoop git commit: HDFS-11900. Hedged reads thread pool creation not synchronized. Contributed by John Zhuge.
HDFS-11900. Hedged reads thread pool creation not synchronized. Contributed by John Zhuge. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/696d7860 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/696d7860 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/696d7860 Branch: refs/heads/branch-3.1 Commit: 696d786063878509e514829357e3838f4f9377a8 Parents: 867135a Author: Wei-Chiu Chuang Authored: Thu Mar 22 11:29:31 2018 -0700 Committer: Arpit Agarwal Committed: Fri Apr 13 10:16:23 2018 -0700 -- .../src/main/java/org/apache/hadoop/hdfs/DFSClient.java| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/696d7860/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index af7b540..0875328 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -2910,7 +2910,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, * @param num Number of threads for hedged reads thread pool. * If zero, skip hedged reads thread pool creation. */ - private synchronized void initThreadsNumForHedgedReads(int num) { + private static synchronized void initThreadsNumForHedgedReads(int num) { if (num <= 0 || HEDGED_READ_THREAD_POOL != null) return; HEDGED_READ_THREAD_POOL = new ThreadPoolExecutor(1, num, 60, TimeUnit.SECONDS, new SynchronousQueue(), - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[09/14] hadoop git commit: MAPREDUCE-7066. TestQueue fails on Java9
MAPREDUCE-7066. TestQueue fails on Java9 Signed-off-by: Akira Ajisaka Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/077eda66 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/077eda66 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/077eda66 Branch: refs/heads/branch-3.1 Commit: 077eda66ade5c552bbf41dd0de70b49dafb957be Parents: 696d7860 Author: Takanobu Asanuma Authored: Mon Mar 19 09:39:49 2018 +0900 Committer: Arpit Agarwal Committed: Fri Apr 13 10:17:34 2018 -0700 -- .../src/test/java/org/apache/hadoop/mapred/TestQueue.java | 7 --- 1 file changed, 4 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/077eda66/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestQueue.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestQueue.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestQueue.java index 796bbee..b2908bf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestQueue.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestQueue.java @@ -153,9 +153,10 @@ public class TestQueue { writer = new StringWriter(); QueueManager.dumpConfiguration(writer, conf); result = writer.toString(); - assertEquals( - "{\"queues\":[{\"name\":\"default\",\"state\":\"running\",\"acl_submit_job\":\"*\",\"acl_administer_jobs\":\"*\",\"properties\":[],\"children\":[]},{\"name\":\"q1\",\"state\":\"running\",\"acl_submit_job\":\" \",\"acl_administer_jobs\":\" \",\"properties\":[],\"children\":[{\"name\":\"q1:q2\",\"state\":\"running\",\"acl_submit_job\":\" \",\"acl_administer_jobs\":\" \",\"properties\":[{\"key\":\"capacity\",\"value\":\"20\"},{\"key\":\"user-limit\",\"value\":\"30\"}],\"children\":[]}]}]}", - result); + assertTrue(result.contains("{\"queues\":[{\"name\":\"default\",\"state\":\"running\",\"acl_submit_job\":\"*\",\"acl_administer_jobs\":\"*\",\"properties\":[],\"children\":[]},{\"name\":\"q1\",\"state\":\"running\",\"acl_submit_job\":\" \",\"acl_administer_jobs\":\" \",\"properties\":[],\"children\":[{\"name\":\"q1:q2\",\"state\":\"running\",\"acl_submit_job\":\" \",\"acl_administer_jobs\":\" \",\"properties\":[")); + assertTrue(result.contains("{\"key\":\"capacity\",\"value\":\"20\"}")); + assertTrue(result.contains("{\"key\":\"user-limit\",\"value\":\"30\"}")); + assertTrue(result.contains("],\"children\":[]}]}]}")); // test constructor QueueAclsInfo QueueAclsInfo qi = new QueueAclsInfo(); assertNull(qi.getQueueName()); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: HDFS-13330. ShortCircuitCache#fetchOrCreate never retries. Contributed by Gabor Bota.
Repository: hadoop Updated Branches: refs/heads/branch-3.1 994c7d66e -> ca8bb322b HDFS-13330. ShortCircuitCache#fetchOrCreate never retries. Contributed by Gabor Bota. (cherry picked from commit e66e287efe2b43e710137a628f03c7df3ebdf498) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ca8bb322 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ca8bb322 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ca8bb322 Branch: refs/heads/branch-3.1 Commit: ca8bb322be5e21ce00bb49cecfc4f515b49d2a37 Parents: 994c7d6 Author: Wei-Chiu Chuang Authored: Fri Apr 13 09:17:34 2018 -0700 Committer: Wei-Chiu Chuang Committed: Fri Apr 13 12:18:49 2018 -0700 -- .../hdfs/shortcircuit/ShortCircuitCache.java| 11 ++--- .../shortcircuit/TestShortCircuitCache.java | 26 2 files changed, 33 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca8bb322/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java index b26652b..c2f0350 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java @@ -664,6 +664,7 @@ public class ShortCircuitCache implements Closeable { unref(replica); } + static final int FETCH_OR_CREATE_RETRY_TIMES = 3; /** * Fetch or create a replica. * @@ -678,11 +679,11 @@ public class ShortCircuitCache implements Closeable { */ public ShortCircuitReplicaInfo fetchOrCreate(ExtendedBlockId key, ShortCircuitReplicaCreator creator) { -Waitable newWaitable = null; +Waitable newWaitable; lock.lock(); try { ShortCircuitReplicaInfo info = null; - do { + for (int i = 0; i < FETCH_OR_CREATE_RETRY_TIMES; i++){ if (closed) { LOG.trace("{}: can't fethchOrCreate {} because the cache is closed.", this, key); @@ -692,11 +693,12 @@ public class ShortCircuitCache implements Closeable { if (waitable != null) { try { info = fetch(key, waitable); +break; } catch (RetriableException e) { LOG.debug("{}: retrying {}", this, e.getMessage()); } } - } while (false); + } if (info != null) return info; // We need to load the replica ourselves. newWaitable = new Waitable<>(lock.newCondition()); @@ -717,7 +719,8 @@ public class ShortCircuitCache implements Closeable { * * @throws RetriableException If the caller needs to retry. */ - private ShortCircuitReplicaInfo fetch(ExtendedBlockId key, + @VisibleForTesting // ONLY for testing + protected ShortCircuitReplicaInfo fetch(ExtendedBlockId key, Waitable waitable) throws RetriableException { // Another thread is already in the process of loading this // ShortCircuitReplica. So we simply wait for it to complete. http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca8bb322/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java index 7ba0edc..5da6a25 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java @@ -65,6 +65,7 @@ import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.ShortCircuitReplica import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.ShmId; import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.Slot; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.ipc.RetriableException; import org.apache.hadoop.net.unix.DomainSocket; import org.apache.hadoop.net.unix.TemporarySocketDirectory; import org.apache.hadoop.security.token.SecretManager.InvalidToken; @@ -793,4 +794,29 @@ public class TestShortCircuitCache { cluster.shutdown(); sockDir.close(); } + + @Test + public void testFetchOrCreateRetries() throws Exception { +tr
hadoop git commit: HDFS-13330. ShortCircuitCache#fetchOrCreate never retries. Contributed by Gabor Bota.
Repository: hadoop Updated Branches: refs/heads/branch-3.0 bce57a139 -> dc01e323e HDFS-13330. ShortCircuitCache#fetchOrCreate never retries. Contributed by Gabor Bota. (cherry picked from commit e66e287efe2b43e710137a628f03c7df3ebdf498) (cherry picked from commit ca8bb322be5e21ce00bb49cecfc4f515b49d2a37) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dc01e323 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dc01e323 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dc01e323 Branch: refs/heads/branch-3.0 Commit: dc01e323ed24c22f9db88af4963654a037f72e75 Parents: bce57a1 Author: Wei-Chiu Chuang Authored: Fri Apr 13 09:17:34 2018 -0700 Committer: Wei-Chiu Chuang Committed: Fri Apr 13 12:20:01 2018 -0700 -- .../hdfs/shortcircuit/ShortCircuitCache.java| 11 ++--- .../shortcircuit/TestShortCircuitCache.java | 26 2 files changed, 33 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc01e323/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java index b26652b..c2f0350 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java @@ -664,6 +664,7 @@ public class ShortCircuitCache implements Closeable { unref(replica); } + static final int FETCH_OR_CREATE_RETRY_TIMES = 3; /** * Fetch or create a replica. * @@ -678,11 +679,11 @@ public class ShortCircuitCache implements Closeable { */ public ShortCircuitReplicaInfo fetchOrCreate(ExtendedBlockId key, ShortCircuitReplicaCreator creator) { -Waitable newWaitable = null; +Waitable newWaitable; lock.lock(); try { ShortCircuitReplicaInfo info = null; - do { + for (int i = 0; i < FETCH_OR_CREATE_RETRY_TIMES; i++){ if (closed) { LOG.trace("{}: can't fethchOrCreate {} because the cache is closed.", this, key); @@ -692,11 +693,12 @@ public class ShortCircuitCache implements Closeable { if (waitable != null) { try { info = fetch(key, waitable); +break; } catch (RetriableException e) { LOG.debug("{}: retrying {}", this, e.getMessage()); } } - } while (false); + } if (info != null) return info; // We need to load the replica ourselves. newWaitable = new Waitable<>(lock.newCondition()); @@ -717,7 +719,8 @@ public class ShortCircuitCache implements Closeable { * * @throws RetriableException If the caller needs to retry. */ - private ShortCircuitReplicaInfo fetch(ExtendedBlockId key, + @VisibleForTesting // ONLY for testing + protected ShortCircuitReplicaInfo fetch(ExtendedBlockId key, Waitable waitable) throws RetriableException { // Another thread is already in the process of loading this // ShortCircuitReplica. So we simply wait for it to complete. http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc01e323/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java index 7ba0edc..5da6a25 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java @@ -65,6 +65,7 @@ import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.ShortCircuitReplica import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.ShmId; import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.Slot; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.ipc.RetriableException; import org.apache.hadoop.net.unix.DomainSocket; import org.apache.hadoop.net.unix.TemporarySocketDirectory; import org.apache.hadoop.security.token.SecretManager.InvalidToken; @@ -793,4 +794,29 @@ public class TestShortCircuitCache { cluster.shutdown(); sockDir.close(); } + + @Test
hadoop git commit: YARN-8142. Improve SIGTERM handling for YARN Service Application Master. Contributed by Billie Rinaldi
Repository: hadoop Updated Branches: refs/heads/trunk e66e287ef -> 9031a76d4 YARN-8142. Improve SIGTERM handling for YARN Service Application Master. Contributed by Billie Rinaldi Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9031a76d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9031a76d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9031a76d Branch: refs/heads/trunk Commit: 9031a76d447f0c5eaa392144fd17c5b9812e1b20 Parents: e66e287 Author: Eric Yang Authored: Fri Apr 13 15:34:33 2018 -0400 Committer: Eric Yang Committed: Fri Apr 13 15:34:33 2018 -0400 -- .../hadoop/yarn/service/ClientAMService.java| 1 + .../hadoop/yarn/service/ServiceScheduler.java | 41 +++ .../hadoop/yarn/service/ServiceTestUtils.java | 11 +++ .../yarn/service/TestYarnNativeServices.java| 71 4 files changed, 110 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9031a76d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java index 08c36f4..3d037e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java @@ -125,6 +125,7 @@ public class ClientAMService extends AbstractService LOG.info("Stop the service by {}", UserGroupInformation.getCurrentUser()); context.scheduler.getDiagnostics() .append("Stopped by user " + UserGroupInformation.getCurrentUser()); +context.scheduler.setGracefulStop(); // Stop the service in 2 seconds delay to make sure this rpc call is completed. // shutdown hook will be executed which will stop AM gracefully. http://git-wip-us.apache.org/repos/asf/hadoop/blob/9031a76d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java index 0fcca16..7eddef9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java @@ -156,6 +156,8 @@ public class ServiceScheduler extends CompositeService { // requests for a single service is not recommended. private boolean hasAtLeastOnePlacementConstraint; + private boolean gracefulStop = false; + public ServiceScheduler(ServiceContext context) { super(context.service.getName()); this.context = context; @@ -199,6 +201,7 @@ public class ServiceScheduler extends CompositeService { addIfService(amRMClient); nmClient = createNMClient(); +nmClient.getClient().cleanupRunningContainersOnStop(false); addIfService(nmClient); dispatcher = new AsyncDispatcher("Component dispatcher"); @@ -252,6 +255,11 @@ public class ServiceScheduler extends CompositeService { .createAMRMClientAsync(1000, new AMRMClientCallback()); } + protected void setGracefulStop() { +this.gracefulStop = true; +nmClient.getClient().cleanupRunningContainersOnStop(true); + } + @Override public void serviceInit(Configuration conf) throws Exception { try { @@ -266,26 +274,31 @@ public class ServiceScheduler extends CompositeService { public void serviceStop() throws Exception { LOG.info("Stopping service scheduler"); -// Mark component-instances/containers as STOPPED -if (YarnConfiguration.timelineServic
hadoop git commit: HDFS-13197. Ozone: Fix ConfServlet#getOzoneTags cmd. Contributed by Ajay Kumar.
Repository: hadoop Updated Branches: refs/heads/HDFS-7240 584c573a5 -> 66610b5fd HDFS-13197. Ozone: Fix ConfServlet#getOzoneTags cmd. Contributed by Ajay Kumar. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/66610b5f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/66610b5f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/66610b5f Branch: refs/heads/HDFS-7240 Commit: 66610b5fd5dc29c1bff006874bf46d426d3a9dfa Parents: 584c573 Author: Xiaoyu Yao Authored: Fri Apr 13 13:42:57 2018 -0700 Committer: Xiaoyu Yao Committed: Fri Apr 13 13:42:57 2018 -0700 -- .../hadoop/hdds/conf/HddsConfServlet.java | 181 + .../common/src/main/resources/ozone-default.xml | 10 + .../hadoop/hdds/server/BaseHttpServer.java | 10 +- .../src/main/resources/webapps/static/ozone.js | 668 ++- .../webapps/static/templates/config.html| 28 +- 5 files changed, 562 insertions(+), 335 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/66610b5f/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/HddsConfServlet.java -- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/HddsConfServlet.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/HddsConfServlet.java new file mode 100644 index 000..068e41f --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/HddsConfServlet.java @@ -0,0 +1,181 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.conf; + +import com.google.gson.Gson; +import java.io.IOException; +import java.io.Writer; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.core.HttpHeaders; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.HttpServer2; + +import com.google.common.annotations.VisibleForTesting; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A servlet to print out the running configuration data. + */ +@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) +@InterfaceStability.Unstable +public class HddsConfServlet extends HttpServlet { + + private static final long serialVersionUID = 1L; + + protected static final String FORMAT_JSON = "json"; + protected static final String FORMAT_XML = "xml"; + private static final String COMMAND = "cmd"; + private static final OzoneConfiguration OZONE_CONFIG = + new OzoneConfiguration(); + transient Logger LOG = LoggerFactory.getLogger(HddsConfServlet.class); + + + /** + * Return the Configuration of the daemon hosting this servlet. + * This is populated when the HttpServer starts. + */ + private Configuration getConfFromContext() { +Configuration conf = (Configuration) getServletContext().getAttribute( +HttpServer2.CONF_CONTEXT_ATTRIBUTE); +assert conf != null; +return conf; + } + + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + +if (!HttpServer2.isInstrumentationAccessAllowed(getServletContext(), +request, response)) { + return; +} + +String format = parseAcceptHeader(request); +if (FORMAT_XML.equals(format)) { + response.setContentType("text/xml; charset=utf-8"); +} else if (FORMAT_JSON.equals(format)) { + response.setContentType("application/json; charset=utf-8"); +} + +String name = request.getParameter("name"); +Writer out = response.getWriter(); +String cmd = request.getParameter(COMMAND); + +processCommand(cmd, format, request, response, out, name); +out.close(); + } +
hadoop git commit: MAPREDUCE-7077. Pipe mapreduce job fails with Permission denied for jobTokenPassword. (Akira Ajisaka via wangda)
Repository: hadoop Updated Branches: refs/heads/trunk 9031a76d4 -> 995cba65f MAPREDUCE-7077. Pipe mapreduce job fails with Permission denied for jobTokenPassword. (Akira Ajisaka via wangda) Change-Id: Ie8f01425d58409fa3661f768205b7616128c8aa4 (cherry picked from commit 035e0f97ea44b0495707949a781d8792dcf6ea6b) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/995cba65 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/995cba65 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/995cba65 Branch: refs/heads/trunk Commit: 995cba65fe29966583e36f9491d9a27b323918ae Parents: 9031a76 Author: Wangda Tan Authored: Thu Apr 12 14:33:33 2018 -0700 Committer: Wangda Tan Committed: Fri Apr 13 13:52:10 2018 -0700 -- .../apache/hadoop/mapred/pipes/Application.java | 5 ++--- .../hadoop/mapred/pipes/TestPipeApplication.java | 19 --- 2 files changed, 10 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/995cba65/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java index 5c8aab9..83d2509 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java @@ -45,7 +45,6 @@ import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapred.TaskAttemptID; import org.apache.hadoop.mapred.TaskLog; -import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.filecache.DistributedCache; import org.apache.hadoop.mapreduce.security.SecureShuffleUtils; @@ -104,8 +103,8 @@ class Applicationhttp://git-wip-us.apache.org/repos/asf/hadoop/blob/995cba65/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java index 13597e0..88d8f95 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java @@ -47,7 +47,6 @@ import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapred.IFile.Writer; -import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.security.TokenCache; import org.apache.hadoop.mapred.Counters; @@ -84,10 +83,10 @@ public class TestPipeApplication { public void testRunner() throws Exception { // clean old password files -JobConf conf = new JobConf(); -File[] psw = cleanTokenPasswordFile(conf); +File[] psw = cleanTokenPasswordFile(); try { RecordReader rReader = new ReaderPipesMapRunner(); + JobConf conf = new JobConf(); conf.set(Submitter.IS_JAVA_RR, "true"); // for stdour and stderror @@ -163,7 +162,7 @@ public class TestPipeApplication { TestTaskReporter reporter = new TestTaskReporter(); -File[] psw = cleanTokenPasswordFile(conf); +File[] psw = cleanTokenPasswordFile(); try { conf.set(MRJobConfig.TASK_ATTEMPT_ID, taskName); @@ -248,7 +247,7 @@ public class TestPipeApplication { JobConf conf = new JobConf(); -File[] psw = cleanTokenPasswordFile(conf); +File[] psw = cleanTokenPasswordFile(); System.setProperty("test.build.data", "target/tmp/build/TEST_SUBMITTER_MAPPER/data"); @@ -389,8 +388,8 @@ public class TestPipeApplication { @Test public void testPipesReduser() throws Exception { +File[] psw = cleanTo
[27/44] hadoop git commit: YARN-8120. JVM can crash with SIGSEGV when exiting due to custom leveldb logger. Contributed by Jason Lowe.
YARN-8120. JVM can crash with SIGSEGV when exiting due to custom leveldb logger. Contributed by Jason Lowe. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6bb128df Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6bb128df Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6bb128df Branch: refs/heads/HDFS-7240 Commit: 6bb128dfb893cf0e4aa2d3ecc65440668a1fc8d7 Parents: d272056 Author: Eric E Payne Authored: Thu Apr 12 16:04:23 2018 + Committer: Eric E Payne Committed: Thu Apr 12 16:04:23 2018 + -- .../v2/hs/HistoryServerLeveldbStateStoreService.java| 11 --- .../java/org/apache/hadoop/mapred/ShuffleHandler.java | 12 .../recovery/NMLeveldbStateStoreService.java| 12 .../resourcemanager/recovery/LeveldbRMStateStore.java | 12 4 files changed, 47 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bb128df/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java index c8741aa..6d2e407 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java @@ -74,7 +74,6 @@ public class HistoryServerLeveldbStateStoreService extends Path storeRoot = createStorageDir(getConfig()); Options options = new Options(); options.createIfMissing(false); -options.logger(new LeveldbLogger()); LOG.info("Using state database at " + storeRoot + " for recovery"); File dbfile = new File(storeRoot.toString()); try { @@ -366,14 +365,4 @@ public class HistoryServerLeveldbStateStoreService extends + getCurrentVersion() + ", but loading version " + loadedVersion); } } - - private static class LeveldbLogger implements org.iq80.leveldb.Logger { -private static final Logger LOG = -LoggerFactory.getLogger(LeveldbLogger.class); - -@Override -public void log(String message) { - LOG.info(message); -} - } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bb128df/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java index ec992fe..aeda9cc 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java @@ -93,7 +93,6 @@ import org.fusesource.leveldbjni.JniDBFactory; import org.fusesource.leveldbjni.internal.NativeDB; import org.iq80.leveldb.DB; import org.iq80.leveldb.DBException; -import org.iq80.leveldb.Logger; import org.iq80.leveldb.Options; import org.jboss.netty.bootstrap.ServerBootstrap; import org.jboss.netty.buffer.ChannelBuffers; @@ -628,7 +627,6 @@ public class ShuffleHandler extends AuxiliaryService { private void startStore(Path recoveryRoot) throws IOException { Options options = new Options(); options.createIfMissing(false); -options.logger(new LevelDBLogger()); Path dbPath = new Path(recoveryRoot, STATE_DB_NAME); LOG.info("Using state database at " + dbPath + " for recovery"); File dbfile = new File(dbPath.toString()); @@ -774,16 +772,6 @@ public class ShuffleHandler extends AuxiliaryService { } } - private static class LevelDBLogger implements Logger { -private static final org.slf4j.Logger LOG = -LoggerFactory.getLogger(LevelDBLogger.class); - -@Override -public void log(String message) { - LOG.info(message); -} - } - static class TimeoutHandler extends IdleSt
[07/44] hadoop git commit: YARN-8133. Doc link broken for yarn-service from overview page. (Rohith Sharma K S via wangda)
YARN-8133. Doc link broken for yarn-service from overview page. (Rohith Sharma K S via wangda) Change-Id: Iacf9a004585dd59e1c0cd8f8c618a38f047cc0fe Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d919eb6e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d919eb6e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d919eb6e Branch: refs/heads/HDFS-7240 Commit: d919eb6efa1072517017c75fb323e391f4418dc8 Parents: 2bf9cc2 Author: Wangda Tan Authored: Tue Apr 10 17:33:15 2018 -0700 Committer: Wangda Tan Committed: Tue Apr 10 17:33:15 2018 -0700 -- .../src/site/markdown/yarn-service/Concepts.md| 6 +++--- .../src/site/markdown/yarn-service/Overview.md| 14 +++--- .../src/site/markdown/yarn-service/QuickStart.md | 12 ++-- .../src/site/markdown/yarn-service/RegistryDNS.md | 4 ++-- .../site/markdown/yarn-service/ServiceDiscovery.md| 4 ++-- 5 files changed, 20 insertions(+), 20 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d919eb6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md index e567d03..5c77f17 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md @@ -29,7 +29,7 @@ affinity and anti-affinity scheduling, log aggregation for services, automatical A restful API server is developed to allow users to deploy/manage their services on YARN via a simple JSON spec. This avoids users from dealing with the low-level APIs, writing complex code to bring their services onto YARN. The REST layer acts as a unified REST based entry for creation and lifecycle management of YARN services. Services here can range from simple single-component apps to the most complex, -multi-component applications needing special orchestration needs. Please refer to this [API doc](YarnServiceAPI.md) for detailed API documentations. +multi-component applications needing special orchestration needs. Please refer to this [API doc](YarnServiceAPI.html) for detailed API documentations. The API-server is stateless, which means users can simply spin up multiple instances, and have a load balancer fronting them to support HA, distribute the load etc. @@ -37,10 +37,10 @@ support HA, distribute the load etc. ### Service Discovery A DNS server is implemented to enable discovering services on YARN via the standard mechanism: DNS lookup. -The framework posts container information such as hostname and ip into the [YARN service registry](../registry/index.md). And the DNS server essentially exposes the +The framework posts container information such as hostname and ip into the [YARN service registry](../registry/index.html). And the DNS server essentially exposes the information in YARN service registry by translating them into DNS records such as A record and SRV record. Clients can then discover the IPs of containers via standard DNS lookup. The previous read mechanisms of YARN Service Registry were limited to a registry specific (java) API and a REST interface and are difficult -to wireup existing clients and services. The DNS based service discovery eliminates this gap. Please refer to this [Service Discovery doc](ServiceDiscovery.md) +to wireup existing clients and services. The DNS based service discovery eliminates this gap. Please refer to this [Service Discovery doc](ServiceDiscovery.html) for more details. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/d919eb6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md index 72c2d3e..8e2bf9a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md @@ -46,16 +46,16 @@ The benefits of combining these workloads are two-fold: *`This feature is in alpha state`* and so APIs, command lines are subject to change. We will continue to update the documents over time. -[QuickStart](QuickStart.md) shows a quick tutorial that walks you throu
[15/44] hadoop git commit: HDFS-13403: libhdfs++ Use hdfs::IoService object rather than asio::io_service. Contributed by James Clampffer.
HDFS-13403: libhdfs++ Use hdfs::IoService object rather than asio::io_service. Contributed by James Clampffer. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eefe2a14 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eefe2a14 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eefe2a14 Branch: refs/heads/HDFS-7240 Commit: eefe2a147c83dbb62c4021b67d59d3b9f065f890 Parents: 7eb783e Author: James Clampffer Authored: Wed Apr 11 10:27:23 2018 -0400 Committer: James Clampffer Committed: Wed Apr 11 10:27:23 2018 -0400 -- .../native/libhdfspp/include/hdfspp/hdfspp.h| 53 +-- .../native/libhdfspp/include/hdfspp/ioservice.h | 140 .../native/libhdfspp/lib/bindings/c/hdfs.cc | 7 +- .../native/libhdfspp/lib/common/CMakeLists.txt | 2 +- .../native/libhdfspp/lib/common/async_stream.h | 13 +- .../libhdfspp/lib/common/continuation/asio.h| 5 - .../libhdfspp/lib/common/hdfs_ioservice.cc | 146 - .../libhdfspp/lib/common/hdfs_ioservice.h | 79 - .../libhdfspp/lib/common/ioservice_impl.cc | 159 +++ .../libhdfspp/lib/common/ioservice_impl.h | 76 + .../main/native/libhdfspp/lib/common/logging.h | 3 - .../libhdfspp/lib/common/namenode_info.cc | 15 +- .../native/libhdfspp/lib/common/namenode_info.h | 8 +- .../main/native/libhdfspp/lib/common/util.cc| 14 +- .../src/main/native/libhdfspp/lib/common/util.h | 25 ++- .../lib/connection/datanodeconnection.cc| 27 +++- .../lib/connection/datanodeconnection.h | 26 +-- .../main/native/libhdfspp/lib/fs/filehandle.cc | 18 +-- .../main/native/libhdfspp/lib/fs/filehandle.h | 12 +- .../main/native/libhdfspp/lib/fs/filesystem.cc | 67 ++-- .../main/native/libhdfspp/lib/fs/filesystem.h | 66 ++-- .../libhdfspp/lib/fs/namenode_operations.h | 4 +- .../native/libhdfspp/lib/reader/block_reader.cc | 18 +-- .../native/libhdfspp/lib/reader/block_reader.h | 10 +- .../native/libhdfspp/lib/reader/datatransfer.h | 4 +- .../libhdfspp/lib/rpc/namenode_tracker.cc | 2 +- .../native/libhdfspp/lib/rpc/namenode_tracker.h | 4 +- .../main/native/libhdfspp/lib/rpc/request.cc| 5 +- .../native/libhdfspp/lib/rpc/rpc_connection.h | 2 +- .../libhdfspp/lib/rpc/rpc_connection_impl.cc| 32 ++-- .../libhdfspp/lib/rpc/rpc_connection_impl.h | 9 +- .../main/native/libhdfspp/lib/rpc/rpc_engine.cc | 14 +- .../main/native/libhdfspp/lib/rpc/rpc_engine.h | 9 +- .../native/libhdfspp/tests/bad_datanode_test.cc | 31 ++-- .../libhdfspp/tests/hdfs_ioservice_test.cc | 10 +- .../native/libhdfspp/tests/mock_connection.h| 4 +- .../libhdfspp/tests/remote_block_reader_test.cc | 4 +- .../native/libhdfspp/tests/rpc_engine_test.cc | 112 ++--- 38 files changed, 681 insertions(+), 554 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h index 2fdeec9..e68a612 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h @@ -19,6 +19,7 @@ #define LIBHDFSPP_HDFSPP_H_ #include "hdfspp/options.h" +#include "hdfspp/ioservice.h" #include "hdfspp/status.h" #include "hdfspp/events.h" #include "hdfspp/block_location.h" @@ -31,62 +32,10 @@ #include #include -#include -#include namespace hdfs { /** - * An IoService manages a queue of asynchronous tasks. All libhdfs++ - * operations are filed against a particular IoService. - * - * When an operation is queued into an IoService, the IoService will - * run the callback handler associated with the operation. Note that - * the IoService must be stopped before destructing the objects that - * post the operations. - * - * From an implementation point of view the hdfs::IoService provides - * a thin wrapper over an asio::io_service object so that additional - * instrumentation and functionality can be added. - **/ - -class IoService : public std::enable_shared_from_this -{ - public: - static IoService *New(); - static std::shared_ptr MakeShared(); - virtual ~IoService(); - - /** - * Start up as many threads as there are logical processors. - * Return number of threads created. - **/ - virtual unsigned int InitDefaultWorkers() = 0; - - /** - * Initia
[14/44] hadoop git commit: HDFS-13403: libhdfs++ Use hdfs::IoService object rather than asio::io_service. Contributed by James Clampffer.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h index 8e579a2..1dd43af 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h @@ -26,6 +26,7 @@ #include "common/logging.h" #include "common/util.h" #include "common/libhdfs_events_impl.h" +#include "hdfspp/ioservice.h" #include #include @@ -76,8 +77,8 @@ template RpcConnectionImpl::RpcConnectionImpl(std::shared_ptr engine) : RpcConnection(engine), options_(engine->options()), - socket_(engine->io_service()), - connect_timer_(engine->io_service()) + socket_(engine->io_service()->GetRaw()), + connect_timer_(engine->io_service()->GetRaw()) { LOG_TRACE(kRPC, << "RpcConnectionImpl::RpcConnectionImpl called &" << (void*)this); } @@ -353,7 +354,7 @@ void RpcConnectionImpl::FlushPendingRequests() { OnSendCompleted(ec, size); }); } else { // Nothing to send for this request, inform the handler immediately -::asio::io_service *service = GetIoService(); +std::shared_ptr service = GetIoService(); if(!service) { LOG_ERROR(kRPC, << "RpcConnectionImpl@" << this << " attempted to access null IoService"); // No easy way to bail out of this context, but the only way to get here is when @@ -361,7 +362,7 @@ void RpcConnectionImpl::FlushPendingRequests() { return; } -service->post( +service->PostTask( // Never hold locks when calling a callback [req]() { req->OnResponseArrived(nullptr, Status::OK()); } ); http://git-wip-us.apache.org/repos/asf/hadoop/blob/eefe2a14/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc index 0ca7c6a..ad6c9b9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc @@ -30,7 +30,7 @@ template using optional = std::experimental::optional; -RpcEngine::RpcEngine(::asio::io_service *io_service, const Options &options, +RpcEngine::RpcEngine(std::shared_ptr io_service, const Options &options, const std::string &client_name, const std::string &user_name, const char *protocol_name, int protocol_version) : io_service_(io_service), @@ -40,7 +40,7 @@ RpcEngine::RpcEngine(::asio::io_service *io_service, const Options &options, protocol_name_(protocol_name), protocol_version_(protocol_version), call_id_(0), - retry_timer(*io_service), + retry_timer(io_service->GetRaw()), event_handlers_(std::make_shared()), connect_canceled_(false) { @@ -86,7 +86,7 @@ bool RpcEngine::CancelPendingConnect() { void RpcEngine::Shutdown() { LOG_DEBUG(kRPC, << "RpcEngine::Shutdown called"); - io_service_->post([this]() { + io_service_->PostLambda([this]() { std::lock_guard state_lock(engine_state_lock_); conn_.reset(); }); @@ -154,7 +154,7 @@ void RpcEngine::AsyncRpc( // In case user-side code isn't checking the status of Connect before doing RPC if(connect_canceled_) { -io_service_->post( +io_service_->PostLambda( [handler](){ handler(Status::Canceled()); } ); return; @@ -190,7 +190,7 @@ void RpcEngine::AsyncRpcCommsError( std::vector> pendingRequests) { LOG_ERROR(kRPC, << "RpcEngine::AsyncRpcCommsError called; status=\"" << status.ToString() << "\" conn=" << failedConnection.get() << " reqs=" << std::to_string(pendingRequests.size())); - io_service().post([this, status, failedConnection, pendingRequests]() { + io_service_->PostLambda([this, status, failedConnection, pendingRequests]() { RpcCommsError(status, failedConnection, pendingRequests); }); } @@ -238,7 +238,7 @@ void RpcEngine::RpcCommsError( //on. There might be a good argument for caching the first error //rather than the last one, that gets messy - io_service().post([req, status]() { + io_service()->PostLambda([req, status]() { req->OnResponseArrived(nullptr, status); // Never call back while h
[02/44] hadoop git commit: HADOOP-15357. Configuration.getPropsWithPrefix no longer does variable substitution. Contributed by Jim Brennan
HADOOP-15357. Configuration.getPropsWithPrefix no longer does variable substitution. Contributed by Jim Brennan Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e8139754 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e8139754 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e8139754 Branch: refs/heads/HDFS-7240 Commit: e81397545a273cf9a090010eb644b836e0ef8c7b Parents: d553799 Author: Jason Lowe Authored: Tue Apr 10 16:44:03 2018 -0500 Committer: Jason Lowe Committed: Tue Apr 10 16:44:55 2018 -0500 -- .../org/apache/hadoop/conf/Configuration.java | 11 +++-- .../apache/hadoop/conf/TestConfiguration.java | 26 +++- 2 files changed, 24 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8139754/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index 78a2e9f..f1e2a9d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -2869,15 +2869,12 @@ public class Configuration implements Iterable>, */ public Map getPropsWithPrefix(String confPrefix) { Properties props = getProps(); -Enumeration e = props.propertyNames(); Map configMap = new HashMap<>(); -String name = null; -while (e.hasMoreElements()) { - name = (String) e.nextElement(); +for (String name : props.stringPropertyNames()) { if (name.startsWith(confPrefix)) { -String value = props.getProperty(name); -name = name.substring(confPrefix.length()); -configMap.put(name, value); +String value = get(name); +String keyName = name.substring(confPrefix.length()); +configMap.put(keyName, value); } } return configMap; http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8139754/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java -- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java index b0bb0d7..33a9880 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java @@ -2320,19 +2320,33 @@ public class TestConfiguration { FileUtil.fullyDelete(tmpDir); } + @Test public void testGettingPropertiesWithPrefix() throws Exception { Configuration conf = new Configuration(); for (int i = 0; i < 10; i++) { - conf.set("prefix" + ".name" + i, "value"); + conf.set("prefix." + "name" + i, "value" + i); } conf.set("different.prefix" + ".name", "value"); -Map props = conf.getPropsWithPrefix("prefix"); -assertEquals(props.size(), 10); +Map prefixedProps = conf.getPropsWithPrefix("prefix."); +assertEquals(prefixedProps.size(), 10); +for (int i = 0; i < 10; i++) { + assertEquals("value" + i, prefixedProps.get("name" + i)); +} +// Repeat test with variable substitution +conf.set("foo", "bar"); +for (int i = 0; i < 10; i++) { + conf.set("subprefix." + "subname" + i, "value_${foo}" + i); +} +prefixedProps = conf.getPropsWithPrefix("subprefix."); +assertEquals(prefixedProps.size(), 10); +for (int i = 0; i < 10; i++) { + assertEquals("value_bar" + i, prefixedProps.get("subname" + i)); +} // test call with no properties for a given prefix -props = conf.getPropsWithPrefix("none"); -assertNotNull(props.isEmpty()); -assertTrue(props.isEmpty()); +prefixedProps = conf.getPropsWithPrefix("none"); +assertNotNull(prefixedProps.isEmpty()); +assertTrue(prefixedProps.isEmpty()); } public static void main(String[] argv) throws Exception { - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[17/44] hadoop git commit: HDFS-13428. RBF: Remove LinkedList From StateStoreFileImpl.java. Contributed by BELUGA BEHR.
HDFS-13428. RBF: Remove LinkedList From StateStoreFileImpl.java. Contributed by BELUGA BEHR. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f7d5bace Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f7d5bace Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f7d5bace Branch: refs/heads/HDFS-7240 Commit: f7d5bace435a8de151b94ccc3599a6c4de8f7daf Parents: 0c93d43 Author: Inigo Goiri Authored: Wed Apr 11 10:40:30 2018 -0700 Committer: Inigo Goiri Committed: Wed Apr 11 10:43:43 2018 -0700 -- .../federation/store/driver/impl/StateStoreFileImpl.java | 11 +++ 1 file changed, 7 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7d5bace/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java index 7d9ddc6..6b288b3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java @@ -26,9 +26,11 @@ import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; -import java.util.LinkedList; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import org.apache.commons.lang.ArrayUtils; import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys; import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord; import org.slf4j.Logger; @@ -134,15 +136,16 @@ public class StateStoreFileImpl extends StateStoreFileBaseImpl { @Override protected List getChildren(String path) { -List ret = new LinkedList<>(); File dir = new File(path); File[] files = dir.listFiles(); -if (files != null) { +if (ArrayUtils.isNotEmpty(files)) { + List ret = new ArrayList<>(files.length); for (File file : files) { String filename = file.getName(); ret.add(filename); } + return ret; } -return ret; +return Collections.emptyList(); } } \ No newline at end of file - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[16/44] hadoop git commit: HDFS-13045. RBF: Improve error message returned from subcluster. Contributed by Inigo Goiri.
HDFS-13045. RBF: Improve error message returned from subcluster. Contributed by Inigo Goiri. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c93d43f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c93d43f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c93d43f Branch: refs/heads/HDFS-7240 Commit: 0c93d43f3d624a4fd17b3b050443d9e7e20d4f0a Parents: eefe2a1 Author: Wei Yan Authored: Wed Apr 11 08:37:43 2018 -0700 Committer: Wei Yan Committed: Wed Apr 11 08:37:43 2018 -0700 -- .../resolver/FederationNamespaceInfo.java | 5 ++ .../federation/resolver/MountTableResolver.java | 4 +- .../federation/resolver/RemoteLocation.java | 35 ++ .../router/RemoteLocationContext.java | 7 ++ .../federation/router/RouterRpcClient.java | 67 +++- .../federation/store/records/MountTable.java| 2 +- .../store/records/impl/pb/MountTablePBImpl.java | 2 +- .../hdfs/server/federation/MockResolver.java| 12 +++- .../federation/router/TestRouterAdmin.java | 4 +- .../server/federation/router/TestRouterRpc.java | 48 ++ .../store/records/TestMountTable.java | 4 +- 11 files changed, 165 insertions(+), 25 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c93d43f/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java index edcd308..33edd30 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java @@ -48,6 +48,11 @@ public class FederationNamespaceInfo extends RemoteLocationContext { return this.nameserviceId; } + @Override + public String getSrc() { +return null; + } + /** * The HDFS cluster id for this namespace. * http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c93d43f/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java index 9713138..3f6efd6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java @@ -388,7 +388,7 @@ public class MountTableResolver } else { // Not found, use default location RemoteLocation remoteLocation = - new RemoteLocation(defaultNameService, path); + new RemoteLocation(defaultNameService, path, path); List locations = Collections.singletonList(remoteLocation); ret = new PathLocation(null, locations); @@ -519,7 +519,7 @@ public class MountTableResolver newPath += Path.SEPARATOR; } newPath += remainingPath; - RemoteLocation remoteLocation = new RemoteLocation(nsId, newPath); + RemoteLocation remoteLocation = new RemoteLocation(nsId, newPath, path); locations.add(remoteLocation); } DestinationOrder order = entry.getDestOrder(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c93d43f/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java index 6aa12ce..77d0500 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java @@ -20,8 +20,8 @@ package org.apache.hadoop.hdfs.server.federation.resolver; import o
[23/44] hadoop git commit: HDFS-7101. Potential null dereference in DFSck#doWork(). Contributed by skrho and Ted Yu.
HDFS-7101. Potential null dereference in DFSck#doWork(). Contributed by skrho and Ted Yu. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/113af12c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/113af12c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/113af12c Branch: refs/heads/HDFS-7240 Commit: 113af12cfb240ea9a7189bb2701693466eb8e993 Parents: 832852c Author: Akira Ajisaka Authored: Thu Apr 12 17:47:37 2018 +0900 Committer: Akira Ajisaka Committed: Thu Apr 12 17:47:37 2018 +0900 -- .../src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/113af12c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java index 96fca24..10b0012 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java @@ -354,7 +354,7 @@ public class DFSck extends Configured implements Tool { BufferedReader input = new BufferedReader(new InputStreamReader( stream, "UTF-8")); String line = null; -String lastLine = null; +String lastLine = NamenodeFsck.CORRUPT_STATUS; int errCode = -1; try { while ((line = input.readLine()) != null) { - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[06/44] hadoop git commit: YARN-8116. Nodemanager fails with NumberFormatException: For input string: . (Chandni Singh via wangda)
YARN-8116. Nodemanager fails with NumberFormatException: For input string: . (Chandni Singh via wangda) Change-Id: Idd30cfca59982d3fc6e47aa1b88f844a78fae94d Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2bf9cc2c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2bf9cc2c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2bf9cc2c Branch: refs/heads/HDFS-7240 Commit: 2bf9cc2c73944c9f7cde56714b8cf6995cfa539b Parents: c467f31 Author: Wangda Tan Authored: Tue Apr 10 17:32:38 2018 -0700 Committer: Wangda Tan Committed: Tue Apr 10 17:32:38 2018 -0700 -- .../containermanager/container/ContainerImpl.java | 3 ++- .../recovery/NMLeveldbStateStoreService.java| 4 +++- .../recovery/TestNMLeveldbStateStoreService.java| 16 3 files changed, 21 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bf9cc2c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index 2115100..c09c7f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -2191,7 +2191,8 @@ public class ContainerImpl implements Container { } private void storeRetryContext() { -if (windowRetryContext.getRestartTimes() != null) { +if (windowRetryContext.getRestartTimes() != null && +!windowRetryContext.getRestartTimes().isEmpty()) { try { stateStore.storeContainerRestartTimes(containerId, windowRetryContext.getRestartTimes()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bf9cc2c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java index bf4c0ad..723dd48 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java @@ -347,7 +347,9 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { value.substring(1, value.length() - 1).split(", "); List restartTimes = new ArrayList<>(); for (String restartTime : unparsedRestartTimes) { - restartTimes.add(Long.parseLong(restartTime)); + if (!restartTime.isEmpty()) { +restartTimes.add(Long.parseLong(restartTime)); + } } rcs.setRestartTimes(restartTimes); } else if (suffix.equals(CONTAINER_WORK_DIR_KEY_SUFFIX)) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bf9cc2c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java index c270199..265b3e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-serv
[03/44] hadoop git commit: HADOOP-14445. Delegation tokens are not shared between KMS instances. Contributed by Xiao Chen and Rushabh S Shah.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/583fa6ed/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java -- diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java index 1517b04..c171143 100644 --- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java +++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java @@ -1,3 +1,4 @@ + /** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -31,26 +32,35 @@ import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersi import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension; import org.apache.hadoop.crypto.key.kms.KMSClientProvider; import org.apache.hadoop.crypto.key.kms.KMSDelegationToken; +import org.apache.hadoop.crypto.key.kms.KMSTokenRenewer; import org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider; +import org.apache.hadoop.crypto.key.kms.TestLoadBalancingKMSClientProvider; import org.apache.hadoop.crypto.key.kms.ValueQueue; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.MultipleIOException; +import org.apache.hadoop.io.Text; import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator; import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.KMSUtil; +import org.apache.hadoop.util.KMSUtilFaultInjector; import org.apache.hadoop.util.Time; import org.apache.http.client.utils.URIBuilder; import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; @@ -71,7 +81,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.Writer; import java.net.InetAddress; -import java.net.InetSocketAddress; import java.net.ServerSocket; import java.net.SocketTimeoutException; import java.net.URI; @@ -96,6 +105,10 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.regex.Matcher; import java.util.regex.Pattern; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH; +import static org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_KIND; +import static org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_LEGACY_KIND; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -113,6 +126,20 @@ public class TestKMS { private SSLFactory sslFactory; + private final KMSUtilFaultInjector oldInjector = + KMSUtilFaultInjector.get(); + + // Injector to create providers with different ports. Can only happen in tests + private final KMSUtilFaultInjector testInjector = + new KMSUtilFaultInjector() { +@Override +public KeyProvider createKeyProviderForTests(String value, +Configuration conf) throws IOException { + return TestLoadBalancingKMSClientProvider + .createKeyProviderForTests(value, conf); +} + }; + // Keep track of all key providers created during a test case, so they can be // closed at test tearDown. private List providersCreated = new LinkedList<>(); @@ -122,7 +149,12 @@ public class TestKMS { @Before public void setUp() throws Exception { -setUpMiniKdc(); +GenericTestUtils.setLogLevel(KMSClientProvider.LOG, Level.TRACE); +GenericTestUtils +.setLogLevel(DelegationTokenAuthenticationHandler.LOG, Level.TRACE); +GenericTestUtils +.setLogLevel(DelegationTokenAuthenticator.LOG, Level.TRACE); +GenericTestUtils.setLogLevel(KMSUtil.LOG, Level.TRACE); // resetting kerberos security Configuration conf = new Configuration(); Us
[10/44] hadoop git commit: HDFS-13056. Expose file-level composite CRCs in HDFS which are comparable across different instances/layouts. Contributed by Dennis Huo.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/datatransfer.proto -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/datatransfer.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/datatransfer.proto index 2356201..384da54 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/datatransfer.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/datatransfer.proto @@ -148,8 +148,9 @@ message OpCopyBlockProto { required BaseHeaderProto header = 1; } -message OpBlockChecksumProto { +message OpBlockChecksumProto { required BaseHeaderProto header = 1; + optional BlockChecksumOptionsProto blockChecksumOptions = 2; } message OpBlockGroupChecksumProto { @@ -160,6 +161,7 @@ message OpBlockGroupChecksumProto { required ErasureCodingPolicyProto ecPolicy = 4; repeated uint32 blockIndices = 5; required uint64 requestedNumBytes = 6; + optional BlockChecksumOptionsProto blockChecksumOptions = 7; } /** @@ -313,8 +315,9 @@ message DNTransferAckProto { message OpBlockChecksumResponseProto { required uint32 bytesPerCrc = 1; required uint64 crcPerBlock = 2; - required bytes md5 = 3; + required bytes blockChecksum = 3; optional ChecksumTypeProto crcType = 4; + optional BlockChecksumOptionsProto blockChecksumOptions = 5; } message OpCustomProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto index 29d0b4e..441b9d6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto @@ -480,6 +480,27 @@ enum ChecksumTypeProto { CHECKSUM_CRC32C = 2; } +enum BlockChecksumTypeProto { + MD5CRC = 1; // BlockChecksum obtained by taking the MD5 digest of chunk CRCs + COMPOSITE_CRC = 2; // Chunk-independent CRC, optionally striped +} + +/** + * Algorithms/types denoting how block-level checksums are computed using + * lower-level chunk checksums/CRCs. + * These options should be kept in sync with + * org.apache.hadoop.hdfs.protocol.BlockChecksumOptions. + */ +message BlockChecksumOptionsProto { + optional BlockChecksumTypeProto blockChecksumType = 1 [default = MD5CRC]; + + // Only used if blockChecksumType specifies a striped format, such as + // COMPOSITE_CRC. If so, then the blockChecksum in the response is expected + // to be the concatenation of N crcs, where + // N == ((requestedLength - 1) / stripedLength) + 1 + optional uint64 stripeLength = 2; +} + /** * HDFS Server Defaults */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java index bab2e8d..5d2d1f8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java @@ -301,8 +301,9 @@ public abstract class Receiver implements DataTransferProtocol { TraceScope traceScope = continueTraceSpan(proto.getHeader(), proto.getClass().getSimpleName()); try { -blockChecksum(PBHelperClient.convert(proto.getHeader().getBlock()), -PBHelperClient.convert(proto.getHeader().getToken())); + blockChecksum(PBHelperClient.convert(proto.getHeader().getBlock()), + PBHelperClient.convert(proto.getHeader().getToken()), + PBHelperClient.convert(proto.getBlockChecksumOptions())); } finally { if (traceScope != null) traceScope.close(); } @@ -325,7 +326,8 @@ public abstract class Receiver implements DataTransferProtocol { try { blockGroupChecksum(stripedBlockInfo, PBHelperClient.convert(proto.getHeader().getToken()), - proto.getRequestedNumBytes()); + proto.getRequestedNumBytes(), + PBHelperClient.convert(proto.getBlockChecksumOptions())); } finally { if (traceScope != null) { traceScope.close(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockChecksumHelper.java -- diff --git a/hadoop-hdfs-project/hadoop-hdf
[18/44] hadoop git commit: YARN-7221. Add security check for privileged docker container. Contributed by Eric Yang
YARN-7221. Add security check for privileged docker container. Contributed by Eric Yang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/933477e9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/933477e9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/933477e9 Branch: refs/heads/HDFS-7240 Commit: 933477e9e0526e2ed81ea454f8806de31981822a Parents: f7d5bac Author: Billie Rinaldi Authored: Wed Apr 11 08:23:20 2018 -0700 Committer: Billie Rinaldi Committed: Wed Apr 11 11:24:23 2018 -0700 -- .../runtime/DockerLinuxContainerRuntime.java| 10 +- .../container-executor/impl/utils/docker-util.c | 100 ++- .../test/utils/test_docker_util.cc | 97 +- .../runtime/TestDockerContainerRuntime.java | 11 +- 4 files changed, 157 insertions(+), 61 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/933477e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 51abeb6..7106aad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -767,7 +767,11 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { throw new ContainerExecutionException(message); } } - dockerRunAsUser = uid + ":" + gid; + if (!allowPrivilegedContainerExecution(container)) { +dockerRunAsUser = uid + ":" + gid; + } else { +dockerRunAsUser = ctx.getExecutionAttribute(USER); + } } //List -> stored as List -> fetched/converted to List @@ -879,7 +883,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { } if(enableUserReMapping) { - runCommand.groupAdd(groups); + if (!allowPrivilegedContainerExecution(container)) { +runCommand.groupAdd(groups); + } } // use plugins to update docker run command. http://git-wip-us.apache.org/repos/asf/hadoop/blob/933477e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c index 3bd94a1..fdeaeea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c @@ -16,6 +16,9 @@ * limitations under the License. */ +#include +#include +#include #include #include #include @@ -25,6 +28,9 @@ #include "docker-util.h" #include "string-utils.h" #include "util.h" +#include +#include +#include static int read_and_verify_command_file(const char *command_file, const char *docker_command, struct configuration *command_config) { @@ -1254,14 +1260,94 @@ static int add_rw_mounts(const struct configuration *command_config, const stru return add_mounts(command_config, conf, "rw-mounts", 0, out, outlen); } +static int check_privileges(const char *user) { + int ngroups = 0; + gid_t *groups = NULL; + struct passwd *pw; + struct group *gr; + int ret = 0; + int waitid = -1; + int statval = 0; + + pw = getpwnam(user); + if (pw == NULL) { +fprintf(ERRORFILE, "User %s does not exist in host OS.\n", user); +exit(INITIALIZE_USER_FAILED); + } + + int rc = getgrouplist(user, pw->pw_gid, groups, &ngroups); + i
[29/44] hadoop git commit: Revert "HDFS-13388. RequestHedgingProxyProvider calls multiple configured NNs all the time. Contributed by Jinglun."
Revert "HDFS-13388. RequestHedgingProxyProvider calls multiple configured NNs all the time. Contributed by Jinglun." This reverts commit ac32b3576da4cc463dff85118163ccfff02215fc. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b5353c75 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b5353c75 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b5353c75 Branch: refs/heads/HDFS-7240 Commit: b5353c75d90b9299f04dba255b9e9af5a8cc19eb Parents: 4571351 Author: Inigo Goiri Authored: Thu Apr 12 09:28:23 2018 -0700 Committer: Inigo Goiri Committed: Thu Apr 12 09:28:23 2018 -0700 -- .../ha/RequestHedgingProxyProvider.java | 3 -- .../ha/TestRequestHedgingProxyProvider.java | 34 2 files changed, 37 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5353c75/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java index 1c38791..7b9cd64 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java @@ -79,9 +79,6 @@ public class RequestHedgingProxyProvider extends public Object invoke(Object proxy, final Method method, final Object[] args) throws Throwable { - if (currentUsedProxy != null) { -return method.invoke(currentUsedProxy.proxy, args); - } Map, ProxyInfo> proxyMap = new HashMap<>(); int numAttempts = 0; http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5353c75/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java index 4b3fdf9..8d6b02d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java @@ -43,13 +43,10 @@ import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import static org.junit.Assert.assertEquals; import org.mockito.Matchers; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import static org.mockito.Mockito.when; -import static org.mockito.Mockito.mock; import com.google.common.collect.Lists; @@ -103,37 +100,6 @@ public class TestRequestHedgingProxyProvider { } @Test - public void testRequestNNAfterOneSuccess() throws Exception { -final AtomicInteger count = new AtomicInteger(0); -final ClientProtocol goodMock = mock(ClientProtocol.class); -when(goodMock.getStats()).thenAnswer(new Answer() { - @Override - public long[] answer(InvocationOnMock invocation) throws Throwable { -count.incrementAndGet(); -Thread.sleep(1000); -return new long[]{1}; - } -}); -final ClientProtocol badMock = mock(ClientProtocol.class); -when(badMock.getStats()).thenAnswer(new Answer() { - @Override - public long[] answer(InvocationOnMock invocation) throws Throwable { -count.incrementAndGet(); -throw new IOException("Bad mock !!"); - } -}); - -RequestHedgingProxyProvider provider = -new RequestHedgingProxyProvider<>(conf, nnUri, ClientProtocol.class, -createFactory(badMock, goodMock, goodMock, badMock)); -ClientProtocol proxy = provider.getProxy().proxy; -proxy.getStats(); -assertEquals(2, count.get()); -proxy.getStats(); -assertEquals(3, count.get()); - } - - @Test public void testHedgingWhenOneIsSlow() throws Exception { final ClientProtocol goodMock = Mockito.mock(ClientProtocol.class); Mockito.when(goodMock.getStats()).thenAnswer(new Answer() { - To unsubscribe, e-ma
[28/44] hadoop git commit: MAPREDUCE-7069. Add ability to specify user environment variables individually. Contributed by Jim Brennan
MAPREDUCE-7069. Add ability to specify user environment variables individually. Contributed by Jim Brennan Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4571351c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4571351c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4571351c Branch: refs/heads/HDFS-7240 Commit: 4571351cccf6d4977469d3d623cf045b06a5f5f0 Parents: 6bb128d Author: Jason Lowe Authored: Thu Apr 12 11:04:22 2018 -0500 Committer: Jason Lowe Committed: Thu Apr 12 11:12:46 2018 -0500 -- .../apache/hadoop/mapred/MapReduceChildJVM.java | 73 +- .../v2/app/job/impl/TaskAttemptImpl.java| 8 +- .../v2/app/job/impl/TestMapReduceChildJVM.java | 24 +++- .../apache/hadoop/mapreduce/v2/util/MRApps.java | 10 ++ .../java/org/apache/hadoop/mapred/JobConf.java | 18 +++ .../src/main/resources/mapred-default.xml | 61 +++-- .../src/site/markdown/MapReduceTutorial.md | 6 + .../org/apache/hadoop/mapred/YARNRunner.java| 11 +- .../apache/hadoop/mapred/TestYARNRunner.java| 26 +++- .../java/org/apache/hadoop/yarn/util/Apps.java | 115 +--- .../org/apache/hadoop/yarn/util/TestApps.java | 136 +++ 11 files changed, 407 insertions(+), 81 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4571351c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java index 936dc5a..d305f9f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java @@ -19,7 +19,6 @@ package org.apache.hadoop.mapred; import java.net.InetSocketAddress; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Vector; @@ -28,7 +27,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.TaskLog.LogName; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.TaskType; -import org.apache.hadoop.mapreduce.TypeConverter; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; @@ -42,50 +40,53 @@ public class MapReduceChildJVM { filter.toString(); } - private static String getChildEnv(JobConf jobConf, boolean isMap) { + private static String getChildEnvProp(JobConf jobConf, boolean isMap) { if (isMap) { - return jobConf.get(JobConf.MAPRED_MAP_TASK_ENV, - jobConf.get(JobConf.MAPRED_TASK_ENV)); + return JobConf.MAPRED_MAP_TASK_ENV; } -return jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV, -jobConf.get(JobConf.MAPRED_TASK_ENV)); +return JobConf.MAPRED_REDUCE_TASK_ENV; + } + + private static String getChildEnvDefaultValue(JobConf jobConf) { +// There is no default value for these - use the fallback value instead. +return jobConf.get(JobConf.MAPRED_TASK_ENV); } public static void setVMEnv(Map environment, Task task) { JobConf conf = task.conf; -// Add the env variables passed by the user -String mapredChildEnv = getChildEnv(conf, task.isMapTask()); -MRApps.setEnvFromInputString(environment, mapredChildEnv, conf); - -// Set logging level in the environment. -// This is so that, if the child forks another "bin/hadoop" (common in -// streaming) it will have the correct loglevel. -environment.put( -"HADOOP_ROOT_LOGGER", -MRApps.getChildLogLevel(conf, task.isMapTask()) + ",console"); - -// TODO: The following is useful for instance in streaming tasks. Should be -// set in ApplicationMaster's env by the RM. -String hadoopClientOpts = System.getenv("HADOOP_CLIENT_OPTS"); -if (hadoopClientOpts == null) { - hadoopClientOpts = ""; -} else { - hadoopClientOpts = hadoopClientOpts + " "; +boolean isMap = task.isMapTask(); + +// Remove these before adding the user variables to prevent +// MRApps.setEnvFromInputProperty() from appending to them. +String hadoopRootLoggerKey = "HADOOP_ROOT_LOGGER"; +String hadoopClientOptsKey = "HADOOP_CLIENT_OPTS"; +environment.remove(hadoopRootLoggerKey
[32/44] hadoop git commit: YARN-7936. Add default service AM Xmx. Contributed by Jian He
YARN-7936. Add default service AM Xmx. Contributed by Jian He Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/53b3e594 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/53b3e594 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/53b3e594 Branch: refs/heads/HDFS-7240 Commit: 53b3e594732e7a567dda4e08b9a8af5f87a4472a Parents: 1884459 Author: Billie Rinaldi Authored: Thu Apr 12 11:35:14 2018 -0700 Committer: Billie Rinaldi Committed: Thu Apr 12 12:38:00 2018 -0700 -- .../apache/hadoop/yarn/service/client/ServiceClient.java| 9 +++-- .../apache/hadoop/yarn/service/conf/YarnServiceConf.java| 2 ++ 2 files changed, 9 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/53b3e594/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index 4c7b72d..21fb075 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -758,8 +758,13 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, Path appRootDir, boolean hasSliderAMLog4j) throws BadConfigException { JavaCommandLineBuilder CLI = new JavaCommandLineBuilder(); CLI.forceIPv4().headless(); -CLI.setJVMOpts(YarnServiceConf.get(YarnServiceConf.JVM_OPTS, null, -app.getConfiguration(), conf)); +String jvmOpts = YarnServiceConf +.get(YarnServiceConf.JVM_OPTS, "", app.getConfiguration(), conf); +if (!jvmOpts.contains("-Xmx")) { + jvmOpts += DEFAULT_AM_JVM_XMX; +} + +CLI.setJVMOpts(jvmOpts); if (hasSliderAMLog4j) { CLI.sysprop(SYSPROP_LOG4J_CONFIGURATION, YARN_SERVICE_LOG4J_FILENAME); CLI.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR); http://git-wip-us.apache.org/repos/asf/hadoop/blob/53b3e594/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java index 14c4d15..3dd5a7e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java @@ -38,6 +38,8 @@ public class YarnServiceConf { public static final String AM_RESOURCE_MEM = "yarn.service.am-resource.memory"; public static final long DEFAULT_KEY_AM_RESOURCE_MEM = 1024; + public static final String DEFAULT_AM_JVM_XMX = " -Xmx768m "; + public static final String YARN_QUEUE = "yarn.service.queue"; public static final String API_SERVER_ADDRESS = "yarn.service.api-server.address"; - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[36/44] hadoop git commit: YARN-8153. Guaranteed containers always stay in SCHEDULED on NM after restart. Contributed by Yang Wang.
YARN-8153. Guaranteed containers always stay in SCHEDULED on NM after restart. Contributed by Yang Wang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/226bedc0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/226bedc0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/226bedc0 Branch: refs/heads/HDFS-7240 Commit: 226bedc0239ba23f3ca0c40dac6aab3777d3ada6 Parents: 375654c Author: Weiwei Yang Authored: Fri Apr 13 13:17:37 2018 +0800 Committer: Weiwei Yang Committed: Fri Apr 13 13:17:37 2018 +0800 -- .../scheduler/ContainerScheduler.java | 7 ++- .../TestContainerManagerRecovery.java | 51 2 files changed, 56 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/226bedc0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java index d9b713f..57368ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java @@ -501,8 +501,11 @@ public class ContainerScheduler extends AbstractService implements private void startContainer(Container container) { LOG.info("Starting container [" + container.getContainerId()+ "]"); -runningContainers.put(container.getContainerId(), container); -this.utilizationTracker.addContainerResources(container); +// Skip to put into runningContainers and addUtilization when recover +if (!runningContainers.containsKey(container.getContainerId())) { + runningContainers.put(container.getContainerId(), container); + this.utilizationTracker.addContainerResources(container); +} if (container.getContainerTokenIdentifier().getExecutionType() == ExecutionType.OPPORTUNISTIC) { this.metrics.startOpportunisticContainer(container.getResource()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/226bedc0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java index cad835c..bf8b500 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java @@ -67,6 +67,7 @@ import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.LogAggregationContext; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceUtilization; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -91,6 +92,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Ap import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationImpl; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState; import org.apach
[05/44] hadoop git commit: YARN-7973. Added ContainerRelaunch feature for Docker containers. Contributed by Shane Kumpf
YARN-7973. Added ContainerRelaunch feature for Docker containers. Contributed by Shane Kumpf Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c467f311 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c467f311 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c467f311 Branch: refs/heads/HDFS-7240 Commit: c467f311d0c7155c09052d93fac12045af925583 Parents: 583fa6e Author: Eric Yang Authored: Tue Apr 10 19:25:00 2018 -0400 Committer: Eric Yang Committed: Tue Apr 10 19:25:00 2018 -0400 -- .../hadoop/yarn/api/ApplicationConstants.java | 10 ++ .../server/nodemanager/ContainerExecutor.java | 11 ++ .../nodemanager/DefaultContainerExecutor.java | 6 + .../nodemanager/LinuxContainerExecutor.java | 124 +++ .../launcher/ContainerLaunch.java | 20 ++- .../launcher/ContainerRelaunch.java | 2 +- .../runtime/DefaultLinuxContainerRuntime.java | 6 + .../DelegatingLinuxContainerRuntime.java| 9 ++ .../runtime/DockerLinuxContainerRuntime.java| 40 +- .../JavaSandboxLinuxContainerRuntime.java | 10 ++ .../runtime/docker/DockerCommandExecutor.java | 14 +++ .../runtime/docker/DockerStartCommand.java | 29 + .../runtime/ContainerRuntime.java | 10 ++ .../impl/container-executor.c | 2 +- .../container-executor/impl/utils/docker-util.c | 40 ++ .../container-executor/impl/utils/docker-util.h | 11 ++ .../test/utils/test_docker_util.cc | 20 +++ .../nodemanager/TestLinuxContainerExecutor.java | 12 ++ .../launcher/TestContainerRelaunch.java | 2 +- .../runtime/TestDockerContainerRuntime.java | 90 ++ .../runtime/docker/TestDockerStartCommand.java | 53 .../TestContainersMonitorResourceChange.java| 5 + 22 files changed, 470 insertions(+), 56 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c467f311/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java index 64bcc44..38ad596 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java @@ -99,6 +99,16 @@ public interface ApplicationConstants { public static final String STDOUT = "stdout"; /** + * The type of launch for the container. + */ + @Public + @Unstable + enum ContainerLaunchType { +LAUNCH, +RELAUNCH + } + + /** * Environment for Applications. * * Some of the environment variables for applications are final http://git-wip-us.apache.org/repos/asf/hadoop/blob/c467f311/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java index f566f48..8e335350 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java @@ -182,6 +182,17 @@ public abstract class ContainerExecutor implements Configurable { IOException, ConfigurationException; /** + * Relaunch the container on the node. This is a blocking call and returns + * only when the container exits. + * @param ctx Encapsulates information necessary for relaunching containers. + * @return the return status of the relaunch + * @throws IOException if the container relaunch fails + * @throws ConfigurationException if config error was found + */ + public abstract int relaunchContainer(ContainerStartContext ctx) throws + IOException, ConfigurationException; + + /** * Signal container with the specified signal. * * @param ctx Encapsulates information necessary fo
[34/44] hadoop git commit: HADOOP-15379. Make IrqHandler.bind() public. Contributed by Ajay Kumar
HADOOP-15379. Make IrqHandler.bind() public. Contributed by Ajay Kumar Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ec1e8c1a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ec1e8c1a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ec1e8c1a Branch: refs/heads/HDFS-7240 Commit: ec1e8c1a8ca3d01b82ca82d73ba1132b6625d659 Parents: 9b0b9f2 Author: Bharat Viswanadham Authored: Thu Apr 12 21:51:20 2018 -0700 Committer: Bharat Viswanadham Committed: Thu Apr 12 21:51:20 2018 -0700 -- .../main/java/org/apache/hadoop/service/launcher/IrqHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec1e8c1a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/IrqHandler.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/IrqHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/IrqHandler.java index 30bb91c..17aa963 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/IrqHandler.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/IrqHandler.java @@ -86,7 +86,7 @@ public final class IrqHandler implements SignalHandler { * Bind to the interrupt handler. * @throws IllegalArgumentException if the exception could not be set */ - void bind() { + public void bind() { Preconditions.checkState(signal == null, "Handler already bound"); try { signal = new Signal(name); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[40/44] hadoop git commit: HDFS-13430. Fix TestEncryptionZonesWithKMS failure due to HADOOP-14445.
HDFS-13430. Fix TestEncryptionZonesWithKMS failure due to HADOOP-14445. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/65035937 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/65035937 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/65035937 Branch: refs/heads/HDFS-7240 Commit: 650359371175fba416331e73aa03d2a96ccb90e5 Parents: fa8b88a Author: Xiao Chen Authored: Fri Apr 13 09:04:52 2018 -0700 Committer: Xiao Chen Committed: Fri Apr 13 09:05:17 2018 -0700 -- .../src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java | 3 +++ 1 file changed, 3 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/65035937/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java index 6f9ef29..51c6c4e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java @@ -110,6 +110,7 @@ import org.junit.Test; import org.junit.rules.Timeout; import org.mockito.Mockito; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_KEY; import static org.junit.Assert.assertNotNull; import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.anyLong; @@ -192,6 +193,8 @@ public class TestEncryptionZones { // Lower the batch size for testing conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES, 2); +// disable kms client copy legacy token logic because it's irrelevant. +conf.setBoolean(KMS_CLIENT_COPY_LEGACY_TOKEN_KEY, false); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); cluster.waitActive(); Logger.getLogger(EncryptionZoneManager.class).setLevel(Level.TRACE); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[20/44] hadoop git commit: HADOOP-12502 SetReplication OutOfMemoryError. Contributed by Vinayakumar B.
HADOOP-12502 SetReplication OutOfMemoryError. Contributed by Vinayakumar B. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0d898b7b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0d898b7b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0d898b7b Branch: refs/heads/HDFS-7240 Commit: 0d898b7bb8d8d616133236da979a4316be4c1a6f Parents: 18de6f2 Author: Aaron Fabbri Authored: Wed Apr 11 17:19:56 2018 -0700 Committer: Aaron Fabbri Committed: Wed Apr 11 17:19:56 2018 -0700 -- .../apache/hadoop/fs/ChecksumFileSystem.java| 9 ++- .../java/org/apache/hadoop/fs/FileSystem.java | 2 +- .../org/apache/hadoop/fs/shell/Command.java | 69 ++-- .../apache/hadoop/fs/shell/CopyCommands.java| 6 ++ .../java/org/apache/hadoop/fs/shell/Ls.java | 26 +++- .../org/apache/hadoop/fs/shell/PathData.java| 27 .../apache/hadoop/fs/shell/find/TestFind.java | 34 +- 7 files changed, 161 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d898b7b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java index 14c1905..663c910 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java @@ -677,7 +677,14 @@ public abstract class ChecksumFileSystem extends FilterFileSystem { public FileStatus[] listStatus(Path f) throws IOException { return fs.listStatus(f, DEFAULT_FILTER); } - + + @Override + public RemoteIterator listStatusIterator(final Path p) + throws IOException { +// Not-using fs#listStatusIterator() since it includes crc files as well +return new DirListingIterator<>(p); + } + /** * List the statuses of the files/directories in the given path if the path is * a directory. http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d898b7b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index facfe03..707b921 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -2147,7 +2147,7 @@ public abstract class FileSystem extends Configured implements Closeable { /** * Generic iterator for implementing {@link #listStatusIterator(Path)}. */ - private class DirListingIterator implements + protected class DirListingIterator implements RemoteIterator { private final Path path; http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d898b7b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java index c292cf6..a4746cf 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java @@ -33,6 +33,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathNotFoundException; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -325,14 +326,9 @@ abstract public class Command extends Configured { */ protected void processPaths(PathData parent, PathData ... items) throws IOException { -// TODO: this really should be iterative for (PathData item : items) { try { -processPath(item); -if (recursive && isPathRecursable(item)) { - recursePath(item); -} -postProcessPath(item); +processPathInternal(item); } catch (IOException e) { displayError(e); } @@ -340,6 +336,59 @@ abstract public class Command extends Configured { } /** + * Iterates over the
[33/44] hadoop git commit: HDFS-13436. Fix javadoc of package-info.java
HDFS-13436. Fix javadoc of package-info.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9b0b9f2a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9b0b9f2a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9b0b9f2a Branch: refs/heads/HDFS-7240 Commit: 9b0b9f2af2f6827d7430f995d3203c4cb7ef7e48 Parents: 53b3e59 Author: Akira Ajisaka Authored: Fri Apr 13 13:23:44 2018 +0900 Committer: Akira Ajisaka Committed: Fri Apr 13 13:23:51 2018 +0900 -- .../hadoop/hdfs/protocol/datatransfer/package-info.java | 8 .../hdfs/server/common/blockaliasmap/package-info.java | 9 + .../hdfs/server/diskbalancer/connectors/package-info.java | 3 +-- .../hdfs/server/diskbalancer/datamodel/package-info.java| 4 ++-- .../hadoop/hdfs/server/diskbalancer/package-info.java | 2 +- .../hdfs/server/diskbalancer/planner/package-info.java | 2 +- 6 files changed, 14 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b0b9f2a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/package-info.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/package-info.java index a13c7d8..13c0c59 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/package-info.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/package-info.java @@ -15,10 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -@InterfaceStability.Evolving -package org.apache.hadoop.hdfs.protocol.datatransfer; -import org.apache.hadoop.classification.InterfaceStability; /** * This package contains classes related to hdfs data transfer protocol. - */ \ No newline at end of file + */ +@InterfaceStability.Evolving +package org.apache.hadoop.hdfs.protocol.datatransfer; +import org.apache.hadoop.classification.InterfaceStability; http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b0b9f2a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/package-info.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/package-info.java index b906791..d088945 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/package-info.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/package-info.java @@ -15,13 +15,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -@InterfaceAudience.Public -@InterfaceStability.Unstable -package org.apache.hadoop.hdfs.server.common.blockaliasmap; /** * The AliasMap defines mapping of PROVIDED HDFS blocks to data in remote * storage systems. */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +package org.apache.hadoop.hdfs.server.common.blockaliasmap; + import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.InterfaceStability; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b0b9f2a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/package-info.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/package-info.java index b4b4437..f118c2e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/package-info.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/package-info.java @@ -15,8 +15,6 @@ * the License. */ -package org.apache.hadoop.hdfs.server.diskbalancer.connectors; - /** * Connectors package is a set of logical connectors that connect * to various data sources to read the hadoop cluster information. @@ -35,3 +33,4 @@ package org.apache.hadoop.hdfs.server.diskbalancer.co
[04/44] hadoop git commit: HADOOP-14445. Delegation tokens are not shared between KMS instances. Contributed by Xiao Chen and Rushabh S Shah.
HADOOP-14445. Delegation tokens are not shared between KMS instances. Contributed by Xiao Chen and Rushabh S Shah. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/583fa6ed Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/583fa6ed Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/583fa6ed Branch: refs/heads/HDFS-7240 Commit: 583fa6ed48ad3df40bcaa9c591d5ccd07ce3ea81 Parents: e813975 Author: Xiao Chen Authored: Tue Apr 10 15:26:33 2018 -0700 Committer: Xiao Chen Committed: Tue Apr 10 15:38:25 2018 -0700 -- .../crypto/key/kms/KMSClientProvider.java | 212 .../crypto/key/kms/KMSDelegationToken.java | 22 +- .../crypto/key/kms/KMSLegacyTokenRenewer.java | 56 ++ .../hadoop/crypto/key/kms/KMSTokenRenewer.java | 103 .../hadoop/crypto/key/kms/package-info.java | 18 + .../fs/CommonConfigurationKeysPublic.java | 10 + .../web/DelegationTokenAuthenticatedURL.java| 21 +- .../DelegationTokenAuthenticationHandler.java | 8 +- .../web/DelegationTokenAuthenticator.java | 2 +- .../java/org/apache/hadoop/util/KMSUtil.java| 45 +- .../hadoop/util/KMSUtilFaultInjector.java | 49 ++ ...apache.hadoop.security.token.TokenIdentifier | 1 + ...rg.apache.hadoop.security.token.TokenRenewer | 3 +- .../src/main/resources/core-default.xml | 20 + .../crypto/key/kms/TestKMSClientProvider.java | 162 ++ .../kms/TestLoadBalancingKMSClientProvider.java | 67 ++- .../org/apache/hadoop/util/TestKMSUtil.java | 65 +++ .../hadoop/crypto/key/kms/server/TestKMS.java | 519 --- 18 files changed, 1180 insertions(+), 203 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/583fa6ed/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java index 2eb2e21..f97fde7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java @@ -36,8 +36,9 @@ import org.apache.hadoop.security.authentication.client.ConnectionConfigurator; import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.security.token.TokenRenewer; +import org.apache.hadoop.security.token.TokenSelector; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector; import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL; import org.apache.hadoop.util.HttpExceptionUtils; import org.apache.hadoop.util.KMSUtil; @@ -82,6 +83,8 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_DEFAULT; import static org.apache.hadoop.util.KMSUtil.checkNotEmpty; import static org.apache.hadoop.util.KMSUtil.checkNotNull; import static org.apache.hadoop.util.KMSUtil.parseJSONEncKeyVersion; @@ -96,16 +99,13 @@ import static org.apache.hadoop.util.KMSUtil.parseJSONMetadata; public class KMSClientProvider extends KeyProvider implements CryptoExtension, KeyProviderDelegationTokenExtension.DelegationTokenExtension { - private static final Logger LOG = + public static final Logger LOG = LoggerFactory.getLogger(KMSClientProvider.class); private static final String INVALID_SIGNATURE = "Invalid signature"; private static final String ANONYMOUS_REQUESTS_DISALLOWED = "Anonymous requests are disallowed"; - public static final String TOKEN_KIND_STR = KMSDelegationToken.TOKEN_KIND_STR; - public static final Text TOKEN_KIND = KMSDelegationToken.TOKEN_KIND; - public static final String SCHEME_NAME = "kms"; private static final String UTF8 = "UTF-8"; @@ -133,12 +133,17 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension, private static final ObjectWriter WRITER = new ObjectMapper().writerWithDefaultPrettyPrinter(); + /* dtService defines the token service value for the kms token. + * The value can be legacy format which is
[31/44] hadoop git commit: YARN-8147. TestClientRMService#testGetApplications sporadically fails. Contributed by Jason Lowe
YARN-8147. TestClientRMService#testGetApplications sporadically fails. Contributed by Jason Lowe Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/18844599 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/18844599 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/18844599 Branch: refs/heads/HDFS-7240 Commit: 18844599aef42f79d2af4500aa2eee472dda95cb Parents: 044341b Author: Eric E Payne Authored: Thu Apr 12 17:53:57 2018 + Committer: Eric E Payne Committed: Thu Apr 12 17:53:57 2018 + -- .../server/resourcemanager/TestClientRMService.java | 14 +- 1 file changed, 9 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/18844599/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 1c50dd3..d66a866 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -1064,7 +1064,7 @@ public class TestClientRMService { } @Test - public void testGetApplications() throws IOException, YarnException { + public void testGetApplications() throws Exception { /** * 1. Submit 3 applications alternately in two queues * 2. Test each of the filters @@ -1113,8 +1113,12 @@ public class TestClientRMService { SubmitApplicationRequest submitRequest = mockSubmitAppRequest( appId, appNames[i], queues[i % queues.length], new HashSet(tags.subList(0, i + 1))); + // make sure each app is submitted at a different time + Thread.sleep(1); rmService.submitApplication(submitRequest); - submitTimeMillis[i] = System.currentTimeMillis(); + submitTimeMillis[i] = rmService.getApplicationReport( + GetApplicationReportRequest.newInstance(appId)) + .getApplicationReport().getStartTime(); } // Test different cases of ClientRMService#getApplications() @@ -1129,19 +1133,19 @@ public class TestClientRMService { // Check start range request = GetApplicationsRequest.newInstance(); -request.setStartRange(submitTimeMillis[0], System.currentTimeMillis()); +request.setStartRange(submitTimeMillis[0] + 1, System.currentTimeMillis()); // 2 applications are submitted after first timeMills assertEquals("Incorrect number of matching start range", 2, rmService.getApplications(request).getApplicationList().size()); // 1 application is submitted after the second timeMills -request.setStartRange(submitTimeMillis[1], System.currentTimeMillis()); +request.setStartRange(submitTimeMillis[1] + 1, System.currentTimeMillis()); assertEquals("Incorrect number of matching start range", 1, rmService.getApplications(request).getApplicationList().size()); // no application is submitted after the third timeMills -request.setStartRange(submitTimeMillis[2], System.currentTimeMillis()); +request.setStartRange(submitTimeMillis[2] + 1, System.currentTimeMillis()); assertEquals("Incorrect number of matching start range", 0, rmService.getApplications(request).getApplicationList().size()); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[11/44] hadoop git commit: HDFS-13056. Expose file-level composite CRCs in HDFS which are comparable across different instances/layouts. Contributed by Dennis Huo.
HDFS-13056. Expose file-level composite CRCs in HDFS which are comparable across different instances/layouts. Contributed by Dennis Huo. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7c9cdad6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7c9cdad6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7c9cdad6 Branch: refs/heads/HDFS-7240 Commit: 7c9cdad6d04c98db5a83e2108219bf6e6c903daf Parents: 6cc59a0 Author: Xiao Chen Authored: Tue Apr 10 20:56:07 2018 -0700 Committer: Xiao Chen Committed: Tue Apr 10 21:31:48 2018 -0700 -- .../hadoop/fs/CompositeCrcFileChecksum.java | 82 + .../main/java/org/apache/hadoop/fs/Options.java | 11 + .../org/apache/hadoop/util/CrcComposer.java | 187 ++ .../java/org/apache/hadoop/util/CrcUtil.java| 220 +++ .../org/apache/hadoop/util/DataChecksum.java| 18 + .../org/apache/hadoop/util/TestCrcComposer.java | 242 .../org/apache/hadoop/util/TestCrcUtil.java | 232 .../main/java/org/apache/hadoop/fs/Hdfs.java| 4 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 56 ++- .../hadoop/hdfs/DistributedFileSystem.java | 5 +- .../apache/hadoop/hdfs/FileChecksumHelper.java | 365 +-- .../hdfs/client/HdfsClientConfigKeys.java | 2 + .../hadoop/hdfs/client/impl/DfsClientConf.java | 27 ++ .../hdfs/protocol/BlockChecksumOptions.java | 54 +++ .../hadoop/hdfs/protocol/BlockChecksumType.java | 30 ++ .../datatransfer/DataTransferProtocol.java | 12 +- .../hdfs/protocol/datatransfer/Sender.java | 11 +- .../hadoop/hdfs/protocolPB/PBHelperClient.java | 44 +++ .../src/main/proto/datatransfer.proto | 7 +- .../src/main/proto/hdfs.proto | 21 ++ .../hdfs/protocol/datatransfer/Receiver.java| 8 +- .../server/datanode/BlockChecksumHelper.java| 289 +-- .../hdfs/server/datanode/DataXceiver.java | 26 +- ...dBlockChecksumCompositeCrcReconstructor.java | 80 ...StripedBlockChecksumMd5CrcReconstructor.java | 74 .../StripedBlockChecksumReconstructor.java | 66 ++-- .../erasurecode/StripedBlockReconstructor.java | 1 + .../src/main/resources/hdfs-default.xml | 11 + .../org/apache/hadoop/hdfs/DFSTestUtil.java | 31 +- .../apache/hadoop/hdfs/TestFileChecksum.java| 101 - .../hdfs/TestFileChecksumCompositeCrc.java | 47 +++ .../hadoop/hdfs/protocolPB/TestPBHelper.java| 14 + .../hadoop/tools/mapred/TestCopyMapper.java | 173 +++-- .../mapred/TestCopyMapperCompositeCrc.java | 50 +++ 34 files changed, 2359 insertions(+), 242 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CompositeCrcFileChecksum.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CompositeCrcFileChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CompositeCrcFileChecksum.java new file mode 100644 index 000..e1ed5cb --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CompositeCrcFileChecksum.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.Options.ChecksumOpt; +import org.apache.hadoop.util.CrcUtil; +import org.apache.hadoop.util.DataChecksum; + +/** Composite CRC. */ +@InterfaceAudience.LimitedPrivate({"HDFS"}) +@InterfaceStability.Unstable +public class CompositeCrcFileChecksum extends FileChecksum { + public static final int LENGTH = Integer.SIZE / Byte.SIZE; + + private int crc; + private DataChecksum.Type crcType; +
[44/44] hadoop git commit: Merge branch 'trunk' into HDFS-7240
Merge branch 'trunk' into HDFS-7240 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/72a3743c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/72a3743c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/72a3743c Branch: refs/heads/HDFS-7240 Commit: 72a3743cc49d9c7b8d2eaec8064a25b8d890c267 Parents: 66610b5 995cba6 Author: Xiaoyu Yao Authored: Fri Apr 13 17:00:19 2018 -0700 Committer: Xiaoyu Yao Committed: Fri Apr 13 17:00:19 2018 -0700 -- .../org/apache/hadoop/conf/Configuration.java | 11 +- .../crypto/key/kms/KMSClientProvider.java | 212 .../crypto/key/kms/KMSDelegationToken.java | 22 +- .../crypto/key/kms/KMSLegacyTokenRenewer.java | 56 ++ .../hadoop/crypto/key/kms/KMSTokenRenewer.java | 103 .../hadoop/crypto/key/kms/package-info.java | 18 + .../apache/hadoop/fs/ChecksumFileSystem.java| 9 +- .../fs/CommonConfigurationKeysPublic.java | 10 + .../hadoop/fs/CompositeCrcFileChecksum.java | 82 +++ .../java/org/apache/hadoop/fs/FileSystem.java | 2 +- .../main/java/org/apache/hadoop/fs/Options.java | 11 + .../org/apache/hadoop/fs/shell/Command.java | 69 ++- .../apache/hadoop/fs/shell/CopyCommands.java| 6 + .../java/org/apache/hadoop/fs/shell/Ls.java | 26 +- .../org/apache/hadoop/fs/shell/PathData.java| 27 + .../web/DelegationTokenAuthenticatedURL.java| 21 +- .../DelegationTokenAuthenticationHandler.java | 8 +- .../web/DelegationTokenAuthenticator.java | 2 +- .../hadoop/service/launcher/IrqHandler.java | 2 +- .../org/apache/hadoop/util/CrcComposer.java | 187 +++ .../java/org/apache/hadoop/util/CrcUtil.java| 220 .../org/apache/hadoop/util/DataChecksum.java| 18 + .../java/org/apache/hadoop/util/KMSUtil.java| 45 +- .../hadoop/util/KMSUtilFaultInjector.java | 49 ++ ...apache.hadoop.security.token.TokenIdentifier | 1 + ...rg.apache.hadoop.security.token.TokenRenewer | 3 +- .../src/main/resources/core-default.xml | 20 + .../apache/hadoop/conf/TestConfiguration.java | 26 +- .../crypto/key/kms/TestKMSClientProvider.java | 162 ++ .../kms/TestLoadBalancingKMSClientProvider.java | 67 ++- .../apache/hadoop/fs/shell/find/TestFind.java | 34 +- .../org/apache/hadoop/util/TestCrcComposer.java | 242 + .../org/apache/hadoop/util/TestCrcUtil.java | 232 + .../org/apache/hadoop/util/TestKMSUtil.java | 65 +++ .../hadoop/crypto/key/kms/server/TestKMS.java | 519 --- .../main/java/org/apache/hadoop/fs/Hdfs.java| 4 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 56 +- .../hadoop/hdfs/DistributedFileSystem.java | 5 +- .../apache/hadoop/hdfs/FileChecksumHelper.java | 365 - .../hdfs/client/HdfsClientConfigKeys.java | 2 + .../hadoop/hdfs/client/impl/DfsClientConf.java | 27 + .../hdfs/protocol/BlockChecksumOptions.java | 54 ++ .../hadoop/hdfs/protocol/BlockChecksumType.java | 30 ++ .../datatransfer/DataTransferProtocol.java | 12 +- .../hdfs/protocol/datatransfer/Sender.java | 11 +- .../hadoop/hdfs/protocolPB/PBHelperClient.java | 44 ++ .../ha/RequestHedgingProxyProvider.java | 3 - .../hdfs/shortcircuit/ShortCircuitCache.java| 11 +- .../src/main/proto/datatransfer.proto | 7 +- .../src/main/proto/hdfs.proto | 21 + .../ha/TestRequestHedgingProxyProvider.java | 34 -- .../native/libhdfspp/include/hdfspp/hdfspp.h| 53 +- .../native/libhdfspp/include/hdfspp/ioservice.h | 140 + .../native/libhdfspp/lib/bindings/c/hdfs.cc | 7 +- .../native/libhdfspp/lib/common/CMakeLists.txt | 2 +- .../native/libhdfspp/lib/common/async_stream.h | 13 +- .../libhdfspp/lib/common/continuation/asio.h| 5 - .../libhdfspp/lib/common/hdfs_ioservice.cc | 146 -- .../libhdfspp/lib/common/hdfs_ioservice.h | 79 --- .../libhdfspp/lib/common/ioservice_impl.cc | 159 ++ .../libhdfspp/lib/common/ioservice_impl.h | 76 +++ .../main/native/libhdfspp/lib/common/logging.h | 3 - .../libhdfspp/lib/common/namenode_info.cc | 15 +- .../native/libhdfspp/lib/common/namenode_info.h | 8 +- .../main/native/libhdfspp/lib/common/util.cc| 14 +- .../src/main/native/libhdfspp/lib/common/util.h | 25 +- .../lib/connection/datanodeconnection.cc| 27 +- .../lib/connection/datanodeconnection.h | 26 +- .../main/native/libhdfspp/lib/fs/filehandle.cc | 18 +- .../main/native/libhdfspp/lib/fs/filehandle.h | 12 +- .../main/native/libhdfspp/lib/fs/filesystem.cc | 67 ++- .../main/native/libhdfspp/lib/fs/filesystem.h | 66 +-- .../libhdfspp/lib/fs/namenode_operations.h | 4 +- .../native/libhdfspp/lib/reader/block_reader.cc | 18 +- .../native/libhdfspp/lib/reader/bl
[42/44] hadoop git commit: YARN-8142. Improve SIGTERM handling for YARN Service Application Master. Contributed by Billie Rinaldi
YARN-8142. Improve SIGTERM handling for YARN Service Application Master. Contributed by Billie Rinaldi Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9031a76d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9031a76d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9031a76d Branch: refs/heads/HDFS-7240 Commit: 9031a76d447f0c5eaa392144fd17c5b9812e1b20 Parents: e66e287 Author: Eric Yang Authored: Fri Apr 13 15:34:33 2018 -0400 Committer: Eric Yang Committed: Fri Apr 13 15:34:33 2018 -0400 -- .../hadoop/yarn/service/ClientAMService.java| 1 + .../hadoop/yarn/service/ServiceScheduler.java | 41 +++ .../hadoop/yarn/service/ServiceTestUtils.java | 11 +++ .../yarn/service/TestYarnNativeServices.java| 71 4 files changed, 110 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9031a76d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java index 08c36f4..3d037e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java @@ -125,6 +125,7 @@ public class ClientAMService extends AbstractService LOG.info("Stop the service by {}", UserGroupInformation.getCurrentUser()); context.scheduler.getDiagnostics() .append("Stopped by user " + UserGroupInformation.getCurrentUser()); +context.scheduler.setGracefulStop(); // Stop the service in 2 seconds delay to make sure this rpc call is completed. // shutdown hook will be executed which will stop AM gracefully. http://git-wip-us.apache.org/repos/asf/hadoop/blob/9031a76d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java index 0fcca16..7eddef9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java @@ -156,6 +156,8 @@ public class ServiceScheduler extends CompositeService { // requests for a single service is not recommended. private boolean hasAtLeastOnePlacementConstraint; + private boolean gracefulStop = false; + public ServiceScheduler(ServiceContext context) { super(context.service.getName()); this.context = context; @@ -199,6 +201,7 @@ public class ServiceScheduler extends CompositeService { addIfService(amRMClient); nmClient = createNMClient(); +nmClient.getClient().cleanupRunningContainersOnStop(false); addIfService(nmClient); dispatcher = new AsyncDispatcher("Component dispatcher"); @@ -252,6 +255,11 @@ public class ServiceScheduler extends CompositeService { .createAMRMClientAsync(1000, new AMRMClientCallback()); } + protected void setGracefulStop() { +this.gracefulStop = true; +nmClient.getClient().cleanupRunningContainersOnStop(true); + } + @Override public void serviceInit(Configuration conf) throws Exception { try { @@ -266,26 +274,31 @@ public class ServiceScheduler extends CompositeService { public void serviceStop() throws Exception { LOG.info("Stopping service scheduler"); -// Mark component-instances/containers as STOPPED -if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) { - for (ContainerId containerId : getLiveInsta
[13/44] hadoop git commit: YARN-8127. Resource leak when async scheduling is enabled. Contributed by Tao Yang.
YARN-8127. Resource leak when async scheduling is enabled. Contributed by Tao Yang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7eb783e2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7eb783e2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7eb783e2 Branch: refs/heads/HDFS-7240 Commit: 7eb783e2634d8c11fb646f1f2fdf597336325312 Parents: b0aff8a Author: Weiwei Yang Authored: Wed Apr 11 17:15:25 2018 +0800 Committer: Weiwei Yang Committed: Wed Apr 11 17:15:25 2018 +0800 -- .../scheduler/common/fica/FiCaSchedulerApp.java | 10 +++ .../TestCapacitySchedulerAsyncScheduling.java | 91 2 files changed, 101 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb783e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index 32b2cad..3ec8191 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -339,6 +339,16 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt { return false; } } +// If allocate from reserved container, make sure node is still reserved +if (allocation.getAllocateFromReservedContainer() != null +&& reservedContainerOnNode == null) { + if (LOG.isDebugEnabled()) { +LOG.debug("Try to allocate from reserved container " + allocation +.getAllocateFromReservedContainer().getRmContainer() +.getContainerId() + ", but node is not reserved"); + } + return false; +} // Do we have enough space on this node? Resource availableResource = Resources.clone( http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb783e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java index 18cd942..338b9f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java @@ -594,6 +594,97 @@ public class TestCapacitySchedulerAsyncScheduling { } } + // Testcase for YARN-8127 + @Test (timeout = 3) + public void testCommitDuplicatedAllocateFromReservedProposals() + throws Exception { +// disable async-scheduling for simulating complex scene +Configuration disableAsyncConf = new Configuration(conf); +disableAsyncConf.setBoolean( +CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, false); + +// init RM & NMs +final MockRM rm = new MockRM(disableAsyncConf); +rm.start(); +final MockNM nm1 = rm.registerNode("192.168.0.1:1234", 8 * GB); +rm.registerNode("192.168.0.2:2234", 8 * GB); + +// init scheduler & nodes +while ( +((CapacityScheduler) rm.getRMContext().getScheduler()).getNodeTracker() +.nodeCount() < 2) { + Thread.sleep(10); +} +Assert.assertEquals(2, +((AbstractYarnScheduler) rm.getRMContext().getScheduler()) +.getNodeTracker().nodeCount()); +CapacityScheduler cs
[37/44] hadoop git commit: HADOOP-14970. MiniHadoopClusterManager doesn't respect lack of format option. Contributed by Erik Krogen
HADOOP-14970. MiniHadoopClusterManager doesn't respect lack of format option. Contributed by Erik Krogen Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1a407bc9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1a407bc9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1a407bc9 Branch: refs/heads/HDFS-7240 Commit: 1a407bc9906306801690bc75ff0f0456f8f265fd Parents: 226bedc Author: Erik Krogen Authored: Thu Apr 12 23:27:51 2018 -0700 Committer: Konstantin V Shvachko Committed: Thu Apr 12 23:27:51 2018 -0700 -- .../java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a407bc9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java index b89cdc0..d29dd34 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java @@ -156,6 +156,7 @@ public class MiniHadoopClusterManager { if (!noDFS) { dfs = new MiniDFSCluster.Builder(conf).nameNodePort(nnPort) .nameNodeHttpPort(nnHttpPort).numDataNodes(numDataNodes) + .format(dfsOpts == StartupOption.FORMAT) .startupOption(dfsOpts).build(); LOG.info("Started MiniDFSCluster -- namenode on port " + dfs.getNameNodePort()); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[41/44] hadoop git commit: HDFS-13330. ShortCircuitCache#fetchOrCreate never retries. Contributed by Gabor Bota.
HDFS-13330. ShortCircuitCache#fetchOrCreate never retries. Contributed by Gabor Bota. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e66e287e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e66e287e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e66e287e Branch: refs/heads/HDFS-7240 Commit: e66e287efe2b43e710137a628f03c7df3ebdf498 Parents: 6503593 Author: Wei-Chiu Chuang Authored: Fri Apr 13 09:17:34 2018 -0700 Committer: Wei-Chiu Chuang Committed: Fri Apr 13 09:17:34 2018 -0700 -- .../hdfs/shortcircuit/ShortCircuitCache.java| 11 ++--- .../shortcircuit/TestShortCircuitCache.java | 26 2 files changed, 33 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e66e287e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java index b26652b..c2f0350 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java @@ -664,6 +664,7 @@ public class ShortCircuitCache implements Closeable { unref(replica); } + static final int FETCH_OR_CREATE_RETRY_TIMES = 3; /** * Fetch or create a replica. * @@ -678,11 +679,11 @@ public class ShortCircuitCache implements Closeable { */ public ShortCircuitReplicaInfo fetchOrCreate(ExtendedBlockId key, ShortCircuitReplicaCreator creator) { -Waitable newWaitable = null; +Waitable newWaitable; lock.lock(); try { ShortCircuitReplicaInfo info = null; - do { + for (int i = 0; i < FETCH_OR_CREATE_RETRY_TIMES; i++){ if (closed) { LOG.trace("{}: can't fethchOrCreate {} because the cache is closed.", this, key); @@ -692,11 +693,12 @@ public class ShortCircuitCache implements Closeable { if (waitable != null) { try { info = fetch(key, waitable); +break; } catch (RetriableException e) { LOG.debug("{}: retrying {}", this, e.getMessage()); } } - } while (false); + } if (info != null) return info; // We need to load the replica ourselves. newWaitable = new Waitable<>(lock.newCondition()); @@ -717,7 +719,8 @@ public class ShortCircuitCache implements Closeable { * * @throws RetriableException If the caller needs to retry. */ - private ShortCircuitReplicaInfo fetch(ExtendedBlockId key, + @VisibleForTesting // ONLY for testing + protected ShortCircuitReplicaInfo fetch(ExtendedBlockId key, Waitable waitable) throws RetriableException { // Another thread is already in the process of loading this // ShortCircuitReplica. So we simply wait for it to complete. http://git-wip-us.apache.org/repos/asf/hadoop/blob/e66e287e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java index 7ba0edc..5da6a25 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java @@ -65,6 +65,7 @@ import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.ShortCircuitReplica import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.ShmId; import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.Slot; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.ipc.RetriableException; import org.apache.hadoop.net.unix.DomainSocket; import org.apache.hadoop.net.unix.TemporarySocketDirectory; import org.apache.hadoop.security.token.SecretManager.InvalidToken; @@ -793,4 +794,29 @@ public class TestShortCircuitCache { cluster.shutdown(); sockDir.close(); } + + @Test + public void testFetchOrCreateRetries() throws Exception { +try(ShortCircuitCache cache = Mockito +.spy(new ShortCircuitCache(10, 1000, 10, 1000, 1, 1, 0))) { + final TestFileDescriptorPair pair
[39/44] hadoop git commit: HDFS-13438. Fix javadoc in FsVolumeList#removeVolume. Contributed by Shashikant Banerjee.
HDFS-13438. Fix javadoc in FsVolumeList#removeVolume. Contributed by Shashikant Banerjee. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fa8b88ab Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fa8b88ab Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fa8b88ab Branch: refs/heads/HDFS-7240 Commit: fa8b88ab2b272b29cf116a5de038d78fc4357b9d Parents: 0725953 Author: Bharat Viswanadham Authored: Fri Apr 13 08:56:02 2018 -0700 Committer: Bharat Viswanadham Committed: Fri Apr 13 08:56:02 2018 -0700 -- .../hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa8b88ab/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java index 75baf84..8f52ea7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java @@ -335,7 +335,7 @@ class FsVolumeList { /** * Dynamically remove volume in the list. - * @param volume the volume to be removed. + * @param storageLocation {@link StorageLocation} of the volume to be removed. * @param clearFailure set true to remove failure info for this volume. */ void removeVolume(StorageLocation storageLocation, boolean clearFailure) { - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[25/44] hadoop git commit: HDFS-13426. Fix javadoc in FsDatasetAsyncDiskService#removeVolume. Contributed by Shashikant Banerjee.
HDFS-13426. Fix javadoc in FsDatasetAsyncDiskService#removeVolume. Contributed by Shashikant Banerjee. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7ed8511a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7ed8511a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7ed8511a Branch: refs/heads/HDFS-7240 Commit: 7ed8511ad8daff19f765e78e4dca07cdebc2c2b2 Parents: b859785 Author: Mukul Kumar Singh Authored: Thu Apr 12 20:12:31 2018 +0530 Committer: Mukul Kumar Singh Committed: Thu Apr 12 20:12:31 2018 +0530 -- .../server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ed8511a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java index 9174cb0..4929b5e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil; import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo; +import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference; import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams; import org.apache.hadoop.hdfs.server.protocol.BlockCommand; @@ -137,7 +138,7 @@ class FsDatasetAsyncDiskService { /** * Stops AsyncDiskService for a volume. - * @param volume the root of the volume. + * @param storageId id of {@link StorageDirectory}. */ synchronized void removeVolume(String storageId) { if (executors == null) { - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[01/44] hadoop git commit: YARN-7984. Improved YARN service stop/destroy and clean up. Contributed by Billie Rinaldi
Repository: hadoop Updated Branches: refs/heads/HDFS-7240 66610b5fd -> 72a3743cc YARN-7984. Improved YARN service stop/destroy and clean up. Contributed by Billie Rinaldi Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d5537990 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d5537990 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d5537990 Branch: refs/heads/HDFS-7240 Commit: d553799030a5a64df328319aceb35734d0b2de20 Parents: 8ab776d Author: Eric Yang Authored: Tue Apr 10 17:40:49 2018 -0400 Committer: Eric Yang Committed: Tue Apr 10 17:40:49 2018 -0400 -- .../hadoop/yarn/service/webapp/ApiServer.java | 47 +++--- .../hadoop/yarn/service/ServiceClientTest.java | 6 ++ .../hadoop/yarn/service/TestApiServer.java | 26 ++ .../yarn/service/client/ServiceClient.java | 93 +++- .../hadoop/yarn/service/ServiceTestUtils.java | 15 +++- .../yarn/service/TestYarnNativeServices.java| 42 - 6 files changed, 191 insertions(+), 38 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5537990/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java index 59ee05d..14c77f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java @@ -231,30 +231,40 @@ public class ApiServer { e.getCause().getMessage()); } catch (YarnException | FileNotFoundException e) { return formatResponse(Status.NOT_FOUND, e.getMessage()); -} catch (IOException | InterruptedException e) { +} catch (Exception e) { LOG.error("Fail to stop service: {}", e); return formatResponse(Status.INTERNAL_SERVER_ERROR, e.getMessage()); } } private Response stopService(String appName, boolean destroy, - final UserGroupInformation ugi) throws IOException, - InterruptedException, YarnException, FileNotFoundException { + final UserGroupInformation ugi) throws Exception { int result = ugi.doAs(new PrivilegedExceptionAction() { @Override - public Integer run() throws IOException, YarnException, - FileNotFoundException { + public Integer run() throws Exception { int result = 0; ServiceClient sc = getServiceClient(); sc.init(YARN_CONFIG); sc.start(); -result = sc.actionStop(appName, destroy); -if (result == EXIT_SUCCESS) { - LOG.info("Successfully stopped service {}", appName); +Exception stopException = null; +try { + result = sc.actionStop(appName, destroy); + if (result == EXIT_SUCCESS) { +LOG.info("Successfully stopped service {}", appName); + } +} catch (Exception e) { + LOG.info("Got exception stopping service", e); + stopException = e; } if (destroy) { result = sc.actionDestroy(appName); - LOG.info("Successfully deleted service {}", appName); + if (result == EXIT_SUCCESS) { +LOG.info("Successfully deleted service {}", appName); + } +} else { + if (stopException != null) { +throw stopException; + } } sc.close(); return result; @@ -262,8 +272,21 @@ public class ApiServer { }); ServiceStatus serviceStatus = new ServiceStatus(); if (destroy) { - serviceStatus.setDiagnostics("Successfully destroyed service " + - appName); + if (result == EXIT_SUCCESS) { +serviceStatus.setDiagnostics("Successfully destroyed service " + +appName); + } else { +if (result == EXIT_NOT_FOUND) { + serviceStatus + .setDiagnostics("Service " + appName + " doesn't exist"); + return formatResponse(Status.BAD_REQUEST, serviceStatus); +} else { + serviceStatus + .setDiagnostics("Service " + appName + " error cleaning up " + + "registry"); + return formatResponse(Status.INTERNAL_SERVER_ERROR, serviceStatus); +} + } } else {
[21/44] hadoop git commit: HDFS-13427. Fix the section titles of transparent encryption document.
HDFS-13427. Fix the section titles of transparent encryption document. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7cd362a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7cd362a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7cd362a Branch: refs/heads/HDFS-7240 Commit: c7cd362afd21add324c3a82c594b133d41cf8d03 Parents: 0d898b7 Author: Akira Ajisaka Authored: Thu Apr 12 10:44:56 2018 +0900 Committer: Akira Ajisaka Committed: Thu Apr 12 10:44:56 2018 +0900 -- .../src/site/markdown/TransparentEncryption.md | 57 ++-- 1 file changed, 29 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7cd362a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md index 3454265..d7a70b3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md @@ -17,12 +17,12 @@ Transparent Encryption in HDFS -Overview +Overview HDFS implements *transparent*, *end-to-end* encryption. Once configured, data read from and written to special HDFS directories is *transparently* encrypted and decrypted without requiring changes to user application code. This encryption is also *end-to-end*, which means the data can only be encrypted and decrypted by the client. HDFS never stores or has access to unencrypted data or unencrypted data encryption keys. This satisfies two typical requirements for encryption: *at-rest encryption* (meaning data on persistent media, such as a disk) as well as *in-transit encryption* (e.g. when data is travelling over the network). -Background +Background -- Encryption can be done at different layers in a traditional data management software/hardware stack. Choosing to encrypt at a given layer comes with different advantages and disadvantages. @@ -39,17 +39,17 @@ HDFS-level encryption fits between database-level and filesystem-level encryptio HDFS-level encryption also prevents attacks at the filesystem-level and below (so-called "OS-level attacks"). The operating system and disk only interact with encrypted bytes, since the data is already encrypted by HDFS. -Use Cases +Use Cases - Data encryption is required by a number of different government, financial, and regulatory entities. For example, the health-care industry has HIPAA regulations, the card payment industry has PCI DSS regulations, and the US government has FISMA regulations. Having transparent encryption built into HDFS makes it easier for organizations to comply with these regulations. Encryption can also be performed at the application-level, but by integrating it into HDFS, existing applications can operate on encrypted data without changes. This integrated architecture implies stronger encrypted file semantics and better coordination with other HDFS functions. -Architecture +Architecture -### Overview +### Overview For transparent encryption, we introduce a new abstraction to HDFS: the *encryption zone*. An encryption zone is a special directory whose contents will be transparently encrypted upon write and transparently decrypted upon read. Each encryption zone is associated with a single *encryption zone key* which is specified when the zone is created. Each file within an encryption zone has its own unique *data encryption key (DEK)*. DEKs are never handled directly by HDFS. Instead, HDFS only ever handles an *encrypted data encryption key (EDEK)*. Clients decrypt an EDEK, and then use the subsequent DEK to read and write data. HDFS datanodes simply see a stream of encrypted bytes. @@ -65,7 +65,7 @@ A new cluster service is required to manage encryption keys: the Hadoop Key Mana The KMS will be described in more detail below. -### Accessing data within an encryption zone +### Accessing data within an encryption zone When creating a new file in an encryption zone, the NameNode asks the KMS to generate a new EDEK encrypted with the encryption zone's key. The EDEK is then stored persistently as part of the file's metadata on the NameNode. @@ -75,7 +75,7 @@ All of the above steps for the read and write path happen automatically through Access to encrypted file data and metadata is controlled by normal HDFS filesystem permissions. This means that if HDFS is compromised (for example, by gaining unauthorized access to an HDFS superuser account), a malicious user only gains access to ciphe
[24/44] hadoop git commit: YARN-7931. [atsv2 read acls] Include domain table creation as part of schema creator. (Vrushali C via Haibo Chen)
YARN-7931. [atsv2 read acls] Include domain table creation as part of schema creator. (Vrushali C via Haibo Chen) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b8597858 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b8597858 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b8597858 Branch: refs/heads/HDFS-7240 Commit: b8597858b17e40a99611e3a384cdd241293af83f Parents: 113af12 Author: Haibo Chen Authored: Thu Apr 12 06:38:30 2018 -0700 Committer: Haibo Chen Committed: Thu Apr 12 06:38:30 2018 -0700 -- .../storage/TimelineSchemaCreator.java | 10 ++ .../storage/domain/DomainTableRW.java | 92 ++ .../storage/domain/package-info.java| 28 +++ .../storage/domain/DomainColumn.java| 111 .../storage/domain/DomainColumnFamily.java | 52 ++ .../storage/domain/DomainRowKey.java| 179 +++ .../storage/domain/DomainTable.java | 45 + .../storage/domain/package-info.java| 28 +++ .../storage/common/TestRowKeys.java | 32 9 files changed, 577 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8597858/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java index e9e4770..37ed50c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java @@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTableR import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityTableRW; import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTableRW; import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationTableRW; +import org.apache.hadoop.yarn.server.timelineservice.storage.domain.DomainTableRW; import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; @@ -357,6 +358,15 @@ public final class TimelineSchemaCreator { throw e; } } + try { +new DomainTableRW().createTable(admin, hbaseConf); + } catch (IOException e) { +if (skipExisting) { + LOG.warn("Skip and continue on: " + e.getMessage()); +} else { + throw e; +} + } } finally { if (conn != null) { conn.close(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8597858/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/DomainTableRW.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/DomainTableRW.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/DomainTableRW.java new file mode 100644 index 000..1d58e40 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/domain/DomainTableRW.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * dist
[30/44] hadoop git commit: HDFS-13386. RBF: Wrong date information in list file(-ls) result. Contributed by Dibyendu Karmakar.
HDFS-13386. RBF: Wrong date information in list file(-ls) result. Contributed by Dibyendu Karmakar. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/044341b4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/044341b4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/044341b4 Branch: refs/heads/HDFS-7240 Commit: 044341b4e4459b23159e94c7ff0601058398fd70 Parents: b5353c7 Author: Inigo Goiri Authored: Thu Apr 12 09:30:11 2018 -0700 Committer: Inigo Goiri Committed: Thu Apr 12 09:30:11 2018 -0700 -- .../federation/router/RouterRpcServer.java | 19 +- .../federation/router/TestRouterMountTable.java | 62 +++- 2 files changed, 79 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/044341b4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java index e6d2f5e..0dc2a69 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java @@ -2328,7 +2328,24 @@ public class RouterRpcServer extends AbstractService */ private Map getMountPointDates(String path) { Map ret = new TreeMap<>(); -// TODO add when we have a Mount Table +if (subclusterResolver instanceof MountTableResolver) { + MountTableResolver mountTable = (MountTableResolver)subclusterResolver; + String srcPath; + try { +final List children = subclusterResolver.getMountPoints(path); +for (String child : children) { + if (path.equals(Path.SEPARATOR)) { +srcPath = Path.SEPARATOR + child; + } else { +srcPath = path + Path.SEPARATOR + child; + } + MountTable entry = mountTable.getMountPoint(srcPath); + ret.put(child, entry.getDateModified()); +} + } catch (IOException e) { +LOG.error("Cannot get mount point: {}", e.getMessage()); + } +} return ret; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/044341b4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java index c9e28b1..b33b998 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java @@ -17,25 +17,33 @@ */ package org.apache.hadoop.hdfs.server.federation.router; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.TreeMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; +import org.apache.hadoop.hdfs.protocol.ClientProtocol; +import org.apache.hadoop.hdfs.protocol.DirectoryListing; +import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext; import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; +import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster; import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager; import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver; import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest; import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse; import org.apache.hadoop.hdfs.server.federation.store.records.MountTable; +import org.apache.hadoop.util.Time; import
[35/44] hadoop git commit: YARN-8154. Fix missing titles in PlacementConstraints document. Contributed by Weiwei Yang.
YARN-8154. Fix missing titles in PlacementConstraints document. Contributed by Weiwei Yang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/375654c3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/375654c3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/375654c3 Branch: refs/heads/HDFS-7240 Commit: 375654c36a8bfa4337c9011fcd86737462dfa61e Parents: ec1e8c1 Author: Weiwei Yang Authored: Fri Apr 13 13:06:47 2018 +0800 Committer: Weiwei Yang Committed: Fri Apr 13 13:06:47 2018 +0800 -- .../src/site/markdown/PlacementConstraints.md.vm | 15 +-- 1 file changed, 9 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/375654c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/PlacementConstraints.md.vm -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/PlacementConstraints.md.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/PlacementConstraints.md.vm index 6af62e7..cb34c3f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/PlacementConstraints.md.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/PlacementConstraints.md.vm @@ -12,6 +12,9 @@ limitations under the License. See accompanying LICENSE file. --> +#set ( $H3 = '###' ) +#set ( $H4 = '' ) + Placement Constraints = @@ -35,7 +38,7 @@ Quick Guide We first describe how to enable scheduling with placement constraints and then provide examples of how to experiment with this feature using the distributed shell, an application that allows to run a given shell command on a set of containers. -### Enabling placement constraints +$H3 Enabling placement constraints To enable placement constraints, the following property has to be set to `placement-processor` or `scheduler` in **conf/yarn-site.xml**: @@ -51,7 +54,7 @@ We now give more details about each of the three placement constraint handlers: The `placement-processor` handler supports a wider range of constraints and can allow more containers to be placed, especially when applications have demanding constraints or the cluster is highly-utilized (due to considering multiple containers at a time). However, if respecting task priority within an application is important for the user and the capacity scheduler is used, then the `scheduler` handler should be used instead. -### Experimenting with placement constraints using distributed shell +$H3 Experimenting with placement constraints using distributed shell Users can experiment with placement constraints by using the distributed shell application through the following command: @@ -89,18 +92,18 @@ The above encodes two constraints: Defining Placement Constraints -- -### Allocation tags +$H3 Allocation tags Allocation tags are string tags that an application can associate with (groups of) its containers. Tags are used to identify components of applications. For example, an HBase Master allocation can be tagged with "hbase-m", and Region Servers with "hbase-rs". Other examples are "latency-critical" to refer to the more general demands of the allocation, or "app_0041" to denote the job ID. Allocation tags play a key role in constraints, as they allow to refer to multiple allocations that share a common tag. Note that instead of using the `ResourceRequest` object to define allocation tags, we use the new `SchedulingRequest` object. This has many similarities with the `ResourceRequest`, but better separates the sizing of the requested allocations (number and size of allocations, priority, execution type, etc.), and the constraints dictating how these allocations should be placed (resource name, relaxed locality). Applications can still use `ResourceRequest` objects, but in order to define allocation tags and constraints, they need to use the `SchedulingRequest` object. Within a single `AllocateRequest`, an application should use either the `ResourceRequest` or the `SchedulingRequest` objects, but not both of them. - Differences between node labels, node attributes and allocation tags +$H4 Differences between node labels, node attributes and allocation tags The difference between allocation tags and node labels or node attributes (YARN-3409), is that allocation tags are attached to allocations and not to nodes. When an allocation gets allocated to a node by the scheduler, the set of tags of that allocation are automatically added to the node for the duration of the allocation. Hence, a node inherits the tags of the allocations that are currently allocated to the node. Likew
[12/44] hadoop git commit: HADOOP-15346 S3ARetryPolicy for 400/BadArgument to be "fail". Contributed by Steve Loughran.
HADOOP-15346 S3ARetryPolicy for 400/BadArgument to be "fail". Contributed by Steve Loughran. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b0aff8a9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b0aff8a9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b0aff8a9 Branch: refs/heads/HDFS-7240 Commit: b0aff8a96221fbf40d1eafe51b4f530b73146a20 Parents: 7c9cdad Author: Aaron Fabbri Authored: Tue Apr 10 23:55:38 2018 -0700 Committer: Aaron Fabbri Committed: Tue Apr 10 23:55:38 2018 -0700 -- .../java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java | 6 +++--- .../java/org/apache/hadoop/fs/s3a/TestInvoker.java| 14 -- 2 files changed, 7 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aff8a9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java -- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java index d857330..2b361fd 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java @@ -175,9 +175,9 @@ public class S3ARetryPolicy implements RetryPolicy { // which isn't going to be recovered from policyMap.put(EOFException.class, retryIdempotentCalls); -// policy on a 400/bad request still ambiguous. Given it -// comes and goes on test runs: try again -policyMap.put(AWSBadRequestException.class, connectivityFailure); +// policy on a 400/bad request still ambiguous. +// Treated as an immediate failure +policyMap.put(AWSBadRequestException.class, fail); // Status 500 error code is also treated as a connectivity problem policyMap.put(AWSStatus500Exception.class, connectivityFailure); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aff8a9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java -- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java index d29e2df..5da665c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java @@ -283,18 +283,12 @@ public class TestInvoker extends Assert { /** * Repeatedly retry until eventually a bad request succeeds. */ - @Test - public void testRetryBadRequestIdempotent() throws Throwable { -final AtomicInteger counter = new AtomicInteger(0); -final int attemptsBeforeSuccess = ACTIVE_RETRY_LIMIT; -invoker.retry("test", null, true, + @Test(expected = AWSBadRequestException.class) + public void testRetryBadRequestNotIdempotent() throws Throwable { +invoker.retry("test", null, false, () -> { - if (counter.incrementAndGet() < attemptsBeforeSuccess) { -throw BAD_REQUEST; - } + throw BAD_REQUEST; }); -assertEquals(attemptsBeforeSuccess, counter.get()); -assertEquals("retry count ", attemptsBeforeSuccess - 1, retryCount); } @Test - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[26/44] hadoop git commit: Revert "HDFS-13386. RBF: Wrong date information in list file(-ls) result. Contributed by Dibyendu Karmakar."
Revert "HDFS-13386. RBF: Wrong date information in list file(-ls) result. Contributed by Dibyendu Karmakar." This reverts commit 18de6f2042b70f9f0d7a2620c60de022768a7b13. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d272056f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d272056f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d272056f Branch: refs/heads/HDFS-7240 Commit: d272056fcb23f6a9252b19d349acd718d7837079 Parents: 7ed8511 Author: Inigo Goiri Authored: Thu Apr 12 08:42:19 2018 -0700 Committer: Inigo Goiri Committed: Thu Apr 12 08:42:19 2018 -0700 -- .../federation/router/RouterRpcServer.java | 19 +- .../federation/router/TestRouterMountTable.java | 62 +--- 2 files changed, 2 insertions(+), 79 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d272056f/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java index 0dc2a69..e6d2f5e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java @@ -2328,24 +2328,7 @@ public class RouterRpcServer extends AbstractService */ private Map getMountPointDates(String path) { Map ret = new TreeMap<>(); -if (subclusterResolver instanceof MountTableResolver) { - MountTableResolver mountTable = (MountTableResolver)subclusterResolver; - String srcPath; - try { -final List children = subclusterResolver.getMountPoints(path); -for (String child : children) { - if (path.equals(Path.SEPARATOR)) { -srcPath = Path.SEPARATOR + child; - } else { -srcPath = path + Path.SEPARATOR + child; - } - MountTable entry = mountTable.getMountPoint(srcPath); - ret.put(child, entry.getDateModified()); -} - } catch (IOException e) { -LOG.error("Cannot get mount point: {}", e.getMessage()); - } -} +// TODO add when we have a Mount Table return ret; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d272056f/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java index b33b998..c9e28b1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java @@ -17,33 +17,25 @@ */ package org.apache.hadoop.hdfs.server.federation.router; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; import java.util.Collections; -import java.util.Iterator; -import java.util.Map; -import java.util.TreeMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.protocol.ClientProtocol; -import org.apache.hadoop.hdfs.protocol.DirectoryListing; -import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext; import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; -import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster; import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager; import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver; import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest; import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse; import org.apache.hadoop.hdfs.server.federatio
[38/44] hadoop git commit: HDFS-13418. NetworkTopology should be configurable when enable DFSNetworkTopology. Contributed by Tao Jie.
HDFS-13418. NetworkTopology should be configurable when enable DFSNetworkTopology. Contributed by Tao Jie. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0725953e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0725953e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0725953e Branch: refs/heads/HDFS-7240 Commit: 0725953efec89b35b7586b846abb01f7c5963b37 Parents: 1a407bc Author: Yiqun Lin Authored: Fri Apr 13 17:55:45 2018 +0800 Committer: Yiqun Lin Committed: Fri Apr 13 17:55:45 2018 +0800 -- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 7 +++ .../hadoop/hdfs/net/DFSNetworkTopology.java | 10 +++- .../src/main/resources/hdfs-default.xml | 14 ++ .../blockmanagement/TestDatanodeManager.java| 52 4 files changed, 81 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0725953e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index b4b9d97..b4dab4d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; +import org.apache.hadoop.hdfs.net.DFSNetworkTopology; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRackFaultTolerant; @@ -1177,6 +1178,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.use.dfs.network.topology"; public static final boolean DFS_USE_DFS_NETWORK_TOPOLOGY_DEFAULT = true; + public static final String DFS_NET_TOPOLOGY_IMPL_KEY = + "dfs.net.topology.impl"; + + public static final Class DFS_NET_TOPOLOGY_IMPL_DEFAULT = + DFSNetworkTopology.class; + // dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry @Deprecated public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY http://git-wip-us.apache.org/repos/asf/hadoop/blob/0725953e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java index e74cdec..7889ef4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java @@ -22,11 +22,13 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.net.Node; import org.apache.hadoop.net.NodeBase; +import org.apache.hadoop.util.ReflectionUtils; import java.util.ArrayList; import java.util.Collection; @@ -44,8 +46,12 @@ public class DFSNetworkTopology extends NetworkTopology { private static final Random RANDOM = new Random(); public static DFSNetworkTopology getInstance(Configuration conf) { -DFSNetworkTopology nt = new DFSNetworkTopology(); -return (DFSNetworkTopology)nt.init(DFSTopologyNodeImpl.FACTORY); + +DFSNetworkTopology nt = ReflectionUtils.newInstance(conf.getClass( +DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_KEY, +DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_DEFAULT, +DFSNetworkTopology.class), conf); +return (DFSNetworkTopology) nt.init(DFSTopologyNodeImpl.FACTORY); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/0725953e/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index f8cce60..4ed4690 100644 --- a/hadoop-h
[43/44] hadoop git commit: MAPREDUCE-7077. Pipe mapreduce job fails with Permission denied for jobTokenPassword. (Akira Ajisaka via wangda)
MAPREDUCE-7077. Pipe mapreduce job fails with Permission denied for jobTokenPassword. (Akira Ajisaka via wangda) Change-Id: Ie8f01425d58409fa3661f768205b7616128c8aa4 (cherry picked from commit 035e0f97ea44b0495707949a781d8792dcf6ea6b) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/995cba65 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/995cba65 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/995cba65 Branch: refs/heads/HDFS-7240 Commit: 995cba65fe29966583e36f9491d9a27b323918ae Parents: 9031a76 Author: Wangda Tan Authored: Thu Apr 12 14:33:33 2018 -0700 Committer: Wangda Tan Committed: Fri Apr 13 13:52:10 2018 -0700 -- .../apache/hadoop/mapred/pipes/Application.java | 5 ++--- .../hadoop/mapred/pipes/TestPipeApplication.java | 19 --- 2 files changed, 10 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/995cba65/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java index 5c8aab9..83d2509 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java @@ -45,7 +45,6 @@ import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapred.TaskAttemptID; import org.apache.hadoop.mapred.TaskLog; -import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.filecache.DistributedCache; import org.apache.hadoop.mapreduce.security.SecureShuffleUtils; @@ -104,8 +103,8 @@ class Applicationhttp://git-wip-us.apache.org/repos/asf/hadoop/blob/995cba65/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java index 13597e0..88d8f95 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java @@ -47,7 +47,6 @@ import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapred.IFile.Writer; -import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.security.TokenCache; import org.apache.hadoop.mapred.Counters; @@ -84,10 +83,10 @@ public class TestPipeApplication { public void testRunner() throws Exception { // clean old password files -JobConf conf = new JobConf(); -File[] psw = cleanTokenPasswordFile(conf); +File[] psw = cleanTokenPasswordFile(); try { RecordReader rReader = new ReaderPipesMapRunner(); + JobConf conf = new JobConf(); conf.set(Submitter.IS_JAVA_RR, "true"); // for stdour and stderror @@ -163,7 +162,7 @@ public class TestPipeApplication { TestTaskReporter reporter = new TestTaskReporter(); -File[] psw = cleanTokenPasswordFile(conf); +File[] psw = cleanTokenPasswordFile(); try { conf.set(MRJobConfig.TASK_ATTEMPT_ID, taskName); @@ -248,7 +247,7 @@ public class TestPipeApplication { JobConf conf = new JobConf(); -File[] psw = cleanTokenPasswordFile(conf); +File[] psw = cleanTokenPasswordFile(); System.setProperty("test.build.data", "target/tmp/build/TEST_SUBMITTER_MAPPER/data"); @@ -389,8 +388,8 @@ public class TestPipeApplication { @Test public void testPipesReduser() throws Exception { +File[] psw = cleanTokenPasswordFile(); JobConf conf = new JobConf(); -File[] psw = clean
[19/44] hadoop git commit: HDFS-13386. RBF: Wrong date information in list file(-ls) result. Contributed by Dibyendu Karmakar.
HDFS-13386. RBF: Wrong date information in list file(-ls) result. Contributed by Dibyendu Karmakar. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/18de6f20 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/18de6f20 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/18de6f20 Branch: refs/heads/HDFS-7240 Commit: 18de6f2042b70f9f0d7a2620c60de022768a7b13 Parents: 933477e Author: Inigo Goiri Authored: Wed Apr 11 12:03:14 2018 -0700 Committer: Inigo Goiri Committed: Wed Apr 11 12:03:14 2018 -0700 -- .../federation/router/RouterRpcServer.java | 19 +- .../federation/router/TestRouterMountTable.java | 62 +++- 2 files changed, 79 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/18de6f20/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java index e6d2f5e..0dc2a69 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java @@ -2328,7 +2328,24 @@ public class RouterRpcServer extends AbstractService */ private Map getMountPointDates(String path) { Map ret = new TreeMap<>(); -// TODO add when we have a Mount Table +if (subclusterResolver instanceof MountTableResolver) { + MountTableResolver mountTable = (MountTableResolver)subclusterResolver; + String srcPath; + try { +final List children = subclusterResolver.getMountPoints(path); +for (String child : children) { + if (path.equals(Path.SEPARATOR)) { +srcPath = Path.SEPARATOR + child; + } else { +srcPath = path + Path.SEPARATOR + child; + } + MountTable entry = mountTable.getMountPoint(srcPath); + ret.put(child, entry.getDateModified()); +} + } catch (IOException e) { +LOG.error("Cannot get mount point: {}", e.getMessage()); + } +} return ret; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/18de6f20/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java index c9e28b1..b33b998 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java @@ -17,25 +17,33 @@ */ package org.apache.hadoop.hdfs.server.federation.router; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.TreeMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; +import org.apache.hadoop.hdfs.protocol.ClientProtocol; +import org.apache.hadoop.hdfs.protocol.DirectoryListing; +import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext; import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; +import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster; import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager; import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver; import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest; import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse; import org.apache.hadoop.hdfs.server.federation.store.records.MountTable; +import org.apache.hadoop.util.Time; import
[08/44] hadoop git commit: YARN-7941. Transitive dependencies for component are not resolved. Contributed by Billie Rinaldi.
YARN-7941. Transitive dependencies for component are not resolved. Contributed by Billie Rinaldi. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c0487110 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c0487110 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c0487110 Branch: refs/heads/HDFS-7240 Commit: c0487110990958fa985d273eb178bdf76002cf3a Parents: d919eb6 Author: Rohith Sharma K S Authored: Wed Apr 11 09:18:50 2018 +0530 Committer: Rohith Sharma K S Committed: Wed Apr 11 09:18:50 2018 +0530 -- .../hadoop/yarn/service/component/Component.java| 1 + .../hadoop/yarn/service/TestYarnNativeServices.java | 16 2 files changed, 13 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0487110/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java index a2127c8..39897f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java @@ -179,6 +179,7 @@ public class Component implements EventHandler { maxContainerFailurePerComp = componentSpec.getConfiguration() .getPropertyInt(CONTAINER_FAILURE_THRESHOLD, 10); createNumCompInstances(component.getNumberOfContainers()); +setDesiredContainers(component.getNumberOfContainers().intValue()); } private void createNumCompInstances(long count) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0487110/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java index 2b44701..5e267bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java @@ -166,7 +166,9 @@ public class TestYarnNativeServices extends ServiceTestUtils { // Create compa with 2 containers // Create compb with 2 containers which depends on compa - // Check containers for compa started before containers for compb + // Create compc with 2 containers which depends on compb + // Check containers for compa started before containers for compb before + // containers for compc @Test (timeout = 20) public void testComponentStartOrder() throws Exception { setupInternal(NUM_NMS); @@ -175,17 +177,23 @@ public class TestYarnNativeServices extends ServiceTestUtils { exampleApp.setName("teststartorder"); exampleApp.setVersion("v1"); exampleApp.addComponent(createComponent("compa", 2, "sleep 1000")); -Component compb = createComponent("compb", 2, "sleep 1000"); -// Let compb depedends on compa; +// Let compb depend on compa +Component compb = createComponent("compb", 2, "sleep 1000"); compb.setDependencies(Collections.singletonList("compa")); exampleApp.addComponent(compb); +// Let compc depend on compb +Component compc = createComponent("compc", 2, "sleep 1000"); +compc.setDependencies(Collections.singletonList("compb")); +exampleApp.addComponent(compc); + client.actionCreate(exampleApp); waitForServiceToBeStable(client, exampleApp); // check that containers for compa are launched before containers for compb -checkContainerLaunchDependencies(client, exampleApp, "compa", "compb"); +
[09/44] hadoop git commit: MAPREDUCE-7062. Update mapreduce.job.tags description for making use for ATSv2 purpose. Contributed by Charan Hebri.
MAPREDUCE-7062. Update mapreduce.job.tags description for making use for ATSv2 purpose. Contributed by Charan Hebri. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6cc59a09 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6cc59a09 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6cc59a09 Branch: refs/heads/HDFS-7240 Commit: 6cc59a09e7330dc893b386d84c8f2aa86c02eace Parents: c048711 Author: Rohith Sharma K S Authored: Wed Apr 11 09:45:39 2018 +0530 Committer: Rohith Sharma K S Committed: Wed Apr 11 09:45:39 2018 +0530 -- .../src/main/resources/mapred-default.xml | 6 ++ 1 file changed, 6 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cc59a09/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index cf8be33..d47c0ff 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -1045,6 +1045,12 @@ Tags for the job that will be passed to YARN at submission time. Queries to YARN for applications can filter on these tags. + If these tags are intended to be used with The YARN Timeline Service v.2, + prefix them with the appropriate tag names for flow name, flow version and + flow run id. Example: + timeline_flow_name_tag:foo, + timeline_flow_version_tag:3df8b0d6100530080d2e0decf9e528e57c42a90a, + timeline_flow_run_id_tag:1465246348599 - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[22/44] hadoop git commit: HADOOP-15350. [JDK10] Update maven plugin tools to fix compile error in hadoop-maven-plugins module
HADOOP-15350. [JDK10] Update maven plugin tools to fix compile error in hadoop-maven-plugins module Signed-off-by: Akira Ajisaka Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/832852ce Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/832852ce Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/832852ce Branch: refs/heads/HDFS-7240 Commit: 832852ce4ff00d4aa698e89e1df39e5bf0df78b9 Parents: c7cd362 Author: Takanobu Asanuma Authored: Thu Apr 12 17:19:35 2018 +0900 Committer: Akira Ajisaka Committed: Thu Apr 12 17:19:35 2018 +0900 -- hadoop-maven-plugins/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/832852ce/hadoop-maven-plugins/pom.xml -- diff --git a/hadoop-maven-plugins/pom.xml b/hadoop-maven-plugins/pom.xml index bd347d6..b31d158 100644 --- a/hadoop-maven-plugins/pom.xml +++ b/hadoop-maven-plugins/pom.xml @@ -27,7 +27,7 @@ Apache Hadoop Maven Plugins 3.0 -3.4 +3.5.1 - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-8156. Increase the default value of yarn.timeline-service.app-collector.linger-period.ms. Contributed by Charan Hebri.
Repository: hadoop Updated Branches: refs/heads/trunk 995cba65f -> 669eb7bde YARN-8156. Increase the default value of yarn.timeline-service.app-collector.linger-period.ms. Contributed by Charan Hebri. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/669eb7bd Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/669eb7bd Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/669eb7bd Branch: refs/heads/trunk Commit: 669eb7bdea34f26e9b9b8a2260ae4356791622e7 Parents: 995cba6 Author: Rohith Sharma K S Authored: Sat Apr 14 10:31:28 2018 +0530 Committer: Rohith Sharma K S Committed: Sat Apr 14 10:31:28 2018 +0530 -- .../main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java | 2 +- .../hadoop-yarn-common/src/main/resources/yarn-default.xml | 2 +- .../collector/TestPerNodeTimelineCollectorsAuxService.java | 2 ++ .../hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md| 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/669eb7bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index d2a71bc..8aa136d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2660,7 +2660,7 @@ public class YarnConfiguration extends Configuration { public static final String ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS = TIMELINE_SERVICE_PREFIX + "app-collector.linger-period.ms"; - public static final int DEFAULT_ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS = 1000; + public static final int DEFAULT_ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS = 6; public static final String NUMBER_OF_ASYNC_ENTITIES_TO_MERGE = TIMELINE_SERVICE_PREFIX http://git-wip-us.apache.org/repos/asf/hadoop/blob/669eb7bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index def0816..85915c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -2499,7 +2499,7 @@ Time period till which the application collector will be alive in NM, after the application master container finishes. yarn.timeline-service.app-collector.linger-period.ms -1000 +6 http://git-wip-us.apache.org/repos/asf/hadoop/blob/669eb7bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java index f27bf63..04b89d6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java @@ -66,6 +66,8 @@ public class TestPerNodeTimelineCollectorsAuxService { conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f); conf.setClass(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS, FileSystemTimelineWriterImpl.class, TimelineWriter.class); +conf.setLong(YarnConfiguration.ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS, +1000L); } @After http://git-wip-us.apache.org/repos/asf/hadoop/blob/669eb7bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-s
hadoop git commit: YARN-8156. Increase the default value of yarn.timeline-service.app-collector.linger-period.ms. Contributed by Charan Hebri.
Repository: hadoop Updated Branches: refs/heads/branch-3.1 ca8bb322b -> eb1026dff YARN-8156. Increase the default value of yarn.timeline-service.app-collector.linger-period.ms. Contributed by Charan Hebri. (cherry picked from commit 669eb7bdea34f26e9b9b8a2260ae4356791622e7) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb1026df Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb1026df Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb1026df Branch: refs/heads/branch-3.1 Commit: eb1026dffaa04d17148f584a10970e97ca6d5916 Parents: ca8bb32 Author: Rohith Sharma K S Authored: Sat Apr 14 10:31:28 2018 +0530 Committer: Rohith Sharma K S Committed: Sat Apr 14 10:57:02 2018 +0530 -- .../main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java | 2 +- .../hadoop-yarn-common/src/main/resources/yarn-default.xml | 2 +- .../collector/TestPerNodeTimelineCollectorsAuxService.java | 2 ++ .../hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md| 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb1026df/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 58c288b..208f85c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2630,7 +2630,7 @@ public class YarnConfiguration extends Configuration { public static final String ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS = TIMELINE_SERVICE_PREFIX + "app-collector.linger-period.ms"; - public static final int DEFAULT_ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS = 1000; + public static final int DEFAULT_ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS = 6; public static final String NUMBER_OF_ASYNC_ENTITIES_TO_MERGE = TIMELINE_SERVICE_PREFIX http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb1026df/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 114ba4b..58f32bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -2491,7 +2491,7 @@ Time period till which the application collector will be alive in NM, after the application master container finishes. yarn.timeline-service.app-collector.linger-period.ms -1000 +6 http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb1026df/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java index f27bf63..04b89d6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java @@ -66,6 +66,8 @@ public class TestPerNodeTimelineCollectorsAuxService { conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f); conf.setClass(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS, FileSystemTimelineWriterImpl.class, TimelineWriter.class); +conf.setLong(YarnConfiguration.ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS, +1000L); } @After http://git-wip-us.apache.
hadoop git commit: YARN-8156. Increase the default value of yarn.timeline-service.app-collector.linger-period.ms. Contributed by Charan Hebri.
Repository: hadoop Updated Branches: refs/heads/branch-3.0 dc01e323e -> 7e3797947 YARN-8156. Increase the default value of yarn.timeline-service.app-collector.linger-period.ms. Contributed by Charan Hebri. (cherry picked from commit 669eb7bdea34f26e9b9b8a2260ae4356791622e7) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7e379794 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7e379794 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7e379794 Branch: refs/heads/branch-3.0 Commit: 7e37979472583710400679660ee5546339e88af8 Parents: dc01e32 Author: Rohith Sharma K S Authored: Sat Apr 14 10:31:28 2018 +0530 Committer: Rohith Sharma K S Committed: Sat Apr 14 11:10:16 2018 +0530 -- .../main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java | 2 +- .../hadoop-yarn-common/src/main/resources/yarn-default.xml | 2 +- .../collector/TestPerNodeTimelineCollectorsAuxService.java | 2 ++ .../hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md| 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e379794/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 8853ae6..b4f2414 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2392,7 +2392,7 @@ public class YarnConfiguration extends Configuration { public static final String ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS = TIMELINE_SERVICE_PREFIX + "app-collector.linger-period.ms"; - public static final int DEFAULT_ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS = 1000; + public static final int DEFAULT_ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS = 6; public static final String NUMBER_OF_ASYNC_ENTITIES_TO_MERGE = TIMELINE_SERVICE_PREFIX http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e379794/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index f3a9b09..6d69a10 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -2377,7 +2377,7 @@ Time period till which the application collector will be alive in NM, after the application master container finishes. yarn.timeline-service.app-collector.linger-period.ms -1000 +6 http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e379794/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java index cb9ced0..f96466f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java @@ -64,6 +64,8 @@ public class TestPerNodeTimelineCollectorsAuxService { conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f); conf.setClass(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS, FileSystemTimelineWriterImpl.class, TimelineWriter.class); +conf.setLong(YarnConfiguration.ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS, +1000L); } @After http://git-wip-us.apache.
hadoop git commit: YARN-8156. Increase the default value of yarn.timeline-service.app-collector.linger-period.ms. Contributed by Charan Hebri.
Repository: hadoop Updated Branches: refs/heads/branch-2 cc2a2a8e0 -> 900e6b435 YARN-8156. Increase the default value of yarn.timeline-service.app-collector.linger-period.ms. Contributed by Charan Hebri. (cherry picked from commit 669eb7bdea34f26e9b9b8a2260ae4356791622e7) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/900e6b43 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/900e6b43 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/900e6b43 Branch: refs/heads/branch-2 Commit: 900e6b435a32bb0726275faf86bb5b7382a5572e Parents: cc2a2a8 Author: Rohith Sharma K S Authored: Sat Apr 14 10:31:28 2018 +0530 Committer: Rohith Sharma K S Committed: Sat Apr 14 11:15:30 2018 +0530 -- .../main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java | 2 +- .../hadoop-yarn-common/src/main/resources/yarn-default.xml | 2 +- .../collector/TestPerNodeTimelineCollectorsAuxService.java | 2 ++ .../hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md.vm | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/900e6b43/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index b058e83..3f0c735 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2338,7 +2338,7 @@ public class YarnConfiguration extends Configuration { public static final String ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS = TIMELINE_SERVICE_PREFIX + "app-collector.linger-period.ms"; - public static final int DEFAULT_ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS = 1000; + public static final int DEFAULT_ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS = 6; public static final String NUMBER_OF_ASYNC_ENTITIES_TO_MERGE = TIMELINE_SERVICE_PREFIX http://git-wip-us.apache.org/repos/asf/hadoop/blob/900e6b43/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 24279f5..3e5e5ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -2380,7 +2380,7 @@ Time period till which the application collector will be alive in NM, after the application master container finishes. yarn.timeline-service.app-collector.linger-period.ms -1000 +6 http://git-wip-us.apache.org/repos/asf/hadoop/blob/900e6b43/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java index 0320739..9d2bb24 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java @@ -70,6 +70,8 @@ public class TestPerNodeTimelineCollectorsAuxService { conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f); conf.setClass(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS, FileSystemTimelineWriterImpl.class, TimelineWriter.class); +conf.setLong(YarnConfiguration.ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS, +1000L); } @After http://git-wip-us.apache.org/