Merge branch 'trunk' into HDFS-12943
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c04e0c0e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c04e0c0e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c04e0c0e Branch: refs/heads/HDFS-12943 Commit: c04e0c0e9951aab88d7e5a4f47bf24a045f6171c Parents: 6c37db9 0cd6346 Author: Konstantin V Shvachko <[email protected]> Authored: Fri Sep 21 18:24:51 2018 -0700 Committer: Konstantin V Shvachko <[email protected]> Committed: Fri Sep 21 18:28:31 2018 -0700 ---------------------------------------------------------------------- dev-support/bin/create-release | 4 +- dev-support/bin/ozone-dist-layout-stitching | 32 +- dev-support/bin/yetus-wrapper | 2 +- dev-support/docker/Dockerfile | 212 +- .../assemblies/hadoop-src-with-hdds.xml | 56 + .../assemblies/hadoop-src-with-hdsl.xml | 56 - .../hadoop-client-minicluster/pom.xml | 17 +- .../hadoop-client-runtime/pom.xml | 11 + .../hadoop-annotations/pom.xml | 24 + hadoop-common-project/hadoop-common/pom.xml | 7 + .../src/main/conf/log4j.properties | 23 - .../apache/hadoop/crypto/CryptoStreamUtils.java | 21 +- .../main/java/org/apache/hadoop/fs/Globber.java | 13 +- .../apache/hadoop/fs/TrashPolicyDefault.java | 10 +- .../main/java/org/apache/hadoop/ha/HAAdmin.java | 7 +- .../org/apache/hadoop/io/nativeio/NativeIO.java | 15 +- .../org/apache/hadoop/ipc/CallQueueManager.java | 5 +- .../apache/hadoop/log/LogThrottlingHelper.java | 358 ++ .../apache/hadoop/security/SecurityUtil.java | 34 +- .../org/apache/hadoop/util/CleanerUtil.java | 199 + .../org/apache/hadoop/util/StringUtils.java | 2 +- .../hadoop/util/curator/ZKCuratorManager.java | 10 +- .../src/main/resources/core-default.xml | 24 +- .../markdown/release/0.1.0/CHANGELOG.0.1.0.md | 101 + .../markdown/release/0.1.0/CHANGES.0.1.0.md | 101 - .../markdown/release/0.1.1/CHANGELOG.0.1.1.md | 39 + .../markdown/release/0.1.1/CHANGES.0.1.1.md | 39 - .../markdown/release/0.10.0/CHANGELOG.0.10.0.md | 101 + .../markdown/release/0.10.0/CHANGES.0.10.0.md | 101 - .../markdown/release/0.10.1/CHANGELOG.0.10.1.md | 49 + .../markdown/release/0.10.1/CHANGES.0.10.1.md | 49 - .../markdown/release/0.11.0/CHANGELOG.0.11.0.md | 96 + .../markdown/release/0.11.0/CHANGES.0.11.0.md | 96 - .../markdown/release/0.11.1/CHANGELOG.0.11.1.md | 34 + .../markdown/release/0.11.1/CHANGES.0.11.1.md | 34 - .../markdown/release/0.11.2/CHANGELOG.0.11.2.md | 33 + .../markdown/release/0.11.2/CHANGES.0.11.2.md | 33 - .../markdown/release/0.12.0/CHANGELOG.0.12.0.md | 113 + .../markdown/release/0.12.0/CHANGES.0.12.0.md | 113 - .../markdown/release/0.12.1/CHANGELOG.0.12.1.md | 59 + .../markdown/release/0.12.1/CHANGES.0.12.1.md | 59 - .../markdown/release/0.12.2/CHANGELOG.0.12.2.md | 34 + .../markdown/release/0.12.2/CHANGES.0.12.2.md | 34 - .../markdown/release/0.12.3/CHANGELOG.0.12.3.md | 38 + .../markdown/release/0.12.3/CHANGES.0.12.3.md | 38 - .../markdown/release/0.13.0/CHANGELOG.0.13.0.md | 173 + .../markdown/release/0.13.0/CHANGES.0.13.0.md | 173 - .../markdown/release/0.14.0/CHANGELOG.0.14.0.md | 214 + .../markdown/release/0.14.0/CHANGES.0.14.0.md | 214 - .../markdown/release/0.14.1/CHANGELOG.0.14.1.md | 33 + .../markdown/release/0.14.1/CHANGES.0.14.1.md | 33 - .../markdown/release/0.14.2/CHANGELOG.0.14.2.md | 40 + .../markdown/release/0.14.2/CHANGES.0.14.2.md | 40 - .../markdown/release/0.14.3/CHANGELOG.0.14.3.md | 34 + .../markdown/release/0.14.3/CHANGES.0.14.3.md | 34 - .../markdown/release/0.14.4/CHANGELOG.0.14.4.md | 39 + .../markdown/release/0.14.4/CHANGES.0.14.4.md | 39 - .../markdown/release/0.15.0/CHANGELOG.0.15.0.md | 190 + .../markdown/release/0.15.0/CHANGES.0.15.0.md | 190 - .../markdown/release/0.15.1/CHANGELOG.0.15.1.md | 49 + .../markdown/release/0.15.1/CHANGES.0.15.1.md | 49 - .../markdown/release/0.15.2/CHANGELOG.0.15.2.md | 51 + .../markdown/release/0.15.2/CHANGES.0.15.2.md | 51 - .../markdown/release/0.15.3/CHANGELOG.0.15.3.md | 35 + .../markdown/release/0.15.3/CHANGES.0.15.3.md | 35 - .../markdown/release/0.15.4/CHANGELOG.0.15.4.md | 31 + .../markdown/release/0.15.4/CHANGES.0.15.4.md | 31 - .../markdown/release/0.16.0/CHANGELOG.0.16.0.md | 225 ++ .../markdown/release/0.16.0/CHANGES.0.16.0.md | 225 -- .../markdown/release/0.16.1/CHANGELOG.0.16.1.md | 94 + .../markdown/release/0.16.1/CHANGES.0.16.1.md | 94 - .../markdown/release/0.16.2/CHANGELOG.0.16.2.md | 59 + .../markdown/release/0.16.2/CHANGES.0.16.2.md | 59 - .../markdown/release/0.16.3/CHANGELOG.0.16.3.md | 37 + .../markdown/release/0.16.3/CHANGES.0.16.3.md | 37 - .../markdown/release/0.16.4/CHANGELOG.0.16.4.md | 34 + .../markdown/release/0.16.4/CHANGES.0.16.4.md | 34 - .../markdown/release/0.17.0/CHANGELOG.0.17.0.md | 259 ++ .../markdown/release/0.17.0/CHANGES.0.17.0.md | 259 -- .../markdown/release/0.17.1/CHANGELOG.0.17.1.md | 44 + .../markdown/release/0.17.1/CHANGES.0.17.1.md | 44 - .../markdown/release/0.17.2/CHANGELOG.0.17.2.md | 43 + .../markdown/release/0.17.2/CHANGES.0.17.2.md | 43 - .../markdown/release/0.17.3/CHANGELOG.0.17.3.md | 42 + .../markdown/release/0.17.3/CHANGES.0.17.3.md | 42 - .../markdown/release/0.18.0/CHANGELOG.0.18.0.md | 331 ++ .../markdown/release/0.18.0/CHANGES.0.18.0.md | 331 -- .../markdown/release/0.18.1/CHANGELOG.0.18.1.md | 46 + .../markdown/release/0.18.1/CHANGES.0.18.1.md | 46 - .../markdown/release/0.18.2/CHANGELOG.0.18.2.md | 70 + .../markdown/release/0.18.2/CHANGES.0.18.2.md | 70 - .../markdown/release/0.18.3/CHANGELOG.0.18.3.md | 103 + .../markdown/release/0.18.3/CHANGES.0.18.3.md | 103 - .../markdown/release/0.18.4/CHANGELOG.0.18.4.md | 45 + .../markdown/release/0.18.4/CHANGES.0.18.4.md | 45 - .../markdown/release/0.19.0/CHANGELOG.0.19.0.md | 422 ++ .../markdown/release/0.19.0/CHANGES.0.19.0.md | 422 -- .../markdown/release/0.19.1/CHANGELOG.0.19.1.md | 94 + .../markdown/release/0.19.1/CHANGES.0.19.1.md | 94 - .../markdown/release/0.19.2/CHANGELOG.0.19.2.md | 80 + .../markdown/release/0.19.2/CHANGES.0.19.2.md | 80 - .../markdown/release/0.2.0/CHANGELOG.0.2.0.md | 103 + .../markdown/release/0.2.0/CHANGES.0.2.0.md | 103 - .../markdown/release/0.2.1/CHANGELOG.0.2.1.md | 33 + .../markdown/release/0.2.1/CHANGES.0.2.1.md | 33 - .../markdown/release/0.20.0/CHANGELOG.0.20.0.md | 341 ++ .../markdown/release/0.20.0/CHANGES.0.20.0.md | 341 -- .../markdown/release/0.20.1/CHANGELOG.0.20.1.md | 127 + .../markdown/release/0.20.1/CHANGES.0.20.1.md | 127 - .../markdown/release/0.20.2/CHANGELOG.0.20.2.md | 91 + .../markdown/release/0.20.2/CHANGES.0.20.2.md | 91 - .../release/0.20.203.0/CHANGELOG.0.20.203.0.md | 73 + .../release/0.20.203.0/CHANGES.0.20.203.0.md | 73 - .../release/0.20.203.1/CHANGELOG.0.20.203.1.md | 32 + .../release/0.20.203.1/CHANGES.0.20.203.1.md | 32 - .../release/0.20.204.0/CHANGELOG.0.20.204.0.md | 121 + .../release/0.20.204.0/CHANGES.0.20.204.0.md | 121 - .../release/0.20.205.0/CHANGELOG.0.20.205.0.md | 191 + .../release/0.20.205.0/CHANGES.0.20.205.0.md | 191 - .../markdown/release/0.20.3/CHANGELOG.0.20.3.md | 91 + .../markdown/release/0.20.3/CHANGES.0.20.3.md | 91 - .../markdown/release/0.21.0/CHANGELOG.0.21.0.md | 1360 +++++++ .../markdown/release/0.21.0/CHANGES.0.21.0.md | 1360 ------- .../markdown/release/0.21.1/CHANGELOG.0.21.1.md | 123 + .../markdown/release/0.21.1/CHANGES.0.21.1.md | 123 - .../markdown/release/0.22.0/CHANGELOG.0.22.0.md | 762 ++++ .../markdown/release/0.22.0/CHANGES.0.22.0.md | 762 ---- .../markdown/release/0.22.1/CHANGELOG.0.22.1.md | 107 + .../markdown/release/0.22.1/CHANGES.0.22.1.md | 107 - .../markdown/release/0.23.0/CHANGELOG.0.23.0.md | 1182 ++++++ .../markdown/release/0.23.0/CHANGES.0.23.0.md | 1182 ------ .../markdown/release/0.23.1/CHANGELOG.0.23.1.md | 484 +++ .../markdown/release/0.23.1/CHANGES.0.23.1.md | 484 --- .../release/0.23.10/CHANGELOG.0.23.10.md | 121 + .../markdown/release/0.23.10/CHANGES.0.23.10.md | 120 - .../release/0.23.11/CHANGELOG.0.23.11.md | 83 + .../markdown/release/0.23.11/CHANGES.0.23.11.md | 83 - .../markdown/release/0.23.2/CHANGELOG.0.23.2.md | 177 + .../markdown/release/0.23.2/CHANGES.0.23.2.md | 177 - .../markdown/release/0.23.3/CHANGELOG.0.23.3.md | 334 ++ .../markdown/release/0.23.3/CHANGES.0.23.3.md | 334 -- .../markdown/release/0.23.4/CHANGELOG.0.23.4.md | 74 + .../markdown/release/0.23.4/CHANGES.0.23.4.md | 74 - .../markdown/release/0.23.5/CHANGELOG.0.23.5.md | 152 + .../markdown/release/0.23.5/CHANGES.0.23.5.md | 152 - .../markdown/release/0.23.6/CHANGELOG.0.23.6.md | 127 + .../markdown/release/0.23.6/CHANGES.0.23.6.md | 127 - .../markdown/release/0.23.7/CHANGELOG.0.23.7.md | 189 + .../markdown/release/0.23.7/CHANGES.0.23.7.md | 189 - .../markdown/release/0.23.8/CHANGELOG.0.23.8.md | 67 + .../markdown/release/0.23.8/CHANGES.0.23.8.md | 67 - .../markdown/release/0.23.9/CHANGELOG.0.23.9.md | 66 + .../markdown/release/0.23.9/CHANGES.0.23.9.md | 66 - .../markdown/release/0.24.0/CHANGELOG.0.24.0.md | 89 + .../markdown/release/0.24.0/CHANGES.0.24.0.md | 89 - .../markdown/release/0.3.0/CHANGELOG.0.3.0.md | 86 + .../markdown/release/0.3.0/CHANGES.0.3.0.md | 86 - .../markdown/release/0.3.1/CHANGELOG.0.3.1.md | 35 + .../markdown/release/0.3.1/CHANGES.0.3.1.md | 35 - .../markdown/release/0.3.2/CHANGELOG.0.3.2.md | 55 + .../markdown/release/0.3.2/CHANGES.0.3.2.md | 55 - .../markdown/release/0.4.0/CHANGELOG.0.4.0.md | 66 + .../markdown/release/0.4.0/CHANGES.0.4.0.md | 66 - .../markdown/release/0.5.0/CHANGELOG.0.5.0.md | 112 + .../markdown/release/0.5.0/CHANGES.0.5.0.md | 112 - .../markdown/release/0.6.0/CHANGELOG.0.6.0.md | 89 + .../markdown/release/0.6.0/CHANGES.0.6.0.md | 89 - .../markdown/release/0.6.1/CHANGELOG.0.6.1.md | 35 + .../markdown/release/0.6.1/CHANGES.0.6.1.md | 35 - .../markdown/release/0.6.2/CHANGELOG.0.6.2.md | 39 + .../markdown/release/0.6.2/CHANGES.0.6.2.md | 39 - .../markdown/release/0.7.0/CHANGELOG.0.7.0.md | 87 + .../markdown/release/0.7.0/CHANGES.0.7.0.md | 87 - .../markdown/release/0.7.1/CHANGELOG.0.7.1.md | 41 + .../markdown/release/0.7.1/CHANGES.0.7.1.md | 41 - .../markdown/release/0.7.2/CHANGELOG.0.7.2.md | 39 + .../markdown/release/0.7.2/CHANGES.0.7.2.md | 39 - .../markdown/release/0.8.0/CHANGELOG.0.8.0.md | 82 + .../markdown/release/0.8.0/CHANGES.0.8.0.md | 82 - .../markdown/release/0.9.0/CHANGELOG.0.9.0.md | 99 + .../markdown/release/0.9.0/CHANGES.0.9.0.md | 99 - .../markdown/release/0.9.1/CHANGELOG.0.9.1.md | 32 + .../markdown/release/0.9.1/CHANGES.0.9.1.md | 32 - .../markdown/release/0.9.2/CHANGELOG.0.9.2.md | 33 + .../markdown/release/0.9.2/CHANGES.0.9.2.md | 33 - .../markdown/release/1.0.0/CHANGELOG.1.0.0.md | 117 + .../markdown/release/1.0.0/CHANGES.1.0.0.md | 117 - .../markdown/release/1.0.1/CHANGELOG.1.0.1.md | 54 + .../markdown/release/1.0.1/CHANGES.1.0.1.md | 54 - .../markdown/release/1.0.2/CHANGELOG.1.0.2.md | 59 + .../markdown/release/1.0.2/CHANGES.1.0.2.md | 59 - .../markdown/release/1.0.3/CHANGELOG.1.0.3.md | 68 + .../markdown/release/1.0.3/CHANGES.1.0.3.md | 68 - .../markdown/release/1.0.4/CHANGELOG.1.0.4.md | 39 + .../markdown/release/1.0.4/CHANGES.1.0.4.md | 39 - .../markdown/release/1.1.0/CHANGELOG.1.1.0.md | 199 + .../markdown/release/1.1.0/CHANGES.1.1.0.md | 199 - .../markdown/release/1.1.1/CHANGELOG.1.1.1.md | 68 + .../markdown/release/1.1.1/CHANGES.1.1.1.md | 68 - .../markdown/release/1.1.2/CHANGELOG.1.1.2.md | 73 + .../markdown/release/1.1.2/CHANGES.1.1.2.md | 73 - .../markdown/release/1.1.3/CHANGELOG.1.1.3.md | 31 + .../markdown/release/1.1.3/CHANGES.1.1.3.md | 31 - .../markdown/release/1.2.0/CHANGELOG.1.2.0.md | 274 ++ .../markdown/release/1.2.0/CHANGES.1.2.0.md | 274 -- .../markdown/release/1.2.1/CHANGELOG.1.2.1.md | 62 + .../markdown/release/1.2.1/CHANGES.1.2.1.md | 62 - .../markdown/release/1.2.2/CHANGELOG.1.2.2.md | 32 + .../markdown/release/1.2.2/CHANGES.1.2.2.md | 32 - .../markdown/release/1.3.0/CHANGELOG.1.3.0.md | 139 + .../markdown/release/1.3.0/CHANGES.1.3.0.md | 139 - .../2.0.0-alpha/CHANGELOG.2.0.0-alpha.md | 398 ++ .../release/2.0.0-alpha/CHANGES.2.0.0-alpha.md | 398 -- .../2.0.1-alpha/CHANGELOG.2.0.1-alpha.md | 36 + .../release/2.0.1-alpha/CHANGES.2.0.1-alpha.md | 36 - .../2.0.2-alpha/CHANGELOG.2.0.2-alpha.md | 714 ++++ .../release/2.0.2-alpha/CHANGES.2.0.2-alpha.md | 714 ---- .../2.0.3-alpha/CHANGELOG.2.0.3-alpha.md | 588 +++ .../release/2.0.3-alpha/CHANGES.2.0.3-alpha.md | 588 --- .../2.0.4-alpha/CHANGELOG.2.0.4-alpha.md | 76 + .../release/2.0.4-alpha/CHANGES.2.0.4-alpha.md | 76 - .../2.0.5-alpha/CHANGELOG.2.0.5-alpha.md | 35 + .../release/2.0.5-alpha/CHANGES.2.0.5-alpha.md | 35 - .../2.0.6-alpha/CHANGELOG.2.0.6-alpha.md | 31 + .../release/2.0.6-alpha/CHANGES.2.0.6-alpha.md | 31 - .../release/2.1.0-beta/CHANGELOG.2.1.0-beta.md | 911 +++++ .../release/2.1.0-beta/CHANGES.2.1.0-beta.md | 911 ----- .../release/2.1.1-beta/CHANGELOG.2.1.1-beta.md | 233 ++ .../release/2.1.1-beta/CHANGES.2.1.1-beta.md | 233 -- .../markdown/release/2.10.0/CHANGELOG.2.10.0.md | 484 +++ .../release/2.10.0/RELEASENOTES.2.10.0.md | 115 + .../markdown/release/2.2.0/CHANGELOG.2.2.0.md | 140 + .../markdown/release/2.2.0/CHANGES.2.2.0.md | 140 - .../markdown/release/2.2.1/CHANGELOG.2.2.1.md | 38 + .../markdown/release/2.2.1/CHANGES.2.2.1.md | 38 - .../markdown/release/2.3.0/CHANGELOG.2.3.0.md | 665 ++++ .../markdown/release/2.3.0/CHANGES.2.3.0.md | 665 ---- .../markdown/release/2.4.0/CHANGELOG.2.4.0.md | 487 +++ .../markdown/release/2.4.0/CHANGES.2.4.0.md | 487 --- .../markdown/release/2.4.1/CHANGELOG.2.4.1.md | 136 + .../markdown/release/2.4.1/CHANGES.2.4.1.md | 136 - .../markdown/release/2.5.0/CHANGELOG.2.5.0.md | 551 +++ .../markdown/release/2.5.0/CHANGES.2.5.0.md | 551 --- .../markdown/release/2.5.1/CHANGELOG.2.5.1.md | 35 + .../markdown/release/2.5.1/CHANGES.2.5.1.md | 35 - .../markdown/release/2.5.2/CHANGELOG.2.5.2.md | 35 + .../markdown/release/2.5.2/CHANGES.2.5.2.md | 35 - .../markdown/release/2.6.0/CHANGELOG.2.6.0.md | 964 +++++ .../markdown/release/2.6.0/CHANGES.2.6.0.md | 964 ----- .../markdown/release/2.6.1/CHANGELOG.2.6.1.md | 202 + .../markdown/release/2.6.1/CHANGES.2.6.1.md | 202 - .../markdown/release/2.6.2/CHANGELOG.2.6.2.md | 61 + .../markdown/release/2.6.2/CHANGES.2.6.2.md | 61 - .../markdown/release/2.6.3/CHANGELOG.2.6.3.md | 77 + .../markdown/release/2.6.3/CHANGES.2.6.3.md | 77 - .../markdown/release/2.6.4/CHANGELOG.2.6.4.md | 98 + .../markdown/release/2.6.4/CHANGES.2.6.4.md | 98 - .../markdown/release/2.6.5/CHANGELOG.2.6.5.md | 132 + .../markdown/release/2.6.5/CHANGES.2.6.5.md | 132 - .../markdown/release/2.6.6/CHANGELOG.2.6.6.md | 57 + .../markdown/release/2.6.6/CHANGES.2.6.6.md | 47 - .../markdown/release/2.7.0/CHANGELOG.2.7.0.md | 988 +++++ .../markdown/release/2.7.0/CHANGES.2.7.0.md | 988 ----- .../release/2.7.0/RELEASENOTES.2.7.0.md | 7 + .../markdown/release/2.7.1/CHANGELOG.2.7.1.md | 175 + .../markdown/release/2.7.1/CHANGES.2.7.1.md | 182 - .../release/2.7.1/RELEASENOTES.2.7.1.md | 7 - .../markdown/release/2.7.2/CHANGELOG.2.7.2.md | 208 + .../markdown/release/2.7.2/CHANGES.2.7.2.md | 208 - .../markdown/release/2.7.3/CHANGELOG.2.7.3.md | 284 ++ .../markdown/release/2.7.3/CHANGES.2.7.3.md | 284 -- .../markdown/release/2.7.4/CHANGELOG.2.7.4.md | 326 ++ .../markdown/release/2.7.4/CHANGES.2.7.4.md | 326 -- .../release/2.7.4/RELEASENOTES.2.7.4.md | 14 + .../markdown/release/2.7.5/CHANGELOG.2.7.5.md | 93 + .../markdown/release/2.7.5/CHANGES.2.7.5.md | 32 - .../release/2.7.5/RELEASENOTES.2.7.5.md | 12 + .../markdown/release/2.7.6/CHANGELOG.2.7.6.md | 92 + .../release/2.7.6/RELEASENOTES.2.7.6.md | 42 + .../markdown/release/2.7.7/CHANGELOG.2.7.7.md | 47 + .../release/2.7.7/RELEASENOTES.2.7.7.md | 21 + .../markdown/release/2.7.8/CHANGELOG.2.7.8.md | 31 + .../release/2.7.8/RELEASENOTES.2.7.8.md | 21 + .../markdown/release/2.8.0/CHANGELOG.2.8.0.md | 2999 ++++++++++++++ .../markdown/release/2.8.0/CHANGES.2.8.0.md | 2993 -------------- .../release/2.8.0/RELEASENOTES.2.8.0.md | 7 + .../markdown/release/2.8.1/CHANGELOG.2.8.1.md | 31 + .../markdown/release/2.8.1/CHANGES.2.8.1.md | 31 - .../markdown/release/2.8.2/CHANGELOG.2.8.2.md | 374 ++ .../markdown/release/2.8.2/CHANGES.2.8.2.md | 385 -- .../release/2.8.2/RELEASENOTES.2.8.2.md | 9 +- .../markdown/release/2.8.3/CHANGELOG.2.8.3.md | 129 + .../markdown/release/2.8.3/CHANGES.2.8.3.md | 69 - .../release/2.8.3/RELEASENOTES.2.8.3.md | 17 + .../markdown/release/2.8.4/CHANGELOG.2.8.4.md | 123 + .../release/2.8.4/RELEASENOTES.2.8.4.md | 21 + .../markdown/release/2.8.5/CHANGELOG.2.8.5.md | 70 + .../release/2.8.5/RELEASENOTES.2.8.5.md | 21 + .../markdown/release/2.9.0/CHANGELOG.2.9.0.md | 2013 ++++++++++ .../markdown/release/2.9.0/CHANGES.2.9.0.md | 1453 ------- .../release/2.9.0/RELEASENOTES.2.9.0.md | 189 +- .../markdown/release/2.9.1/CHANGELOG.2.9.1.md | 279 ++ .../markdown/release/2.9.1/CHANGES.2.9.1.md | 277 -- .../release/2.9.1/RELEASENOTES.2.9.1.md | 2 +- .../markdown/release/2.9.2/CHANGELOG.2.9.2.md | 202 + .../release/2.9.2/RELEASENOTES.2.9.2.md | 21 + .../3.0.0-alpha1/CHANGELOG.3.0.0-alpha1.md | 3751 ++++++++++++++++++ .../3.0.0-alpha1/CHANGES.3.0.0-alpha1.md | 3750 ----------------- .../3.0.0-alpha1/RELEASENOTES.3.0.0-alpha1.md | 14 + .../3.0.0-alpha2/CHANGELOG.3.0.0-alpha2.md | 922 +++++ .../3.0.0-alpha2/CHANGES.3.0.0-alpha2.md | 919 ----- .../3.0.0-alpha2/RELEASENOTES.3.0.0-alpha2.md | 17 +- .../3.0.0-alpha3/CHANGELOG.3.0.0-alpha3.md | 46 + .../3.0.0-alpha3/CHANGES.3.0.0-alpha3.md | 46 - .../3.0.0-alpha4/CHANGELOG.3.0.0-alpha4.md | 878 ++++ .../3.0.0-alpha4/CHANGES.3.0.0-alpha4.md | 887 ----- .../3.0.0-alpha4/RELEASENOTES.3.0.0-alpha4.md | 12 +- .../3.0.0-beta1/CHANGELOG.3.0.0-beta1.md | 649 +++ .../release/3.0.0-beta1/CHANGES.3.0.0-beta1.md | 646 --- .../3.0.0-beta1/RELEASENOTES.3.0.0-beta1.md | 13 + .../markdown/release/3.0.0/CHANGELOG.3.0.0.md | 363 ++ .../markdown/release/3.0.0/CHANGES.3.0.0.md | 360 -- .../release/3.0.0/RELEASENOTES.3.0.0.md | 7 - .../markdown/release/3.0.1/CHANGELOG.3.0.1.md | 231 ++ .../markdown/release/3.0.1/CHANGES.3.0.1.md | 241 -- .../markdown/release/3.0.2/CHANGELOG.3.0.2.md | 31 + .../markdown/release/3.0.2/CHANGES.3.0.2.md | 31 - .../markdown/release/3.0.3/CHANGELOG.3.0.3.md | 312 ++ .../markdown/release/3.0.3/CHANGES.3.0.3.md | 309 -- .../release/3.0.3/RELEASENOTES.3.0.3.md | 7 + .../markdown/release/3.0.4/CHANGELOG.3.0.4.md | 189 + .../release/3.0.4/RELEASENOTES.3.0.4.md | 50 + .../markdown/release/3.1.0/CHANGELOG.3.1.0.md | 1042 +++++ .../markdown/release/3.1.0/CHANGES.3.1.0.md | 1022 ----- .../release/3.1.0/RELEASENOTES.3.1.0.md | 7 + .../markdown/release/3.1.1/CHANGELOG.3.1.1.md | 502 +++ .../markdown/release/3.1.1/CHANGES.3.1.1.md | 498 --- .../release/3.1.1/RELEASENOTES.3.1.1.md | 539 +-- .../markdown/release/3.1.2/CHANGELOG.3.1.2.md | 158 + .../release/3.1.2/RELEASENOTES.3.1.2.md | 28 + .../markdown/release/3.2.0/CHANGELOG.3.2.0.md | 881 ++++ .../release/3.2.0/RELEASENOTES.3.2.0.md | 134 + .../java/org/apache/hadoop/fs/TestTrash.java | 6 + .../hadoop/log/TestLogThrottlingHelper.java | 172 + .../org/apache/hadoop/util/TestStringUtils.java | 9 + .../util/curator/TestZKCuratorManager.java | 23 + hadoop-dist/src/main/compose/ozone-hdfs/.env | 17 + .../main/compose/ozone-hdfs/docker-compose.yaml | 60 + .../src/main/compose/ozone-hdfs/docker-config | 76 + .../src/main/compose/ozone/docker-config | 5 + .../main/compose/ozonefs/docker-compose.yaml | 59 + .../src/main/compose/ozonefs/docker-config | 35 + .../src/main/compose/ozoneperf/docker-config | 4 + .../compose/ozonescripts/.ssh/authorized_keys | 16 + .../src/main/compose/ozonescripts/.ssh/config | 18 + .../main/compose/ozonescripts/.ssh/environment | 16 + .../src/main/compose/ozonescripts/.ssh/id_rsa | 42 + .../main/compose/ozonescripts/.ssh/id_rsa.pub | 16 + .../src/main/compose/ozonescripts/Dockerfile | 33 + .../src/main/compose/ozonescripts/README.md | 38 + .../compose/ozonescripts/docker-compose.yaml | 42 + .../src/main/compose/ozonescripts/docker-config | 38 + hadoop-dist/src/main/compose/ozonescripts/ps.sh | 17 + .../src/main/compose/ozonescripts/start.sh | 24 + .../src/main/compose/ozonescripts/stop.sh | 17 + hadoop-dist/src/main/ozone/README.txt | 51 + hadoop-dist/src/main/smoketest/README.md | 30 + .../src/main/smoketest/basic/basic.robot | 47 + .../src/main/smoketest/basic/ozone-shell.robot | 82 + hadoop-dist/src/main/smoketest/commonlib.robot | 24 + .../src/main/smoketest/ozonefs/ozonefs.robot | 35 + hadoop-dist/src/main/smoketest/test.sh | 101 + hadoop-hdds/client/pom.xml | 5 +- .../apache/hadoop/hdds/scm/XceiverClient.java | 209 - .../hadoop/hdds/scm/XceiverClientHandler.java | 202 - .../hdds/scm/XceiverClientInitializer.java | 74 - .../hadoop/hdds/scm/XceiverClientRatis.java | 101 +- .../hadoop/hdds/scm/client/HddsClientUtils.java | 4 +- .../hdds/scm/storage/ChunkInputStream.java | 30 +- .../hdds/scm/storage/ChunkOutputStream.java | 13 +- hadoop-hdds/common/pom.xml | 11 +- .../common/src/main/conf/log4j.properties | 157 + .../org/apache/hadoop/hdds/HddsConfigKeys.java | 19 + .../java/org/apache/hadoop/hdds/HddsUtils.java | 8 +- .../org/apache/hadoop/hdds/cli/GenericCli.java | 26 +- .../hadoop/hdds/cli/GenericParentCommand.java | 25 + .../hdds/cli/MissingSubcommandException.java | 35 + .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 19 + .../hadoop/hdds/scm/container/ContainerID.java | 26 +- .../container/common/helpers/ContainerInfo.java | 11 +- .../scm/container/common/helpers/Pipeline.java | 46 +- .../container/common/helpers/PipelineID.java | 13 +- .../scm/storage/ContainerProtocolCalls.java | 62 +- .../apache/hadoop/ozone/OzoneConfigKeys.java | 20 + .../org/apache/hadoop/ozone/OzoneConsts.java | 2 + .../container/common/helpers/BlockData.java | 255 ++ .../ozone/container/common/helpers/KeyData.java | 253 -- .../apache/hadoop/ozone/lock/ActiveLock.java | 101 + .../apache/hadoop/ozone/lock/LockManager.java | 101 + .../hadoop/ozone/lock/PooledLockFactory.java | 43 + .../apache/hadoop/ozone/lock/package-info.java | 21 + .../hadoop/utils/db/DBConfigFromFile.java | 34 +- .../main/java/org/apache/ratis/RatisHelper.java | 75 +- .../main/proto/DatanodeContainerProtocol.proto | 74 +- .../main/proto/ScmBlockLocationProtocol.proto | 6 +- hadoop-hdds/common/src/main/proto/hdds.proto | 14 + .../common/src/main/resources/ozone-default.xml | 63 + .../hadoop/ozone/lock/TestLockManager.java | 64 + .../apache/hadoop/ozone/lock/package-info.java | 21 + hadoop-hdds/container-service/pom.xml | 6 +- .../apache/hadoop/hdds/scm/HddsServerUtil.java | 21 + .../common/impl/OpenContainerBlockMap.java | 46 +- .../common/report/PipelineReportPublisher.java | 73 + .../common/report/ReportPublisherFactory.java | 4 + .../statemachine/DatanodeStateMachine.java | 19 +- .../statemachine/SCMConnectionManager.java | 7 +- .../common/statemachine/StateContext.java | 44 +- .../CloseContainerCommandHandler.java | 17 +- .../DeleteBlocksCommandHandler.java | 4 +- .../ReplicateContainerCommandHandler.java | 129 +- .../states/datanode/InitDatanodeState.java | 22 +- .../states/endpoint/RegisterEndpointTask.java | 8 +- .../common/transport/server/XceiverServer.java | 140 - .../transport/server/XceiverServerGrpc.java | 28 +- .../transport/server/XceiverServerHandler.java | 82 - .../server/XceiverServerInitializer.java | 64 - .../transport/server/XceiverServerSpi.java | 9 + .../server/ratis/ContainerStateMachine.java | 42 +- .../server/ratis/XceiverServerRatis.java | 132 +- .../keyvalue/KeyValueBlockIterator.java | 16 +- .../container/keyvalue/KeyValueContainer.java | 5 +- .../container/keyvalue/KeyValueHandler.java | 124 +- .../container/keyvalue/helpers/BlockUtils.java | 199 + .../container/keyvalue/helpers/KeyUtils.java | 199 - .../keyvalue/helpers/KeyValueContainerUtil.java | 12 +- .../keyvalue/helpers/SmallFileUtils.java | 2 +- .../keyvalue/impl/BlockManagerImpl.java | 229 ++ .../container/keyvalue/impl/KeyManagerImpl.java | 227 -- .../container/keyvalue/impl/package-info.java | 5 +- .../keyvalue/interfaces/BlockManager.java | 84 + .../keyvalue/interfaces/KeyManager.java | 84 - .../keyvalue/interfaces/package-info.java | 21 + .../background/BlockDeletingService.java | 10 +- .../container/ozoneimpl/OzoneContainer.java | 126 +- .../replication/ContainerReplicator.java | 27 + .../DownloadAndImportReplicator.java | 136 + .../replication/GrpcReplicationClient.java | 2 +- .../replication/ReplicationSupervisor.java | 142 + .../container/replication/ReplicationTask.java | 102 + .../StorageContainerDatanodeProtocol.java | 10 +- .../protocol/StorageContainerNodeProtocol.java | 6 +- .../commands/CloseContainerCommand.java | 23 +- ...rDatanodeProtocolClientSideTranslatorPB.java | 6 +- ...rDatanodeProtocolServerSideTranslatorPB.java | 5 +- .../StorageContainerDatanodeProtocol.proto | 10 + .../ozone/container/common/ScmTestMock.java | 8 +- .../common/TestDatanodeStateMachine.java | 42 + .../TestReplicateContainerCommandHandler.java | 146 - .../keyvalue/TestBlockManagerImpl.java | 211 + .../keyvalue/TestChunkManagerImpl.java | 2 +- .../container/keyvalue/TestKeyManagerImpl.java | 191 - .../keyvalue/TestKeyValueBlockIterator.java | 30 +- .../keyvalue/TestKeyValueContainer.java | 26 +- .../container/keyvalue/TestKeyValueHandler.java | 38 +- .../replication/TestReplicationSupervisor.java | 143 + .../container/replication/package-info.java | 22 + hadoop-hdds/framework/pom.xml | 5 +- .../hadoop/hdds/server/events/EventQueue.java | 18 + hadoop-hdds/pom.xml | 12 +- hadoop-hdds/server-scm/pom.xml | 8 +- .../org/apache/hadoop/hdds/scm/ScmUtils.java | 45 + .../hadoop/hdds/scm/block/BlockManagerImpl.java | 339 +- .../container/CloseContainerEventHandler.java | 90 +- .../hdds/scm/container/ContainerMapping.java | 54 +- .../scm/container/ContainerReportHandler.java | 55 +- .../scm/container/ContainerStateManager.java | 67 +- .../hadoop/hdds/scm/container/Mapping.java | 15 +- .../replication/ReplicationActivityStatus.java | 55 +- .../replication/ReplicationManager.java | 18 +- .../scm/container/states/ContainerQueryKey.java | 110 + .../scm/container/states/ContainerStateMap.java | 235 +- .../hadoop/hdds/scm/events/SCMEvents.java | 27 +- .../hdds/scm/exceptions/SCMException.java | 3 +- .../hadoop/hdds/scm/node/DeadNodeHandler.java | 17 + .../hadoop/hdds/scm/node/SCMNodeManager.java | 5 +- .../hadoop/hdds/scm/node/StaleNodeHandler.java | 21 +- .../hdds/scm/node/states/Node2ContainerMap.java | 123 +- .../hdds/scm/node/states/Node2ObjectsMap.java | 162 + .../hdds/scm/node/states/ReportResult.java | 105 +- .../hdds/scm/pipelines/Node2PipelineMap.java | 45 +- .../pipelines/PipelineActionEventHandler.java | 2 + .../scm/pipelines/PipelineCloseHandler.java | 24 +- .../hdds/scm/pipelines/PipelineManager.java | 180 +- .../scm/pipelines/PipelineReportHandler.java | 59 + .../hdds/scm/pipelines/PipelineSelector.java | 340 +- .../scm/pipelines/PipelineStateManager.java | 136 + .../scm/pipelines/ratis/RatisManagerImpl.java | 57 +- .../standalone/StandaloneManagerImpl.java | 52 +- .../hdds/scm/server/ChillModePrecheck.java | 56 + .../apache/hadoop/hdds/scm/server/Precheck.java | 29 + .../hdds/scm/server/SCMChillModeManager.java | 245 ++ .../scm/server/SCMClientProtocolServer.java | 53 +- .../server/SCMDatanodeHeartbeatDispatcher.java | 23 + .../scm/server/SCMDatanodeProtocolServer.java | 33 +- .../scm/server/StorageContainerManager.java | 137 +- .../apache/hadoop/hdds/scm/HddsTestUtils.java | 85 + .../org/apache/hadoop/hdds/scm/TestUtils.java | 55 +- .../hadoop/hdds/scm/block/TestBlockManager.java | 89 +- .../hdds/scm/container/MockNodeManager.java | 4 +- .../scm/container/TestContainerMapping.java | 54 +- .../container/TestContainerReportHandler.java | 23 +- .../container/TestContainerStateManager.java | 98 + .../TestReplicationActivityStatus.java | 63 + .../replication/TestReplicationManager.java | 104 +- .../hdds/scm/node/TestDeadNodeHandler.java | 99 +- .../hadoop/hdds/scm/node/TestNodeManager.java | 6 +- .../scm/node/states/TestNode2ContainerMap.java | 35 +- .../scm/server/TestSCMChillModeManager.java | 125 + .../scm/server/TestSCMClientProtocolServer.java | 60 + .../ozone/container/common/TestEndPoint.java | 5 +- .../testutils/ReplicationNodeManagerMock.java | 5 +- hadoop-hdds/tools/pom.xml | 4 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 7 +- .../hdfs/protocol/NoECPolicySetException.java | 37 + .../apache/hadoop/hdfs/web/JsonUtilClient.java | 4 + .../hadoop/fs/http/client/HttpFSFileSystem.java | 2 +- .../hadoop/fs/http/server/FSOperations.java | 3 + .../fs/http/client/BaseTestHttpFSWith.java | 35 +- .../federation/router/RouterAdminServer.java | 23 + .../src/site/markdown/HDFSRouterFederation.md | 26 + .../store/driver/TestStateStoreZK.java | 53 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 + .../qjournal/server/JournaledEditsCache.java | 2 +- .../blockmanagement/BlockManagerSafeMode.java | 20 +- .../BlockPlacementPolicyDefault.java | 29 +- .../apache/hadoop/hdfs/server/common/Util.java | 3 +- .../hdfs/server/datanode/BlockSender.java | 7 +- .../hdfs/server/namenode/CacheManager.java | 42 +- .../server/namenode/FSDirErasureCodingOp.java | 4 + .../hdfs/server/namenode/FSEditLogAsync.java | 61 +- .../hdfs/server/namenode/FSNamesystemLock.java | 46 +- .../hdfs/server/namenode/ha/EditLogTailer.java | 34 +- .../snapshot/DirectorySnapshottableFeature.java | 2 +- .../org/apache/hadoop/hdfs/tools/ECAdmin.java | 7 + .../PBImageDelimitedTextWriter.java | 18 +- .../src/main/resources/hdfs-default.xml | 17 +- .../src/main/webapps/datanode/datanode.html | 3 + .../src/main/webapps/static/dfs-dust.js | 4 +- .../site/markdown/CentralizedCacheManagement.md | 5 + .../src/site/markdown/HDFSDiskbalancer.md | 5 +- .../apache/hadoop/hdfs/MiniDFSNNTopology.java | 17 + .../TestDFSStripedOutputStreamWithFailure.java | 41 +- .../TestUnsetAndChangeDirectoryEcPolicy.java | 23 +- .../TestBlockManagerSafeMode.java | 81 +- .../blockmanagement/TestReplicationPolicy.java | 28 + .../TestDataNodeMultipleRegistrations.java | 4 +- .../server/namenode/TestCacheDirectives.java | 49 + .../hdfs/server/namenode/TestEditLogRace.java | 158 +- .../server/namenode/ha/TestEditLogTailer.java | 98 +- .../snapshot/TestSnapshotDiffReport.java | 6 + .../TestOfflineImageViewer.java | 17 + .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 18 + .../test/resources/testErasureCodingConf.xml | 24 + .../v2/app/speculate/DefaultSpeculator.java | 4 +- .../mapreduce/v2/app/webapp/AMWebServices.java | 9 +- .../v2/app/webapp/JAXBContextResolver.java | 7 +- .../mapreduce/v2/app/webapp/TaskPage.java | 3 +- .../v2/app/webapp/dao/MapTaskAttemptInfo.java | 39 + .../app/webapp/dao/ReduceTaskAttemptInfo.java | 11 +- .../v2/app/webapp/dao/TaskAttemptInfo.java | 14 +- .../v2/app/webapp/dao/TaskAttemptsInfo.java | 18 +- .../hadoop/mapreduce/v2/app/MRAppBenchmark.java | 20 +- .../impl/TestTaskAttemptContainerRequest.java | 11 +- .../v2/app/metrics/TestMRAppMetrics.java | 7 + .../app/webapp/TestAMWebServicesAttempts.java | 3 + .../src/site/markdown/MapredAppMasterRest.md | 2 +- .../mapreduce/v2/hs/webapp/HsTaskPage.java | 3 +- .../mapreduce/v2/hs/webapp/HsTasksBlock.java | 5 +- .../mapreduce/v2/hs/webapp/HsWebServices.java | 9 +- .../v2/hs/webapp/JAXBContextResolver.java | 4 +- .../hadoop/mapred/ResourceMgrDelegate.java | 22 + .../hadoop/mapred/TestClientRedirect.java | 25 + hadoop-ozone/acceptance-test/README.md | 48 - .../dev-support/bin/robot-all.sh | 18 - .../dev-support/bin/robot-dnd-all.sh | 57 - .../acceptance-test/dev-support/bin/robot.sh | 38 - .../dev-support/docker/Dockerfile | 21 - .../dev-support/docker/docker-compose.yaml | 23 - hadoop-ozone/acceptance-test/pom.xml | 59 - .../src/test/acceptance/basic/.env | 17 - .../src/test/acceptance/basic/basic.robot | 50 - .../test/acceptance/basic/docker-compose.yaml | 50 - .../src/test/acceptance/basic/docker-config | 33 - .../src/test/acceptance/basic/ozone-shell.robot | 86 - .../src/test/acceptance/commonlib.robot | 78 - .../src/test/acceptance/ozonefs/.env | 17 - .../test/acceptance/ozonefs/docker-compose.yaml | 59 - .../src/test/acceptance/ozonefs/docker-config | 34 - .../src/test/acceptance/ozonefs/ozonefs.robot | 39 - .../acceptance/ozonefs/ozonesinglenode.robot | 49 - hadoop-ozone/client/pom.xml | 4 +- .../ozone/client/io/ChunkGroupInputStream.java | 41 +- .../ozone/client/io/OzoneInputStream.java | 5 + .../hadoop/ozone/client/rpc/RpcClient.java | 3 +- hadoop-ozone/common/pom.xml | 4 +- hadoop-ozone/common/src/main/bin/ozone | 21 +- .../common/src/main/bin/ozone-config.sh | 51 + hadoop-ozone/common/src/main/bin/start-ozone.sh | 90 +- hadoop-ozone/common/src/main/bin/stop-ozone.sh | 22 +- .../hadoop/ozone/om/helpers/OmBucketInfo.java | 2 +- .../hadoop/ozone/om/helpers/OmVolumeArgs.java | 2 +- .../src/main/proto/OzoneManagerProtocol.proto | 2 +- hadoop-ozone/docs/README.md | 13 +- hadoop-ozone/docs/archetypes/default.md | 13 +- hadoop-ozone/docs/config.toml | 23 - hadoop-ozone/docs/config.yaml | 41 + hadoop-ozone/docs/content/BucketCommands.md | 122 + hadoop-ozone/docs/content/BuildingSources.md | 54 + hadoop-ozone/docs/content/CommandShell.md | 245 +- hadoop-ozone/docs/content/Concepts.md | 108 + hadoop-ozone/docs/content/Dozone.md | 110 + hadoop-ozone/docs/content/Freon.md | 64 + hadoop-ozone/docs/content/GettingStarted.md | 369 -- hadoop-ozone/docs/content/Hdds.md | 65 + hadoop-ozone/docs/content/JavaApi.md | 172 + hadoop-ozone/docs/content/KeyCommands.md | 127 + hadoop-ozone/docs/content/Metrics.md | 170 - hadoop-ozone/docs/content/OzoneFS.md | 80 + hadoop-ozone/docs/content/OzoneManager.md | 77 + hadoop-ozone/docs/content/RealCluster.md | 74 + hadoop-ozone/docs/content/Rest.md | 45 +- hadoop-ozone/docs/content/RunningViaDocker.md | 73 + hadoop-ozone/docs/content/RunningWithHDFS.md | 77 + hadoop-ozone/docs/content/SCMCLI.md | 29 + hadoop-ozone/docs/content/Settings.md | 142 + hadoop-ozone/docs/content/VolumeCommands.md | 116 + hadoop-ozone/docs/content/_index.md | 99 +- hadoop-ozone/docs/pom.xml | 13 +- hadoop-ozone/docs/static/NOTES.md | 13 +- .../ozonedoc/layouts/_default/single.html | 16 +- .../docs/themes/ozonedoc/layouts/index.html | 16 + .../ozonedoc/layouts/partials/footer.html | 13 +- .../ozonedoc/layouts/partials/header.html | 17 +- .../ozonedoc/layouts/partials/navbar.html | 13 +- .../ozonedoc/layouts/partials/sidebar.html | 21 +- .../themes/ozonedoc/static/css/ozonedoc.css | 14 +- hadoop-ozone/integration-test/pom.xml | 9 +- .../container/TestContainerStateManager.java | 415 -- .../TestContainerStateManagerIntegration.java | 417 ++ .../hdds/scm/pipeline/TestNode2PipelineMap.java | 22 +- .../hdds/scm/pipeline/TestPipelineClose.java | 15 +- .../hdds/scm/pipeline/TestSCMRestart.java | 119 + .../apache/hadoop/ozone/MiniOzoneCluster.java | 28 +- .../hadoop/ozone/MiniOzoneClusterImpl.java | 101 +- .../apache/hadoop/ozone/RatisTestHelper.java | 22 + .../hadoop/ozone/TestMiniOzoneCluster.java | 4 +- .../ozone/TestStorageContainerManager.java | 224 +- .../TestStorageContainerManagerHelper.java | 8 +- .../ozone/client/rest/TestOzoneRestClient.java | 8 +- .../rpc/TestCloseContainerHandlingByClient.java | 5 +- .../ozone/client/rpc/TestOzoneRpcClient.java | 8 +- .../ozone/container/ContainerTestHelper.java | 84 +- .../container/TestContainerReplication.java | 24 +- .../common/TestBlockDeletingService.java | 12 +- .../container/common/helpers/TestBlockData.java | 127 + .../container/common/helpers/TestKeyData.java | 119 - .../common/impl/TestCloseContainerHandler.java | 51 +- .../common/impl/TestContainerPersistence.java | 154 +- .../commandhandler/TestBlockDeletion.java | 9 +- .../TestCloseContainerByPipeline.java | 5 +- .../transport/server/ratis/TestCSMMetrics.java | 16 +- .../container/metrics/TestContainerMetrics.java | 21 +- .../container/ozoneimpl/TestOzoneContainer.java | 100 +- .../container/server/TestContainerServer.java | 67 +- .../server/TestContainerStateMachine.java | 2 +- .../hadoop/ozone/freon/TestDataValidate.java | 119 +- .../apache/hadoop/ozone/freon/TestFreon.java | 129 - .../ozone/freon/TestRandomKeyGenerator.java | 106 + ...TestGenerateOzoneRequiredConfigurations.java | 131 - .../hadoop/ozone/om/TestOzoneManager.java | 4 +- .../hadoop/ozone/om/TestScmChillMode.java | 171 + .../hadoop/ozone/ozShell/TestOzoneShell.java | 453 ++- .../hadoop/ozone/scm/TestContainerSQLCli.java | 15 +- .../ozone/scm/TestContainerSmallFile.java | 4 +- .../TestGetCommittedBlockLengthAndPutKey.java | 12 +- .../hadoop/ozone/web/client/TestKeys.java | 45 +- .../hadoop/ozone/web/client/TestKeysRatis.java | 2 - .../src/test/resources/log4j.properties | 5 +- hadoop-ozone/objectstore-service/pom.xml | 56 +- .../org/apache/hadoop/ozone/TestErrorCode.java | 53 - .../apache/hadoop/ozone/web/TestErrorCode.java | 53 + hadoop-ozone/ozone-manager/pom.xml | 6 +- .../hadoop/ozone/om/BucketManagerImpl.java | 45 +- .../apache/hadoop/ozone/om/KeyManagerImpl.java | 134 +- .../hadoop/ozone/om/OMMetadataManager.java | 14 +- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 35 +- .../apache/hadoop/ozone/om/OzoneManager.java | 35 +- .../hadoop/ozone/om/OzoneManagerLock.java | 181 + .../hadoop/ozone/om/VolumeManagerImpl.java | 41 +- .../hadoop/ozone/om/exceptions/OMException.java | 3 +- .../hadoop/ozone/web/ozShell/Handler.java | 49 +- .../apache/hadoop/ozone/web/ozShell/Shell.java | 408 +- .../web/ozShell/bucket/BucketCommands.java | 60 + .../web/ozShell/bucket/CreateBucketHandler.java | 46 +- .../web/ozShell/bucket/DeleteBucketHandler.java | 45 +- .../web/ozShell/bucket/InfoBucketHandler.java | 42 +- .../web/ozShell/bucket/ListBucketHandler.java | 89 +- .../web/ozShell/bucket/UpdateBucketHandler.java | 66 +- .../web/ozShell/keys/DeleteKeyHandler.java | 48 +- .../ozone/web/ozShell/keys/GetKeyHandler.java | 77 +- .../ozone/web/ozShell/keys/InfoKeyHandler.java | 45 +- .../ozone/web/ozShell/keys/KeyCommands.java | 60 + .../ozone/web/ozShell/keys/ListKeyHandler.java | 92 +- .../ozone/web/ozShell/keys/PutKeyHandler.java | 80 +- .../web/ozShell/volume/CreateVolumeHandler.java | 69 +- .../web/ozShell/volume/DeleteVolumeHandler.java | 35 +- .../web/ozShell/volume/InfoVolumeHandler.java | 37 +- .../web/ozShell/volume/ListVolumeHandler.java | 82 +- .../web/ozShell/volume/UpdateVolumeHandler.java | 49 +- .../web/ozShell/volume/VolumeCommands.java | 61 + .../ozone/om/ScmBlockLocationTestIngClient.java | 2 +- .../hadoop/ozone/om/TestChunkStreams.java | 14 +- .../hadoop/ozone/om/TestKeyDeletingService.java | 47 +- .../hadoop/ozone/om/TestKeyManagerImpl.java | 165 + .../hadoop/ozone/om/TestOzoneManagerLock.java | 192 + hadoop-ozone/ozonefs/pom.xml | 18 +- .../ozonefs/src/test/resources/log4j.properties | 1 + hadoop-ozone/pom.xml | 46 +- hadoop-ozone/tools/pom.xml | 6 +- .../org/apache/hadoop/ozone/freon/Freon.java | 1136 +----- .../hadoop/ozone/freon/RandomKeyGenerator.java | 1039 +++++ .../GenerateOzoneRequiredConfigurations.java | 12 +- .../genesis/BenchMarkContainerStateMap.java | 24 +- .../genesis/BenchMarkDatanodeDispatcher.java | 84 +- .../apache/hadoop/ozone/genesis/Genesis.java | 9 +- ...TestGenerateOzoneRequiredConfigurations.java | 152 + .../hadoop/ozone/genconf/package-info.java | 22 + hadoop-project-dist/pom.xml | 34 +- hadoop-project/pom.xml | 14 +- hadoop-project/src/site/site.xml | 1 + hadoop-tools/hadoop-aws/pom.xml | 4 + .../fs/s3a/BasicAWSCredentialsProvider.java | 62 - .../org/apache/hadoop/fs/s3a/Constants.java | 47 +- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 32 +- .../hadoop/fs/s3a/S3AInstrumentation.java | 5 +- .../apache/hadoop/fs/s3a/S3ARetryPolicy.java | 24 +- .../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 60 +- .../fs/s3a/SimpleAWSCredentialsProvider.java | 27 +- .../fs/s3a/TemporaryAWSCredentialsProvider.java | 24 +- .../fs/s3a/s3guard/DynamoDBMetadataStore.java | 430 +- .../fs/s3a/s3guard/LocalMetadataStore.java | 26 +- .../s3guard/S3GuardDataAccessRetryPolicy.java | 47 + .../hadoop/fs/s3a/s3guard/S3GuardTool.java | 35 + .../hadoop/fs/s3native/S3xLoginHelper.java | 121 +- .../src/site/markdown/tools/hadoop-aws/index.md | 32 +- .../site/markdown/tools/hadoop-aws/s3guard.md | 165 +- .../site/markdown/tools/hadoop-aws/testing.md | 95 +- .../tools/hadoop-aws/troubleshooting_s3a.md | 44 +- .../fs/s3a/ITestS3AAWSCredentialsProvider.java | 20 +- .../hadoop/fs/s3a/ITestS3AConfiguration.java | 51 +- .../hadoop/fs/s3a/ITestS3ACredentialsInURL.java | 164 - .../fs/s3a/ITestS3AFileSystemContract.java | 5 + .../fs/s3a/TestS3AAWSCredentialsProvider.java | 50 +- .../hadoop/fs/s3a/TestSSEConfiguration.java | 31 +- .../s3guard/AbstractS3GuardToolTestBase.java | 27 +- .../s3a/s3guard/ITestDynamoDBMetadataStore.java | 54 +- .../ITestDynamoDBMetadataStoreScale.java | 595 ++- .../s3a/s3guard/ITestS3GuardToolDynamoDB.java | 103 +- .../AbstractITestS3AMetadataStoreScale.java | 24 +- .../hadoop/fs/s3native/TestS3xLoginHelper.java | 70 +- .../hadoop-aws/src/test/resources/core-site.xml | 10 + .../hadoop/fs/azure/PageBlobOutputStream.java | 21 +- .../fs/azure/ITestOutputStreamSemantics.java | 43 + .../hadoop/yarn/sls/nodemanager/NodeInfo.java | 7 + .../yarn/sls/scheduler/RMNodeWrapper.java | 6 + hadoop-yarn-project/hadoop-yarn/bin/yarn | 5 + .../yarn/api/ApplicationClientProtocol.java | 55 + .../GetAttributesToNodesRequest.java | 74 + .../GetAttributesToNodesResponse.java | 65 + .../GetClusterNodeAttributesRequest.java | 47 + .../GetClusterNodeAttributesResponse.java | 73 + .../GetNodesToAttributesRequest.java | 65 + .../GetNodesToAttributesResponse.java | 63 + .../hadoop/yarn/api/records/NodeAttribute.java | 92 + .../yarn/api/records/NodeAttributeInfo.java | 62 + .../yarn/api/records/NodeAttributeKey.java | 66 + .../yarn/api/records/NodeAttributeOpCode.java | 43 + .../yarn/api/records/NodeAttributeType.java | 35 + .../hadoop/yarn/api/records/NodeReport.java | 13 + .../yarn/api/records/NodeToAttributeValue.java | 57 + .../hadoop/yarn/api/records/Resource.java | 2 +- .../yarn/api/resource/PlacementConstraint.java | 40 +- .../yarn/api/resource/PlacementConstraints.java | 19 + .../hadoop/yarn/conf/YarnConfiguration.java | 68 +- .../ResourceManagerAdministrationProtocol.java | 13 +- .../AttributeMappingOperationType.java | 42 + .../api/protocolrecords/NodeToAttributes.java | 59 + .../NodesToAttributesMappingRequest.java | 69 + .../NodesToAttributesMappingResponse.java | 31 + .../constraint/PlacementConstraintParser.java | 156 +- .../main/proto/applicationclient_protocol.proto | 3 + ...esourcemanager_administration_protocol.proto | 1 + ..._server_resourcemanager_service_protos.proto | 16 + .../src/main/proto/yarn_protos.proto | 44 + .../src/main/proto/yarn_service_protos.proto | 23 + .../resource/TestPlacementConstraintParser.java | 83 +- .../distributedshell/ApplicationMaster.java | 51 +- .../applications/distributedshell/Client.java | 9 +- .../distributedshell/PlacementSpec.java | 19 +- .../distributedshell/TestDistributedShell.java | 12 +- .../yarn/service/api/records/Artifact.java | 1 - .../yarn/service/api/records/Component.java | 1 - .../yarn/service/api/records/ConfigFile.java | 1 - .../yarn/service/api/records/Configuration.java | 1 - .../yarn/service/api/records/Container.java | 1 - .../hadoop/yarn/service/api/records/Error.java | 1 - .../service/api/records/KerberosPrincipal.java | 2 - .../api/records/PlacementConstraint.java | 3 - .../service/api/records/PlacementPolicy.java | 3 - .../service/api/records/PlacementScope.java | 3 - .../yarn/service/api/records/PlacementType.java | 3 - .../service/api/records/ReadinessCheck.java | 1 - .../yarn/service/api/records/Resource.java | 1 - .../api/records/ResourceInformation.java | 2 - .../yarn/service/api/records/Service.java | 1 - .../yarn/service/api/records/ServiceState.java | 1 - .../yarn/service/api/records/ServiceStatus.java | 1 - .../yarn/submarine/client/cli/CliConstants.java | 7 + .../yarn/submarine/client/cli/CliUtils.java | 10 +- .../yarn/submarine/client/cli/RunJobCli.java | 51 +- .../submarine/client/cli/param/Quicklink.java | 71 + .../client/cli/param/RunJobParameters.java | 70 +- .../fs/DefaultRemoteDirectoryManager.java | 21 +- .../common/fs/RemoteDirectoryManager.java | 4 +- .../common/FSBasedSubmarineStorageImpl.java | 4 +- .../yarnservice/YarnServiceJobSubmitter.java | 208 +- .../runtimes/yarnservice/YarnServiceUtils.java | 63 +- .../yarnservice/TestYarnServiceRunJobCli.java | 362 +- .../common/fs/MockRemoteDirectoryManager.java | 7 +- .../hadoop/yarn/client/api/YarnClient.java | 60 +- .../yarn/client/api/impl/YarnClientImpl.java | 33 +- .../hadoop/yarn/client/cli/ClusterCLI.java | 17 + .../yarn/client/cli/NodeAttributesCLI.java | 715 ++++ .../apache/hadoop/yarn/client/cli/NodeCLI.java | 13 +- .../hadoop/yarn/client/cli/TestClusterCLI.java | 32 +- .../yarn/client/cli/TestNodeAttributesCLI.java | 537 +++ .../hadoop/yarn/client/cli/TestYarnCLI.java | 31 +- .../ApplicationClientProtocolPBClientImpl.java | 55 + .../ApplicationClientProtocolPBServiceImpl.java | 65 + .../PlacementConstraintFromProtoConverter.java | 10 +- .../pb/PlacementConstraintToProtoConverter.java | 11 + .../pb/GetAttributesToNodesRequestPBImpl.java | 176 + .../pb/GetAttributesToNodesResponsePBImpl.java | 207 + .../GetClusterNodeAttributesRequestPBImpl.java | 75 + .../GetClusterNodeAttributesResponsePBImpl.java | 160 + .../pb/GetNodesToAttributesRequestPBImpl.java | 132 + .../pb/GetNodesToAttributesResponsePBImpl.java | 181 + .../impl/pb/NodeAttributeInfoPBImpl.java | 147 + .../records/impl/pb/NodeAttributeKeyPBImpl.java | 140 + .../records/impl/pb/NodeAttributePBImpl.java | 170 + .../api/records/impl/pb/NodeReportPBImpl.java | 44 +- .../impl/pb/NodeToAttributeValuePBImpl.java | 137 + .../hadoop/yarn/nodelabels/AbstractLabel.java | 71 + .../AttributeExpressionOperation.java | 26 + .../hadoop/yarn/nodelabels/AttributeValue.java | 53 + .../nodelabels/CommonNodeLabelsManager.java | 32 +- .../nodelabels/FileSystemNodeLabelsStore.java | 270 +- .../yarn/nodelabels/NodeAttributeStore.java | 77 + .../yarn/nodelabels/NodeAttributesManager.java | 137 + .../hadoop/yarn/nodelabels/NodeLabelUtil.java | 155 + .../hadoop/yarn/nodelabels/NodeLabelsStore.java | 32 +- .../NonAppendableFSNodeLabelStore.java | 46 +- .../hadoop/yarn/nodelabels/RMNodeAttribute.java | 98 + .../hadoop/yarn/nodelabels/RMNodeLabel.java | 110 +- .../yarn/nodelabels/StringAttributeValue.java | 61 + .../nodelabels/store/AbstractFSNodeStore.java | 213 + .../yarn/nodelabels/store/FSStoreOpHandler.java | 131 + .../hadoop/yarn/nodelabels/store/StoreOp.java | 49 + .../nodelabels/store/op/AddClusterLabelOp.java | 73 + .../store/op/AddNodeToAttributeLogOp.java | 71 + .../nodelabels/store/op/FSNodeStoreLogOp.java | 52 + .../store/op/NodeAttributeMirrorOp.java | 64 + .../nodelabels/store/op/NodeLabelMirrorOp.java | 85 + .../yarn/nodelabels/store/op/NodeToLabelOp.java | 75 + .../store/op/RemoveClusterLabelOp.java | 75 + .../store/op/RemoveNodeToAttributeLogOp.java | 71 + .../store/op/ReplaceNodeToAttributeLogOp.java | 73 + .../yarn/nodelabels/store/op/package-info.java | 21 + .../yarn/nodelabels/store/package-info.java | 21 + ...nagerAdministrationProtocolPBClientImpl.java | 26 +- ...agerAdministrationProtocolPBServiceImpl.java | 31 +- .../impl/pb/NodeToAttributesPBImpl.java | 164 + .../NodesToAttributesMappingRequestPBImpl.java | 197 + .../NodesToAttributesMappingResponsePBImpl.java | 50 + ...emoveFromClusterNodeLabelsRequestPBImpl.java | 24 +- .../src/main/resources/yarn-default.xml | 86 + .../hadoop/yarn/api/TestPBImplRecords.java | 114 +- .../DummyCommonNodeLabelsManager.java | 8 +- .../TestFileSystemNodeLabelsStore.java | 16 +- .../yarn/nodelabels/TestNodeLabelUtil.java | 51 + .../hadoop/yarn/server/AMRMClientRelayer.java | 412 +- .../protocolrecords/NodeHeartbeatRequest.java | 17 + .../impl/pb/NodeHeartbeatRequestPBImpl.java | 52 + .../metrics/AMRMClientRelayerMetrics.java | 368 ++ .../yarn/server/metrics/package-info.java | 18 + .../yarn/server/uam/UnmanagedAMPoolManager.java | 30 +- .../server/uam/UnmanagedApplicationManager.java | 16 +- .../hadoop/yarn/server/utils/BuilderUtils.java | 6 +- .../yarn_server_common_service_protos.proto | 5 + .../yarn/server/MockResourceManagerFacade.java | 46 +- .../yarn/server/TestAMRMClientRelayer.java | 2 +- .../protocolrecords/TestProtocolRecords.java | 12 + .../metrics/TestAMRMClientRelayerMetrics.java | 513 +++ .../uam/TestUnmanagedApplicationManager.java | 2 +- .../server/nodemanager/ContainerExecutor.java | 3 +- .../nodemanager/LinuxContainerExecutor.java | 15 +- .../yarn/server/nodemanager/NodeManager.java | 70 +- .../server/nodemanager/NodeStatusUpdater.java | 14 + .../nodemanager/NodeStatusUpdaterImpl.java | 92 +- .../amrmproxy/FederationInterceptor.java | 19 +- .../containermanager/ContainerManagerImpl.java | 29 +- .../launcher/ContainerLaunch.java | 5 +- .../linux/resources/ResourceHandlerModule.java | 15 + .../TrafficControlBandwidthHandlerImpl.java | 2 +- .../runtime/DockerLinuxContainerRuntime.java | 95 +- .../JavaSandboxLinuxContainerRuntime.java | 28 +- .../runtime/docker/DockerCommandExecutor.java | 73 +- .../runtime/docker/DockerInspectCommand.java | 16 +- .../linux/runtime/docker/DockerRmCommand.java | 11 +- .../localizer/ResourceLocalizationService.java | 87 +- .../AbstractNodeDescriptorsProvider.java | 197 + .../nodelabels/AbstractNodeLabelsProvider.java | 149 - .../ConfigurationNodeAttributesProvider.java | 156 + .../ConfigurationNodeLabelsProvider.java | 13 +- .../nodelabels/NodeAttributesProvider.java | 32 + .../nodelabels/NodeDescriptorsProvider.java | 45 + .../nodelabels/NodeDescriptorsScriptRunner.java | 84 + .../nodelabels/NodeLabelsProvider.java | 22 +- .../ScriptBasedNodeAttributesProvider.java | 158 + .../ScriptBasedNodeLabelsProvider.java | 126 +- .../nodemanager/nodelabels/package-info.java | 28 + .../recovery/NMLeveldbStateStoreService.java | 173 +- .../recovery/NMStateStoreService.java | 29 +- .../impl/container-executor.c | 153 +- .../impl/container-executor.h | 8 +- .../main/native/container-executor/impl/main.c | 12 +- .../container-executor/impl/utils/docker-util.c | 5 +- .../container-executor/impl/utils/docker-util.h | 30 +- .../test/test-container-executor.c | 147 + .../server/nodemanager/TestNodeManager.java | 2 +- .../TestNodeStatusUpdaterForLabels.java | 52 +- .../TestableFederationInterceptor.java | 5 +- .../launcher/TestContainerLaunch.java | 81 + .../runtime/TestDockerContainerRuntime.java | 47 +- .../docker/TestDockerCommandExecutor.java | 23 +- .../runtime/docker/TestDockerRmCommand.java | 35 +- ...TestConfigurationNodeAttributesProvider.java | 262 ++ .../TestConfigurationNodeLabelsProvider.java | 26 +- .../TestScriptBasedNodeAttributesProvider.java | 251 ++ .../TestScriptBasedNodeLabelsProvider.java | 18 +- .../recovery/NMMemoryStateStoreService.java | 18 +- .../TestNMLeveldbStateStoreService.java | 269 +- .../server/resourcemanager/AdminService.java | 106 + .../server/resourcemanager/ClientRMService.java | 71 +- .../resourcemanager/DefaultAMSProcessor.java | 3 +- .../resourcemanager/RMActiveServiceContext.java | 14 + .../server/resourcemanager/RMAppManager.java | 5 +- .../yarn/server/resourcemanager/RMContext.java | 5 + .../server/resourcemanager/RMContextImpl.java | 11 + .../server/resourcemanager/ResourceManager.java | 14 +- .../resourcemanager/ResourceTrackerService.java | 31 +- .../ProportionalCapacityPreemptionPolicy.java | 3 + .../FileSystemNodeAttributeStore.java | 102 + .../nodelabels/NodeAttributesManagerImpl.java | 754 ++++ .../nodelabels/NodeAttributesStoreEvent.java | 52 + .../NodeAttributesStoreEventType.java | 26 + .../nodelabels/NodeLabelsUtils.java | 19 + .../server/resourcemanager/rmnode/RMNode.java | 6 + .../resourcemanager/rmnode/RMNodeImpl.java | 11 + .../scheduler/SchedulerNode.java | 11 + .../scheduler/capacity/CapacityScheduler.java | 42 +- .../allocator/RegularContainerAllocator.java | 3 +- .../constraint/PlacementConstraintsUtil.java | 148 +- .../NodeAttributesUpdateSchedulerEvent.java | 41 + .../scheduler/event/SchedulerEventType.java | 1 + .../LocalityAppPlacementAllocator.java | 4 + .../webapp/dao/NodeAttributeInfo.java | 65 + .../webapp/dao/NodeAttributesInfo.java | 49 + .../resourcemanager/webapp/dao/NodeInfo.java | 10 + .../yarn/server/resourcemanager/MockNodes.java | 11 + .../resourcemanager/NodeAttributeTestUtils.java | 54 + .../server/resourcemanager/TestAppManager.java | 3 + .../resourcemanager/TestClientRMService.java | 246 +- .../resourcemanager/TestRMAdminService.java | 171 +- .../TestResourceTrackerService.java | 88 + ...alCapacityPreemptionPolicyMockFramework.java | 5 + ...ionalCapacityPreemptionPolicyIntraQueue.java | 90 +- .../nodelabels/NullRMNodeLabelsManager.java | 7 + .../TestFileSystemNodeAttributeStore.java | 279 ++ .../nodelabels/TestNodeAttributesManager.java | 364 ++ .../capacity/TestContainerAllocation.java | 68 + .../scheduler/capacity/TestUtils.java | 20 +- ...stSingleConstraintAppPlacementAllocator.java | 192 +- .../webapp/TestRMWebServicesNodeLabels.java | 2 +- .../webapp/TestRMWebServicesNodes.java | 2 +- .../server/router/clientrm/ClientMethod.java | 71 + .../DefaultClientRequestInterceptor.java | 25 + .../clientrm/FederationClientInterceptor.java | 121 +- .../router/clientrm/RouterClientRMService.java | 28 + .../router/clientrm/RouterYarnClientUtils.java | 55 + .../DefaultRMAdminRequestInterceptor.java | 9 + .../router/rmadmin/RouterRMAdminService.java | 10 + .../PassThroughClientRequestInterceptor.java | 25 + .../TestFederationClientInterceptor.java | 37 +- .../clientrm/TestRouterYarnClientUtils.java | 57 + .../PassThroughRMAdminRequestInterceptor.java | 9 + .../reader/TimelineReaderWebServices.java | 4 +- .../TestTimelineReaderWebServicesBasicAcl.java | 11 +- .../src/site/markdown/NodeAttributes.md | 156 + .../site/markdown/PlacementConstraints.md.vm | 2 + .../main/webapp/app/controllers/application.js | 5 +- .../app/controllers/yarn-app/components.js | 2 +- .../controllers/yarn-component-instance/info.js | 5 +- .../yarn-component-instances/info.js | 3 +- .../src/main/webapp/app/initializers/loader.js | 3 + .../src/main/webapp/app/routes/application.js | 4 +- .../app/routes/yarn-component-instance/info.js | 4 +- .../app/serializers/yarn-component-instance.js | 1 - .../webapp/app/serializers/yarn-container.js | 2 +- .../app/serializers/yarn-service-component.js | 2 +- .../app/serializers/yarn-timeline-container.js | 2 +- .../webapp/app/templates/yarn-app/configs.hbs | 7 +- .../templates/yarn-component-instance/info.hbs | 4 - pom.xml | 5 +- 1034 files changed, 72254 insertions(+), 47687 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c04e0c0e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c04e0c0e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c04e0c0e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c04e0c0e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournaledEditsCache.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournaledEditsCache.java index 2693301,0000000..387caa1 mode 100644,000000..100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournaledEditsCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournaledEditsCache.java @@@ -1,412 -1,0 +1,412 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.qjournal.server; + +import com.google.common.annotations.VisibleForTesting; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.TreeMap; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream; +import org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader; +import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp; +import org.apache.hadoop.util.AutoCloseableLock; + +/** + * An in-memory cache of edits in their serialized form. This is used to serve + * the {@link Journal#getJournaledEdits(long, int)} call, used by the + * QJM when {@value DFSConfigKeys#DFS_HA_TAILEDITS_INPROGRESS_KEY} is + * enabled. + * + * <p>When a batch of edits is received by the JournalNode, it is put into this + * cache via {@link #storeEdits(byte[], long, long, int)}. Edits must be + * stored contiguously; if a batch of edits is stored that does not align with + * the previously stored edits, the cache will be cleared before storing new + * edits to avoid gaps. This decision is made because gaps are only handled + * when in recovery mode, which the cache is not intended to be used for. + * + * <p>Batches of edits are stored in a {@link TreeMap} mapping the starting + * transaction ID of the batch to the data buffer. Upon retrieval, the + * relevant data buffers are concatenated together and a header is added + * to construct a fully-formed edit data stream. + * + * <p>The cache is of a limited size capacity determined by + * {@value DFSConfigKeys#DFS_JOURNALNODE_EDIT_CACHE_SIZE_KEY}. If the capacity + * is exceeded after adding a new batch of edits, batches of edits are removed + * until the total size is less than the capacity, starting from the ones + * containing the oldest transactions. Transactions range in size, but a + * decent rule of thumb is that 200 bytes are needed per transaction. Monitoring + * the {@link JournalMetrics#rpcRequestCacheMissAmount} metric is recommended + * to determine if the cache is too small; it will indicate both how many + * cache misses occurred, and how many more transactions would have been + * needed in the cache to serve the request. + */ +class JournaledEditsCache { + + private static final int INVALID_LAYOUT_VERSION = 0; + private static final long INVALID_TXN_ID = -1; + + /** The capacity, in bytes, of this cache. */ + private final int capacity; + + /** + * Read/write lock pair wrapped in AutoCloseable; these refer to the same + * underlying lock. + */ + private final AutoCloseableLock readLock; + private final AutoCloseableLock writeLock; + + // ** Start lock-protected fields ** + + /** + * Stores the actual data as a mapping of the StartTxnId of a batch of edits + * to the serialized batch of edits. Stores only contiguous ranges; that is, + * the last transaction ID in one batch is always one less than the first + * transaction ID in the next batch. Though the map is protected by the lock, + * individual data buffers are immutable and can be accessed without locking. + */ + private final NavigableMap<Long, byte[]> dataMap = new TreeMap<>(); + /** Stores the layout version currently present in the cache. */ + private int layoutVersion = INVALID_LAYOUT_VERSION; + /** Stores the serialized version of the header for the current version. */ + private ByteBuffer layoutHeader; + + /** + * The lowest/highest transaction IDs present in the cache. + * {@value INVALID_TXN_ID} if there are no transactions in the cache. + */ + private long lowestTxnId; + private long highestTxnId; + /** + * The lowest transaction ID that was ever present in the cache since last + * being reset (i.e. since initialization or since reset due to being out of + * sync with the Journal). Until the cache size goes above capacity, this is + * equal to lowestTxnId. + */ + private long initialTxnId; + /** The current total size of all buffers in this cache. */ + private int totalSize; + + // ** End lock-protected fields ** + + JournaledEditsCache(Configuration conf) { + capacity = conf.getInt(DFSConfigKeys.DFS_JOURNALNODE_EDIT_CACHE_SIZE_KEY, + DFSConfigKeys.DFS_JOURNALNODE_EDIT_CACHE_SIZE_DEFAULT); + if (capacity > 0.9 * Runtime.getRuntime().maxMemory()) { + Journal.LOG.warn(String.format("Cache capacity is set at %d bytes but " + + "maximum JVM memory is only %d bytes. It is recommended that you " + + "decrease the cache size or increase the heap size.", + capacity, Runtime.getRuntime().maxMemory())); + } + Journal.LOG.info("Enabling the journaled edits cache with a capacity " + + "of bytes: " + capacity); + ReadWriteLock lock = new ReentrantReadWriteLock(true); + readLock = new AutoCloseableLock(lock.readLock()); + writeLock = new AutoCloseableLock(lock.writeLock()); + initialize(INVALID_TXN_ID); + } + + /** + * Fetch the data for edits starting at the specific transaction ID, fetching + * up to {@code maxTxns} transactions. Populates a list of output buffers + * which contains a serialized version of the edits, and returns the count of + * edits contained within the serialized buffers. The serialized edits are + * prefixed with a standard edit log header containing information about the + * layout version. The transactions returned are guaranteed to have contiguous + * transaction IDs. + * + * If {@code requestedStartTxn} is higher than the highest transaction which + * has been added to this cache, a response with an empty buffer and a + * transaction count of 0 will be returned. If {@code requestedStartTxn} is + * lower than the lowest transaction currently contained in this cache, or no + * transactions have yet been added to the cache, an exception will be thrown. + * + * @param requestedStartTxn The ID of the first transaction to return. If any + * transactions are returned, it is guaranteed that + * the first one will have this ID. + * @param maxTxns The maximum number of transactions to return. + * @param outputBuffers A list to populate with output buffers. When + * concatenated, these form a full response. + * @return The number of transactions contained within the set of output + * buffers. + * @throws IOException If transactions are requested which cannot be served + * by this cache. + */ + int retrieveEdits(long requestedStartTxn, int maxTxns, + List<ByteBuffer> outputBuffers) throws IOException { + int txnCount = 0; + + try (AutoCloseableLock l = readLock.acquire()) { + if (lowestTxnId == INVALID_TXN_ID || requestedStartTxn < lowestTxnId) { + throw getCacheMissException(requestedStartTxn); + } else if (requestedStartTxn > highestTxnId) { + return 0; + } + outputBuffers.add(layoutHeader); + Iterator<Map.Entry<Long, byte[]>> incrBuffIter = + dataMap.tailMap(dataMap.floorKey(requestedStartTxn), true) + .entrySet().iterator(); + long prevTxn = requestedStartTxn; + byte[] prevBuf = null; + // Stop when maximum transactions reached... + while ((txnCount < maxTxns) && + // ... or there are no more entries ... + (incrBuffIter.hasNext() || prevBuf != null)) { + long currTxn; + byte[] currBuf; + if (incrBuffIter.hasNext()) { + Map.Entry<Long, byte[]> ent = incrBuffIter.next(); + currTxn = ent.getKey(); + currBuf = ent.getValue(); + } else { + // This accounts for the trailing entry + currTxn = highestTxnId + 1; + currBuf = null; + } + if (prevBuf != null) { // True except for the first loop iteration + outputBuffers.add(ByteBuffer.wrap(prevBuf)); + // if prevTxn < requestedStartTxn, the extra transactions will get + // removed after the loop, so don't include them in the txn count + txnCount += currTxn - Math.max(requestedStartTxn, prevTxn); + } + prevTxn = currTxn; + prevBuf = currBuf; + } + // Release the lock before doing operations on the buffers (deserializing + // to find transaction boundaries, and copying into an output buffer) + } + // Remove extra leading transactions in the first buffer + ByteBuffer firstBuf = outputBuffers.get(1); // 0th is the header + firstBuf.position( + findTransactionPosition(firstBuf.array(), requestedStartTxn)); + // Remove trailing transactions in the last buffer if necessary + if (txnCount > maxTxns) { + ByteBuffer lastBuf = outputBuffers.get(outputBuffers.size() - 1); + int limit = + findTransactionPosition(lastBuf.array(), requestedStartTxn + maxTxns); + lastBuf.limit(limit); + txnCount = maxTxns; + } + + return txnCount; + } + + /** + * Store a batch of serialized edits into this cache. Removes old batches + * as necessary to keep the total size of the cache below the capacity. + * See the class Javadoc for more info. + * + * This attempts to always handle malformed inputs gracefully rather than + * throwing an exception, to allow the rest of the Journal's operations + * to proceed normally. + * + * @param inputData A buffer containing edits in serialized form + * @param newStartTxn The txn ID of the first edit in {@code inputData} + * @param newEndTxn The txn ID of the last edit in {@code inputData} + * @param newLayoutVersion The version of the layout used to serialize + * the edits + */ + void storeEdits(byte[] inputData, long newStartTxn, long newEndTxn, + int newLayoutVersion) { + if (newStartTxn < 0 || newEndTxn < newStartTxn) { + Journal.LOG.error(String.format("Attempted to cache data of length %d " + + "with newStartTxn %d and newEndTxn %d", + inputData.length, newStartTxn, newEndTxn)); + return; + } + try (AutoCloseableLock l = writeLock.acquire()) { + if (newLayoutVersion != layoutVersion) { + try { + updateLayoutVersion(newLayoutVersion, newStartTxn); + } catch (IOException ioe) { + Journal.LOG.error(String.format("Unable to save new edits [%d, %d] " + + "due to exception when updating to new layout version %d", + newStartTxn, newEndTxn, newLayoutVersion), ioe); + return; + } + } else if (lowestTxnId == INVALID_TXN_ID) { + Journal.LOG.info("Initializing edits cache starting from txn ID " + + newStartTxn); + initialize(newStartTxn); + } else if (highestTxnId + 1 != newStartTxn) { + // Cache is out of sync; clear to avoid storing noncontiguous regions + Journal.LOG.error(String.format("Edits cache is out of sync; " + + "looked for next txn id at %d but got start txn id for " + + "cache put request at %d. Reinitializing at new request.", + highestTxnId + 1, newStartTxn)); + initialize(newStartTxn); + } + + while ((totalSize + inputData.length) > capacity && !dataMap.isEmpty()) { + Map.Entry<Long, byte[]> lowest = dataMap.firstEntry(); + dataMap.remove(lowest.getKey()); + totalSize -= lowest.getValue().length; + } + if (inputData.length > capacity) { + initialize(INVALID_TXN_ID); + Journal.LOG.warn(String.format("A single batch of edits was too " + + "large to fit into the cache: startTxn = %d, endTxn = %d, " + + "input length = %d. The capacity of the cache (%s) must be " + + "increased for it to work properly (current capacity %d)." + + "Cache is now empty.", + newStartTxn, newEndTxn, inputData.length, + DFSConfigKeys.DFS_JOURNALNODE_EDIT_CACHE_SIZE_KEY, capacity)); + return; + } + if (dataMap.isEmpty()) { + lowestTxnId = newStartTxn; + } else { + lowestTxnId = dataMap.firstKey(); + } + + dataMap.put(newStartTxn, inputData); + highestTxnId = newEndTxn; + totalSize += inputData.length; + } + } + + /** + * Skip through a given stream of edits until the given transaction ID is + * found. Return the number of bytes that appear prior to the given + * transaction. + * + * @param buf A buffer containing a stream of serialized edits + * @param txnId The transaction ID to search for + * @return The number of bytes appearing in {@code buf} <i>before</i> + * the start of the transaction with ID {@code txnId}. + */ + private int findTransactionPosition(byte[] buf, long txnId) + throws IOException { + ByteArrayInputStream bais = new ByteArrayInputStream(buf); + FSEditLogLoader.PositionTrackingInputStream tracker = + new FSEditLogLoader.PositionTrackingInputStream(bais); + FSEditLogOp.Reader reader = FSEditLogOp.Reader.create( + new DataInputStream(tracker), tracker, layoutVersion); + long previousPos = 0; + while (reader.scanOp() < txnId) { + previousPos = tracker.getPos(); + } + // tracker is backed by a byte[]; position cannot go above an integer + return (int) previousPos; + } + + /** + * Update the layout version of the cache. This clears out all existing + * entries, and populates the new layout version and header for that version. + * + * @param newLayoutVersion The new layout version to be stored in the cache + * @param newStartTxn The new lowest transaction in the cache + */ + private void updateLayoutVersion(int newLayoutVersion, long newStartTxn) + throws IOException { + StringBuilder logMsg = new StringBuilder() + .append("Updating edits cache to use layout version ") + .append(newLayoutVersion) + .append(" starting from txn ID ") + .append(newStartTxn); + if (layoutVersion != INVALID_LAYOUT_VERSION) { + logMsg.append("; previous version was ").append(layoutVersion) + .append("; old entries will be cleared."); + } - Journal.LOG.info(logMsg); ++ Journal.LOG.info(logMsg.toString()); + initialize(newStartTxn); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + EditLogFileOutputStream.writeHeader(newLayoutVersion, + new DataOutputStream(baos)); + layoutVersion = newLayoutVersion; + layoutHeader = ByteBuffer.wrap(baos.toByteArray()); + } + + /** + * Initialize the cache back to a clear state. + * + * @param newInitialTxnId The new lowest transaction ID stored in the cache. + * This should be {@value INVALID_TXN_ID} if the cache + * is to remain empty at this time. + */ + private void initialize(long newInitialTxnId) { + dataMap.clear(); + totalSize = 0; + initialTxnId = newInitialTxnId; + lowestTxnId = initialTxnId; + highestTxnId = INVALID_TXN_ID; // this will be set later + } + + /** + * Return the underlying data buffer used to store information about the + * given transaction ID. + * + * @param txnId Transaction ID whose containing buffer should be fetched. + * @return The data buffer for the transaction + */ + @VisibleForTesting + byte[] getRawDataForTests(long txnId) { + try (AutoCloseableLock l = readLock.acquire()) { + return dataMap.floorEntry(txnId).getValue(); + } + } + + private CacheMissException getCacheMissException(long requestedTxnId) { + if (lowestTxnId == INVALID_TXN_ID) { + return new CacheMissException(0, "Cache is empty; either it was never " + + "written to or the last write overflowed the cache capacity."); + } else if (requestedTxnId < initialTxnId) { + return new CacheMissException(initialTxnId - requestedTxnId, + "Cache started at txn ID %d but requested txns starting at %d.", + initialTxnId, requestedTxnId); + } else { + return new CacheMissException(lowestTxnId - requestedTxnId, + "Oldest txn ID available in the cache is %d, but requested txns " + + "starting at %d. The cache size (%s) may need to be increased " + + "to hold more transactions (currently %d bytes containing %d " + + "transactions)", lowestTxnId, requestedTxnId, + DFSConfigKeys.DFS_JOURNALNODE_EDIT_CACHE_SIZE_KEY, capacity, + highestTxnId - lowestTxnId + 1); + } + } + + static class CacheMissException extends IOException { + + private static final long serialVersionUID = 0L; + + private final long cacheMissAmount; + + CacheMissException(long cacheMissAmount, String msgFormat, + Object... msgArgs) { + super(String.format(msgFormat, msgArgs)); + this.cacheMissAmount = cacheMissAmount; + } + + long getCacheMissAmount() { + return cacheMissAmount; + } + + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c04e0c0e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c04e0c0e/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c04e0c0e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
