Merge branch 'trunk' into hdfs-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3100002d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3100002d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3100002d Branch: refs/heads/hdfs-7240 Commit: 3100002deac6781bd15a5e1a46e2007243bf5186 Parents: b14a70e 5667129 Author: Anu Engineer <aengin...@apache.org> Authored: Wed Nov 4 14:19:34 2015 -0800 Committer: Anu Engineer <aengin...@apache.org> Committed: Wed Nov 4 14:19:34 2015 -0800 ---------------------------------------------------------------------- .gitignore | 1 + LICENSE.txt | 59 + dev-support/docker/Dockerfile | 7 +- dev-support/test-patch.sh | 10 +- .../main/resources/assemblies/hadoop-dist.xml | 4 +- .../assemblies/hadoop-hdfs-nfs-dist.xml | 4 +- .../resources/assemblies/hadoop-httpfs-dist.xml | 4 +- .../resources/assemblies/hadoop-kms-dist.xml | 4 +- .../assemblies/hadoop-mapreduce-dist.xml | 4 +- .../resources/assemblies/hadoop-nfs-dist.xml | 4 +- .../main/resources/assemblies/hadoop-sls.xml | 4 +- .../main/resources/assemblies/hadoop-src.xml | 4 +- .../main/resources/assemblies/hadoop-tools.xml | 4 +- .../resources/assemblies/hadoop-yarn-dist.xml | 4 +- hadoop-client/pom.xml | 6 +- .../JWTRedirectAuthenticationHandler.java | 7 +- .../server/KerberosAuthenticationHandler.java | 4 +- .../TestJWTRedirectAuthentictionHandler.java | 42 +- .../hadoop-common/CHANGES-HDFS-EC-7285.txt | 74 - hadoop-common-project/hadoop-common/CHANGES.txt | 270 +- hadoop-common-project/hadoop-common/pom.xml | 5 + .../hadoop-common/src/main/bin/hadoop | 15 +- .../hadoop-common/src/main/bin/hadoop-daemon.sh | 6 +- .../src/main/bin/hadoop-daemons.sh | 6 +- .../src/main/bin/hadoop-functions.sh | 109 +- .../src/main/bin/hadoop-layout.sh.example | 16 +- .../hadoop-common/src/main/bin/rcc | 4 +- .../hadoop-common/src/main/bin/slaves.sh | 6 +- .../hadoop-common/src/main/bin/start-all.sh | 6 +- .../hadoop-common/src/main/bin/stop-all.sh | 6 +- .../main/conf/hadoop-user-functions.sh.example | 10 +- .../org/apache/hadoop/conf/Configuration.java | 2 +- .../fs/CommonConfigurationKeysPublic.java | 11 + .../java/org/apache/hadoop/fs/FileSystem.java | 26 +- .../java/org/apache/hadoop/fs/FileUtil.java | 29 - .../org/apache/hadoop/fs/FilterFileSystem.java | 8 +- .../java/org/apache/hadoop/fs/GlobFilter.java | 2 +- .../java/org/apache/hadoop/fs/GlobPattern.java | 7 +- .../main/java/org/apache/hadoop/fs/Globber.java | 2 +- .../org/apache/hadoop/fs/HarFileSystem.java | 6 + .../java/org/apache/hadoop/fs/HardLink.java | 8 + .../org/apache/hadoop/fs/LocalDirAllocator.java | 6 +- .../apache/hadoop/fs/shell/CopyCommands.java | 6 +- .../java/org/apache/hadoop/fs/shell/Delete.java | 2 +- .../apache/hadoop/ha/ActiveStandbyElector.java | 53 +- .../org/apache/hadoop/ha/HAServiceTarget.java | 50 +- .../org/apache/hadoop/ha/HealthMonitor.java | 2 +- .../org/apache/hadoop/http/HttpServer2.java | 2 + .../java/org/apache/hadoop/io/SequenceFile.java | 15 +- .../org/apache/hadoop/io/WritableUtils.java | 8 +- .../apache/hadoop/io/erasurecode/CodecUtil.java | 2 + .../apache/hadoop/io/erasurecode/ECBlock.java | 3 + .../hadoop/io/erasurecode/ECBlockGroup.java | 3 + .../apache/hadoop/io/erasurecode/ECChunk.java | 3 + .../apache/hadoop/io/erasurecode/ECSchema.java | 5 + .../erasurecode/codec/AbstractErasureCodec.java | 2 + .../io/erasurecode/codec/ErasureCodec.java | 2 + .../io/erasurecode/codec/RSErasureCodec.java | 2 + .../io/erasurecode/codec/XORErasureCodec.java | 2 + .../erasurecode/coder/AbstractErasureCoder.java | 2 + .../coder/AbstractErasureCodingStep.java | 2 + .../coder/AbstractErasureDecoder.java | 25 +- .../coder/AbstractErasureEncoder.java | 2 + .../io/erasurecode/coder/ErasureCoder.java | 2 + .../io/erasurecode/coder/ErasureCodingStep.java | 2 + .../erasurecode/coder/ErasureDecodingStep.java | 2 + .../erasurecode/coder/ErasureEncodingStep.java | 2 + .../io/erasurecode/coder/RSErasureDecoder.java | 2 + .../io/erasurecode/coder/RSErasureEncoder.java | 2 + .../io/erasurecode/coder/XORErasureDecoder.java | 2 + .../io/erasurecode/coder/XORErasureEncoder.java | 2 + .../io/erasurecode/grouper/BlockGrouper.java | 2 + .../rawcoder/AbstractRawErasureCoder.java | 114 +- .../rawcoder/AbstractRawErasureDecoder.java | 10 +- .../rawcoder/AbstractRawErasureEncoder.java | 10 +- .../io/erasurecode/rawcoder/CoderOption.java | 43 + .../erasurecode/rawcoder/DummyRawDecoder.java | 47 + .../erasurecode/rawcoder/DummyRawEncoder.java | 46 + .../rawcoder/DummyRawErasureCoderFactory.java | 36 + .../io/erasurecode/rawcoder/RSRawDecoder.java | 75 +- .../io/erasurecode/rawcoder/RSRawEncoder.java | 50 +- .../rawcoder/RSRawErasureCoderFactory.java | 3 + .../erasurecode/rawcoder/RawErasureCoder.java | 25 +- .../rawcoder/RawErasureCoderFactory.java | 3 + .../erasurecode/rawcoder/RawErasureDecoder.java | 42 +- .../erasurecode/rawcoder/RawErasureEncoder.java | 37 +- .../io/erasurecode/rawcoder/XORRawDecoder.java | 4 +- .../io/erasurecode/rawcoder/XORRawEncoder.java | 4 +- .../rawcoder/XORRawErasureCoderFactory.java | 3 + .../io/erasurecode/rawcoder/util/DumpUtil.java | 2 + .../erasurecode/rawcoder/util/GaloisField.java | 3 + .../io/erasurecode/rawcoder/util/RSUtil.java | 3 + .../hadoop/io/retry/RetryInvocationHandler.java | 4 + .../apache/hadoop/io/retry/RetryPolicies.java | 46 +- .../org/apache/hadoop/ipc/CallerContext.java | 150 + .../main/java/org/apache/hadoop/ipc/Client.java | 83 +- .../main/java/org/apache/hadoop/ipc/Server.java | 134 +- .../apache/hadoop/metrics/ContextFactory.java | 3 + .../apache/hadoop/metrics/MetricsContext.java | 3 + .../apache/hadoop/metrics/MetricsException.java | 2 + .../apache/hadoop/metrics/MetricsRecord.java | 3 + .../apache/hadoop/metrics/MetricsServlet.java | 3 + .../org/apache/hadoop/metrics/MetricsUtil.java | 2 + .../java/org/apache/hadoop/metrics/Updater.java | 3 + .../hadoop/metrics/ganglia/GangliaContext.java | 5 +- .../metrics/ganglia/GangliaContext31.java | 4 + .../apache/hadoop/metrics/jvm/EventCounter.java | 3 + .../apache/hadoop/metrics/jvm/JvmMetrics.java | 3 + .../metrics/spi/AbstractMetricsContext.java | 3 + .../hadoop/metrics/spi/CompositeContext.java | 4 + .../apache/hadoop/metrics/spi/MetricValue.java | 1 + .../hadoop/metrics/spi/MetricsRecordImpl.java | 4 + .../metrics/spi/NoEmitMetricsContext.java | 3 + .../apache/hadoop/metrics/spi/NullContext.java | 2 + .../spi/NullContextWithUpdateThread.java | 2 + .../apache/hadoop/metrics/spi/OutputRecord.java | 3 + .../org/apache/hadoop/metrics/spi/Util.java | 3 + .../apache/hadoop/metrics/util/MBeanUtil.java | 2 + .../apache/hadoop/metrics/util/MetricsBase.java | 2 + .../metrics/util/MetricsDynamicMBeanBase.java | 3 +- .../hadoop/metrics/util/MetricsIntValue.java | 2 + .../hadoop/metrics/util/MetricsLongValue.java | 2 + .../hadoop/metrics/util/MetricsRegistry.java | 2 + .../metrics/util/MetricsTimeVaryingInt.java | 3 +- .../metrics/util/MetricsTimeVaryingLong.java | 3 +- .../metrics/util/MetricsTimeVaryingRate.java | 2 + .../metrics2/filter/AbstractPatternFilter.java | 4 +- .../hadoop/metrics2/filter/GlobFilter.java | 2 +- .../hadoop/metrics2/filter/RegexFilter.java | 2 +- .../hadoop/metrics2/impl/MetricsSystemImpl.java | 3 +- .../hadoop/metrics2/lib/MetricsRegistry.java | 9 +- .../hadoop/metrics2/source/JvmMetrics.java | 15 +- .../org/apache/hadoop/net/NetworkTopology.java | 34 +- .../net/NetworkTopologyWithNodeGroup.java | 2 +- .../apache/hadoop/net/unix/DomainSocket.java | 3 + .../HttpCrossOriginFilterInitializer.java | 74 + .../apache/hadoop/security/SaslRpcClient.java | 75 +- .../hadoop/security/UserGroupInformation.java | 9 +- .../authorize/DefaultImpersonationProvider.java | 4 + .../hadoop/security/http/CrossOriginFilter.java | 259 ++ .../org/apache/hadoop/util/DataChecksum.java | 2 +- .../org/apache/hadoop/util/MachineList.java | 5 +- .../hadoop/util/NativeLibraryChecker.java | 25 +- .../java/org/apache/hadoop/util/ProtoUtil.java | 13 + .../main/java/org/apache/hadoop/util/Shell.java | 773 ++-- .../org/apache/hadoop/util/SysInfoWindows.java | 5 +- .../org/apache/hadoop/net/unix/DomainSocket.c | 63 +- .../JniBasedUnixGroupsNetgroupMapping.c | 18 +- .../src/main/proto/RpcHeader.proto | 9 + .../src/main/resources/core-default.xml | 36 + .../src/site/markdown/FileSystemShell.md | 4 + .../src/site/markdown/HttpAuthentication.md | 14 + .../hadoop-common/src/site/markdown/Metrics.md | 1 + .../hadoop-common/src/site/markdown/Tracing.md | 59 +- .../markdown/release/0.1.0/CHANGES.0.1.0.md | 117 + .../release/0.1.0/RELEASENOTES.0.1.0.md | 24 + .../markdown/release/0.1.1/CHANGES.0.1.1.md | 67 + .../release/0.1.1/RELEASENOTES.0.1.1.md | 24 + .../markdown/release/0.10.0/CHANGES.0.10.0.md | 123 + .../release/0.10.0/RELEASENOTES.0.10.0.md | 24 + .../markdown/release/0.10.1/CHANGES.0.10.1.md | 77 + .../release/0.10.1/RELEASENOTES.0.10.1.md | 24 + .../markdown/release/0.11.0/CHANGES.0.11.0.md | 118 + .../release/0.11.0/RELEASENOTES.0.11.0.md | 24 + .../markdown/release/0.11.1/CHANGES.0.11.1.md | 68 + .../release/0.11.1/RELEASENOTES.0.11.1.md | 24 + .../markdown/release/0.11.2/CHANGES.0.11.2.md | 67 + .../release/0.11.2/RELEASENOTES.0.11.2.md | 24 + .../markdown/release/0.12.0/CHANGES.0.12.0.md | 129 + .../release/0.12.0/RELEASENOTES.0.12.0.md | 24 + .../markdown/release/0.12.1/CHANGES.0.12.1.md | 87 + .../release/0.12.1/RELEASENOTES.0.12.1.md | 24 + .../markdown/release/0.12.2/CHANGES.0.12.2.md | 68 + .../release/0.12.2/RELEASENOTES.0.12.2.md | 24 + .../markdown/release/0.12.3/CHANGES.0.12.3.md | 72 + .../release/0.12.3/RELEASENOTES.0.12.3.md | 24 + .../markdown/release/0.13.0/CHANGES.0.13.0.md | 195 + .../release/0.13.0/RELEASENOTES.0.13.0.md | 31 + .../markdown/release/0.13.1/CHANGES.0.13.1.md | 64 + .../release/0.13.1/RELEASENOTES.0.13.1.md | 24 + .../markdown/release/0.14.0/CHANGES.0.14.0.md | 224 ++ .../release/0.14.0/RELEASENOTES.0.14.0.md | 24 + .../markdown/release/0.14.1/CHANGES.0.14.1.md | 67 + .../release/0.14.1/RELEASENOTES.0.14.1.md | 24 + .../markdown/release/0.14.2/CHANGES.0.14.2.md | 74 + .../release/0.14.2/RELEASENOTES.0.14.2.md | 24 + .../markdown/release/0.14.3/CHANGES.0.14.3.md | 68 + .../release/0.14.3/RELEASENOTES.0.14.3.md | 24 + .../markdown/release/0.14.4/CHANGES.0.14.4.md | 67 + .../release/0.14.4/RELEASENOTES.0.14.4.md | 24 + .../markdown/release/0.15.0/CHANGES.0.15.0.md | 206 ++ .../release/0.15.0/RELEASENOTES.0.15.0.md | 24 + .../markdown/release/0.15.1/CHANGES.0.15.1.md | 71 + .../release/0.15.1/RELEASENOTES.0.15.1.md | 24 + .../markdown/release/0.15.2/CHANGES.0.15.2.md | 79 + .../release/0.15.2/RELEASENOTES.0.15.2.md | 24 + .../markdown/release/0.15.3/CHANGES.0.15.3.md | 69 + .../release/0.15.3/RELEASENOTES.0.15.3.md | 24 + .../markdown/release/0.15.4/CHANGES.0.15.4.md | 65 + .../release/0.15.4/RELEASENOTES.0.15.4.md | 24 + .../markdown/release/0.16.0/CHANGES.0.16.0.md | 241 ++ .../release/0.16.0/RELEASENOTES.0.16.0.md | 24 + .../markdown/release/0.16.1/CHANGES.0.16.1.md | 104 + .../release/0.16.1/RELEASENOTES.0.16.1.md | 24 + .../markdown/release/0.16.2/CHANGES.0.16.2.md | 87 + .../release/0.16.2/RELEASENOTES.0.16.2.md | 31 + .../markdown/release/0.16.3/CHANGES.0.16.3.md | 71 + .../release/0.16.3/RELEASENOTES.0.16.3.md | 31 + .../markdown/release/0.16.4/CHANGES.0.16.4.md | 68 + .../release/0.16.4/RELEASENOTES.0.16.4.md | 31 + .../markdown/release/0.17.0/CHANGES.0.17.0.md | 265 ++ .../release/0.17.0/RELEASENOTES.0.17.0.md | 604 ++++ .../markdown/release/0.17.1/CHANGES.0.17.1.md | 74 + .../release/0.17.1/RELEASENOTES.0.17.1.md | 38 + .../markdown/release/0.17.2/CHANGES.0.17.2.md | 77 + .../release/0.17.2/RELEASENOTES.0.17.2.md | 52 + .../markdown/release/0.17.3/CHANGES.0.17.3.md | 70 + .../release/0.17.3/RELEASENOTES.0.17.3.md | 45 + .../markdown/release/0.18.0/CHANGES.0.18.0.md | 337 ++ .../release/0.18.0/RELEASENOTES.0.18.0.md | 545 +++ .../markdown/release/0.18.1/CHANGES.0.18.1.md | 74 + .../release/0.18.1/RELEASENOTES.0.18.1.md | 38 + .../markdown/release/0.18.2/CHANGES.0.18.2.md | 88 + .../release/0.18.2/RELEASENOTES.0.18.2.md | 59 + .../markdown/release/0.18.3/CHANGES.0.18.3.md | 115 + .../release/0.18.3/RELEASENOTES.0.18.3.md | 104 + .../markdown/release/0.18.4/CHANGES.0.18.4.md | 73 + .../release/0.18.4/RELEASENOTES.0.18.4.md | 31 + .../markdown/release/0.19.0/CHANGES.0.19.0.md | 422 +++ .../release/0.19.0/RELEASENOTES.0.19.0.md | 529 +++ .../markdown/release/0.19.1/CHANGES.0.19.1.md | 112 + .../release/0.19.1/RELEASENOTES.0.19.1.md | 101 + .../markdown/release/0.19.2/CHANGES.0.19.2.md | 104 + .../release/0.19.2/RELEASENOTES.0.19.2.md | 38 + .../markdown/release/0.2.0/CHANGES.0.2.0.md | 119 + .../release/0.2.0/RELEASENOTES.0.2.0.md | 24 + .../markdown/release/0.2.1/CHANGES.0.2.1.md | 67 + .../release/0.2.1/RELEASENOTES.0.2.1.md | 24 + .../markdown/release/0.20.0/CHANGES.0.20.0.md | 347 ++ .../release/0.20.0/RELEASENOTES.0.20.0.md | 347 ++ .../markdown/release/0.20.1/CHANGES.0.20.1.md | 139 + .../release/0.20.1/RELEASENOTES.0.20.1.md | 218 ++ .../markdown/release/0.20.2/CHANGES.0.20.2.md | 109 + .../release/0.20.2/RELEASENOTES.0.20.2.md | 150 + .../release/0.20.203.0/CHANGES.0.20.203.0.md | 95 + .../0.20.203.0/RELEASENOTES.0.20.203.0.md | 117 + .../release/0.20.203.1/CHANGES.0.20.203.1.md | 66 + .../0.20.203.1/RELEASENOTES.0.20.203.1.md | 24 + .../release/0.20.204.0/CHANGES.0.20.204.0.md | 127 + .../0.20.204.0/RELEASENOTES.0.20.204.0.md | 83 + .../release/0.20.204.1/CHANGES.0.20.204.1.md | 64 + .../0.20.204.1/RELEASENOTES.0.20.204.1.md | 24 + .../release/0.20.205.0/CHANGES.0.20.205.0.md | 191 + .../0.20.205.0/RELEASENOTES.0.20.205.0.md | 195 + .../markdown/release/0.20.3/CHANGES.0.20.3.md | 103 + .../release/0.20.3/RELEASENOTES.0.20.3.md | 45 + .../markdown/release/0.21.0/CHANGES.0.21.0.md | 1360 +++++++ .../release/0.21.0/RELEASENOTES.0.21.0.md | 2232 ++++++++++++ .../markdown/release/0.21.1/CHANGES.0.21.1.md | 123 + .../release/0.21.1/RELEASENOTES.0.21.1.md | 94 + .../markdown/release/0.22.0/CHANGES.0.22.0.md | 762 ++++ .../release/0.22.0/RELEASENOTES.0.22.0.md | 630 ++++ .../markdown/release/0.22.1/CHANGES.0.22.1.md | 113 + .../release/0.22.1/RELEASENOTES.0.22.1.md | 64 + .../markdown/release/0.23.0/CHANGES.0.23.0.md | 1182 +++++++ .../release/0.23.0/RELEASENOTES.0.23.0.md | 1175 ++++++ .../markdown/release/0.23.1/CHANGES.0.23.1.md | 484 +++ .../release/0.23.1/RELEASENOTES.0.23.1.md | 677 ++++ .../markdown/release/0.23.10/CHANGES.0.23.10.md | 132 + .../release/0.23.10/RELEASENOTES.0.23.10.md | 31 + .../markdown/release/0.23.11/CHANGES.0.23.11.md | 105 + .../release/0.23.11/RELEASENOTES.0.23.11.md | 24 + .../markdown/release/0.23.2/CHANGES.0.23.2.md | 177 + .../release/0.23.2/RELEASENOTES.0.23.2.md | 199 ++ .../markdown/release/0.23.3/CHANGES.0.23.3.md | 334 ++ .../release/0.23.3/RELEASENOTES.0.23.3.md | 115 + .../markdown/release/0.23.4/CHANGES.0.23.4.md | 96 + .../release/0.23.4/RELEASENOTES.0.23.4.md | 24 + .../markdown/release/0.23.5/CHANGES.0.23.5.md | 164 + .../release/0.23.5/RELEASENOTES.0.23.5.md | 38 + .../markdown/release/0.23.6/CHANGES.0.23.6.md | 137 + .../release/0.23.6/RELEASENOTES.0.23.6.md | 24 + .../markdown/release/0.23.7/CHANGES.0.23.7.md | 195 + .../release/0.23.7/RELEASENOTES.0.23.7.md | 45 + .../markdown/release/0.23.8/CHANGES.0.23.8.md | 89 + .../release/0.23.8/RELEASENOTES.0.23.8.md | 24 + .../markdown/release/0.23.9/CHANGES.0.23.9.md | 82 + .../release/0.23.9/RELEASENOTES.0.23.9.md | 31 + .../markdown/release/0.24.0/CHANGES.0.24.0.md | 95 + .../release/0.24.0/RELEASENOTES.0.24.0.md | 52 + .../markdown/release/0.3.0/CHANGES.0.3.0.md | 102 + .../release/0.3.0/RELEASENOTES.0.3.0.md | 24 + .../markdown/release/0.3.1/CHANGES.0.3.1.md | 69 + .../release/0.3.1/RELEASENOTES.0.3.1.md | 24 + .../markdown/release/0.3.2/CHANGES.0.3.2.md | 77 + .../release/0.3.2/RELEASENOTES.0.3.2.md | 24 + .../markdown/release/0.4.0/CHANGES.0.4.0.md | 88 + .../release/0.4.0/RELEASENOTES.0.4.0.md | 24 + .../markdown/release/0.5.0/CHANGES.0.5.0.md | 122 + .../release/0.5.0/RELEASENOTES.0.5.0.md | 24 + .../markdown/release/0.6.0/CHANGES.0.6.0.md | 105 + .../release/0.6.0/RELEASENOTES.0.6.0.md | 24 + .../markdown/release/0.6.1/CHANGES.0.6.1.md | 69 + .../release/0.6.1/RELEASENOTES.0.6.1.md | 24 + .../markdown/release/0.6.2/CHANGES.0.6.2.md | 67 + .../release/0.6.2/RELEASENOTES.0.6.2.md | 24 + .../markdown/release/0.7.0/CHANGES.0.7.0.md | 103 + .../release/0.7.0/RELEASENOTES.0.7.0.md | 24 + .../markdown/release/0.7.1/CHANGES.0.7.1.md | 69 + .../release/0.7.1/RELEASENOTES.0.7.1.md | 24 + .../markdown/release/0.7.2/CHANGES.0.7.2.md | 67 + .../release/0.7.2/RELEASENOTES.0.7.2.md | 24 + .../markdown/release/0.8.0/CHANGES.0.8.0.md | 104 + .../release/0.8.0/RELEASENOTES.0.8.0.md | 24 + .../markdown/release/0.9.0/CHANGES.0.9.0.md | 115 + .../release/0.9.0/RELEASENOTES.0.9.0.md | 24 + .../markdown/release/0.9.1/CHANGES.0.9.1.md | 66 + .../release/0.9.1/RELEASENOTES.0.9.1.md | 24 + .../markdown/release/0.9.2/CHANGES.0.9.2.md | 67 + .../release/0.9.2/RELEASENOTES.0.9.2.md | 24 + .../markdown/release/1.0.0/CHANGES.1.0.0.md | 123 + .../release/1.0.0/RELEASENOTES.1.0.0.md | 72 + .../markdown/release/1.0.1/CHANGES.1.0.1.md | 78 + .../release/1.0.1/RELEASENOTES.1.0.1.md | 52 + .../markdown/release/1.0.2/CHANGES.1.0.2.md | 81 + .../release/1.0.2/RELEASENOTES.1.0.2.md | 49 + .../markdown/release/1.0.3/CHANGES.1.0.3.md | 90 + .../release/1.0.3/RELEASENOTES.1.0.3.md | 39 + .../markdown/release/1.0.4/CHANGES.1.0.4.md | 67 + .../release/1.0.4/RELEASENOTES.1.0.4.md | 24 + .../markdown/release/1.1.0/CHANGES.1.1.0.md | 199 ++ .../release/1.1.0/RELEASENOTES.1.1.0.md | 216 ++ .../markdown/release/1.1.1/CHANGES.1.1.1.md | 84 + .../release/1.1.1/RELEASENOTES.1.1.1.md | 24 + .../markdown/release/1.1.2/CHANGES.1.1.2.md | 89 + .../release/1.1.2/RELEASENOTES.1.1.2.md | 55 + .../markdown/release/1.1.3/CHANGES.1.1.3.md | 65 + .../release/1.1.3/RELEASENOTES.1.1.3.md | 24 + .../markdown/release/1.2.0/CHANGES.1.2.0.md | 280 ++ .../release/1.2.0/RELEASENOTES.1.2.0.md | 176 + .../markdown/release/1.2.1/CHANGES.1.2.1.md | 84 + .../release/1.2.1/RELEASENOTES.1.2.1.md | 38 + .../markdown/release/1.2.2/CHANGES.1.2.2.md | 66 + .../release/1.2.2/RELEASENOTES.1.2.2.md | 31 + .../markdown/release/1.3.0/CHANGES.1.3.0.md | 151 + .../release/1.3.0/RELEASENOTES.1.3.0.md | 69 + .../release/2.0.0-alpha/CHANGES.2.0.0-alpha.md | 398 +++ .../2.0.0-alpha/RELEASENOTES.2.0.0-alpha.md | 157 + .../release/2.0.1-alpha/CHANGES.2.0.1-alpha.md | 66 + .../2.0.1-alpha/RELEASENOTES.2.0.1-alpha.md | 31 + .../release/2.0.2-alpha/CHANGES.2.0.2-alpha.md | 714 ++++ .../2.0.2-alpha/RELEASENOTES.2.0.2-alpha.md | 273 ++ .../release/2.0.3-alpha/CHANGES.2.0.3-alpha.md | 588 +++ .../2.0.3-alpha/RELEASENOTES.2.0.3-alpha.md | 222 ++ .../release/2.0.4-alpha/CHANGES.2.0.4-alpha.md | 98 + .../2.0.4-alpha/RELEASENOTES.2.0.4-alpha.md | 38 + .../release/2.0.5-alpha/CHANGES.2.0.5-alpha.md | 69 + .../2.0.5-alpha/RELEASENOTES.2.0.5-alpha.md | 24 + .../release/2.0.6-alpha/CHANGES.2.0.6-alpha.md | 65 + .../2.0.6-alpha/RELEASENOTES.2.0.6-alpha.md | 24 + .../release/2.1.0-beta/CHANGES.2.1.0-beta.md | 912 +++++ .../2.1.0-beta/RELEASENOTES.2.1.0-beta.md | 757 ++++ .../release/2.1.1-beta/CHANGES.2.1.1-beta.md | 239 ++ .../2.1.1-beta/RELEASENOTES.2.1.1-beta.md | 52 + .../markdown/release/2.2.0/CHANGES.2.2.0.md | 152 + .../release/2.2.0/RELEASENOTES.2.2.0.md | 45 + .../markdown/release/2.2.1/CHANGES.2.2.1.md | 66 + .../release/2.2.1/RELEASENOTES.2.2.1.md | 24 + .../markdown/release/2.3.0/CHANGES.2.3.0.md | 664 ++++ .../release/2.3.0/RELEASENOTES.2.3.0.md | 138 + .../markdown/release/2.4.0/CHANGES.2.4.0.md | 486 +++ .../release/2.4.0/RELEASENOTES.2.4.0.md | 145 + .../markdown/release/2.4.1/CHANGES.2.4.1.md | 152 + .../release/2.4.1/RELEASENOTES.2.4.1.md | 31 + .../markdown/release/2.5.0/CHANGES.2.5.0.md | 551 +++ .../release/2.5.0/RELEASENOTES.2.5.0.md | 143 + .../markdown/release/2.5.1/CHANGES.2.5.1.md | 69 + .../release/2.5.1/RELEASENOTES.2.5.1.md | 24 + .../markdown/release/2.5.2/CHANGES.2.5.2.md | 69 + .../release/2.5.2/RELEASENOTES.2.5.2.md | 24 + .../markdown/release/2.6.0/CHANGES.2.6.0.md | 964 +++++ .../release/2.6.0/RELEASENOTES.2.6.0.md | 184 + .../markdown/release/2.7.0/CHANGES.2.7.0.md | 988 ++++++ .../release/2.7.0/RELEASENOTES.2.7.0.md | 295 ++ .../markdown/release/2.7.1/CHANGES.2.7.1.md | 195 + .../release/2.7.1/RELEASENOTES.2.7.1.md | 47 + .../org/apache/hadoop/fs/SymlinkBaseTest.java | 30 +- .../java/org/apache/hadoop/fs/TestFileUtil.java | 8 +- .../org/apache/hadoop/fs/TestGlobPattern.java | 9 +- .../apache/hadoop/fs/TestLocalDirAllocator.java | 17 + .../apache/hadoop/fs/TestSymlinkLocalFS.java | 11 - .../org/apache/hadoop/ha/DummyHAService.java | 47 +- .../org/apache/hadoop/ha/TestHealthMonitor.java | 10 +- ...HealthMonitorWithDedicatedHealthAddress.java | 37 + .../org/apache/hadoop/io/TestSequenceFile.java | 21 + .../hadoop/io/erasurecode/TestCoderBase.java | 82 +- .../erasurecode/coder/TestErasureCoderBase.java | 8 +- .../erasurecode/coder/TestRSErasureCoder.java | 7 +- .../erasurecode/rawcoder/TestDummyRawCoder.java | 83 + .../erasurecode/rawcoder/TestRawCoderBase.java | 45 +- .../erasurecode/rawcoder/TestXORRawCoder.java | 1 + .../apache/hadoop/io/retry/TestRetryProxy.java | 27 +- .../io/retry/UnreliableImplementation.java | 17 + .../hadoop/io/retry/UnreliableInterface.java | 3 + .../java/org/apache/hadoop/ipc/TestIPC.java | 37 +- .../hadoop/ipc/TestIPCServerResponder.java | 161 +- .../java/org/apache/hadoop/ipc/TestRPC.java | 11 +- .../java/org/apache/hadoop/ipc/TestSaslRPC.java | 114 +- .../hadoop/metrics/TestMetricsServlet.java | 1 + .../metrics/ganglia/TestGangliaContext.java | 1 + .../hadoop/metrics/spi/TestOutputRecord.java | 1 + .../metrics2/lib/TestMetricsRegistry.java | 16 + .../org/apache/hadoop/net/ServerSocketUtil.java | 4 +- .../java/org/apache/hadoop/net/TestDNS.java | 17 +- .../org/apache/hadoop/net/TestNetUtils.java | 26 +- .../hadoop/net/unix/TestDomainSocket.java | 7 +- .../TestHttpCrossOriginFilterInitializer.java | 58 + .../security/TestUserGroupInformation.java | 3 +- .../security/authorize/TestProxyUsers.java | 39 + .../security/http/TestCrossOriginFilter.java | 330 ++ .../apache/hadoop/test/GenericTestUtils.java | 33 +- .../hadoop/test/TestGenericTestUtils.java | 78 + .../apache/hadoop/tracing/TestTraceUtils.java | 18 + .../org/apache/hadoop/util/TestMachineList.java | 8 + .../java/org/apache/hadoop/util/TestShell.java | 276 +- .../org/apache/hadoop/util/TestWinUtils.java | 124 +- .../src/test/resources/testConf.xml | 22 +- .../scripts/hadoop_add_common_to_classpath.bats | 4 +- .../hadoop_add_to_classpath_toolspath.bats | 74 + .../hadoop_add_to_classpath_userpath.bats | 8 +- .../src/test/scripts/hadoop_basic_init.bats | 6 +- .../src/test/scripts/hadoop_ssh.bats | 8 +- .../src/test/scripts/hadoop_stop_daemon.bats | 31 + .../test/scripts/hadoop_stop_secure_daemon.bats | 37 + .../hadoop-kms/src/main/sbin/kms.sh | 6 +- .../hadoop/crypto/key/kms/server/MiniKMS.java | 12 +- hadoop-dist/pom.xml | 2 +- .../dev-support/findbugsExcludeFile.xml | 3 + hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 21 + .../java/org/apache/hadoop/fs/CacheFlag.java | 2 +- .../org/apache/hadoop/fs/HdfsBlockLocation.java | 9 +- .../main/java/org/apache/hadoop/fs/XAttr.java | 4 +- .../hadoop/hdfs/BlockMissingException.java | 9 +- .../org/apache/hadoop/hdfs/BlockReader.java | 6 +- .../apache/hadoop/hdfs/BlockReaderFactory.java | 39 +- .../apache/hadoop/hdfs/BlockReaderLocal.java | 68 +- .../hadoop/hdfs/BlockReaderLocalLegacy.java | 117 +- .../org/apache/hadoop/hdfs/BlockReaderUtil.java | 2 +- .../org/apache/hadoop/hdfs/ClientContext.java | 12 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 1463 ++++---- .../hadoop/hdfs/DFSClientFaultInjector.java | 2 +- .../hadoop/hdfs/DFSInotifyEventInputStream.java | 25 +- .../org/apache/hadoop/hdfs/DFSInputStream.java | 194 +- .../org/apache/hadoop/hdfs/DFSOutputStream.java | 166 +- .../java/org/apache/hadoop/hdfs/DFSPacket.java | 12 +- .../hadoop/hdfs/DFSStripedInputStream.java | 49 +- .../hadoop/hdfs/DFSStripedOutputStream.java | 136 +- .../org/apache/hadoop/hdfs/DFSUtilClient.java | 13 +- .../org/apache/hadoop/hdfs/DataStreamer.java | 143 +- .../hadoop/hdfs/DistributedFileSystem.java | 455 ++- .../org/apache/hadoop/hdfs/ExtendedBlockId.java | 5 +- .../apache/hadoop/hdfs/ExternalBlockReader.java | 2 +- .../apache/hadoop/hdfs/HdfsConfiguration.java | 152 + .../hadoop/hdfs/HdfsConfigurationLoader.java | 44 - .../apache/hadoop/hdfs/KeyProviderCache.java | 18 +- .../hadoop/hdfs/NameNodeProxiesClient.java | 4 +- .../java/org/apache/hadoop/hdfs/PeerCache.java | 39 +- .../apache/hadoop/hdfs/RemoteBlockReader.java | 148 +- .../apache/hadoop/hdfs/RemoteBlockReader2.java | 126 +- .../apache/hadoop/hdfs/RemotePeerFactory.java | 2 +- .../apache/hadoop/hdfs/StripedDataStreamer.java | 5 +- .../org/apache/hadoop/hdfs/XAttrHelper.java | 39 +- .../hdfs/client/HdfsClientConfigKeys.java | 107 +- .../hadoop/hdfs/client/HdfsDataInputStream.java | 10 +- .../hdfs/client/HdfsDataOutputStream.java | 31 +- .../client/impl/CorruptFileBlockIterator.java | 6 +- .../hadoop/hdfs/client/impl/DfsClientConf.java | 22 +- .../hadoop/hdfs/client/impl/LeaseRenewer.java | 20 +- .../hadoop/hdfs/client/impl/package-info.java | 2 +- .../org/apache/hadoop/hdfs/inotify/Event.java | 49 +- .../apache/hadoop/hdfs/net/BasicInetPeer.java | 8 +- .../org/apache/hadoop/hdfs/net/DomainPeer.java | 2 +- .../apache/hadoop/hdfs/net/EncryptedPeer.java | 6 +- .../org/apache/hadoop/hdfs/net/NioInetPeer.java | 4 +- .../java/org/apache/hadoop/hdfs/net/Peer.java | 36 +- .../org/apache/hadoop/hdfs/protocol/Block.java | 44 +- .../hdfs/protocol/BlockLocalPathInfo.java | 6 +- .../hdfs/protocol/BlockStoragePolicy.java | 8 +- .../hdfs/protocol/CacheDirectiveEntry.java | 2 +- .../hdfs/protocol/CacheDirectiveInfo.java | 21 +- .../hdfs/protocol/CacheDirectiveIterator.java | 11 +- .../hdfs/protocol/CacheDirectiveStats.java | 26 +- .../hadoop/hdfs/protocol/CachePoolInfo.java | 22 +- .../hadoop/hdfs/protocol/CachePoolIterator.java | 5 +- .../hadoop/hdfs/protocol/CachePoolStats.java | 14 +- .../hdfs/protocol/ClientDatanodeProtocol.java | 44 +- .../hadoop/hdfs/protocol/ClientProtocol.java | 6 +- .../hdfs/protocol/DSQuotaExceededException.java | 6 +- .../apache/hadoop/hdfs/protocol/DatanodeID.java | 12 +- .../hadoop/hdfs/protocol/DatanodeInfo.java | 77 +- .../hdfs/protocol/DatanodeInfoWithStorage.java | 3 +- .../hadoop/hdfs/protocol/DatanodeLocalInfo.java | 8 +- .../hdfs/protocol/EncryptionZoneIterator.java | 7 +- .../hdfs/protocol/ErasureCodingPolicy.java | 23 +- .../hadoop/hdfs/protocol/HdfsConstants.java | 2 +- .../hadoop/hdfs/protocol/HdfsFileStatus.java | 5 +- .../hdfs/protocol/HdfsLocatedFileStatus.java | 6 +- .../hdfs/protocol/LastBlockWithStatus.java | 3 +- .../hadoop/hdfs/protocol/LocatedBlock.java | 10 +- .../hadoop/hdfs/protocol/LocatedBlocks.java | 49 +- .../hdfs/protocol/LocatedStripedBlock.java | 1 + .../hdfs/protocol/NSQuotaExceededException.java | 2 +- .../hdfs/protocol/RollingUpgradeInfo.java | 8 +- .../hdfs/protocol/SnapshotDiffReport.java | 55 +- .../protocol/SnapshottableDirectoryStatus.java | 74 +- .../hdfs/protocol/UnresolvedPathException.java | 16 +- .../datatransfer/BlockConstructionStage.java | 8 +- .../datatransfer/DataTransferProtoUtil.java | 18 +- .../datatransfer/DataTransferProtocol.java | 56 +- .../protocol/datatransfer/IOStreamPair.java | 4 +- .../hadoop/hdfs/protocol/datatransfer/Op.java | 6 +- .../protocol/datatransfer/PacketHeader.java | 42 +- .../protocol/datatransfer/PacketReceiver.java | 42 +- .../hdfs/protocol/datatransfer/PipelineAck.java | 12 +- .../datatransfer/ReplaceDatanodeOnFailure.java | 118 +- .../hdfs/protocol/datatransfer/Sender.java | 119 +- .../datatransfer/TrustedChannelResolver.java | 9 +- .../datatransfer/sasl/DataTransferSaslUtil.java | 119 +- .../sasl/SaslDataTransferClient.java | 108 +- .../datatransfer/sasl/SaslParticipant.java | 12 +- .../SaslResponseWithNegotiatedCipherOption.java | 6 +- .../protocolPB/ClientDatanodeProtocolPB.java | 2 +- .../ClientDatanodeProtocolTranslatorPB.java | 36 +- .../protocolPB/ClientNamenodeProtocolPB.java | 6 +- .../ClientNamenodeProtocolTranslatorPB.java | 346 +- .../hadoop/hdfs/protocolPB/PBHelperClient.java | 690 ++-- .../delegation/DelegationTokenIdentifier.java | 23 +- .../delegation/DelegationTokenSelector.java | 8 +- .../server/datanode/BlockMetadataHeader.java | 32 +- .../hdfs/server/datanode/CachingStrategy.java | 4 +- .../datanode/ReplicaNotFoundException.java | 20 +- .../hdfs/server/namenode/SafeModeException.java | 6 +- .../ha/AbstractNNFailoverProxyProvider.java | 9 +- .../ha/WrappedFailoverProxyProvider.java | 4 +- .../hdfs/server/protocol/DatanodeStorage.java | 27 +- .../server/protocol/DatanodeStorageReport.java | 2 +- .../hdfs/server/protocol/StorageReport.java | 2 +- .../hadoop/hdfs/shortcircuit/ClientMmap.java | 6 +- .../hdfs/shortcircuit/DfsClientShmManager.java | 48 +- .../hdfs/shortcircuit/DomainSocketFactory.java | 5 +- .../hdfs/shortcircuit/ShortCircuitCache.java | 87 +- .../hdfs/shortcircuit/ShortCircuitReplica.java | 24 +- .../shortcircuit/ShortCircuitReplicaInfo.java | 9 +- .../hdfs/shortcircuit/ShortCircuitShm.java | 58 +- .../hdfs/util/ByteBufferOutputStream.java | 2 +- .../hadoop/hdfs/util/ExactSizeInputStream.java | 14 +- .../apache/hadoop/hdfs/util/LongBitFormat.java | 5 +- .../hadoop/hdfs/util/StripedBlockUtil.java | 44 +- .../apache/hadoop/hdfs/util/package-info.java | 2 +- .../hadoop/hdfs/web/ByteRangeInputStream.java | 32 +- .../apache/hadoop/hdfs/web/JsonUtilClient.java | 65 +- .../hadoop/hdfs/web/URLConnectionFactory.java | 43 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 136 +- .../hdfs/web/oauth2/AccessTokenProvider.java | 4 +- .../hdfs/web/oauth2/AccessTokenTimer.java | 14 +- .../ConfCredentialBasedAccessTokenProvider.java | 4 +- ...onfRefreshTokenBasedAccessTokenProvider.java | 26 +- .../CredentialBasedAccessTokenProvider.java | 22 +- .../oauth2/OAuth2ConnectionConfigurator.java | 24 +- .../hadoop/hdfs/web/oauth2/OAuth2Constants.java | 4 +- .../apache/hadoop/hdfs/web/oauth2/Utils.java | 6 +- .../hadoop/hdfs/web/oauth2/package-info.java | 2 +- .../hdfs/web/resources/AccessTimeParam.java | 2 +- .../hdfs/web/resources/AclPermissionParam.java | 2 +- .../hdfs/web/resources/BlockSizeParam.java | 2 +- .../hdfs/web/resources/BufferSizeParam.java | 2 +- .../hdfs/web/resources/ConcatSourcesParam.java | 3 +- .../hdfs/web/resources/CreateFlagParam.java | 2 +- .../hdfs/web/resources/CreateParentParam.java | 2 +- .../hdfs/web/resources/DelegationParam.java | 2 +- .../hdfs/web/resources/DeleteOpParam.java | 6 +- .../hadoop/hdfs/web/resources/DoAsParam.java | 2 +- .../hadoop/hdfs/web/resources/EnumParam.java | 3 +- .../hadoop/hdfs/web/resources/EnumSetParam.java | 3 +- .../web/resources/ExcludeDatanodesParam.java | 2 +- .../hadoop/hdfs/web/resources/GetOpParam.java | 8 +- .../hadoop/hdfs/web/resources/GroupParam.java | 2 +- .../hadoop/hdfs/web/resources/HttpOpParam.java | 20 +- .../hadoop/hdfs/web/resources/LengthParam.java | 2 +- .../hadoop/hdfs/web/resources/LongParam.java | 3 +- .../web/resources/ModificationTimeParam.java | 2 +- .../hdfs/web/resources/NewLengthParam.java | 2 +- .../hadoop/hdfs/web/resources/OffsetParam.java | 2 +- .../hdfs/web/resources/OverwriteParam.java | 2 +- .../hadoop/hdfs/web/resources/OwnerParam.java | 2 +- .../apache/hadoop/hdfs/web/resources/Param.java | 16 +- .../hdfs/web/resources/PermissionParam.java | 2 +- .../hadoop/hdfs/web/resources/PostOpParam.java | 6 +- .../hadoop/hdfs/web/resources/PutOpParam.java | 8 +- .../hdfs/web/resources/RecursiveParam.java | 2 +- .../web/resources/RenameOptionSetParam.java | 4 +- .../hadoop/hdfs/web/resources/RenewerParam.java | 2 +- .../hdfs/web/resources/ReplicationParam.java | 2 +- .../hadoop/hdfs/web/resources/UserParam.java | 6 +- .../hdfs/web/resources/XAttrEncodingParam.java | 2 +- .../hdfs/web/resources/XAttrNameParam.java | 6 +- .../hdfs/web/resources/XAttrSetFlagParam.java | 2 +- .../src/main/proto/ClientNamenodeProtocol.proto | 2 + .../hadoop/fs/TestUrlStreamHandlerFactory.java | 80 + .../java/org/apache/hadoop/fs/TestXAttr.java | 94 + .../org/apache/hadoop/hdfs/TestDFSPacket.java | 94 + .../hadoop/hdfs/TestDefaultNameNodePort.java | 66 + .../org/apache/hadoop/hdfs/TestPeerCache.java | 292 ++ .../hdfs/client/impl/TestLeaseRenewer.java | 207 ++ .../hadoop/hdfs/protocol/TestExtendedBlock.java | 77 + .../hdfs/shortcircuit/TestShortCircuitShm.java | 109 + .../hadoop/hdfs/util/TestByteArrayManager.java | 644 ++++ .../hdfs/util/TestExactSizeInputStream.java | 129 + .../hdfs/web/TestByteRangeInputStream.java | 292 ++ .../hdfs/web/TestOffsetUrlInputStream.java | 64 + .../apache/hadoop/hdfs/web/TestTokenAspect.java | 316 ++ .../hdfs/web/TestURLConnectionFactory.java | 50 + .../hadoop/hdfs/web/TestWebHDFSOAuth2.java | 217 ++ .../hdfs/web/TestWebHdfsContentLength.java | 197 ++ .../hdfs/web/oauth2/TestAccessTokenTimer.java | 63 + ...ClientCredentialTimeBasedTokenRefresher.java | 139 + ...TestRefreshTokenTimeBasedTokenRefresher.java | 138 + .../hadoop/fs/http/server/FSOperations.java | 2 +- .../hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh | 6 +- .../fs/http/client/BaseTestHttpFSWith.java | 3 +- .../hadoop/fs/http/server/TestHttpFSServer.java | 3 +- .../fs/http/server/TestHttpFSServerNoACLs.java | 5 +- .../hadoop-hdfs-native-client/pom.xml | 252 ++ .../src/CMakeLists.txt | 110 + .../src/config.h.cmake | 27 + .../src/contrib/libwebhdfs/CMakeLists.txt | 88 + .../libwebhdfs/resources/FindJansson.cmake | 43 + .../contrib/libwebhdfs/src/hdfs_http_client.c | 490 +++ .../contrib/libwebhdfs/src/hdfs_http_client.h | 294 ++ .../contrib/libwebhdfs/src/hdfs_http_query.c | 402 +++ .../contrib/libwebhdfs/src/hdfs_http_query.h | 240 ++ .../contrib/libwebhdfs/src/hdfs_json_parser.c | 654 ++++ .../contrib/libwebhdfs/src/hdfs_json_parser.h | 178 + .../src/contrib/libwebhdfs/src/hdfs_web.c | 1538 ++++++++ .../libwebhdfs/src/test_libwebhdfs_ops.c | 552 +++ .../libwebhdfs/src/test_libwebhdfs_read.c | 78 + .../libwebhdfs/src/test_libwebhdfs_threaded.c | 247 ++ .../libwebhdfs/src/test_libwebhdfs_write.c | 111 + .../src/main/native/fuse-dfs/CMakeLists.txt | 90 + .../src/main/native/fuse-dfs/doc/README | 131 + .../src/main/native/fuse-dfs/fuse_connect.c | 658 ++++ .../src/main/native/fuse-dfs/fuse_connect.h | 76 + .../main/native/fuse-dfs/fuse_context_handle.h | 40 + .../src/main/native/fuse-dfs/fuse_dfs.c | 136 + .../src/main/native/fuse-dfs/fuse_dfs.h | 81 + .../main/native/fuse-dfs/fuse_dfs_wrapper.sh | 46 + .../src/main/native/fuse-dfs/fuse_file_handle.h | 46 + .../src/main/native/fuse-dfs/fuse_impls.h | 63 + .../main/native/fuse-dfs/fuse_impls_access.c | 29 + .../src/main/native/fuse-dfs/fuse_impls_chmod.c | 57 + .../src/main/native/fuse-dfs/fuse_impls_chown.c | 87 + .../main/native/fuse-dfs/fuse_impls_create.c | 27 + .../src/main/native/fuse-dfs/fuse_impls_flush.c | 54 + .../main/native/fuse-dfs/fuse_impls_getattr.c | 75 + .../src/main/native/fuse-dfs/fuse_impls_mkdir.c | 70 + .../src/main/native/fuse-dfs/fuse_impls_mknod.c | 27 + .../src/main/native/fuse-dfs/fuse_impls_open.c | 172 + .../src/main/native/fuse-dfs/fuse_impls_read.c | 163 + .../main/native/fuse-dfs/fuse_impls_readdir.c | 122 + .../main/native/fuse-dfs/fuse_impls_release.c | 66 + .../main/native/fuse-dfs/fuse_impls_rename.c | 66 + .../src/main/native/fuse-dfs/fuse_impls_rmdir.c | 76 + .../main/native/fuse-dfs/fuse_impls_statfs.c | 70 + .../main/native/fuse-dfs/fuse_impls_symlink.c | 30 + .../main/native/fuse-dfs/fuse_impls_truncate.c | 79 + .../main/native/fuse-dfs/fuse_impls_unlink.c | 65 + .../main/native/fuse-dfs/fuse_impls_utimens.c | 70 + .../src/main/native/fuse-dfs/fuse_impls_write.c | 83 + .../src/main/native/fuse-dfs/fuse_init.c | 192 + .../src/main/native/fuse-dfs/fuse_init.h | 33 + .../src/main/native/fuse-dfs/fuse_options.c | 188 + .../src/main/native/fuse-dfs/fuse_options.h | 44 + .../src/main/native/fuse-dfs/fuse_stat_struct.c | 112 + .../src/main/native/fuse-dfs/fuse_stat_struct.h | 36 + .../src/main/native/fuse-dfs/fuse_trash.c | 243 ++ .../src/main/native/fuse-dfs/fuse_trash.h | 26 + .../src/main/native/fuse-dfs/fuse_users.c | 213 ++ .../src/main/native/fuse-dfs/fuse_users.h | 70 + .../main/native/fuse-dfs/test/TestFuseDFS.java | 410 +++ .../main/native/fuse-dfs/test/fuse_workload.c | 403 +++ .../main/native/fuse-dfs/test/fuse_workload.h | 36 + .../main/native/fuse-dfs/test/test_fuse_dfs.c | 378 ++ .../src/main/native/fuse-dfs/util/posix_util.c | 155 + .../src/main/native/fuse-dfs/util/posix_util.h | 58 + .../src/main/native/fuse-dfs/util/tree.h | 765 ++++ .../main/native/libhdfs-tests/CMakeLists.txt | 41 + .../src/main/native/libhdfs-tests/expect.c | 68 + .../src/main/native/libhdfs-tests/expect.h | 179 + .../src/main/native/libhdfs-tests/hdfs_test.h | 64 + .../main/native/libhdfs-tests/native_mini_dfs.c | 357 ++ .../main/native/libhdfs-tests/native_mini_dfs.h | 129 + .../src/main/native/libhdfs-tests/test_htable.c | 100 + .../native/libhdfs-tests/test_libhdfs_ops.c | 540 +++ .../native/libhdfs-tests/test_libhdfs_read.c | 72 + .../libhdfs-tests/test_libhdfs_threaded.c | 360 ++ .../native/libhdfs-tests/test_libhdfs_write.c | 99 + .../libhdfs-tests/test_libhdfs_zerocopy.c | 280 ++ .../native/libhdfs-tests/test_native_mini_dfs.c | 41 + .../src/main/native/libhdfs-tests/vecsum.c | 825 +++++ .../src/main/native/libhdfs/CMakeLists.txt | 74 + .../src/main/native/libhdfs/common/htable.c | 287 ++ .../src/main/native/libhdfs/common/htable.h | 161 + .../src/main/native/libhdfs/exception.c | 239 ++ .../src/main/native/libhdfs/exception.h | 157 + .../src/main/native/libhdfs/hdfs.c | 3342 ++++++++++++++++++ .../src/main/native/libhdfs/include/hdfs/hdfs.h | 939 +++++ .../src/main/native/libhdfs/jni_helper.c | 595 ++++ .../src/main/native/libhdfs/jni_helper.h | 161 + .../src/main/native/libhdfs/os/mutexes.h | 55 + .../src/main/native/libhdfs/os/posix/mutexes.c | 43 + .../src/main/native/libhdfs/os/posix/platform.h | 34 + .../src/main/native/libhdfs/os/posix/thread.c | 52 + .../libhdfs/os/posix/thread_local_storage.c | 80 + .../src/main/native/libhdfs/os/thread.h | 54 + .../native/libhdfs/os/thread_local_storage.h | 75 + .../main/native/libhdfs/os/windows/inttypes.h | 28 + .../main/native/libhdfs/os/windows/mutexes.c | 52 + .../main/native/libhdfs/os/windows/platform.h | 86 + .../src/main/native/libhdfs/os/windows/thread.c | 66 + .../libhdfs/os/windows/thread_local_storage.c | 172 + .../src/main/native/libhdfs/os/windows/unistd.h | 29 + hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 5 - .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 455 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 829 ++++- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 216 +- .../hadoop-hdfs/src/CMakeLists.txt | 206 -- .../hadoop-hdfs/src/config.h.cmake | 27 - .../hadoop-hdfs/src/contrib/bkjournal/pom.xml | 5 - .../src/contrib/libwebhdfs/CMakeLists.txt | 78 - .../libwebhdfs/resources/FindJansson.cmake | 43 - .../contrib/libwebhdfs/src/hdfs_http_client.c | 490 --- .../contrib/libwebhdfs/src/hdfs_http_client.h | 294 -- .../contrib/libwebhdfs/src/hdfs_http_query.c | 402 --- .../contrib/libwebhdfs/src/hdfs_http_query.h | 240 -- .../contrib/libwebhdfs/src/hdfs_json_parser.c | 654 ---- .../contrib/libwebhdfs/src/hdfs_json_parser.h | 178 - .../src/contrib/libwebhdfs/src/hdfs_web.c | 1538 -------- .../libwebhdfs/src/test_libwebhdfs_ops.c | 552 --- .../libwebhdfs/src/test_libwebhdfs_read.c | 78 - .../libwebhdfs/src/test_libwebhdfs_threaded.c | 247 -- .../libwebhdfs/src/test_libwebhdfs_write.c | 111 - .../src/main/bin/distribute-exclude.sh | 4 +- .../hadoop-hdfs/src/main/bin/hdfs | 9 +- .../src/main/bin/refresh-namenodes.sh | 6 +- .../hadoop-hdfs/src/main/bin/start-balancer.sh | 6 +- .../hadoop-hdfs/src/main/bin/start-dfs.sh | 6 +- .../src/main/bin/start-secure-dns.sh | 6 +- .../hadoop-hdfs/src/main/bin/stop-balancer.sh | 6 +- .../hadoop-hdfs/src/main/bin/stop-dfs.sh | 6 +- .../hadoop-hdfs/src/main/bin/stop-secure-dns.sh | 6 +- .../main/java/org/apache/hadoop/fs/Hdfs.java | 2 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 105 +- .../java/org/apache/hadoop/hdfs/DFSUtil.java | 42 + .../apache/hadoop/hdfs/HdfsConfiguration.java | 149 - .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 21 +- .../hadoop/hdfs/server/balancer/Dispatcher.java | 76 +- .../hdfs/server/blockmanagement/BlockInfo.java | 6 +- .../blockmanagement/BlockInfoStriped.java | 2 + .../server/blockmanagement/BlockManager.java | 346 +- .../blockmanagement/BlockPlacementPolicies.java | 2 + .../blockmanagement/BlockPlacementPolicy.java | 113 +- .../BlockPlacementPolicyDefault.java | 194 +- .../BlockPlacementPolicyRackFaultTolerant.java | 18 + .../BlockPlacementPolicyWithNodeGroup.java | 113 +- .../BlockPlacementPolicyWithUpgradeDomain.java | 311 ++ .../BlockPlacementStatusWithUpgradeDomain.java | 88 + .../hdfs/server/blockmanagement/BlocksMap.java | 14 +- .../blockmanagement/DatanodeDescriptor.java | 155 +- .../server/blockmanagement/DatanodeManager.java | 24 +- .../server/blockmanagement/DatanodeStats.java | 212 ++ .../blockmanagement/DecommissionManager.java | 4 +- .../blockmanagement/HeartbeatManager.java | 255 +- .../server/blockmanagement/NumberReplicas.java | 18 +- .../ReplicaUnderConstruction.java | 2 +- .../server/blockmanagement/ReplicationWork.java | 3 + .../blockmanagement/UnderReplicatedBlocks.java | 205 +- .../hdfs/server/common/HdfsServerConstants.java | 6 +- .../hadoop/hdfs/server/common/Storage.java | 15 + .../hdfs/server/datanode/BPOfferService.java | 3 +- .../hdfs/server/datanode/BPServiceActor.java | 4 +- .../server/datanode/BlockPoolSliceStorage.java | 36 +- .../hdfs/server/datanode/BlockReceiver.java | 57 +- .../server/datanode/BlockRecoveryWorker.java | 330 ++ .../hdfs/server/datanode/BlockSender.java | 2 +- .../hadoop/hdfs/server/datanode/DataNode.java | 157 +- .../server/datanode/DataNodeFaultInjector.java | 8 + .../hdfs/server/datanode/DataStorage.java | 1 + .../hdfs/server/datanode/DataXceiver.java | 2 +- .../erasurecode/ErasureCodingWorker.java | 24 +- .../datanode/fsdataset/impl/FsVolumeList.java | 109 +- .../apache/hadoop/hdfs/server/mover/Mover.java | 11 + .../hdfs/server/namenode/CacheManager.java | 2 +- .../hdfs/server/namenode/Checkpointer.java | 2 +- .../hdfs/server/namenode/FSDirAttrOp.java | 15 +- .../hdfs/server/namenode/FSDirConcatOp.java | 9 +- .../hdfs/server/namenode/FSDirDeleteOp.java | 5 + .../hdfs/server/namenode/FSDirRenameOp.java | 6 + .../hdfs/server/namenode/FSDirSnapshotOp.java | 38 + .../server/namenode/FSDirStatAndListingOp.java | 16 + .../hdfs/server/namenode/FSDirSymlinkOp.java | 3 +- .../hdfs/server/namenode/FSDirWriteFileOp.java | 7 + .../hdfs/server/namenode/FSDirectory.java | 63 +- .../hadoop/hdfs/server/namenode/FSImage.java | 4 + .../server/namenode/FSImageFormatPBINode.java | 4 + .../hdfs/server/namenode/FSNamesystem.java | 213 +- .../hdfs/server/namenode/HdfsAuditLogger.java | 7 +- .../hadoop/hdfs/server/namenode/INode.java | 3 +- .../hdfs/server/namenode/ImageServlet.java | 38 +- .../hadoop/hdfs/server/namenode/NNStorage.java | 27 +- .../hdfs/server/namenode/NNUpgradeUtil.java | 48 +- .../hadoop/hdfs/server/namenode/NameNode.java | 71 +- .../hdfs/server/namenode/NameNodeMXBean.java | 12 + .../hdfs/server/namenode/NameNodeRpcServer.java | 74 +- .../server/namenode/NameNodeStatusMXBean.java | 6 + .../hdfs/server/namenode/NamenodeFsck.java | 172 +- .../hdfs/server/namenode/SecondaryNameNode.java | 89 +- .../hdfs/server/namenode/TransferFsImage.java | 5 +- .../server/namenode/ha/BootstrapStandby.java | 2 +- .../hdfs/server/namenode/ha/EditLogTailer.java | 4 +- .../hdfs/server/namenode/snapshot/Snapshot.java | 8 + .../web/resources/NamenodeWebHdfsMethods.java | 4 + .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 18 +- .../org/apache/hadoop/hdfs/tools/DFSck.java | 3 +- .../hdfs/tools/DelegationTokenFetcher.java | 2 +- .../hadoop/hdfs/tools/NNHAServiceTarget.java | 11 + .../hadoop/hdfs/tools/erasurecode/ECCli.java | 2 + .../offlineEditsViewer/OfflineEditsViewer.java | 23 +- .../OfflineImageViewerPB.java | 23 +- .../offlineImageViewer/PBImageXmlWriter.java | 2 +- .../src/main/native/fuse-dfs/CMakeLists.txt | 102 - .../src/main/native/fuse-dfs/doc/README | 131 - .../src/main/native/fuse-dfs/fuse_connect.c | 644 ---- .../src/main/native/fuse-dfs/fuse_connect.h | 90 - .../main/native/fuse-dfs/fuse_context_handle.h | 40 - .../src/main/native/fuse-dfs/fuse_dfs.c | 136 - .../src/main/native/fuse-dfs/fuse_dfs.h | 81 - .../main/native/fuse-dfs/fuse_dfs_wrapper.sh | 46 - .../src/main/native/fuse-dfs/fuse_file_handle.h | 46 - .../src/main/native/fuse-dfs/fuse_impls.h | 63 - .../main/native/fuse-dfs/fuse_impls_access.c | 29 - .../src/main/native/fuse-dfs/fuse_impls_chmod.c | 57 - .../src/main/native/fuse-dfs/fuse_impls_chown.c | 87 - .../main/native/fuse-dfs/fuse_impls_create.c | 27 - .../src/main/native/fuse-dfs/fuse_impls_flush.c | 54 - .../main/native/fuse-dfs/fuse_impls_getattr.c | 75 - .../src/main/native/fuse-dfs/fuse_impls_mkdir.c | 70 - .../src/main/native/fuse-dfs/fuse_impls_mknod.c | 27 - .../src/main/native/fuse-dfs/fuse_impls_open.c | 172 - .../src/main/native/fuse-dfs/fuse_impls_read.c | 163 - .../main/native/fuse-dfs/fuse_impls_readdir.c | 122 - .../main/native/fuse-dfs/fuse_impls_release.c | 66 - .../main/native/fuse-dfs/fuse_impls_rename.c | 66 - .../src/main/native/fuse-dfs/fuse_impls_rmdir.c | 76 - .../main/native/fuse-dfs/fuse_impls_statfs.c | 70 - .../main/native/fuse-dfs/fuse_impls_symlink.c | 30 - .../main/native/fuse-dfs/fuse_impls_truncate.c | 79 - .../main/native/fuse-dfs/fuse_impls_unlink.c | 65 - .../main/native/fuse-dfs/fuse_impls_utimens.c | 70 - .../src/main/native/fuse-dfs/fuse_impls_write.c | 83 - .../src/main/native/fuse-dfs/fuse_init.c | 192 - .../src/main/native/fuse-dfs/fuse_init.h | 33 - .../src/main/native/fuse-dfs/fuse_options.c | 188 - .../src/main/native/fuse-dfs/fuse_options.h | 44 - .../src/main/native/fuse-dfs/fuse_stat_struct.c | 112 - .../src/main/native/fuse-dfs/fuse_stat_struct.h | 36 - .../src/main/native/fuse-dfs/fuse_trash.c | 244 -- .../src/main/native/fuse-dfs/fuse_trash.h | 26 - .../src/main/native/fuse-dfs/fuse_users.c | 213 -- .../src/main/native/fuse-dfs/fuse_users.h | 70 - .../main/native/fuse-dfs/test/TestFuseDFS.java | 410 --- .../main/native/fuse-dfs/test/fuse_workload.c | 403 --- .../main/native/fuse-dfs/test/fuse_workload.h | 36 - .../main/native/fuse-dfs/test/test_fuse_dfs.c | 378 -- .../src/main/native/libhdfs/common/htable.c | 287 -- .../src/main/native/libhdfs/common/htable.h | 161 - .../src/main/native/libhdfs/exception.c | 239 -- .../src/main/native/libhdfs/exception.h | 157 - .../src/main/native/libhdfs/expect.c | 68 - .../src/main/native/libhdfs/expect.h | 179 - .../hadoop-hdfs/src/main/native/libhdfs/hdfs.c | 3342 ------------------ .../hadoop-hdfs/src/main/native/libhdfs/hdfs.h | 939 ----- .../src/main/native/libhdfs/hdfs_test.h | 64 - .../src/main/native/libhdfs/jni_helper.c | 595 ---- .../src/main/native/libhdfs/jni_helper.h | 161 - .../src/main/native/libhdfs/native_mini_dfs.c | 357 -- .../src/main/native/libhdfs/native_mini_dfs.h | 129 - .../src/main/native/libhdfs/os/mutexes.h | 55 - .../src/main/native/libhdfs/os/posix/mutexes.c | 43 - .../src/main/native/libhdfs/os/posix/platform.h | 34 - .../src/main/native/libhdfs/os/posix/thread.c | 52 - .../libhdfs/os/posix/thread_local_storage.c | 80 - .../src/main/native/libhdfs/os/thread.h | 54 - .../native/libhdfs/os/thread_local_storage.h | 75 - .../main/native/libhdfs/os/windows/inttypes.h | 28 - .../main/native/libhdfs/os/windows/mutexes.c | 52 - .../main/native/libhdfs/os/windows/platform.h | 86 - .../src/main/native/libhdfs/os/windows/thread.c | 66 - .../libhdfs/os/windows/thread_local_storage.c | 172 - .../src/main/native/libhdfs/os/windows/unistd.h | 29 - .../src/main/native/libhdfs/test/test_htable.c | 100 - .../main/native/libhdfs/test/test_libhdfs_ops.c | 540 --- .../native/libhdfs/test/test_libhdfs_read.c | 72 - .../native/libhdfs/test/test_libhdfs_write.c | 99 - .../native/libhdfs/test/test_libhdfs_zerocopy.c | 280 -- .../src/main/native/libhdfs/test/vecsum.c | 825 ----- .../main/native/libhdfs/test_libhdfs_threaded.c | 360 -- .../main/native/libhdfs/test_native_mini_dfs.c | 41 - .../src/main/native/util/posix_util.c | 155 - .../src/main/native/util/posix_util.h | 58 - .../hadoop-hdfs/src/main/native/util/tree.h | 765 ---- .../src/main/resources/hdfs-default.xml | 93 +- .../src/main/webapps/hdfs/dfshealth.html | 10 +- .../src/main/webapps/hdfs/dfshealth.js | 2 +- .../src/main/webapps/hdfs/explorer.html | 1 + .../src/main/webapps/hdfs/explorer.js | 3 +- .../src/main/webapps/static/hadoop.css | 4 +- .../src/main/webapps/static/json-bignum.js | 379 ++ .../src/site/markdown/HDFSCommands.md | 6 +- .../hadoop-hdfs/src/site/markdown/HdfsDesign.md | 19 + .../src/site/markdown/HdfsEditsViewer.md | 2 + .../src/site/markdown/HdfsImageViewer.md | 2 +- .../src/site/markdown/HdfsNfsGateway.md | 8 + .../org/apache/hadoop/fs/TestGlobPaths.java | 4 - .../TestSWebHdfsFileContextMainOperations.java | 11 +- .../hadoop/fs/TestUrlStreamHandlerFactory.java | 80 - .../java/org/apache/hadoop/fs/TestXAttr.java | 94 - .../org/apache/hadoop/hdfs/DFSTestUtil.java | 38 +- .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 231 +- .../apache/hadoop/hdfs/StripedFileTestUtil.java | 66 +- .../apache/hadoop/hdfs/TestAclsEndToEnd.java | 1657 +++++++++ .../hadoop/hdfs/TestBlockMissingException.java | 15 +- .../TestClientProtocolForPipelineRecovery.java | 63 + .../hadoop/hdfs/TestClientReportBadBlock.java | 29 +- .../apache/hadoop/hdfs/TestCrcCorruption.java | 120 +- .../hadoop/hdfs/TestDFSClientSocketSize.java | 96 + .../org/apache/hadoop/hdfs/TestDFSPacket.java | 94 - .../org/apache/hadoop/hdfs/TestDFSShell.java | 239 +- .../hadoop/hdfs/TestDFSStripedInputStream.java | 13 +- .../hadoop/hdfs/TestDFSStripedOutputStream.java | 8 +- .../TestDFSStripedOutputStreamWithFailure.java | 85 +- ...estDFSStripedOutputStreamWithFailure020.java | 22 + ...estDFSStripedOutputStreamWithFailure030.java | 22 + ...estDFSStripedOutputStreamWithFailure040.java | 22 + ...estDFSStripedOutputStreamWithFailure050.java | 22 + ...estDFSStripedOutputStreamWithFailure060.java | 22 + ...estDFSStripedOutputStreamWithFailure070.java | 22 + ...estDFSStripedOutputStreamWithFailure080.java | 22 + ...estDFSStripedOutputStreamWithFailure090.java | 22 + ...estDFSStripedOutputStreamWithFailure100.java | 22 + ...estDFSStripedOutputStreamWithFailure110.java | 22 + ...estDFSStripedOutputStreamWithFailure120.java | 22 + ...estDFSStripedOutputStreamWithFailure130.java | 22 + ...estDFSStripedOutputStreamWithFailure140.java | 22 + ...estDFSStripedOutputStreamWithFailure150.java | 22 + ...estDFSStripedOutputStreamWithFailure160.java | 22 + ...estDFSStripedOutputStreamWithFailure170.java | 22 + ...estDFSStripedOutputStreamWithFailure180.java | 22 + ...estDFSStripedOutputStreamWithFailure190.java | 22 + ...estDFSStripedOutputStreamWithFailure200.java | 22 + ...estDFSStripedOutputStreamWithFailure210.java | 23 + .../apache/hadoop/hdfs/TestDecommission.java | 47 +- .../hadoop/hdfs/TestDefaultNameNodePort.java | 68 - .../hadoop/hdfs/TestDistributedFileSystem.java | 28 + .../TestErasureCodingPolicyWithSnapshot.java | 199 ++ .../org/apache/hadoop/hdfs/TestFileAppend2.java | 37 + .../org/apache/hadoop/hdfs/TestFileAppend3.java | 12 +- .../apache/hadoop/hdfs/TestFileCorruption.java | 65 +- .../apache/hadoop/hdfs/TestFileCreation.java | 14 +- .../hadoop/hdfs/TestFileStatusWithECPolicy.java | 17 + .../apache/hadoop/hdfs/TestHDFSServerPorts.java | 1 + .../apache/hadoop/hdfs/TestLeaseRecovery.java | 35 +- .../apache/hadoop/hdfs/TestMiniDFSCluster.java | 47 + .../hadoop/hdfs/TestMissingBlocksAlert.java | 4 +- .../org/apache/hadoop/hdfs/TestPeerCache.java | 293 -- .../org/apache/hadoop/hdfs/TestPipelines.java | 6 +- .../hdfs/TestReadStripedFileWithDecoding.java | 1 + .../TestReadStripedFileWithMissingBlocks.java | 6 +- .../hadoop/hdfs/TestRecoverStripedFile.java | 152 +- .../org/apache/hadoop/hdfs/TestReplication.java | 307 +- .../hadoop/hdfs/TestReservedRawPaths.java | 13 +- .../hdfs/TestSafeModeWithStripedFile.java | 5 +- .../org/apache/hadoop/hdfs/TestSetTimes.java | 31 + .../hadoop/hdfs/TestWriteReadStripedFile.java | 129 +- .../hdfs/TestWriteStripedFileWithFailure.java | 6 +- .../hdfs/client/impl/TestLeaseRenewer.java | 209 -- .../hadoop/hdfs/protocol/TestExtendedBlock.java | 77 - .../hadoop/hdfs/protocol/TestLocatedBlock.java | 46 + .../sasl/SaslDataTransferTestCase.java | 6 + .../hdfs/qjournal/TestSecureNNWithQJM.java | 6 + .../hdfs/security/TestDelegationToken.java | 9 + .../hdfs/server/balancer/TestBalancer.java | 646 ++-- .../BaseReplicationPolicyTest.java | 159 + .../blockmanagement/TestBlockManager.java | 25 +- .../TestBlockTokenWithDFSStriped.java | 4 +- .../TestBlockUnderConstructionFeature.java | 4 +- .../TestBlocksWithNotEnoughRacks.java | 2 +- .../blockmanagement/TestDatanodeDescriptor.java | 6 +- .../TestOverReplicatedBlocks.java | 2 +- .../TestRBWBlockInvalidation.java | 18 +- .../blockmanagement/TestReplicationPolicy.java | 401 ++- .../TestReplicationPolicyConsiderLoad.java | 124 +- .../TestReplicationPolicyWithNodeGroup.java | 257 +- .../TestReplicationPolicyWithUpgradeDomain.java | 459 +++ .../TestUnderReplicatedBlockQueues.java | 14 +- .../TestUnderReplicatedBlocks.java | 5 + .../hdfs/server/datanode/DataNodeTestUtils.java | 14 - .../datanode/FsDatasetImplTestUtilsFactory.java | 32 + .../server/datanode/FsDatasetTestUtils.java | 216 ++ .../hdfs/server/datanode/TestBlockRecovery.java | 74 +- .../hdfs/server/datanode/TestBlockScanner.java | 10 +- .../datanode/TestBpServiceActorScheduler.java | 22 + .../hdfs/server/datanode/TestDataNodeExit.java | 8 +- .../server/datanode/TestDataNodeMetrics.java | 5 +- .../TestDataNodeVolumeFailureToleration.java | 4 +- .../server/datanode/TestDirectoryScanner.java | 17 +- .../fsdataset/impl/FsDatasetImplTestUtils.java | 325 ++ .../fsdataset/impl/TestFsDatasetImpl.java | 24 +- .../impl/TestInterDatanodeProtocol.java | 7 +- .../fsdataset/impl/TestScrLazyPersistFiles.java | 7 +- .../fsdataset/impl/TestWriteToReplica.java | 160 +- .../server/namenode/TestAuditLogAtDebug.java | 2 +- .../hdfs/server/namenode/TestAuditLogger.java | 191 + .../hdfs/server/namenode/TestCheckpoint.java | 2 +- .../TestCommitBlockWithInvalidGenStamp.java | 100 + .../hdfs/server/namenode/TestFSImage.java | 26 +- .../server/namenode/TestFSImageWithAcl.java | 29 + .../hdfs/server/namenode/TestFSNamesystem.java | 57 + .../hdfs/server/namenode/TestFileTruncate.java | 5 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 164 +- .../namenode/TestListCorruptFileBlocks.java | 68 + .../server/namenode/TestNameNodeHttpServer.java | 4 + .../server/namenode/TestNameNodeMXBean.java | 64 +- .../TestNameNodeMetadataConsistency.java | 186 + .../TestNameNodeRespectsBindHostKeys.java | 54 +- .../namenode/TestProcessCorruptBlocks.java | 6 +- .../namenode/TestQuotaWithStripedBlocks.java | 4 +- .../namenode/TestRecoverStripedBlocks.java | 70 + .../namenode/ha/TestBootstrapStandby.java | 104 +- .../hdfs/server/namenode/ha/TestDNFencing.java | 14 +- .../server/namenode/ha/TestEditLogTailer.java | 24 +- .../hdfs/server/namenode/ha/TestHAAppend.java | 2 +- .../server/namenode/ha/TestNNHealthCheck.java | 93 +- .../namenode/ha/TestSeveralNameNodes.java | 4 +- .../shortcircuit/TestShortCircuitLocalRead.java | 6 +- .../hdfs/shortcircuit/TestShortCircuitShm.java | 109 - .../TestOfflineEditsViewer.java | 31 + .../TestOfflineImageViewer.java | 34 + .../hadoop/hdfs/util/TestByteArrayManager.java | 645 ---- .../hdfs/util/TestExactSizeInputStream.java | 129 - .../hdfs/web/TestByteRangeInputStream.java | 292 -- .../hadoop/hdfs/web/TestHttpsFileSystem.java | 4 + .../hdfs/web/TestOffsetUrlInputStream.java | 64 - .../apache/hadoop/hdfs/web/TestTokenAspect.java | 316 -- .../hdfs/web/TestURLConnectionFactory.java | 50 - .../hadoop/hdfs/web/TestWebHDFSOAuth2.java | 216 -- .../hdfs/web/TestWebHdfsContentLength.java | 197 -- .../hadoop/hdfs/web/TestWebHdfsTokens.java | 6 +- .../hdfs/web/oauth2/TestAccessTokenTimer.java | 63 - ...ClientCredentialTimeBasedTokenRefresher.java | 138 - ...TestRefreshTokenTimeBasedTokenRefresher.java | 138 - .../hadoop/tools/TestHdfsConfigFields.java | 4 +- .../TestTracingShortCircuitLocalRead.java | 4 +- .../src/test/resources/testHDFSConf.xml | 20 +- hadoop-hdfs-project/pom.xml | 1 + hadoop-mapreduce-project/CHANGES.txt | 101 +- hadoop-mapreduce-project/bin/mapred | 15 +- .../bin/mr-jobhistory-daemon.sh | 6 +- .../jobhistory/JobHistoryEventHandler.java | 2 +- .../apache/hadoop/mapreduce/v2/app/job/Job.java | 2 + .../mapreduce/v2/app/job/impl/JobImpl.java | 13 +- .../v2/app/job/impl/TaskAttemptImpl.java | 7 +- .../mapreduce/v2/app/job/impl/TaskImpl.java | 20 +- .../v2/app/rm/RMContainerAllocator.java | 166 +- .../v2/app/rm/RMContainerRequestor.java | 3 +- .../jobhistory/TestJobHistoryEventHandler.java | 10 +- .../hadoop/mapreduce/v2/app/MockJobs.java | 6 + .../mapreduce/v2/app/TestRuntimeEstimators.java | 6 + .../mapreduce/v2/app/job/impl/TestJobImpl.java | 35 + .../mapreduce/v2/app/job/impl/TestTaskImpl.java | 94 +- .../v2/app/rm/TestRMContainerAllocator.java | 88 +- .../apache/hadoop/mapreduce/TypeConverter.java | 34 +- .../mapreduce/v2/api/records/JobReport.java | 4 + .../v2/api/records/impl/pb/JobReportPBImpl.java | 44 +- .../mapreduce/v2/util/MRBuilderUtils.java | 15 +- .../src/main/proto/mr_protos.proto | 1 + .../hadoop/mapreduce/TestTypeConverter.java | 8 +- .../org/apache/hadoop/mapred/JobPriority.java | 5 +- .../java/org/apache/hadoop/mapred/Task.java | 70 +- .../apache/hadoop/mapreduce/JobPriority.java | 4 +- .../apache/hadoop/mapreduce/MRJobConfig.java | 21 +- .../org/apache/hadoop/mapreduce/tools/CLI.java | 2 +- .../src/main/resources/mapred-default.xml | 43 +- .../src/site/markdown/MapredCommands.md | 13 +- .../hadoop/mapred/TestTaskProgressReporter.java | 80 + .../hadoop-mapreduce-client-hs/pom.xml | 2 +- .../hadoop/mapreduce/v2/hs/CompletedJob.java | 7 + .../mapreduce/v2/hs/HistoryFileManager.java | 13 +- .../hadoop/mapreduce/v2/hs/PartialJob.java | 7 + .../v2/hs/webapp/TestHsWebServicesAcls.java | 5 + .../hadoop/mapred/ResourceMgrDelegate.java | 13 + .../hadoop/mapred/TestClientRedirect.java | 16 + .../apache/hadoop/mapred/TestNetworkedJob.java | 16 - .../hadoop/mapreduce/JobHistoryFileParser.java | 53 + .../mapreduce/JobHistoryFileReplayHelper.java | 196 + .../mapreduce/JobHistoryFileReplayMapperV1.java | 158 + .../hadoop/mapreduce/SimpleEntityWriterV1.java | 120 + .../mapreduce/TimelineEntityConverterV1.java | 167 + .../mapreduce/TimelineServicePerformance.java | 197 ++ .../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 43 +- .../apache/hadoop/test/MapredTestDriver.java | 43 +- .../src/main/native/src/lib/Buffers.h | 21 +- .../src/main/native/src/lib/FileSystem.cc | 9 +- .../src/main/native/src/lib/MemoryBlock.cc | 4 +- .../src/main/native/src/lib/MemoryBlock.h | 2 +- .../src/main/native/src/util/Checksum.cc | 15 +- .../src/main/native/src/util/Random.cc | 4 +- .../src/main/native/src/util/Timer.cc | 2 +- .../src/main/native/src/util/WritableUtils.cc | 3 +- .../src/main/native/test/TestPrimitives.cc | 2 +- .../apache/hadoop/mapred/ShuffleHandler.java | 1 + .../hadoop/mapred/TestShuffleHandler.java | 59 + .../hadoop-mapreduce-examples/pom.xml | 2 +- .../hadoop/maven/plugin/protoc/ProtocMojo.java | 6 +- .../apache/hadoop/maven/plugin/util/Exec.java | 19 +- hadoop-project/pom.xml | 18 +- hadoop-project/src/site/site.xml | 1 + hadoop-tools/hadoop-ant/pom.xml | 2 +- .../java/org/apache/hadoop/ant/DfsTask.java | 9 +- .../apache/hadoop/tools/HadoopArchiveLogs.java | 99 +- .../src/site/markdown/HadoopArchiveLogs.md | 85 + .../src/site/resources/css/site.css | 30 + .../hadoop/tools/TestHadoopArchiveLogs.java | 42 +- hadoop-tools/hadoop-archives/pom.xml | 12 +- .../fs/azure/AzureNativeFileSystemStore.java | 160 +- .../hadoop/fs/azure/NativeAzureFileSystem.java | 443 ++- .../hadoop/fs/azure/SelfRenewingLease.java | 5 +- .../hadoop-azure/src/site/markdown/index.md | 29 +- .../fs/azure/AzureBlobStorageTestAccount.java | 4 +- ...estFileSystemOperationExceptionHandling.java | 131 + ...perationsExceptionHandlingMultiThreaded.java | 185 + .../fs/azure/TestNativeAzureFileSystemLive.java | 113 + .../TestAzureFileSystemInstrumentation.java | 25 +- .../src/test/resources/azure-test.xml | 14 +- hadoop-tools/hadoop-datajoin/pom.xml | 12 +- hadoop-tools/hadoop-distcp/pom.xml | 7 +- .../org/apache/hadoop/tools/DistCpSync.java | 8 +- .../tools/mapred/lib/DynamicInputChunk.java | 137 +- .../mapred/lib/DynamicInputChunkContext.java | 113 + .../tools/mapred/lib/DynamicInputFormat.java | 31 +- .../tools/mapred/lib/DynamicRecordReader.java | 13 +- .../org/apache/hadoop/tools/StubContext.java | 4 + .../mapred/lib/TestDynamicInputFormat.java | 33 +- hadoop-tools/hadoop-extras/pom.xml | 12 +- hadoop-tools/hadoop-gridmix/pom.xml | 7 +- hadoop-tools/hadoop-kafka/pom.xml | 134 + .../apache/hadoop/metrics2/sink/KafkaSink.java | 193 + .../hadoop/metrics2/impl/TestKafkaMetrics.java | 184 + .../hadoop/fs/swift/http/SwiftRestClient.java | 20 +- hadoop-tools/hadoop-rumen/pom.xml | 2 +- .../hadoop-sls/src/main/assemblies/sls.xml | 4 +- .../hadoop-sls/src/main/bin/rumen2sls.sh | 9 +- hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 12 +- .../sls/scheduler/ResourceSchedulerWrapper.java | 2 +- hadoop-tools/hadoop-streaming/pom.xml | 12 +- hadoop-tools/pom.xml | 1 + hadoop-yarn-project/CHANGES.txt | 274 +- .../hadoop-yarn/bin/start-yarn.sh | 6 +- .../hadoop-yarn/bin/stop-yarn.sh | 6 +- hadoop-yarn-project/hadoop-yarn/bin/yarn | 6 +- .../hadoop-yarn/bin/yarn-daemon.sh | 6 +- .../hadoop-yarn/bin/yarn-daemons.sh | 6 +- .../dev-support/findbugs-exclude.xml | 5 + .../yarn/api/ApplicationClientProtocol.java | 61 +- .../api/protocolrecords/AllocateResponse.java | 14 + .../FailApplicationAttemptRequest.java | 62 + .../FailApplicationAttemptResponse.java | 46 + .../protocolrecords/SignalContainerRequest.java | 78 + .../SignalContainerResponse.java | 36 + .../records/ApplicationResourceUsageReport.java | 38 +- .../hadoop/yarn/api/records/NodeLabel.java | 5 +- .../api/records/SignalContainerCommand.java | 45 + .../hadoop/yarn/conf/YarnConfiguration.java | 168 +- .../main/proto/applicationclient_protocol.proto | 2 + .../src/main/proto/yarn_protos.proto | 9 + .../src/main/proto/yarn_service_protos.proto | 16 + .../applications/distributedshell/Client.java | 6 +- .../distributedshell/TestDistributedShell.java | 12 +- .../hadoop/yarn/client/api/YarnClient.java | 34 +- .../yarn/client/api/impl/YarnClientImpl.java | 24 +- .../hadoop/yarn/client/cli/ApplicationCLI.java | 69 +- .../apache/hadoop/yarn/client/cli/YarnCLI.java | 8 +- .../api/impl/TestAMRMClientOnRMRestart.java | 4 +- .../yarn/client/api/impl/TestYarnClient.java | 25 + .../hadoop/yarn/client/cli/TestYarnCLI.java | 69 +- .../ApplicationClientProtocolPBClientImpl.java | 37 + .../ApplicationClientProtocolPBServiceImpl.java | 38 + .../impl/pb/AllocateResponsePBImpl.java | 40 +- .../pb/FailApplicationAttemptRequestPBImpl.java | 135 + .../FailApplicationAttemptResponsePBImpl.java | 70 + .../impl/pb/SignalContainerRequestPBImpl.java | 169 + .../impl/pb/SignalContainerResponsePBImpl.java | 44 + .../ApplicationResourceUsageReportPBImpl.java | 24 + .../org/apache/hadoop/yarn/client/RMProxy.java | 6 +- .../nodelabels/CommonNodeLabelsManager.java | 22 +- .../yarn/util/WindowsBasedProcessTree.java | 10 +- .../org/apache/hadoop/yarn/webapp/WebApps.java | 18 +- .../src/main/resources/yarn-default.xml | 140 +- .../yarn/nodelabels/NodeLabelTestBase.java | 116 +- .../nodelabels/TestCommonNodeLabelsManager.java | 6 +- ...pplicationHistoryManagerOnTimelineStore.java | 5 +- .../ApplicationHistoryServer.java | 58 +- .../timeline/webapp/CrossOriginFilter.java | 259 -- .../webapp/CrossOriginFilterInitializer.java | 40 +- .../TestApplicationHistoryServer.java | 53 +- .../timeline/webapp/TestCrossOriginFilter.java | 324 -- .../TestCrossOriginFilterInitializer.java | 57 - .../hadoop-yarn-server-common/pom.xml | 5 + .../protocolrecords/NodeHeartbeatResponse.java | 3 + .../impl/pb/NodeHeartbeatResponsePBImpl.java | 78 +- .../hadoop/yarn/server/utils/BuilderUtils.java | 12 + .../hadoop/yarn/server/webapp/WebPageUtils.java | 2 +- .../yarn_server_common_service_protos.proto | 2 + .../nodemanager/CMgrSignalContainersEvent.java | 37 + .../server/nodemanager/ContainerExecutor.java | 2 +- .../nodemanager/ContainerManagerEventType.java | 3 +- .../nodemanager/DefaultContainerExecutor.java | 9 +- .../server/nodemanager/DirectoryCollection.java | 68 +- .../nodemanager/DockerContainerExecutor.java | 9 +- .../nodemanager/LinuxContainerExecutor.java | 275 +- .../nodemanager/LocalDirsHandlerService.java | 30 +- .../yarn/server/nodemanager/NodeManager.java | 26 +- .../nodemanager/NodeStatusUpdaterImpl.java | 87 +- .../WindowsSecureContainerExecutor.java | 3 +- .../containermanager/ContainerManagerImpl.java | 20 + .../launcher/ContainerLaunch.java | 109 +- .../launcher/ContainersLauncher.java | 18 +- .../launcher/ContainersLauncherEventType.java | 1 + .../launcher/SignalContainersLauncherEvent.java | 38 + .../privileged/PrivilegedOperationExecutor.java | 11 +- .../runtime/DockerLinuxContainerRuntime.java | 89 +- .../linux/runtime/docker/DockerRunCommand.java | 18 + .../localizer/LocalResourcesTrackerImpl.java | 10 + .../localizer/ResourceLocalizationService.java | 13 + .../monitor/ContainerMetrics.java | 38 +- .../monitor/ContainersMonitorImpl.java | 16 +- .../nodelabels/AbstractNodeLabelsProvider.java | 30 +- .../ConfigurationNodeLabelsProvider.java | 18 +- .../nodelabels/NodeLabelsProvider.java | 7 +- .../ScriptBasedNodeLabelsProvider.java | 186 + .../server/nodemanager/webapp/WebServer.java | 10 +- .../container-executor/impl/configuration.c | 4 +- .../container-executor/impl/configuration.h | 4 + .../impl/container-executor.c | 307 +- .../test/test-container-executor.c | 11 + .../nodemanager/TestContainerExecutor.java | 1 + .../TestContainerManagerWithLCE.java | 36 + .../nodemanager/TestDirectoryCollection.java | 68 +- .../nodemanager/TestNodeManagerReboot.java | 13 +- .../nodemanager/TestNodeManagerResync.java | 22 +- .../nodemanager/TestNodeManagerShutdown.java | 27 +- .../nodemanager/TestNodeStatusUpdater.java | 136 +- .../TestNodeStatusUpdaterForLabels.java | 118 +- .../amrmproxy/MockResourceManagerFacade.java | 16 + .../BaseContainerManagerTest.java | 11 +- .../containermanager/TestContainerManager.java | 111 +- .../TestContainerManagerRecovery.java | 3 +- .../launcher/TestContainerLaunch.java | 31 +- .../runtime/TestDockerContainerRuntime.java | 230 +- .../TestLocalResourcesTrackerImpl.java | 6 +- .../TestResourceLocalizationService.java | 293 +- .../TestLogAggregationService.java | 32 +- .../monitor/TestContainerMetrics.java | 45 +- .../monitor/TestContainersMonitor.java | 1 - .../TestConfigurationNodeLabelsProvider.java | 34 +- .../TestScriptBasedNodeLabelsProvider.java | 209 ++ .../server/resourcemanager/AdminService.java | 26 +- .../ApplicationMasterService.java | 4 + .../server/resourcemanager/ClientRMService.java | 150 +- .../resourcemanager/EmbeddedElectorService.java | 9 +- .../resourcemanager/RMActiveServiceContext.java | 15 + .../server/resourcemanager/RMAppManager.java | 12 +- .../server/resourcemanager/RMAuditLogger.java | 2 + .../yarn/server/resourcemanager/RMContext.java | 6 + .../server/resourcemanager/RMContextImpl.java | 13 + .../server/resourcemanager/ResourceManager.java | 40 +- .../resourcemanager/ResourceTrackerService.java | 31 +- .../resourcemanager/amlauncher/AMLauncher.java | 5 +- .../amlauncher/ApplicationMasterLauncher.java | 4 +- .../metrics/SystemMetricsPublisher.java | 4 +- .../nodelabels/NodeLabelsUtils.java | 59 + .../RMDelegatedNodeLabelsUpdater.java | 211 ++ .../nodelabels/RMNodeLabelsManager.java | 42 +- .../nodelabels/RMNodeLabelsMappingProvider.java | 46 + .../recovery/FileSystemRMStateStore.java | 2 +- .../resourcemanager/recovery/RMStateStore.java | 261 +- .../recovery/ZKRMStateStore.java | 25 +- .../reservation/AbstractReservationSystem.java | 93 +- .../AbstractSchedulerPlanFollower.java | 76 +- .../CapacitySchedulerPlanFollower.java | 16 - .../reservation/FairSchedulerPlanFollower.java | 15 - .../reservation/InMemoryPlan.java | 38 +- .../resourcemanager/reservation/PlanEdit.java | 6 +- .../reservation/PlanFollower.java | 4 +- .../reservation/ReservationSystem.java | 7 +- .../reservation/planning/PlanningAlgorithm.java | 2 +- .../resourcemanager/rmapp/RMAppEvent.java | 11 + .../rmapp/RMAppFailedAttemptEvent.java | 8 +- .../rmapp/RMAppFinishedAttemptEvent.java | 35 - .../server/resourcemanager/rmapp/RMAppImpl.java | 50 +- .../rmapp/RMAppRejectedEvent.java | 35 - .../rmapp/attempt/RMAppAttemptEvent.java | 11 + .../rmapp/attempt/RMAppAttemptEventType.java | 1 + .../rmapp/attempt/RMAppAttemptImpl.java | 103 +- .../RMAppAttemptContainerAllocatedEvent.java | 31 - .../event/RMAppAttemptLaunchFailedEvent.java | 38 - .../event/RMAppAttemptRegistrationEvent.java | 2 - .../event/RMAppAttemptUnregistrationEvent.java | 11 +- .../rmcontainer/RMContainerImpl.java | 7 +- .../resourcemanager/rmnode/RMNodeEventType.java | 3 + .../resourcemanager/rmnode/RMNodeImpl.java | 21 + .../rmnode/RMNodeSignalContainerEvent.java | 38 + .../scheduler/AbstractYarnScheduler.java | 5 +- .../resourcemanager/scheduler/Allocation.java | 8 +- .../scheduler/AppSchedulingInfo.java | 50 +- .../scheduler/QueueInvalidException.java | 32 + .../scheduler/QueueNotFoundException.java | 32 - .../scheduler/SchedulerApplicationAttempt.java | 31 +- .../scheduler/YarnScheduler.java | 9 + .../scheduler/capacity/AbstractCSQueue.java | 4 +- .../scheduler/capacity/CSQueueUtils.java | 5 + .../scheduler/capacity/CapacityScheduler.java | 159 +- .../CapacitySchedulerConfiguration.java | 19 +- .../scheduler/capacity/LeafQueue.java | 248 +- .../scheduler/capacity/QueueCapacities.java | 15 +- .../scheduler/capacity/UserInfo.java | 10 +- .../allocator/IncreaseContainerAllocator.java | 4 +- .../scheduler/common/fica/FiCaSchedulerApp.java | 46 +- .../scheduler/fair/AllocationConfiguration.java | 26 +- .../fair/AllocationFileLoaderService.java | 15 +- .../scheduler/fair/FSAppAttempt.java | 105 +- .../scheduler/fair/FairScheduler.java | 66 +- .../fair/FairSchedulerConfiguration.java | 11 +- .../scheduler/fair/QueuePlacementRule.java | 6 +- .../security/DelegationTokenRenewer.java | 73 +- .../webapp/CapacitySchedulerPage.java | 102 +- .../resourcemanager/webapp/RMAppsBlock.java | 40 +- .../resourcemanager/webapp/RMWebServices.java | 28 +- .../resourcemanager/webapp/dao/AppInfo.java | 6 +- .../webapp/dao/CapacitySchedulerInfo.java | 40 +- .../dao/CapacitySchedulerLeafQueueInfo.java | 10 +- .../webapp/dao/CapacitySchedulerQueueInfo.java | 37 +- .../dao/PartitionQueueCapacitiesInfo.java | 110 + .../webapp/dao/PartitionResourceUsageInfo.java | 89 + .../webapp/dao/QueueCapacitiesInfo.java | 93 + .../webapp/dao/ResourceUsageInfo.java | 79 + .../yarn/server/resourcemanager/MockRM.java | 67 +- .../server/resourcemanager/RMHATestBase.java | 16 +- .../TestApplicationMasterService.java | 43 + .../resourcemanager/TestClientRMService.java | 24 +- .../resourcemanager/TestRMAdminService.java | 8 +- .../server/resourcemanager/TestRMRestart.java | 86 +- .../TestReservationSystemWithRMHA.java | 518 +++ .../TestResourceTrackerService.java | 5 +- .../resourcemanager/TestSignalContainer.java | 113 + .../TestWorkPreservingRMRestart.java | 305 +- .../applicationsmanager/MockAsm.java | 2 +- .../applicationsmanager/TestAMRestart.java | 119 +- .../metrics/TestSystemMetricsPublisher.java | 2 +- .../TestRMDelegatedNodeLabelsUpdater.java | 163 + .../nodelabels/TestRMNodeLabelsManager.java | 131 +- .../recovery/TestMemoryRMStateStore.java | 65 + .../recovery/TestZKRMStateStore.java | 68 + .../reservation/ReservationSystemTestUtil.java | 39 +- .../reservation/TestCapacityOverTimePolicy.java | 81 +- .../reservation/TestInMemoryPlan.java | 36 +- .../reservation/TestNoOverCommitPolicy.java | 53 +- .../TestSchedulerPlanFollowerBase.java | 30 +- .../planning/TestAlignedPlanner.java | 16 +- .../planning/TestGreedyReservationAgent.java | 44 +- .../planning/TestSimpleCapacityReplanner.java | 37 +- .../rmapp/TestRMAppTransitions.java | 83 +- .../attempt/TestRMAppAttemptTransitions.java | 111 +- .../TestSchedulerApplicationAttempt.java | 64 +- .../capacity/TestApplicationLimits.java | 5 + .../TestApplicationLimitsByPartition.java | 511 +++ .../capacity/TestCapacityScheduler.java | 24 +- .../TestCapacitySchedulerNodeLabelUpdate.java | 103 +- .../capacity/TestContainerResizing.java | 72 + .../TestNodeLabelContainerAllocation.java | 116 +- .../scheduler/capacity/TestUtils.java | 6 +- .../scheduler/fair/FairSchedulerTestBase.java | 2 + .../fair/TestAllocationFileLoaderService.java | 18 +- .../scheduler/fair/TestFairScheduler.java | 192 + .../fair/TestFairSchedulerConfiguration.java | 4 + .../fair/TestQueuePlacementPolicy.java | 18 + .../scheduler/fifo/TestFifoScheduler.java | 8 +- .../webapp/TestRMWebServicesApps.java | 9 +- .../webapp/TestRMWebServicesCapacitySched.java | 6 +- .../TestRMWebServicesForCSWithPartitions.java | 639 ++++ .../webapp/TestRMWebServicesNodeLabels.java | 4 +- .../hadoop/yarn/server/MiniYARNCluster.java | 15 +- .../hadoop/yarn/server/TestMiniYarnCluster.java | 9 + .../src/site/markdown/FairScheduler.md | 3 + .../src/site/markdown/NodeManagerRest.md | 10 +- .../src/site/markdown/ResourceManagerRest.md | 8 + .../src/site/markdown/TimelineServer.md | 8 + 1416 files changed, 94305 insertions(+), 35992 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3100002d/hadoop-hdfs-project/hadoop-hdfs/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3100002d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3100002d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 41b572f,3c6c4d3..27b292a --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@@ -297,15 -293,14 +294,21 @@@ public class BlockManager implements Bl /** Check whether there are any non-EC blocks using StripedID */ private boolean hasNonEcBlockUsingStripedID = false; + /** Keeps track of how many bytes are in Future Generation blocks. */ + private AtomicLong numberOfBytesInFutureBlocks; + + /** Reports if Name node was started with Rollback option. */ + private boolean inRollBack = false; + public BlockManager(final Namesystem namesystem, final Configuration conf) - throws IOException { + throws IOException { + // Compute the map capacity by allocating 2% of total memory + this(namesystem, conf, new BlocksMap( + LightWeightGSet.computeCapacity(2.0, "BlocksMap"))); + } + + public BlockManager(final Namesystem namesystem, final Configuration conf, + BlocksMap blocksMap) throws IOException { this.namesystem = namesystem; datanodeManager = new DatanodeManager(this, namesystem, conf); heartbeatManager = datanodeManager.getHeartbeatManager(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3100002d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3100002d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3100002d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3100002d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3100002d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index e4df760,575e7cc..c33d160 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@@ -541,8 -538,9 +541,9 @@@ class BPServiceActor implements Runnabl HeartbeatResponse sendHeartBeat(boolean requestBlockReportLease) throws IOException { + scheduler.scheduleNextHeartbeat(); StorageReport[] reports = - dn.getFSDataset().getStorageReports(bpos.getBlockPoolId()); + dataset.getStorageReports(bpos.getBlockPoolId()); if (LOG.isDebugEnabled()) { LOG.debug("Sending heartbeat with " + reports.length + " storage reports from service actor: " + this); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3100002d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3100002d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java ----------------------------------------------------------------------
