[50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into HDFS-12943
Merge branch 'trunk' into HDFS-12943 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/375ac944 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/375ac944 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/375ac944 Branch: refs/heads/HDFS-12943 Commit: 375ac94462947afa1f5c073e14d8aa6fbfe7e7f6 Parents: 21f1432 aa12859 Author: Konstantin V Shvachko Authored: Fri Dec 14 10:53:54 2018 -0800 Committer: Konstantin V Shvachko Committed: Fri Dec 14 10:53:54 2018 -0800 -- LICENSE.txt | 33 + NOTICE.txt |7 + .../hadoop-client-minicluster/pom.xml |8 + .../hadoop/security/LdapGroupsMapping.java | 23 +- .../hadoop/security/token/DtFileOperations.java | 25 +- .../org/apache/hadoop/security/token/Token.java | 44 +- .../src/main/resources/core-default.xml | 10 + .../java/org/apache/hadoop/ipc/TestSaslRPC.java |4 +- .../hadoop/security/TestLdapGroupsMapping.java | 42 +- .../apache/hadoop/security/token/TestToken.java | 22 +- .../hadoop/hdds/scm/XceiverClientGrpc.java |6 +- .../hadoop/hdds/scm/XceiverClientRatis.java | 110 +- .../hdds/scm/storage/ChunkOutputStream.java | 426 +- .../hdds/scm/XceiverClientAsyncReply.java | 44 +- .../hadoop/hdds/scm/XceiverClientSpi.java | 19 +- .../scm/storage/ContainerProtocolCalls.java |1 - .../apache/hadoop/ozone/OzoneConfigKeys.java|8 + .../org/apache/hadoop/ozone/OzoneConsts.java| 13 + .../java/org/apache/hadoop/utils/db/Codec.java |3 + .../apache/hadoop/utils/db/CodecRegistry.java | 19 + .../org/apache/hadoop/utils/db/DBStore.java | 13 +- .../apache/hadoop/utils/db/DBStoreBuilder.java |9 +- .../org/apache/hadoop/utils/db/RDBStore.java| 106 +- .../org/apache/hadoop/utils/db/TypedTable.java |3 + .../common/src/main/resources/ozone-default.xml | 123 +- .../apache/hadoop/hdds/scm/HddsServerUtil.java | 15 +- .../server/ratis/DispatcherContext.java |4 +- .../datanode/ReplicaNotFoundException.java |8 +- .../hdfs/util/CombinedHostsFileReader.java | 41 +- .../apache/hadoop/hdfs/web/JsonUtilClient.java | 11 + .../hadoop/hdfs/web/WebHdfsFileSystem.java | 33 + .../hadoop/hdfs/web/resources/GetOpParam.java |2 + .../hadoop/hdfs/web/resources/PostOpParam.java |2 + .../hadoop/hdfs/web/resources/PutOpParam.java |1 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java |4 + .../hadoop/hdfs/protocol/CacheDirective.java|3 +- .../hdfs/server/aliasmap/InMemoryAliasMap.java | 216 +- .../aliasmap/InMemoryLevelDBAliasMapServer.java |8 + .../hdfs/server/namenode/CacheManager.java | 38 +- .../namenode/ErasureCodingPolicyManager.java| 13 + .../hdfs/server/namenode/ImageServlet.java | 32 +- .../hadoop/hdfs/server/namenode/NameNode.java | 11 + .../server/namenode/NameNodeHttpServer.java | 15 + .../hdfs/server/namenode/TransferFsImage.java | 21 +- .../server/namenode/ha/BootstrapStandby.java| 91 +- .../namenode/metrics/NameNodeMetrics.java |6 + .../web/resources/NamenodeWebHdfsMethods.java | 15 +- .../src/main/resources/hdfs-default.xml |8 + .../hadoop-hdfs/src/site/markdown/WebHDFS.md| 89 +- .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 23 + .../hadoop/hdfs/TestErasureCodingAddConfig.java | 79 + .../hadoop/hdfs/TestMaintenanceState.java |2 + .../java/org/apache/hadoop/hdfs/TestPread.java |3 +- .../server/aliasmap/ITestInMemoryAliasMap.java | 30 + .../server/aliasmap/TestInMemoryAliasMap.java | 45 - .../impl/TestInMemoryLevelDBAliasMapClient.java |9 - .../namenode/TestListCorruptFileBlocks.java | 24 +- .../namenode/ha/TestBootstrapAliasmap.java | 99 + .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 18 +- .../v2/app/job/impl/TestTaskAttempt.java| 38 +- .../PluggableShuffleAndPluggableSort.md |4 +- .../apache/hadoop/mapred/TestYARNRunner.java| 44 +- .../hadoop-mapreduce-client-uploader/pom.xml|6 + .../mapred/uploader/FrameworkUploader.java | 52 + .../mapred/uploader/TestFrameworkUploader.java | 58 + .../apache/hadoop/ozone/client/OzoneBucket.java | 37 + .../ozone/client/io/ChunkGroupOutputStream.java | 166 +- .../ozone/client/io/OzoneOutputStream.java | 10 + .../ozone/client/protocol/ClientProtocol.java | 22 + .../hadoop/ozone/client/rest/RestClient.java| 24 + .../hadoop/ozone/client/rpc/RpcClient.java | 73 + .../org/apache/hadoop/ozone/audit/OMAction.java |4 +- .../apache/hadoop/ozone/om/OMConfigKeys.java| 60 + .../hadoop/ozone/om/OMMetadataManager.java | 58 +- .../ozone/om/codec/OmBucketInfoCodec.java
[50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into HDFS-12943
Merge branch 'trunk' into HDFS-12943 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/47d72601 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/47d72601 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/47d72601 Branch: refs/heads/HDFS-12943 Commit: 47d726015810d091f58387a0fc6bcc0ad30f4b02 Parents: b549707 1dabb31 Author: Konstantin V Shvachko Authored: Wed Dec 5 13:38:45 2018 -0800 Committer: Konstantin V Shvachko Committed: Wed Dec 5 13:38:45 2018 -0800 -- .../resources/assemblies/hadoop-yarn-dist.xml | 23 +- .../authentication/util/KerberosName.java | 9 +- .../TestKerberosAuthenticationHandler.java | 7 +- .../authentication/util/TestKerberosName.java | 17 +- .../hadoop-common/src/main/conf/hadoop-env.sh | 13 - .../hadoop/fs/FileSystemMultipartUploader.java | 36 +- .../org/apache/hadoop/fs/MultipartUploader.java | 88 ++- .../hadoop/fs/MultipartUploaderFactory.java | 7 + .../apache/hadoop/fs/shell/CopyCommands.java| 7 +- .../java/org/apache/hadoop/security/KDiag.java | 46 +- .../hadoop/security/LdapGroupsMapping.java | 163 ++-- .../src/main/resources/core-default.xml | 28 +- .../src/site/markdown/GroupsMapping.md | 54 +- .../src/site/markdown/SecureMode.md | 6 - .../src/site/markdown/SingleCluster.md.vm | 3 - .../src/site/markdown/filesystem/index.md | 1 + .../markdown/filesystem/multipartuploader.md| 235 ++ .../AbstractContractMultipartUploaderTest.java | 565 +++--- .../TestLocalFSContractMultipartUploader.java | 10 + .../hadoop/fs/shell/TestCopyPreserveFlag.java | 10 + .../org/apache/hadoop/security/TestKDiag.java | 16 - .../hadoop/security/TestLdapGroupsMapping.java | 80 +- .../security/TestLdapGroupsMappingBase.java | 76 +- .../TestLdapGroupsMappingWithFailover.java | 142 .../TestLdapGroupsMappingWithOneQuery.java | 16 +- .../TestLdapGroupsMappingWithPosixGroup.java| 10 +- .../security/TestUserGroupInformation.java | 27 +- .../hdds/scm/storage/ChunkInputStream.java | 11 + .../hdds/scm/storage/ChunkOutputStream.java | 21 +- .../org/apache/hadoop/hdds/HddsConfigKeys.java | 8 + .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 8 + .../hdds/scm/container/ContainerInfo.java | 29 +- .../scm/storage/ContainerProtocolCalls.java | 14 +- .../apache/hadoop/ozone/OzoneConfigKeys.java| 27 +- .../org/apache/hadoop/ozone/audit/DNAction.java | 44 +- .../apache/hadoop/ozone/common/Checksum.java| 249 ++ .../hadoop/ozone/common/ChecksumData.java | 190 + .../ozone/common/OzoneChecksumException.java| 66 ++ .../container/common/helpers/ChunkInfo.java | 33 +- .../hadoop/utils/db/ByteArrayKeyValue.java | 67 ++ .../java/org/apache/hadoop/utils/db/Codec.java | 38 + .../apache/hadoop/utils/db/CodecRegistry.java | 70 ++ .../org/apache/hadoop/utils/db/DBStore.java | 12 +- .../org/apache/hadoop/utils/db/RDBStore.java| 12 +- .../hadoop/utils/db/RDBStoreIterator.java | 20 +- .../org/apache/hadoop/utils/db/RDBTable.java| 7 +- .../org/apache/hadoop/utils/db/StringCodec.java | 45 ++ .../java/org/apache/hadoop/utils/db/Table.java | 63 +- .../apache/hadoop/utils/db/TableIterator.java | 6 +- .../org/apache/hadoop/utils/db/TypedTable.java | 184 + .../main/proto/DatanodeContainerProtocol.proto | 21 +- hadoop-hdds/common/src/main/proto/hdds.proto| 20 +- .../common/src/main/resources/ozone-default.xml | 54 ++ .../apache/hadoop/ozone/audit/DummyAction.java | 36 +- .../hadoop/ozone/common/TestChecksum.java | 101 +++ .../hadoop/utils/db/TestDBStoreBuilder.java | 4 +- .../apache/hadoop/utils/db/TestRDBStore.java| 31 +- .../hadoop/utils/db/TestRDBTableStore.java | 4 +- .../hadoop/utils/db/TestTypedRDBTableStore.java | 235 ++ .../container/common/impl/HddsDispatcher.java | 10 +- .../common/interfaces/ContainerDispatcher.java | 5 +- .../container/common/interfaces/Handler.java| 4 +- .../transport/server/GrpcXceiverService.java| 3 +- .../transport/server/XceiverServerGrpc.java | 2 +- .../server/ratis/ContainerStateMachine.java | 234 +++--- .../server/ratis/DispatcherContext.java | 133 .../server/ratis/XceiverServerRatis.java| 14 +- .../container/keyvalue/KeyValueHandler.java | 91 ++- .../container/keyvalue/helpers/BlockUtils.java | 8 +- .../container/keyvalue/helpers/ChunkUtils.java | 54 +- .../keyvalue/helpers/SmallFileUtils.java| 10 +- .../keyvalue/impl/ChunkManagerImpl.java | 52 +- .../keyvalue/interfaces/ChunkManager.java | 12 +- .../common/impl/TestHddsDispatcher.java | 16 +- .../keyvalue/TestChunkManagerImpl.java |
[50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into HDFS-12943
Merge branch 'trunk' into HDFS-12943 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e49fb63b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e49fb63b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e49fb63b Branch: refs/heads/HDFS-12943 Commit: e49fb63be36f5b2f5e1cb7ba1eaee4e10fdc56d3 Parents: 481b618 ad5256e Author: Erik Krogen Authored: Fri Nov 16 09:57:13 2018 -0800 Committer: Erik Krogen Committed: Fri Nov 16 09:57:13 2018 -0800 -- .gitignore | 2 +- BUILDING.txt| 30 +- LICENSE.txt | 2 +- NOTICE.txt | 8 +- dev-support/bin/create-release | 19 +- ...ClusterStorageCapacityExceededException.java | 51 ++ .../org/apache/hadoop/http/HttpServer2.java | 3 + .../io/compress/BlockDecompressorStream.java| 4 +- .../hadoop/metrics2/lib/MutableGaugeFloat.java | 7 + .../hadoop/metrics2/lib/MutableGaugeInt.java| 7 + .../hadoop/metrics2/lib/MutableGaugeLong.java | 6 + .../org/apache/hadoop/security/Credentials.java | 15 + .../hadoop/security/token/DtFileOperations.java | 28 +- .../hadoop/security/token/DtUtilShell.java | 37 +- .../AbstractDelegationTokenSecretManager.java | 12 +- .../src/site/markdown/CommandsManual.md | 1 + .../hadoop-common/src/site/markdown/Metrics.md | 10 + .../compress/TestBlockDecompressorStream.java | 29 + .../apache/hadoop/security/TestCredentials.java | 57 +- .../hadoop/security/token/TestDtUtilShell.java | 44 ++ .../hadoop/hdds/scm/XceiverClientGrpc.java | 29 +- .../hadoop/hdds/scm/XceiverClientManager.java | 1 + .../hadoop/hdds/scm/XceiverClientRatis.java | 67 ++- .../scm/client/ContainerOperationClient.java| 30 +- .../hdds/scm/storage/ChunkInputStream.java | 2 +- .../hdds/scm/storage/ChunkOutputStream.java | 448 +++ hadoop-hdds/common/pom.xml | 6 + .../org/apache/hadoop/hdds/HddsConfigKeys.java | 7 + .../java/org/apache/hadoop/hdds/HddsUtils.java | 31 +- .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 25 +- .../hdds/scm/XceiverClientAsyncReply.java | 98 .../hadoop/hdds/scm/XceiverClientSpi.java | 12 +- .../hadoop/hdds/scm/client/ScmClient.java | 18 +- .../hadoop/hdds/scm/container/ContainerID.java | 2 +- .../hdds/scm/container/ContainerInfo.java | 18 +- .../common/helpers/AllocatedBlock.java | 19 +- .../helpers/ContainerNotOpenException.java | 36 ++ .../helpers/InvalidContainerStateException.java | 35 ++ .../hadoop/hdds/scm/pipeline/Pipeline.java | 11 + .../hdds/scm/protocol/LocatedContainer.java | 2 +- .../hdds/scm/protocol/ScmLocatedBlock.java | 2 +- .../StorageContainerLocationProtocol.java | 2 +- ...kLocationProtocolClientSideTranslatorPB.java | 3 +- .../scm/storage/ContainerProtocolCalls.java | 69 ++- .../apache/hadoop/ozone/OzoneConfigKeys.java| 46 +- .../org/apache/hadoop/ozone/OzoneConsts.java| 2 + .../apache/hadoop/ozone/audit/Auditable.java| 2 +- .../apache/hadoop/ozone/common/StorageInfo.java | 1 - ...kLocationProtocolServerSideTranslatorPB.java | 1 - .../org/apache/hadoop/utils/db/RDBStore.java| 7 +- .../main/proto/DatanodeContainerProtocol.proto | 3 +- .../main/proto/ScmBlockLocationProtocol.proto | 3 +- hadoop-hdds/common/src/main/proto/hdds.proto| 24 +- .../common/src/main/resources/ozone-default.xml | 101 +++- .../apache/hadoop/ozone/audit/DummyAction.java | 2 +- .../hadoop/ozone/lease/TestLeaseManager.java| 17 +- .../apache/hadoop/utils/TestHddsIdFactory.java | 2 +- .../apache/hadoop/utils/TestMetadataStore.java | 96 ++-- .../hadoop/utils/TestRocksDBStoreMBean.java | 6 +- .../hadoop/utils/db/TestDBStoreBuilder.java | 4 +- .../hadoop/utils/db/TestRDBTableStore.java | 6 +- .../org/apache/hadoop/utils/package-info.java | 22 + .../apache/hadoop/hdds/scm/HddsServerUtil.java | 33 +- .../container/common/impl/HddsDispatcher.java | 179 -- .../common/interfaces/ContainerDispatcher.java | 10 + .../statemachine/DatanodeStateMachine.java | 17 +- .../common/statemachine/StateContext.java | 13 +- .../CloseContainerCommandHandler.java | 96 ++-- .../states/endpoint/HeartbeatEndpointTask.java | 12 +- .../states/endpoint/VersionEndpointTask.java| 79 +-- .../server/ratis/ContainerStateMachine.java | 128 +++-- .../server/ratis/XceiverServerRatis.java| 25 + .../container/keyvalue/KeyValueHandler.java | 96 +--- .../container/keyvalue/helpers/BlockUtils.java | 2 +- .../container/keyvalue/helpers/ChunkUtils.java | 30 +-
