Merge branch 'trunk' into HDFS-12943
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/47d72601 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/47d72601 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/47d72601 Branch: refs/heads/HDFS-12943 Commit: 47d726015810d091f58387a0fc6bcc0ad30f4b02 Parents: b549707 1dabb31 Author: Konstantin V Shvachko <s...@apache.org> Authored: Wed Dec 5 13:38:45 2018 -0800 Committer: Konstantin V Shvachko <s...@apache.org> Committed: Wed Dec 5 13:38:45 2018 -0800 ---------------------------------------------------------------------- .../resources/assemblies/hadoop-yarn-dist.xml | 23 +- .../authentication/util/KerberosName.java | 9 +- .../TestKerberosAuthenticationHandler.java | 7 +- .../authentication/util/TestKerberosName.java | 17 +- .../hadoop-common/src/main/conf/hadoop-env.sh | 13 - .../hadoop/fs/FileSystemMultipartUploader.java | 36 +- .../org/apache/hadoop/fs/MultipartUploader.java | 88 ++- .../hadoop/fs/MultipartUploaderFactory.java | 7 + .../apache/hadoop/fs/shell/CopyCommands.java | 7 +- .../java/org/apache/hadoop/security/KDiag.java | 46 +- .../hadoop/security/LdapGroupsMapping.java | 163 ++-- .../src/main/resources/core-default.xml | 28 +- .../src/site/markdown/GroupsMapping.md | 54 +- .../src/site/markdown/SecureMode.md | 6 - .../src/site/markdown/SingleCluster.md.vm | 3 - .../src/site/markdown/filesystem/index.md | 1 + .../markdown/filesystem/multipartuploader.md | 235 ++++++ .../AbstractContractMultipartUploaderTest.java | 565 +++++++++++--- .../TestLocalFSContractMultipartUploader.java | 10 + .../hadoop/fs/shell/TestCopyPreserveFlag.java | 10 + .../org/apache/hadoop/security/TestKDiag.java | 16 - .../hadoop/security/TestLdapGroupsMapping.java | 80 +- .../security/TestLdapGroupsMappingBase.java | 76 +- .../TestLdapGroupsMappingWithFailover.java | 142 ++++ .../TestLdapGroupsMappingWithOneQuery.java | 16 +- .../TestLdapGroupsMappingWithPosixGroup.java | 10 +- .../security/TestUserGroupInformation.java | 27 +- .../hdds/scm/storage/ChunkInputStream.java | 11 + .../hdds/scm/storage/ChunkOutputStream.java | 21 +- .../org/apache/hadoop/hdds/HddsConfigKeys.java | 8 + .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 8 + .../hdds/scm/container/ContainerInfo.java | 29 +- .../scm/storage/ContainerProtocolCalls.java | 14 +- .../apache/hadoop/ozone/OzoneConfigKeys.java | 27 +- .../org/apache/hadoop/ozone/audit/DNAction.java | 44 +- .../apache/hadoop/ozone/common/Checksum.java | 249 ++++++ .../hadoop/ozone/common/ChecksumData.java | 190 +++++ .../ozone/common/OzoneChecksumException.java | 66 ++ .../container/common/helpers/ChunkInfo.java | 33 +- .../hadoop/utils/db/ByteArrayKeyValue.java | 67 ++ .../java/org/apache/hadoop/utils/db/Codec.java | 38 + .../apache/hadoop/utils/db/CodecRegistry.java | 70 ++ .../org/apache/hadoop/utils/db/DBStore.java | 12 +- .../org/apache/hadoop/utils/db/RDBStore.java | 12 +- .../hadoop/utils/db/RDBStoreIterator.java | 20 +- .../org/apache/hadoop/utils/db/RDBTable.java | 7 +- .../org/apache/hadoop/utils/db/StringCodec.java | 45 ++ .../java/org/apache/hadoop/utils/db/Table.java | 63 +- .../apache/hadoop/utils/db/TableIterator.java | 6 +- .../org/apache/hadoop/utils/db/TypedTable.java | 184 +++++ .../main/proto/DatanodeContainerProtocol.proto | 21 +- hadoop-hdds/common/src/main/proto/hdds.proto | 20 +- .../common/src/main/resources/ozone-default.xml | 54 ++ .../apache/hadoop/ozone/audit/DummyAction.java | 36 +- .../hadoop/ozone/common/TestChecksum.java | 101 +++ .../hadoop/utils/db/TestDBStoreBuilder.java | 4 +- .../apache/hadoop/utils/db/TestRDBStore.java | 31 +- .../hadoop/utils/db/TestRDBTableStore.java | 4 +- .../hadoop/utils/db/TestTypedRDBTableStore.java | 235 ++++++ .../container/common/impl/HddsDispatcher.java | 10 +- .../common/interfaces/ContainerDispatcher.java | 5 +- .../container/common/interfaces/Handler.java | 4 +- .../transport/server/GrpcXceiverService.java | 3 +- .../transport/server/XceiverServerGrpc.java | 2 +- .../server/ratis/ContainerStateMachine.java | 234 +++--- .../server/ratis/DispatcherContext.java | 133 ++++ .../server/ratis/XceiverServerRatis.java | 14 +- .../container/keyvalue/KeyValueHandler.java | 91 ++- .../container/keyvalue/helpers/BlockUtils.java | 8 +- .../container/keyvalue/helpers/ChunkUtils.java | 54 +- .../keyvalue/helpers/SmallFileUtils.java | 10 +- .../keyvalue/impl/ChunkManagerImpl.java | 52 +- .../keyvalue/interfaces/ChunkManager.java | 12 +- .../common/impl/TestHddsDispatcher.java | 16 +- .../keyvalue/TestChunkManagerImpl.java | 57 +- .../container/keyvalue/TestKeyValueHandler.java | 48 +- hadoop-hdds/docs/config.yaml | 5 + hadoop-hdds/docs/content/Prometheus.md | 94 +++ .../hadoop/hdds/server/BaseHttpServer.java | 20 + .../hdds/server/PrometheusMetricsSink.java | 133 ++++ .../hadoop/hdds/server/PrometheusServlet.java | 46 ++ .../hdds/server/TestPrometheusMetricsSink.java | 96 +++ hadoop-hdds/pom.xml | 1 + .../scm/chillmode/PipelineChillModeRule.java | 108 +++ .../hdds/scm/chillmode/SCMChillModeManager.java | 19 +- .../scm/container/ContainerReportHandler.java | 50 +- .../scm/container/ContainerStateManager.java | 58 +- .../IncrementalContainerReportHandler.java | 24 +- .../hdds/scm/container/ReportHandlerHelper.java | 368 +++++++++ .../hadoop/hdds/scm/server/SCMMXBean.java | 5 + .../scm/server/StorageContainerManager.java | 16 +- .../scm/chillmode/TestSCMChillModeManager.java | 81 +- .../hdds/scm/container/MockNodeManager.java | 1 + .../container/TestContainerReportHandler.java | 772 +++++++++++++++---- .../container/TestContainerReportHelper.java | 113 +++ .../TestIncrementalContainerReportHandler.java | 218 ++++++ .../hadoop/hdfs/DFSOpsCountStatistics.java | 9 + .../hadoop/hdfs/DistributedFileSystem.java | 18 + .../federation/metrics/NamenodeBeanMetrics.java | 5 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 + .../server/blockmanagement/BlockManager.java | 30 +- .../BlockPlacementPolicyDefault.java | 22 +- .../server/blockmanagement/DatanodeManager.java | 7 + .../hdfs/server/common/ECTopologyVerifier.java | 124 +++ .../diskbalancer/planner/GreedyPlanner.java | 24 +- .../namenode/ECTopologyVerifierResult.java | 45 ++ .../namenode/ErasureCodingPolicyManager.java | 9 + .../server/namenode/FSDirStatAndListingOp.java | 9 + .../hdfs/server/namenode/FSDirectory.java | 7 + .../hadoop/hdfs/server/namenode/FSEditLog.java | 11 +- .../hdfs/server/namenode/FSNamesystem.java | 19 + .../hadoop/hdfs/server/namenode/JournalSet.java | 10 +- .../hdfs/server/namenode/NameNodeMXBean.java | 7 + .../hdfs/server/namenode/NamenodeFsck.java | 49 +- .../org/apache/hadoop/hdfs/tools/ECAdmin.java | 51 +- .../OfflineImageViewerPB.java | 27 +- .../offlineImageViewer/PBImageCorruption.java | 107 +++ .../PBImageCorruptionDetector.java | 344 +++++++++ .../PBImageDelimitedTextWriter.java | 38 +- .../offlineImageViewer/PBImageTextWriter.java | 220 +++++- .../src/main/resources/hdfs-default.xml | 10 + .../src/site/markdown/HdfsImageViewer.md | 36 +- .../hadoop-hdfs/src/site/markdown/WebHDFS.md | 24 + .../hdfs/TestHDFSContractMultipartUploader.java | 15 + .../org/apache/hadoop/hdfs/DFSTestUtil.java | 35 + .../hadoop/hdfs/TestDistributedFileSystem.java | 63 +- .../hdfs/TestErasureCodingMultipleRacks.java | 18 +- .../java/org/apache/hadoop/hdfs/TestQuota.java | 8 + .../hadoop/hdfs/qjournal/TestNNWithQJM.java | 15 +- .../hdfs/server/namenode/TestEditLog.java | 8 +- .../namenode/TestEditLogJournalFailures.java | 44 ++ .../server/namenode/TestNameNodeMXBean.java | 19 + .../apache/hadoop/hdfs/tools/TestECAdmin.java | 157 ++++ .../TestOfflineImageViewer.java | 219 +++++- .../TestPBImageCorruption.java | 55 ++ .../test/resources/testMultipleCorruption.csv | 22 + .../resources/testMultipleFileCorruption.csv | 21 + .../test/resources/testSingleFileCorruption.csv | 18 + .../resources/testSingleFolderCorruption.csv | 18 + .../mapreduce/v2/app/TaskHeartbeatHandler.java | 34 +- .../v2/app/TestTaskHeartbeatHandler.java | 53 +- .../apache/hadoop/mapreduce/MRJobConfig.java | 8 + .../mapreduce/jobhistory/EventWriter.java | 1 + .../lib/output/FileOutputCommitter.java | 28 +- .../src/main/resources/mapred-default.xml | 9 + .../lib/output/TestFileOutputCommitter.java | 33 + .../ozone/client/io/ChunkGroupOutputStream.java | 30 +- .../hadoop/ozone/client/rpc/RpcClient.java | 26 + .../org/apache/hadoop/ozone/audit/OMAction.java | 54 +- .../hadoop/ozone/om/OMMetadataManager.java | 17 +- .../dist/src/main/compose/ozoneperf/README.md | 49 +- .../src/main/compose/ozoneperf/compose-all.sh | 18 - .../compose/ozoneperf/docker-compose-freon.yaml | 26 - .../main/compose/ozoneperf/docker-compose.yaml | 23 +- .../src/main/compose/ozoneperf/docker-config | 5 +- .../dist/src/main/compose/ozoneperf/init.sh | 21 - .../src/main/compose/ozoneperf/prometheus.yml | 11 +- .../ozone/client/rpc/TestOzoneRpcClient.java | 98 ++- .../ozone/container/ContainerTestHelper.java | 17 +- .../common/TestBlockDeletingService.java | 3 +- .../container/common/helpers/TestBlockData.java | 7 +- .../common/impl/TestContainerPersistence.java | 73 +- .../transport/server/ratis/TestCSMMetrics.java | 3 +- .../container/server/TestContainerServer.java | 6 +- .../hadoop/ozone/om/TestOzoneManager.java | 4 +- .../apache/hadoop/ozone/scm/TestSCMMXBean.java | 81 +- .../web/storage/DistributedStorageHandler.java | 26 + .../hadoop/ozone/om/OmMetadataManagerImpl.java | 82 +- .../hadoop/ozone/web/ozShell/OzoneAddress.java | 6 + .../ozone/web/ozShell/TestOzoneAddress.java | 3 +- hadoop-ozone/pom.xml | 3 +- .../ozone/s3/endpoint/ObjectEndpoint.java | 11 +- .../hadoop/ozone/s3/endpoint/TestObjectGet.java | 2 + .../genesis/BenchMarkDatanodeDispatcher.java | 16 +- hadoop-project/pom.xml | 4 +- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 16 +- .../hadoop/fs/s3a/S3AMultipartUploader.java | 31 +- .../fs/s3a/s3guard/DynamoDBMetadataStore.java | 2 +- .../fs/s3a/s3guard/LocalMetadataEntry.java | 3 + .../fs/s3a/s3guard/LocalMetadataStore.java | 29 +- .../hadoop/fs/s3a/s3guard/S3GuardTool.java | 38 +- .../s3a/ITestS3AContractMultipartUploader.java | 64 +- .../fs/s3a/ITestS3ACopyFromLocalFile.java | 15 +- .../fs/s3a/ITestS3GuardListConsistency.java | 18 +- .../fs/s3a/TestS3AMultipartUploaderSupport.java | 2 +- .../s3a/s3guard/ITestDynamoDBMetadataStore.java | 18 +- .../fs/s3a/s3guard/MetadataStoreTestBase.java | 15 + .../fs/azure/AzureNativeFileSystemStore.java | 62 +- .../hadoop/fs/azurebfs/AzureBlobFileSystem.java | 39 +- .../fs/azurebfs/AzureBlobFileSystemStore.java | 13 +- .../azure/ITestNativeAzureFileSystemLive.java | 4 +- .../fs/azure/TestKeyPageBlobDirectories.java | 170 ++++ .../hadoop/yarn/service/webapp/ApiServer.java | 209 +++-- .../hadoop/yarn/service/ServiceClientTest.java | 2 +- .../hadoop/yarn/service/ServiceScheduler.java | 6 +- .../yarn/service/client/ServiceClient.java | 1 + .../yarn/service/component/Component.java | 4 +- .../component/instance/ComponentInstance.java | 57 +- .../instance/ComponentInstanceState.java | 3 +- .../yarn/service/monitor/ServiceMonitor.java | 5 +- .../hadoop/yarn/service/TestServiceManager.java | 22 +- .../yarn/service/TestYarnNativeServices.java | 3 +- .../yarn/service/component/TestComponent.java | 8 + .../instance/TestComponentInstance.java | 39 +- .../installation/install.conf | 74 -- .../installation/install.sh | 116 --- .../package/calico/calico-node.service | 50 -- .../installation/package/calico/calicoctl.cfg | 22 - .../installation/package/docker/daemon.json | 23 - .../installation/package/docker/docker.service | 35 - .../installation/package/etcd/etcd.service | 40 - .../package/hadoop/container-executor.cfg | 41 - .../installation/package/submarine/submarine.sh | 25 - .../installation/scripts/calico.sh | 224 ------ .../installation/scripts/docker.sh | 166 ---- .../installation/scripts/download-server.sh | 42 - .../installation/scripts/environment.sh | 213 ----- .../installation/scripts/etcd.sh | 152 ---- .../installation/scripts/hadoop.sh | 117 --- .../installation/scripts/menu.sh | 444 ----------- .../installation/scripts/nvidia-docker.sh | 99 --- .../installation/scripts/nvidia.sh | 120 --- .../installation/scripts/submarine.sh | 38 - .../installation/scripts/utils.sh | 123 --- .../yarnservice/YarnServiceJobSubmitter.java | 4 +- .../hadoop/yarn/client/cli/ApplicationCLI.java | 17 + .../hadoop/yarn/client/cli/ClusterCLI.java | 2 + .../apache/hadoop/yarn/client/cli/LogsCLI.java | 105 ++- .../apache/hadoop/yarn/client/cli/NodeCLI.java | 2 + .../apache/hadoop/yarn/client/cli/QueueCLI.java | 2 +- .../apache/hadoop/yarn/client/cli/TopCLI.java | 1 + .../apache/hadoop/yarn/client/cli/YarnCLI.java | 9 +- .../hadoop/yarn/client/cli/TestClusterCLI.java | 41 +- .../hadoop/yarn/client/cli/TestLogsCLI.java | 37 + .../hadoop/yarn/client/cli/TestYarnCLI.java | 89 ++- .../api/impl/TimelineReaderClientImpl.java | 2 + .../LogAggregationFileController.java | 74 +- .../hadoop-yarn/hadoop-yarn-csi/pom.xml | 78 +- .../utils/FederationStateStoreFacade.java | 5 +- .../utils/FederationStateStoreTestUtil.java | 2 +- .../utils/TestFederationStateStoreFacade.java | 17 + .../launcher/ContainerLaunch.java | 2 +- .../linux/resources/CGroupsHandlerImpl.java | 3 +- .../runtime/DockerLinuxContainerRuntime.java | 70 +- .../linux/runtime/docker/DockerRunCommand.java | 6 + .../resourceplugin/ResourcePluginManager.java | 14 +- .../deviceframework/DeviceMappingManager.java | 324 ++++++++ .../deviceframework/DevicePluginAdapter.java | 20 +- .../DeviceResourceHandlerImpl.java | 145 ++++ .../impl/modules/fpga/fpga-module.c | 5 +- .../impl/modules/gpu/gpu-module.c | 5 +- .../container-executor/impl/utils/docker-util.c | 72 ++ .../container-executor/impl/utils/docker-util.h | 1 + .../test/utils/test_docker_util.cc | 94 ++- .../launcher/TestContainerLaunch.java | 2 +- .../linux/resources/TestCGroupsHandlerImpl.java | 38 + .../resources/gpu/TestGpuResourceHandler.java | 2 +- .../runtime/TestDockerContainerRuntime.java | 15 +- .../runtime/docker/TestDockerRunCommand.java | 9 +- .../TestDeviceMappingManager.java | 366 +++++++++ .../TestDevicePluginAdapter.java | 388 +++++++++- .../placement/AppNameMappingPlacementRule.java | 12 +- .../placement/PlacementRule.java | 4 +- .../UserGroupMappingPlacementRule.java | 11 +- .../scheduler/fair/FSPreemptionThread.java | 55 +- .../resourcemanager/webapp/NodesPage.java | 4 +- .../webapp/dao/SchedulerInfo.java | 2 + .../fair/TestFairSchedulerPreemption.java | 155 ++-- .../src/site/markdown/DockerContainers.md | 1 + 269 files changed, 10395 insertions(+), 4277 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/47d72601/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/47d72601/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/47d72601/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
