Merge branch 'trunk' into HADOOP-13345
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cf36cbd3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cf36cbd3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cf36cbd3 Branch: refs/heads/HADOOP-13345 Commit: cf36cbd35677cdd18b0f07ed52e1a78215113299 Parents: 886d680 3be2659 Author: Steve Loughran <[email protected]> Authored: Fri Jun 30 15:35:20 2017 +0100 Committer: Steve Loughran <[email protected]> Committed: Fri Jun 30 15:35:20 2017 +0100 ---------------------------------------------------------------------- dev-support/bin/create-release | 36 +++- hadoop-assemblies/pom.xml | 4 +- hadoop-build-tools/pom.xml | 2 +- hadoop-client-modules/hadoop-client-api/pom.xml | 4 +- .../hadoop-client-check-invariants/pom.xml | 4 +- .../hadoop-client-check-test-invariants/pom.xml | 4 +- .../hadoop-client-integration-tests/pom.xml | 4 +- .../hadoop-client-minicluster/pom.xml | 4 +- .../hadoop-client-runtime/pom.xml | 4 +- hadoop-client-modules/hadoop-client/pom.xml | 4 +- hadoop-client-modules/pom.xml | 2 +- .../hadoop-cloud-storage/pom.xml | 4 +- hadoop-cloud-storage-project/pom.xml | 4 +- .../hadoop-annotations/pom.xml | 4 +- .../hadoop-auth-examples/pom.xml | 4 +- hadoop-common-project/hadoop-auth/pom.xml | 4 +- hadoop-common-project/hadoop-common/pom.xml | 4 +- .../java/org/apache/hadoop/fs/shell/Ls.java | 63 ++++-- .../erasurecode/coder/ErasureEncodingStep.java | 2 +- .../org/apache/hadoop/net/NetworkTopology.java | 1 - .../hadoop/io/erasurecode/jni_xor_decoder.c | 5 +- .../hadoop/io/erasurecode/jni_xor_encoder.c | 2 +- .../src/site/markdown/FileSystemShell.md | 4 +- .../erasurecode/TestCodecRawCoderMapping.java | 15 +- .../src/test/resources/testConf.xml | 6 +- hadoop-common-project/hadoop-kms/pom.xml | 4 +- .../src/main/conf/kms-log4j.properties | 5 +- hadoop-common-project/hadoop-minikdc/pom.xml | 4 +- hadoop-common-project/hadoop-nfs/pom.xml | 4 +- hadoop-common-project/pom.xml | 4 +- hadoop-dist/pom.xml | 4 +- hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 4 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 19 +- .../org/apache/hadoop/hdfs/DFSInputStream.java | 5 +- .../hadoop/hdfs/DistributedFileSystem.java | 22 +++ .../apache/hadoop/hdfs/client/HdfsAdmin.java | 22 +++ .../hadoop/hdfs/protocol/ClientProtocol.java | 17 ++ .../ClientNamenodeProtocolTranslatorPB.java | 30 +++ .../hadoop/hdfs/protocolPB/PBHelperClient.java | 5 +- .../hdfs/server/protocol/SlowDiskReports.java | 5 +- .../src/main/proto/ClientNamenodeProtocol.proto | 4 + .../src/main/proto/erasurecoding.proto | 14 ++ hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 4 +- .../hadoop-hdfs-native-client/pom.xml | 4 +- hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 4 +- .../dev-support/findbugsExcludeFile.xml | 5 - hadoop-hdfs-project/hadoop-hdfs/pom.xml | 4 +- ...tNamenodeProtocolServerSideTranslatorPB.java | 28 +++ .../hdfs/qjournal/server/JournalNode.java | 16 +- .../token/block/BlockTokenSecretManager.java | 1 + .../server/blockmanagement/BlockManager.java | 13 +- .../blockmanagement/DatanodeDescriptor.java | 4 +- .../server/blockmanagement/DatanodeManager.java | 1 + .../PendingReconstructionBlocks.java | 8 +- .../hdfs/server/common/HdfsServerConstants.java | 7 +- .../hdfs/server/datanode/DataStorage.java | 12 +- .../hdfs/server/datanode/DataXceiver.java | 17 +- .../hdfs/server/datanode/DiskBalancer.java | 12 +- .../namenode/ErasureCodingPolicyManager.java | 56 ++++++ .../server/namenode/FSDirErasureCodingOp.java | 12 ++ .../hdfs/server/namenode/FSNamesystem.java | 56 +++++- .../namenode/NNStorageRetentionManager.java | 27 ++- .../hdfs/server/namenode/NameNodeRpcServer.java | 16 ++ .../namenode/metrics/NameNodeMetrics.java | 18 ++ .../namenode/snapshot/SnapshotDiffInfo.java | 5 +- .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 6 +- .../org/apache/hadoop/hdfs/tools/ECAdmin.java | 99 +++++++++- .../offlineImageViewer/ImageLoaderCurrent.java | 10 +- .../src/site/markdown/HDFSCommands.md | 4 + .../src/site/markdown/HDFSErasureCoding.md | 20 +- .../hadoop/hdfs/TestDistributedFileSystem.java | 48 +++++ .../hadoop/hdfs/TestSnapshotCommands.java | 50 +++++ .../security/token/block/TestBlockToken.java | 35 ++++ .../blockmanagement/TestDatanodeManager.java | 51 +++++ .../TestPendingReconstruction.java | 86 +++++++- .../fsdataset/impl/TestFsDatasetImpl.java | 2 +- .../test/resources/testErasureCodingConf.xml | 196 +++++++++++++++++++ hadoop-hdfs-project/pom.xml | 4 +- hadoop-mapreduce-project/bin/mapred-config.sh | 2 +- .../hadoop-mapreduce-client-app/pom.xml | 4 +- .../v2/app/rm/RMContainerAllocator.java | 6 +- .../v2/app/rm/TestRMContainerAllocator.java | 73 ++++++- .../hadoop-mapreduce-client-common/pom.xml | 4 +- .../hadoop-mapreduce-client-core/pom.xml | 4 +- .../hadoop-mapreduce-client-hs-plugins/pom.xml | 4 +- .../hadoop-mapreduce-client-hs/pom.xml | 4 +- .../hadoop-mapreduce-client-jobclient/pom.xml | 4 +- .../hadoop-mapreduce-client-nativetask/pom.xml | 4 +- .../hadoop-mapreduce-client-shuffle/pom.xml | 4 +- .../hadoop-mapreduce-client/pom.xml | 4 +- .../hadoop-mapreduce-examples/pom.xml | 4 +- hadoop-mapreduce-project/pom.xml | 4 +- hadoop-maven-plugins/pom.xml | 2 +- hadoop-minicluster/pom.xml | 4 +- hadoop-project-dist/pom.xml | 4 +- hadoop-project/pom.xml | 6 +- hadoop-tools/hadoop-aliyun/pom.xml | 2 +- hadoop-tools/hadoop-archive-logs/pom.xml | 4 +- hadoop-tools/hadoop-archives/pom.xml | 4 +- hadoop-tools/hadoop-aws/pom.xml | 4 +- .../apache/hadoop/fs/s3a/S3AInputStream.java | 26 ++- .../src/site/markdown/tools/hadoop-aws/index.md | 109 ++++++++--- .../fs/s3a/ITestS3AFileOperationCost.java | 16 +- .../hadoop-aws/src/test/resources/core-site.xml | 63 +++++- hadoop-tools/hadoop-azure-datalake/pom.xml | 2 +- hadoop-tools/hadoop-azure/pom.xml | 2 +- .../fs/azure/AzureNativeFileSystemStore.java | 14 +- .../hadoop/fs/azure/NativeAzureFileSystem.java | 8 +- .../fs/azure/RemoteSASKeyGeneratorImpl.java | 10 +- .../fs/azure/RemoteWasbAuthorizerImpl.java | 7 +- .../hadoop/fs/azure/SelfRenewingLease.java | 4 +- .../hadoop/fs/azure/SendRequestIntercept.java | 91 +-------- .../fs/azure/AzureBlobStorageTestAccount.java | 7 + .../azure/TestAzureConcurrentOutOfBandIo.java | 4 +- ...zureConcurrentOutOfBandIoWithSecureMode.java | 50 +++++ .../hadoop/fs/azure/TestBlobDataValidation.java | 6 +- .../TestFileSystemOperationsWithThreads.java | 164 +++++++++++----- .../TestNativeAzureFileSystemClientLogging.java | 26 ++- hadoop-tools/hadoop-datajoin/pom.xml | 4 +- hadoop-tools/hadoop-distcp/pom.xml | 4 +- hadoop-tools/hadoop-extras/pom.xml | 4 +- hadoop-tools/hadoop-gridmix/pom.xml | 4 +- hadoop-tools/hadoop-kafka/pom.xml | 4 +- hadoop-tools/hadoop-openstack/pom.xml | 4 +- hadoop-tools/hadoop-pipes/pom.xml | 8 +- hadoop-tools/hadoop-rumen/pom.xml | 4 +- hadoop-tools/hadoop-sls/pom.xml | 4 +- hadoop-tools/hadoop-streaming/pom.xml | 4 +- hadoop-tools/hadoop-tools-dist/pom.xml | 4 +- hadoop-tools/pom.xml | 4 +- .../hadoop-yarn/hadoop-yarn-api/pom.xml | 4 +- .../hadoop/yarn/conf/YarnConfiguration.java | 4 + .../pom.xml | 4 +- .../pom.xml | 4 +- .../hadoop-yarn-applications/pom.xml | 4 +- .../hadoop-yarn/hadoop-yarn-client/pom.xml | 4 +- .../hadoop-yarn/hadoop-yarn-common/pom.xml | 4 +- .../src/main/resources/yarn-default.xml | 12 +- .../hadoop-yarn/hadoop-yarn-registry/pom.xml | 4 +- .../pom.xml | 4 +- .../hadoop-yarn-server-common/pom.xml | 4 +- .../hadoop-yarn-server-nodemanager/pom.xml | 4 +- .../hadoop-yarn-server-resourcemanager/pom.xml | 4 +- .../resourcemanager/scheduler/fair/FSQueue.java | 7 + .../resourcemanager/webapp/DeSelectFields.java | 127 ++++++++++++ .../webapp/FairSchedulerPage.java | 6 +- .../webapp/RMWebServiceProtocol.java | 7 +- .../resourcemanager/webapp/RMWebServices.java | 18 +- .../resourcemanager/webapp/dao/AppInfo.java | 29 ++- .../webapp/dao/FairSchedulerQueueInfo.java | 6 + .../webapp/TestRMWebServices.java | 4 +- .../webapp/TestRMWebServicesApps.java | 64 ++++++ .../pom.xml | 4 +- .../hadoop-yarn-server-tests/pom.xml | 4 +- .../pom.xml | 4 +- .../timeline/LevelDBCacheTimelineStore.java | 4 +- .../pom.xml | 4 +- .../pom.xml | 2 +- .../hadoop-yarn-server-timelineservice/pom.xml | 4 +- .../hadoop-yarn-server-web-proxy/pom.xml | 4 +- .../hadoop-yarn/hadoop-yarn-server/pom.xml | 4 +- .../hadoop-yarn/hadoop-yarn-site/pom.xml | 4 +- .../src/site/markdown/GracefulDecommission.md | 168 ++++++++++++++++ .../src/site/markdown/ResourceManagerRest.md | 180 ++++++++++++++++- .../hadoop-yarn/hadoop-yarn-ui/pom.xml | 4 +- hadoop-yarn-project/hadoop-yarn/pom.xml | 4 +- hadoop-yarn-project/pom.xml | 4 +- pom.xml | 2 +- 168 files changed, 2321 insertions(+), 530 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf36cbd3/hadoop-common-project/hadoop-common/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf36cbd3/hadoop-project/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf36cbd3/hadoop-tools/hadoop-aws/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf36cbd3/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf36cbd3/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf36cbd3/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml ---------------------------------------------------------------------- diff --cc hadoop-tools/hadoop-aws/src/test/resources/core-site.xml index fa7e270,d424aa4..e8200da --- a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml +++ b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml @@@ -36,25 -36,20 +36,39 @@@ <description>The endpoint for s3a://landsat-pds URLs</description> </property> + <!-- Make sure S3Guard is disabled for read-only bucket tests. --> + <property> + <name>fs.s3a.bucket.landsat-pds.metadatastore.impl</name> + <value>${s3guard.null}</value> + <description>The read-only landsat-pds repository isn't + managed by s3guard</description> + </property> + + <!-- Convenience definitions. --> + <property> + <name>s3guard.null</name> + <value>org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore</value> + </property> + + <property> + <name>s3guard.dynamo</name> + <value>org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore</value> + </property> + + <!-- + This is the default endpoint, which can be used to interact + with any v2 region. + --> + <property> + <name>central.endpoint</name> + <value>s3.amazonaws.com</value> + </property> + + <property> + <name>canada.endpoint</name> + <value>s3.ca-central-1.amazonaws.com</value> + </property> + <property> <name>frankfurt.endpoint</name> <value>s3.eu-central-1.amazonaws.com</value> @@@ -66,17 -61,55 +80,62 @@@ </property> <property> - <name>central.endpoint</name> - <value>s3.amazonaws.com</value> + <name>london.endpoint</name> + <value>s3.eu-west-2.amazonaws.com</value> + </property> + + <property> + <name>mumbai.endpoint</name> + <value>s3.ap-south-1.amazonaws.com</value> + </property> + + <property> + <name>ohio.endpoint</name> + <value>s3.us-east-2.amazonaws.com</value> + </property> + + <property> + <name>oregon.endpoint</name> + <value>s3-us-west-2.amazonaws.com</value> + </property> + + <property> + <name>sao-paolo.endpoint</name> + <value>s3-sa-east-1.amazonaws.com</value> + </property> + + <property> + <name>seoul.endpoint</name> + <value>s3.ap-northeast-2.amazonaws.com</value> + </property> + + <property> + <name>singapore.endpoint</name> + <value>s3-ap-southeast-1.amazonaws.com</value> + </property> + + <property> + <name>sydney.endpoint</name> + <value>s3-ap-southeast-2.amazonaws.com</value> + </property> + + <property> + <name>tokyo.endpoint</name> + <value>s3-ap-northeast-1.amazonaws.com</value> + </property> + + <property> + <name>virginia.endpoint</name> + <value>${central.endpoint}</value> </property> + <!-- Scale integration tests may time out on slower connections + you can reduce the operation count like so to mitigate this. + <property> + <name>scale.test.operation.count</name> + <value>500</value> + </property> + --> <!-- Turn security off for tests by default --> <property> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
