Merge branch 'trunk' into HADOOP-13345
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c41f9f3b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c41f9f3b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c41f9f3b Branch: refs/heads/HADOOP-13345 Commit: c41f9f3b71da1d37a9875cf6fcf4eb4f06ad9b5f Parents: 13fafee b053fdc Author: Mingliang Liu <[email protected]> Authored: Wed Apr 12 15:12:11 2017 -0700 Committer: Mingliang Liu <[email protected]> Committed: Wed Apr 12 15:12:11 2017 -0700 ---------------------------------------------------------------------- BUILDING.txt | 2 +- LICENSE.txt | 9 +- dev-support/bin/yetus-wrapper | 44 +- dev-support/docker/Dockerfile | 16 +- .../hadoop-client-integration-tests/pom.xml | 5 + .../hadoop-client-minicluster/pom.xml | 4 - .../util/RandomSignerSecretProvider.java | 9 +- .../util/ZKSignerSecretProvider.java | 10 +- .../util/TestRandomSignerSecretProvider.java | 68 +- .../util/TestZKSignerSecretProvider.java | 154 +- .../jdiff/Apache_Hadoop_Common_2.8.0.xml | 37921 +++++++++++++++++ hadoop-common-project/hadoop-common/pom.xml | 15 +- .../hadoop-common/src/main/bin/hadoop | 69 +- .../hadoop-common/src/main/bin/hadoop-config.sh | 10 +- .../src/main/bin/hadoop-functions.sh | 217 +- .../hadoop-common/src/main/conf/hadoop-env.sh | 13 +- .../src/main/conf/log4j.properties | 1 + .../org/apache/hadoop/conf/Configuration.java | 395 +- .../apache/hadoop/fs/AbstractFileSystem.java | 16 +- .../java/org/apache/hadoop/fs/ChecksumFs.java | 13 +- .../apache/hadoop/fs/DelegateToFileSystem.java | 6 + .../hadoop/fs/FSDataOutputStreamBuilder.java | 142 + .../java/org/apache/hadoop/fs/FileStatus.java | 23 +- .../java/org/apache/hadoop/fs/FileSystem.java | 12 +- .../org/apache/hadoop/fs/FilterFileSystem.java | 5 + .../java/org/apache/hadoop/fs/FilterFs.java | 8 +- .../org/apache/hadoop/fs/FsServerDefaults.java | 30 +- .../org/apache/hadoop/fs/HarFileSystem.java | 5 + .../hadoop/fs/MD5MD5CRC32FileChecksum.java | 62 - .../org/apache/hadoop/fs/ftp/FtpConfigKeys.java | 4 +- .../java/org/apache/hadoop/fs/ftp/FtpFs.java | 7 + .../apache/hadoop/fs/local/LocalConfigKeys.java | 5 +- .../org/apache/hadoop/fs/local/RawLocalFs.java | 8 + .../hadoop/fs/permission/FsPermission.java | 7 + .../hadoop/fs/permission/PermissionStatus.java | 13 - .../apache/hadoop/fs/shell/SetReplication.java | 30 +- .../apache/hadoop/fs/shell/XAttrCommands.java | 2 +- .../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 9 +- .../org/apache/hadoop/fs/viewfs/ViewFs.java | 20 +- .../apache/hadoop/ha/ZKFailoverController.java | 3 +- .../apache/hadoop/io/erasurecode/CodecUtil.java | 135 +- .../apache/hadoop/metrics2/AbstractMetric.java | 3 +- .../org/apache/hadoop/metrics2/MetricsTag.java | 3 +- .../metrics2/impl/AbstractMetricsRecord.java | 3 +- .../org/apache/hadoop/metrics2/impl/MsInfo.java | 4 +- .../hadoop/metrics2/lib/MetricsInfoImpl.java | 3 +- .../hadoop/metrics2/lib/MetricsRegistry.java | 30 +- .../hadoop/metrics2/lib/MutableGaugeFloat.java | 80 + .../metrics2/lib/MutableMetricsFactory.java | 3 + .../hadoop/metrics2/source/JvmMetricsInfo.java | 6 +- .../hadoop/metrics2/util/MetricsCache.java | 4 +- .../org/apache/hadoop/net/NetworkTopology.java | 2 +- .../java/org/apache/hadoop/net/NodeBase.java | 2 +- .../hadoop/net/unix/DomainSocketWatcher.java | 12 +- .../alias/CredentialProviderFactory.java | 16 +- .../security/ssl/SSLHostnameVerifier.java | 21 +- .../java/org/apache/hadoop/util/KMSUtil.java | 20 +- .../apache/hadoop/util/LightWeightCache.java | 20 +- .../org/apache/hadoop/util/Progressable.java | 2 +- .../org/apache/hadoop/util/StringUtils.java | 8 +- .../main/java/org/apache/hadoop/util/Time.java | 10 + .../main/java/org/apache/hadoop/util/Timer.java | 10 + .../apache/hadoop/util/UTF8ByteArrayUtils.java | 4 +- .../org/apache/hadoop/util/VersionInfo.java | 27 +- .../apache/hadoop/util/curator/ChildReaper.java | 4 +- .../src/main/native/gtest/gtest-all.cc | 2 +- .../src/main/native/gtest/include/gtest/gtest.h | 2 +- .../src/main/resources/core-default.xml | 99 +- .../src/site/markdown/FileSystemShell.md | 2 +- .../src/site/markdown/SecureMode.md | 20 +- .../src/site/markdown/UnixShellGuide.md | 31 +- .../src/site/markdown/filesystem/filesystem.md | 14 +- .../markdown/release/2.8.0/CHANGES.2.8.0.md | 2986 ++ .../release/2.8.0/RELEASENOTES.2.8.0.md | 1105 + .../conf/TestCommonConfigurationFields.java | 2 +- .../apache/hadoop/conf/TestConfiguration.java | 62 +- .../fs/FileContextMainOperationsBaseTest.java | 21 +- .../org/apache/hadoop/fs/TestAfsCheckPath.java | 1 + .../org/apache/hadoop/fs/TestFileStatus.java | 10 +- .../java/org/apache/hadoop/fs/TestFilterFs.java | 12 + .../apache/hadoop/fs/TestLocalFileSystem.java | 54 + .../fs/contract/AbstractContractMkdirTest.java | 42 + .../hadoop/fs/contract/ContractTestUtils.java | 27 + .../hadoop/fs/viewfs/TestViewfsFileStatus.java | 10 +- .../apache/hadoop/fs/viewfs/ViewFsBaseTest.java | 49 + .../erasurecode/TestCodecRawCoderMapping.java | 71 +- .../coder/TestHHXORErasureCoder.java | 2 +- .../erasurecode/coder/TestRSErasureCoder.java | 2 +- .../metrics2/lib/TestMetricsAnnotations.java | 3 + .../metrics2/lib/TestMetricsRegistry.java | 4 +- .../hadoop/metrics2/lib/TestMutableMetrics.java | 2 + .../org/apache/hadoop/test/MetricsAsserts.java | 9 +- .../hadoop/test/MultithreadedTestUtil.java | 2 +- .../java/org/apache/hadoop/util/FakeTimer.java | 25 +- .../hadoop/util/TestLightWeightCache.java | 19 +- .../src/test/resources/testConf.xml | 2 +- .../scripts/hadoop_build_custom_subcmd_var.bats | 21 + .../test/scripts/hadoop_detect_priv_subcmd.bats | 34 + .../test/scripts/hadoop_get_verify_uservar.bats | 21 - .../src/test/scripts/hadoop_verify_user.bats | 53 - .../test/scripts/hadoop_verify_user_perm.bats | 53 + .../scripts/hadoop_verify_user_resolves.bats | 44 + .../main/java/org/apache/hadoop/fs/Hdfs.java | 6 + .../java/org/apache/hadoop/hdfs/DFSClient.java | 101 +- .../hadoop/hdfs/DFSInotifyEventInputStream.java | 4 +- .../org/apache/hadoop/hdfs/DFSInputStream.java | 2 +- .../org/apache/hadoop/hdfs/DFSOutputStream.java | 44 +- .../hadoop/hdfs/DFSStripedInputStream.java | 22 +- .../hadoop/hdfs/DFSStripedOutputStream.java | 1 + .../org/apache/hadoop/hdfs/DFSUtilClient.java | 15 +- .../hadoop/hdfs/DistributedFileSystem.java | 114 +- .../org/apache/hadoop/hdfs/HAUtilClient.java | 55 + .../apache/hadoop/hdfs/KeyProviderCache.java | 17 +- .../hadoop/hdfs/NameNodeProxiesClient.java | 15 +- .../hdfs/client/HdfsClientConfigKeys.java | 3 + .../hadoop/hdfs/protocol/ClientProtocol.java | 6 +- .../hadoop/hdfs/protocol/DatanodeInfo.java | 1 + .../hdfs/protocol/FsPermissionExtension.java | 17 +- .../hadoop/hdfs/protocol/HdfsConstants.java | 6 - .../protocol/SystemErasureCodingPolicies.java | 121 + .../protocol/datatransfer/PacketReceiver.java | 2 +- .../ClientNamenodeProtocolTranslatorPB.java | 5 +- .../hadoop/hdfs/protocolPB/PBHelperClient.java | 38 +- .../namenode/ha/ClientHAProxyFactory.java | 44 + .../ha/ConfiguredFailoverProxyProvider.java | 183 + .../hdfs/server/namenode/ha/HAProxyFactory.java | 44 + .../namenode/ha/IPFailoverProxyProvider.java | 126 + .../ha/RequestHedgingProxyProvider.java | 234 + .../hdfs/server/protocol/SlowDiskReports.java | 146 + .../apache/hadoop/hdfs/web/JsonUtilClient.java | 13 +- .../src/main/proto/ClientNamenodeProtocol.proto | 1 + .../src/main/proto/hdfs.proto | 8 +- .../ha/TestRequestHedgingProxyProvider.java | 476 + hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 5 + .../hadoop/fs/http/client/HttpFSFileSystem.java | 7 +- .../hadoop/fs/http/server/FSOperations.java | 39 +- .../src/site/markdown/index.md | 8 +- .../fs/http/client/BaseTestHttpFSWith.java | 24 +- .../hadoop/fs/http/server/TestHttpFSServer.java | 18 + .../org/apache/hadoop/test/TestHdfsHelper.java | 17 +- hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 10 +- .../jdiff/Apache_Hadoop_HDFS_2.8.0.xml | 312 + hadoop-hdfs-project/hadoop-hdfs/pom.xml | 10 +- .../hadoop-hdfs/src/main/bin/hdfs | 97 +- .../hadoop-hdfs/src/main/bin/hdfs-config.sh | 5 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 +- .../java/org/apache/hadoop/hdfs/DFSUtil.java | 15 +- .../java/org/apache/hadoop/hdfs/HAUtil.java | 57 +- .../org/apache/hadoop/hdfs/NameNodeProxies.java | 3 +- .../hadoop/hdfs/net/DFSNetworkTopology.java | 76 +- ...tNamenodeProtocolServerSideTranslatorPB.java | 3 +- .../DatanodeProtocolClientSideTranslatorPB.java | 7 +- .../DatanodeProtocolServerSideTranslatorPB.java | 3 +- .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 68 + .../server/blockmanagement/DatanodeManager.java | 46 +- .../server/blockmanagement/SlowDiskTracker.java | 294 + .../server/blockmanagement/SlowPeerTracker.java | 2 +- .../hadoop/hdfs/server/common/Storage.java | 6 +- .../hdfs/server/datanode/BPOfferService.java | 19 +- .../hdfs/server/datanode/BPServiceActor.java | 65 +- .../hdfs/server/datanode/BlockPoolManager.java | 8 +- .../hdfs/server/datanode/BlockReceiver.java | 61 +- .../hdfs/server/datanode/DataStorage.java | 4 +- .../hdfs/server/datanode/DataXceiver.java | 4 +- .../datanode/IncrementalBlockReportManager.java | 41 +- .../datanode/checker/ThrottledAsyncChecker.java | 25 +- .../StripedBlockChecksumReconstructor.java | 48 +- .../erasurecode/StripedBlockReconstructor.java | 1 + .../erasurecode/StripedReconstructor.java | 6 + .../datanode/fsdataset/impl/BlockPoolSlice.java | 2 +- .../datanode/fsdataset/impl/FsVolumeImpl.java | 2 +- .../datanode/metrics/DataNodeDiskMetrics.java | 55 +- .../datanode/metrics/DataNodeMetrics.java | 35 + .../diskbalancer/DiskBalancerException.java | 1 + .../server/diskbalancer/command/Command.java | 29 +- .../diskbalancer/command/ReportCommand.java | 6 +- .../apache/hadoop/hdfs/server/mover/Mover.java | 11 +- .../hdfs/server/namenode/AclTransformation.java | 4 +- .../hadoop/hdfs/server/namenode/DfsServlet.java | 50 +- .../namenode/ErasureCodingPolicyManager.java | 78 +- .../server/namenode/FSDirErasureCodingOp.java | 60 +- .../server/namenode/FSDirStatAndListingOp.java | 10 +- .../hdfs/server/namenode/FSDirWriteFileOp.java | 17 +- .../server/namenode/FSImageFormatPBINode.java | 3 +- .../hdfs/server/namenode/FSNamesystem.java | 31 +- .../hadoop/hdfs/server/namenode/INodeFile.java | 16 +- .../hdfs/server/namenode/INodesInPath.java | 6 +- .../hadoop/hdfs/server/namenode/JournalSet.java | 2 +- .../hadoop/hdfs/server/namenode/NameNode.java | 6 + .../hdfs/server/namenode/NameNodeRpcServer.java | 12 +- .../server/namenode/NameNodeStatusMXBean.java | 8 + .../ha/ConfiguredFailoverProxyProvider.java | 207 - .../namenode/ha/IPFailoverProxyProvider.java | 132 - .../namenode/ha/NameNodeHAProxyFactory.java | 45 + .../ha/RequestHedgingProxyProvider.java | 241 - .../snapshot/DirectoryWithSnapshotFeature.java | 5 + .../hdfs/server/protocol/DatanodeProtocol.java | 3 +- .../OfflineImageReconstructor.java | 6 +- .../offlineImageViewer/PBImageXmlWriter.java | 6 +- .../org/apache/hadoop/hdfs/web/JsonUtil.java | 6 + .../src/main/proto/DatanodeProtocol.proto | 15 + .../src/main/webapps/hdfs/robots.txt | 2 + .../src/main/webapps/journal/robots.txt | 2 + .../src/main/webapps/secondary/robots.txt | 2 + .../src/site/markdown/ArchivalStorage.md | 2 +- .../src/site/markdown/HDFSCommands.md | 2 +- .../src/site/markdown/HDFSDiskbalancer.md | 4 +- .../src/site/markdown/HDFSErasureCoding.md | 1 + .../src/site/markdown/HdfsNfsGateway.md | 14 +- .../src/site/markdown/TransparentEncryption.md | 1 + .../org/apache/hadoop/hdfs/DFSTestUtil.java | 29 +- .../apache/hadoop/hdfs/StripedFileTestUtil.java | 6 +- .../hadoop/hdfs/TestDFSClientFailover.java | 4 +- .../hadoop/hdfs/TestDFSClientRetries.java | 5 +- .../apache/hadoop/hdfs/TestDFSOutputStream.java | 126 + .../TestDFSRSDefault10x4StripedInputStream.java | 7 +- ...TestDFSRSDefault10x4StripedOutputStream.java | 7 +- ...fault10x4StripedOutputStreamWithFailure.java | 7 +- .../hadoop/hdfs/TestDFSStripedInputStream.java | 2 +- .../hadoop/hdfs/TestDFSStripedOutputStream.java | 2 +- .../TestDFSStripedOutputStreamWithFailure.java | 2 +- .../org/apache/hadoop/hdfs/TestDFSUtil.java | 2 +- .../hdfs/TestDFSXORStripedInputStream.java | 7 +- .../hdfs/TestDFSXORStripedOutputStream.java | 7 +- ...estDFSXORStripedOutputStreamWithFailure.java | 7 +- .../hadoop/hdfs/TestDataTransferProtocol.java | 2 +- .../apache/hadoop/hdfs/TestDecommission.java | 50 + .../hadoop/hdfs/TestDistributedFileSystem.java | 35 +- .../apache/hadoop/hdfs/TestEncryptionZones.java | 208 +- .../hadoop/hdfs/TestErasureCodingPolicies.java | 71 +- .../TestErasureCodingPolicyWithSnapshot.java | 26 + .../apache/hadoop/hdfs/TestFileCreation.java | 4 +- .../org/apache/hadoop/hdfs/TestFileStatus.java | 18 +- .../hadoop/hdfs/TestFileStatusWithECPolicy.java | 22 +- .../hadoop/hdfs/TestKeyProviderCache.java | 21 +- .../java/org/apache/hadoop/hdfs/TestLease.java | 3 +- .../hadoop/hdfs/TestReconstructStripedFile.java | 2 +- .../hadoop/hdfs/TestSetrepIncreasing.java | 44 + .../TestUnsetAndChangeDirectoryEcPolicy.java | 13 +- .../hadoop/hdfs/net/TestDFSNetworkTopology.java | 57 + .../hadoop/hdfs/protocolPB/TestPBHelper.java | 144 +- .../hdfs/qjournal/client/TestQJMWithFaults.java | 2 +- .../client/TestQuorumJournalManager.java | 2 +- .../TestNameNodePrunesMissingStorages.java | 4 +- .../blockmanagement/TestSlowDiskTracker.java | 439 + .../datanode/InternalDataNodeTestUtils.java | 4 +- .../server/datanode/TestBPOfferService.java | 6 +- .../datanode/TestBlockCountersInPendingIBR.java | 146 + .../server/datanode/TestBlockPoolManager.java | 4 +- .../hdfs/server/datanode/TestBlockRecovery.java | 8 +- .../datanode/TestBpServiceActorScheduler.java | 20 +- .../server/datanode/TestDataNodeLifeline.java | 8 +- .../server/datanode/TestDataNodeMXBean.java | 2 +- .../hdfs/server/datanode/TestDataStorage.java | 2 +- .../TestDatanodeProtocolRetryPolicy.java | 4 +- .../server/datanode/TestFsDatasetCache.java | 4 +- .../hdfs/server/datanode/TestStorageReport.java | 4 +- .../TestDatasetVolumeCheckerTimeout.java | 2 +- .../checker/TestThrottledAsyncChecker.java | 118 +- .../TestThrottledAsyncCheckerTimeout.java | 45 +- .../command/TestDiskBalancerCommand.java | 71 +- .../hadoop/hdfs/server/mover/TestMover.java | 42 + .../hdfs/server/namenode/FSAclBaseTest.java | 25 +- .../server/namenode/NNThroughputBenchmark.java | 23 +- .../hdfs/server/namenode/NameNodeAdapter.java | 3 +- .../hdfs/server/namenode/TestAddBlockRetry.java | 4 +- ...stBlockPlacementPolicyRackFaultTolerant.java | 4 +- .../hdfs/server/namenode/TestDeadDatanode.java | 4 +- .../TestDefaultBlockPlacementPolicy.java | 2 +- .../server/namenode/TestEnabledECPolicies.java | 17 +- .../hdfs/server/namenode/TestFSImage.java | 13 +- .../namenode/TestFavoredNodesEndToEnd.java | 23 + .../hdfs/server/namenode/TestINodeFile.java | 7 +- .../namenode/TestNameNodeStatusMXBean.java | 148 + .../server/namenode/TestNamenodeRetryCache.java | 17 +- .../hdfs/server/namenode/TestStartup.java | 57 +- .../server/namenode/TestStripedINodeFile.java | 8 +- .../namenode/ha/TestDelegationTokensWithHA.java | 4 +- .../ha/TestRequestHedgingProxyProvider.java | 470 - .../namenode/ha/TestRetryCacheWithHA.java | 5 +- .../snapshot/TestRenameWithSnapshots.java | 6 +- .../namenode/snapshot/TestSnapshotDeletion.java | 75 + .../TestOfflineImageViewer.java | 11 +- .../hadoop/hdfs/util/TestStripedBlockUtil.java | 4 +- .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 72 + hadoop-mapreduce-project/bin/mapred | 64 +- hadoop-mapreduce-project/bin/mapred-config.sh | 1 + .../Apache_Hadoop_MapReduce_Common_2.8.0.xml | 113 + .../Apache_Hadoop_MapReduce_Core_2.8.0.xml | 27490 ++++++++++++ .../Apache_Hadoop_MapReduce_JobClient_2.8.0.xml | 16 + .../v2/app/job/impl/TaskAttemptImpl.java | 285 +- .../mapreduce/v2/app/webapp/ConfBlock.java | 2 +- .../v2/app/rm/TestRMContainerAllocator.java | 453 +- .../java/org/apache/hadoop/mapred/JobConf.java | 8 +- .../mapred/lib/FieldSelectionMapReduce.java | 2 +- .../mapred/lib/KeyFieldBasedComparator.java | 2 +- .../hadoop/mapreduce/JobResourceUploader.java | 137 +- .../apache/hadoop/mapreduce/JobSubmitter.java | 2 +- .../apache/hadoop/mapreduce/MRJobConfig.java | 8 +- .../lib/fieldsel/FieldSelectionHelper.java | 7 +- .../lib/fieldsel/FieldSelectionMapper.java | 2 +- .../lib/fieldsel/FieldSelectionReducer.java | 2 +- .../lib/input/KeyValueLineRecordReader.java | 13 +- .../mapreduce/lib/jobcontrol/JobControl.java | 2 +- .../mapreduce/lib/output/TextOutputFormat.java | 9 +- .../lib/partition/KeyFieldBasedComparator.java | 4 +- .../lib/partition/KeyFieldBasedPartitioner.java | 2 +- .../hadoop/mapreduce/util/ConfigUtil.java | 8 +- .../mapreduce/TestJobResourceUploader.java | 509 +- .../org/apache/hadoop/mapred/YARNRunner.java | 4 +- .../apache/hadoop/fs/slive/OperationOutput.java | 4 +- .../hadoop/mapred/TestFieldSelection.java | 2 +- .../mapred/TestMRTimelineEventHandling.java | 2 +- .../apache/hadoop/mapred/TestNetworkedJob.java | 3 + .../apache/hadoop/mapred/TestYARNRunner.java | 2 +- .../mapred/lib/TestKeyFieldBasedComparator.java | 4 +- .../lib/fieldsel/TestMRFieldSelection.java | 2 +- .../TestMRKeyFieldBasedComparator.java | 4 +- .../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 6 +- .../apache/hadoop/mapred/ShuffleHandler.java | 68 +- .../hadoop/mapred/TestShuffleHandler.java | 31 + .../hadoop-mapreduce-client/pom.xml | 5 + hadoop-mapreduce-project/pom.xml | 1 + hadoop-project-dist/pom.xml | 2 +- hadoop-project/pom.xml | 36 +- .../hadoop/fs/s3a/DefaultS3ClientFactory.java | 4 +- .../hadoop/fs/s3a/S3ABlockOutputStream.java | 5 +- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 11 +- .../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 42 +- .../apache/hadoop/fs/s3a/S3ClientFactory.java | 3 +- ...haredInstanceProfileCredentialsProvider.java | 67 - .../fs/s3a/SimpleAWSCredentialsProvider.java | 3 +- .../fs/s3a/TemporaryAWSCredentialsProvider.java | 3 +- .../fs/s3a/s3guard/DynamoDBClientFactory.java | 2 +- .../src/site/markdown/tools/hadoop-aws/index.md | 36 +- .../site/markdown/tools/hadoop-aws/testing.md | 1 + .../fs/s3a/ITestS3AAWSCredentialsProvider.java | 43 +- .../fs/s3a/ITestS3ATemporaryCredentials.java | 2 +- .../hadoop/fs/s3a/MockS3ClientFactory.java | 2 +- .../fs/s3a/TestS3AAWSCredentialsProvider.java | 20 +- .../hadoop/fs/s3a/TestS3AGetFileStatus.java | 9 + .../s3a/s3guard/DynamoDBLocalClientFactory.java | 2 +- .../org/apache/hadoop/fs/adl/AdlConfKeys.java | 40 +- .../org/apache/hadoop/fs/adl/AdlFileSystem.java | 7 +- .../src/site/markdown/index.md | 32 +- .../apache/hadoop/fs/adl/AdlMockWebServer.java | 3 + .../hadoop/fs/adl/TestAzureADTokenProvider.java | 8 +- .../hadoop/fs/adl/TestCustomTokenProvider.java | 4 + .../apache/hadoop/fs/adl/TestGetFileStatus.java | 13 +- .../fs/adl/TestRelativePathFormation.java | 4 + .../fs/adl/TestValidateConfiguration.java | 83 +- .../adl/live/TestAdlFileSystemContractLive.java | 3 +- .../fs/azure/AzureNativeFileSystemStore.java | 2 +- .../hadoop/fs/azure/BlockBlobAppendStream.java | 2 +- .../hadoop/fs/azure/NativeAzureFileSystem.java | 69 +- .../fs/azure/RemoteSASKeyGeneratorImpl.java | 191 +- .../fs/azure/RemoteWasbAuthorizerImpl.java | 155 +- .../fs/azure/SecureStorageInterfaceImpl.java | 11 +- .../fs/azure/WasbAuthorizerInterface.java | 5 +- .../hadoop/fs/azure/WasbRemoteCallHelper.java | 3 +- .../hadoop/fs/azure/security/Constants.java | 54 + .../hadoop/fs/azure/security/SecurityUtils.java | 86 + .../security/WasbDelegationTokenIdentifier.java | 48 + .../fs/azure/security/WasbTokenRenewer.java | 150 + .../hadoop/fs/azure/security/package-info.java | 22 + ...apache.hadoop.security.token.TokenIdentifier | 16 + ...rg.apache.hadoop.security.token.TokenRenewer | 16 + .../hadoop/fs/azure/MockWasbAuthorizerImpl.java | 4 +- .../org/apache/hadoop/tools/CopyListing.java | 73 +- .../hadoop/tools/CopyListingFileStatus.java | 91 +- .../java/org/apache/hadoop/tools/DistCp.java | 96 +- .../org/apache/hadoop/tools/DistCpContext.java | 198 + .../apache/hadoop/tools/DistCpOptionSwitch.java | 12 +- .../org/apache/hadoop/tools/DistCpOptions.java | 913 +- .../org/apache/hadoop/tools/DistCpSync.java | 42 +- .../hadoop/tools/FileBasedCopyListing.java | 12 +- .../apache/hadoop/tools/GlobbedCopyListing.java | 17 +- .../org/apache/hadoop/tools/OptionsParser.java | 271 +- .../apache/hadoop/tools/SimpleCopyListing.java | 190 +- .../hadoop/tools/mapred/CopyCommitter.java | 189 +- .../apache/hadoop/tools/mapred/CopyMapper.java | 40 +- .../tools/mapred/RetriableFileCopyCommand.java | 29 +- .../tools/mapred/UniformSizeInputFormat.java | 5 +- .../apache/hadoop/tools/util/DistCpUtils.java | 126 +- .../src/site/markdown/DistCp.md.vm | 3 +- .../apache/hadoop/tools/TestCopyListing.java | 51 +- .../hadoop/tools/TestCopyListingFileStatus.java | 1 + .../apache/hadoop/tools/TestDistCpOptions.java | 500 + .../org/apache/hadoop/tools/TestDistCpSync.java | 68 +- .../hadoop/tools/TestDistCpSyncReverseBase.java | 44 +- .../apache/hadoop/tools/TestDistCpSystem.java | 368 +- .../apache/hadoop/tools/TestDistCpViewFs.java | 10 +- .../hadoop/tools/TestFileBasedCopyListing.java | 9 +- .../hadoop/tools/TestGlobbedCopyListing.java | 11 +- .../apache/hadoop/tools/TestIntegration.java | 20 +- .../apache/hadoop/tools/TestOptionsParser.java | 81 +- .../contract/AbstractContractDistCpTest.java | 6 +- .../hadoop/tools/mapred/TestCopyCommitter.java | 39 +- .../mapred/TestUniformSizeInputFormat.java | 15 +- .../mapred/lib/TestDynamicInputFormat.java | 17 +- .../hadoop/tools/util/TestDistCpUtils.java | 123 +- hadoop-tools/hadoop-kafka/pom.xml | 26 - .../hadoop/metrics2/impl/TestKafkaMetrics.java | 4 +- .../org/apache/hadoop/yarn/sls/SLSRunner.java | 22 +- .../hadoop/yarn/sls/appmaster/AMSimulator.java | 29 +- .../sls/scheduler/FairSchedulerMetrics.java | 308 +- .../sls/scheduler/ResourceSchedulerWrapper.java | 972 - .../sls/scheduler/SLSCapacityScheduler.java | 686 +- .../yarn/sls/scheduler/SLSFairScheduler.java | 339 + .../yarn/sls/scheduler/SchedulerMetrics.java | 602 +- .../yarn/sls/scheduler/SchedulerWrapper.java | 23 +- .../hadoop/yarn/sls/scheduler/Tracker.java | 46 + .../apache/hadoop/yarn/sls/utils/SLSUtils.java | 1 + .../apache/hadoop/yarn/sls/web/SLSWebApp.java | 12 +- .../yarn/sls/appmaster/TestAMSimulator.java | 87 +- .../yarn/sls/nodemanager/TestNMSimulator.java | 32 +- .../hadoop/streaming/StreamKeyValUtil.java | 8 +- hadoop-yarn-project/hadoop-yarn/bin/yarn | 62 +- .../hadoop-yarn/bin/yarn-config.sh | 1 + .../jdiff/Apache_Hadoop_YARN_Client_2.8.0.xml | 2316 + .../jdiff/Apache_Hadoop_YARN_Common_2.8.0.xml | 2665 ++ .../Apache_Hadoop_YARN_Server_Common_2.8.0.xml | 829 + .../records/ApplicationSubmissionContext.java | 57 +- .../org/apache/hadoop/yarn/conf/HAUtil.java | 30 +- .../hadoop/yarn/conf/YarnConfiguration.java | 31 +- .../src/main/proto/yarn_protos.proto | 2 +- .../pom.xml | 5 + .../distributedshell/TestDistributedShell.java | 30 +- .../apache/hadoop/yarn/client/cli/LogsCLI.java | 6 + .../hadoop/yarn/client/ProtocolHATestBase.java | 6 +- .../hadoop/yarn/client/TestRMFailover.java | 39 +- .../api/impl/TestAMRMClientOnRMRestart.java | 59 +- .../hadoop/yarn/client/cli/TestLogsCLI.java | 58 +- .../hadoop/yarn/client/cli/TestRMAdminCLI.java | 32 +- .../pb/ApplicationSubmissionContextPBImpl.java | 66 +- .../impl/pb/ContainerLaunchContextPBImpl.java | 13 + .../yarn/api/records/impl/pb/ProtoUtils.java | 6 +- .../client/api/impl/TimelineV2ClientImpl.java | 9 +- .../logaggregation/AggregatedLogFormat.java | 29 +- .../nodelabels/CommonNodeLabelsManager.java | 22 + .../org/apache/hadoop/yarn/webapp/WebApp.java | 4 +- .../src/main/resources/yarn-default.xml | 9 + .../TestApplicationClientProtocolRecords.java | 29 + .../org/apache/hadoop/yarn/conf/TestHAUtil.java | 86 +- .../logaggregation/TestAggregatedLogFormat.java | 123 +- .../logaggregation/TestAggregatedLogsBlock.java | 25 +- .../logaggregation/TestContainerLogsUtils.java | 15 +- .../timeline/RollingLevelDBTimelineStore.java | 263 +- .../webapp/TestTimelineWebServices.java | 27 - .../server/nodemanager/DirectoryCollection.java | 3 +- .../yarn/server/nodemanager/NodeManager.java | 26 +- .../nodemanager/NodeStatusUpdaterImpl.java | 4 +- .../containermanager/ContainerManagerImpl.java | 18 +- .../localizer/ContainerLocalizer.java | 3 +- .../localizer/ResourceLocalizationService.java | 16 +- .../logaggregation/AppLogAggregatorImpl.java | 72 +- .../nodemanager/metrics/NodeManagerMetrics.java | 41 + .../timelineservice/NMTimelinePublisher.java | 8 + .../nodemanager/DummyContainerManager.java | 5 +- .../TestContainerManagerWithLCE.java | 299 +- .../nodemanager/TestNodeManagerResync.java | 2 +- .../nodemanager/TestNodeStatusUpdater.java | 2 +- .../containermanager/TestContainerManager.java | 45 + .../TestContainerManagerRecovery.java | 11 +- .../launcher/TestContainerLaunch.java | 12 +- .../localizer/TestLocalCacheCleanup.java | 17 +- .../TestLocalCacheDirectoryManager.java | 8 +- .../TestResourceLocalizationService.java | 412 +- .../TestAppLogAggregatorImpl.java | 8 +- .../TestNMTimelinePublisher.java | 10 +- .../conf/capacity-scheduler.xml | 24 +- .../hadoop-yarn-server-resourcemanager/pom.xml | 5 + .../server/resourcemanager/ClientRMService.java | 290 +- .../server/resourcemanager/RMAppManager.java | 110 +- .../server/resourcemanager/RMAuditLogger.java | 5 + .../server/resourcemanager/RMServerUtils.java | 68 +- .../server/resourcemanager/ResourceManager.java | 2 +- .../metrics/TimelineServiceV2Publisher.java | 11 +- .../nodelabels/RMNodeLabelsManager.java | 12 + .../recovery/FileSystemRMStateStore.java | 8 +- .../recovery/LeveldbRMStateStore.java | 8 + .../server/resourcemanager/rmapp/RMApp.java | 3 +- .../server/resourcemanager/rmapp/RMAppImpl.java | 54 +- .../rmapp/attempt/RMAppAttemptImpl.java | 35 +- .../rmcontainer/RMContainer.java | 3 +- .../rmcontainer/RMContainerImpl.java | 2 +- .../scheduler/AbstractYarnScheduler.java | 5 + .../scheduler/AppSchedulingInfo.java | 47 +- .../scheduler/ClusterNodeTracker.java | 65 +- .../scheduler/ResourceScheduler.java | 9 + .../scheduler/SchedulerApplicationAttempt.java | 5 + .../scheduler/SchedulerNode.java | 6 +- .../scheduler/capacity/AbstractCSQueue.java | 20 +- .../scheduler/capacity/CSQueue.java | 14 - .../scheduler/capacity/CSQueueMetrics.java | 21 + .../scheduler/capacity/CSQueueUtils.java | 32 +- .../CapacitySchedulerConfiguration.java | 12 + .../capacity/CapacitySchedulerQueueManager.java | 33 + .../scheduler/capacity/LeafQueue.java | 18 +- .../scheduler/capacity/ParentQueue.java | 19 +- .../scheduler/capacity/PlanQueue.java | 4 +- .../scheduler/capacity/ReservationQueue.java | 2 +- .../allocator/RegularContainerAllocator.java | 41 +- .../scheduler/common/fica/FiCaSchedulerApp.java | 10 +- .../fair/AllocationFileLoaderService.java | 2 +- .../scheduler/fair/FSAppAttempt.java | 20 +- .../scheduler/fair/FSOpDurations.java | 8 - .../scheduler/fair/FSPreemptionThread.java | 25 +- .../scheduler/fair/FSSchedulerNode.java | 133 +- .../scheduler/fair/FairScheduler.java | 49 +- .../scheduler/fair/QueueManager.java | 2 +- .../LocalitySchedulingPlacementSet.java | 8 +- .../resourcemanager/webapp/dao/AppInfo.java | 2 +- .../server/resourcemanager/ACLsTestBase.java | 10 - .../server/resourcemanager/Application.java | 2 +- .../yarn/server/resourcemanager/MockRM.java | 45 +- .../server/resourcemanager/RMHATestBase.java | 20 +- .../ReservationACLsTestBase.java | 5 +- .../server/resourcemanager/TestAppManager.java | 195 +- .../resourcemanager/TestApplicationCleanup.java | 44 +- .../TestApplicationMasterLauncher.java | 11 +- .../TestApplicationMasterService.java | 19 +- .../resourcemanager/TestClientRMService.java | 8 +- .../TestNodeBlacklistingOnAMFailures.java | 213 +- .../yarn/server/resourcemanager/TestRM.java | 10 +- .../resourcemanager/TestRMAdminService.java | 95 + .../resourcemanager/TestRMServerUtils.java | 297 + .../TestReservationSystemWithRMHA.java | 5 +- .../resourcemanager/TestSignalContainer.java | 13 +- .../applicationsmanager/MockAsm.java | 6 +- .../TestAMRMRPCNodeUpdates.java | 18 +- .../TestRMAppLogAggregationStatus.java | 2 + .../metrics/TestSystemMetricsPublisher.java | 3 +- .../recovery/TestLeveldbRMStateStore.java | 19 +- .../TestCapacitySchedulerPlanFollower.java | 7 + .../TestFairSchedulerPlanFollower.java | 4 + .../TestSchedulerPlanFollowerBase.java | 9 +- .../resourcetracker/TestNMReconnect.java | 14 +- .../server/resourcemanager/rmapp/MockRMApp.java | 11 +- .../rmapp/TestRMAppTransitions.java | 9 +- .../attempt/TestRMAppAttemptTransitions.java | 19 +- .../rmcontainer/TestRMContainerImpl.java | 1 - .../capacity/TestApplicationLimits.java | 4 +- .../TestApplicationLimitsByPartition.java | 4 +- .../capacity/TestApplicationPriority.java | 29 +- .../capacity/TestCapacityScheduler.java | 264 +- .../TestCapacitySchedulerDynamicBehavior.java | 11 + .../scheduler/capacity/TestLeafQueue.java | 170 +- .../scheduler/capacity/TestParentQueue.java | 18 +- .../fair/TestContinuousScheduling.java | 36 + .../scheduler/fair/TestFSSchedulerNode.java | 403 + .../scheduler/fair/TestFairScheduler.java | 78 + .../fair/TestFairSchedulerPreemption.java | 19 + .../scheduler/fair/TestSchedulingPolicy.java | 1 - .../security/TestClientToAMTokens.java | 23 +- .../webapp/TestRMWebServicesApps.java | 131 +- .../hadoop/yarn/server/MiniYARNCluster.java | 22 +- .../hadoop/yarn/server/TestMiniYarnCluster.java | 34 +- .../pom.xml | 5 + .../pom.xml | 19 + .../collector/TimelineCollector.java | 37 +- .../collector/TimelineCollectorManager.java | 7 +- .../collector/TimelineCollectorWebService.java | 12 +- .../collector/TestTimelineCollector.java | 63 + .../src/site/markdown/DockerContainers.md | 4 +- .../hadoop-yarn/hadoop-yarn-ui/pom.xml | 7 +- .../webapp/app/components/app-attempt-table.js | 7 - .../main/webapp/app/components/timeline-view.js | 199 +- .../main/webapp/app/helpers/prepend-protocol.js | 29 + .../src/main/webapp/app/services/hosts.js | 6 +- .../templates/components/app-attempt-table.hbs | 36 +- .../templates/components/container-table.hbs | 22 +- .../app/templates/components/timeline-view.hbs | 61 +- .../tests/unit/helpers/prepend-protocol-test.js | 28 + 574 files changed, 93155 insertions(+), 8494 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c41f9f3b/hadoop-common-project/hadoop-common/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c41f9f3b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c41f9f3b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c41f9f3b/hadoop-project/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c41f9f3b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java ---------------------------------------------------------------------- diff --cc hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java index c411fdd,0000000..a329580 mode 100644,000000..100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java @@@ -1,233 -1,0 +1,233 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.S3ClientOptions; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.util.VersionInfo; +import org.slf4j.Logger; + +import java.io.IOException; +import java.net.URI; + +import static org.apache.hadoop.fs.s3a.Constants.*; +import static org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet; +import static org.apache.hadoop.fs.s3a.S3AUtils.intOption; + +/** + * The default factory implementation, which calls the AWS SDK to configure + * and create an {@link AmazonS3Client} that communicates with the S3 service. + */ +public class DefaultS3ClientFactory extends Configured implements + S3ClientFactory { + + private static final Logger LOG = S3AFileSystem.LOG; + + @Override - public AmazonS3 createS3Client(URI name, URI uri) throws IOException { ++ public AmazonS3 createS3Client(URI name) throws IOException { + Configuration conf = getConf(); + AWSCredentialsProvider credentials = - createAWSCredentialProviderSet(name, conf, uri); ++ createAWSCredentialProviderSet(name, conf); + final ClientConfiguration awsConf = createAwsConf(getConf()); + AmazonS3 s3 = newAmazonS3Client(credentials, awsConf); + return createAmazonS3Client(s3, conf, credentials, awsConf); + } + + /** + * Create a new {@link ClientConfiguration}. + * @param conf The Hadoop configuration + * @return new AWS client configuration + */ + public static ClientConfiguration createAwsConf(Configuration conf) { + final ClientConfiguration awsConf = new ClientConfiguration(); + initConnectionSettings(conf, awsConf); + initProxySupport(conf, awsConf); + initUserAgent(conf, awsConf); + return awsConf; + } + + /** + * Wrapper around constructor for {@link AmazonS3} client. Override this to + * provide an extended version of the client + * @param credentials credentials to use + * @param awsConf AWS configuration + * @return new AmazonS3 client + */ + protected AmazonS3 newAmazonS3Client( + AWSCredentialsProvider credentials, ClientConfiguration awsConf) { + return new AmazonS3Client(credentials, awsConf); + } + + /** + * Initializes all AWS SDK settings related to connection management. + * + * @param conf Hadoop configuration + * @param awsConf AWS SDK configuration + */ + private static void initConnectionSettings(Configuration conf, + ClientConfiguration awsConf) { + awsConf.setMaxConnections(intOption(conf, MAXIMUM_CONNECTIONS, + DEFAULT_MAXIMUM_CONNECTIONS, 1)); + boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS, + DEFAULT_SECURE_CONNECTIONS); + awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP); + awsConf.setMaxErrorRetry(intOption(conf, MAX_ERROR_RETRIES, + DEFAULT_MAX_ERROR_RETRIES, 0)); + awsConf.setConnectionTimeout(intOption(conf, ESTABLISH_TIMEOUT, + DEFAULT_ESTABLISH_TIMEOUT, 0)); + awsConf.setSocketTimeout(intOption(conf, SOCKET_TIMEOUT, + DEFAULT_SOCKET_TIMEOUT, 0)); + int sockSendBuffer = intOption(conf, SOCKET_SEND_BUFFER, + DEFAULT_SOCKET_SEND_BUFFER, 2048); + int sockRecvBuffer = intOption(conf, SOCKET_RECV_BUFFER, + DEFAULT_SOCKET_RECV_BUFFER, 2048); + awsConf.setSocketBufferSizeHints(sockSendBuffer, sockRecvBuffer); + String signerOverride = conf.getTrimmed(SIGNING_ALGORITHM, ""); + if (!signerOverride.isEmpty()) { + LOG.debug("Signer override = {}", signerOverride); + awsConf.setSignerOverride(signerOverride); + } + } + + /** + * Initializes AWS SDK proxy support if configured. + * + * @param conf Hadoop configuration + * @param awsConf AWS SDK configuration + * @throws IllegalArgumentException if misconfigured + */ + private static void initProxySupport(Configuration conf, + ClientConfiguration awsConf) throws IllegalArgumentException { + String proxyHost = conf.getTrimmed(PROXY_HOST, ""); + int proxyPort = conf.getInt(PROXY_PORT, -1); + if (!proxyHost.isEmpty()) { + awsConf.setProxyHost(proxyHost); + if (proxyPort >= 0) { + awsConf.setProxyPort(proxyPort); + } else { + if (conf.getBoolean(SECURE_CONNECTIONS, DEFAULT_SECURE_CONNECTIONS)) { + LOG.warn("Proxy host set without port. Using HTTPS default 443"); + awsConf.setProxyPort(443); + } else { + LOG.warn("Proxy host set without port. Using HTTP default 80"); + awsConf.setProxyPort(80); + } + } + String proxyUsername = conf.getTrimmed(PROXY_USERNAME); + String proxyPassword = conf.getTrimmed(PROXY_PASSWORD); + if ((proxyUsername == null) != (proxyPassword == null)) { + String msg = "Proxy error: " + PROXY_USERNAME + " or " + + PROXY_PASSWORD + " set without the other."; + LOG.error(msg); + throw new IllegalArgumentException(msg); + } + awsConf.setProxyUsername(proxyUsername); + awsConf.setProxyPassword(proxyPassword); + awsConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN)); + awsConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION)); + if (LOG.isDebugEnabled()) { + LOG.debug("Using proxy server {}:{} as user {} with password {} on " + + "domain {} as workstation {}", awsConf.getProxyHost(), + awsConf.getProxyPort(), + String.valueOf(awsConf.getProxyUsername()), + awsConf.getProxyPassword(), awsConf.getProxyDomain(), + awsConf.getProxyWorkstation()); + } + } else if (proxyPort >= 0) { + String msg = + "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST; + LOG.error(msg); + throw new IllegalArgumentException(msg); + } + } + + /** + * Initializes the User-Agent header to send in HTTP requests to the S3 + * back-end. We always include the Hadoop version number. The user also + * may set an optional custom prefix to put in front of the Hadoop version + * number. The AWS SDK interally appends its own information, which seems + * to include the AWS SDK version, OS and JVM version. + * + * @param conf Hadoop configuration + * @param awsConf AWS SDK configuration + */ + private static void initUserAgent(Configuration conf, + ClientConfiguration awsConf) { + String userAgent = "Hadoop " + VersionInfo.getVersion(); + String userAgentPrefix = conf.getTrimmed(USER_AGENT_PREFIX, ""); + if (!userAgentPrefix.isEmpty()) { + userAgent = userAgentPrefix + ", " + userAgent; + } + LOG.debug("Using User-Agent: {}", userAgent); + awsConf.setUserAgentPrefix(userAgent); + } + + /** + * Creates an {@link AmazonS3Client} from the established configuration. + * + * @param conf Hadoop configuration + * @param credentials AWS credentials + * @param awsConf AWS SDK configuration + * @return S3 client + * @throws IllegalArgumentException if misconfigured + */ + private static AmazonS3 createAmazonS3Client(AmazonS3 s3, Configuration conf, + AWSCredentialsProvider credentials, ClientConfiguration awsConf) + throws IllegalArgumentException { + String endPoint = conf.getTrimmed(ENDPOINT, ""); + if (!endPoint.isEmpty()) { + try { + s3.setEndpoint(endPoint); + } catch (IllegalArgumentException e) { + String msg = "Incorrect endpoint: " + e.getMessage(); + LOG.error(msg); + throw new IllegalArgumentException(msg, e); + } + } + enablePathStyleAccessIfRequired(s3, conf); + return s3; + } + + /** + * Enables path-style access to S3 buckets if configured. By default, the + * behavior is to use virtual hosted-style access with URIs of the form + * http://bucketname.s3.amazonaws.com. Enabling path-style access and a + * region-specific endpoint switches the behavior to use URIs of the form + * http://s3-eu-west-1.amazonaws.com/bucketname. + * + * @param s3 S3 client + * @param conf Hadoop configuration + */ + private static void enablePathStyleAccessIfRequired(AmazonS3 s3, + Configuration conf) { + final boolean pathStyleAccess = conf.getBoolean(PATH_STYLE_ACCESS, false); + if (pathStyleAccess) { + LOG.debug("Enabling path style access!"); + s3.setS3ClientOptions(S3ClientOptions.builder() + .setPathStyleAccess(true) + .build()); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c41f9f3b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c41f9f3b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java ---------------------------------------------------------------------- diff --cc hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index f56259c,9eb5575..29f84b0 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@@ -1643,11 -1525,8 +1643,9 @@@ public class S3AFileSystem extends File * @throws IOException other IO problems * @throws AmazonClientException on failures inside the AWS SDK */ - // TODO: If we have created an empty file at /foo/bar and we then call - // mkdirs for /foo/bar/baz/roo what happens to the empty file /foo/bar/? - private boolean innerMkdirs(Path f, FsPermission permission) + private boolean innerMkdirs(Path p, FsPermission permission) throws IOException, FileAlreadyExistsException, AmazonClientException { + Path f = qualify(p); LOG.debug("Making directory: {}", f); incrementStatistic(INVOCATION_MKDIRS); FileStatus fileStatus; @@@ -1694,7 -1559,7 +1692,8 @@@ String key = pathToKey(f); createFakeDirectory(key); + S3Guard.makeDirsOrdered(metadataStore, metadataStoreDirs, username); + deleteUnnecessaryFakeDirectories(f.getParent()); return true; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c41f9f3b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c41f9f3b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java ---------------------------------------------------------------------- diff --cc hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java index 387eb43,d4e09e3..e7603d9 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java @@@ -42,7 -55,179 +42,6 @@@ public interface S3ClientFactory * @return S3 client * @throws IOException IO problem */ - AmazonS3 createS3Client(URI name, URI uri) throws IOException; + AmazonS3 createS3Client(URI name) throws IOException; - /** - * The default factory implementation, which calls the AWS SDK to configure - * and create an {@link AmazonS3Client} that communicates with the S3 service. - */ - static class DefaultS3ClientFactory extends Configured - implements S3ClientFactory { - - private static final Logger LOG = S3AFileSystem.LOG; - - @Override - public AmazonS3 createS3Client(URI name) throws IOException { - Configuration conf = getConf(); - AWSCredentialsProvider credentials = - createAWSCredentialProviderSet(name, conf); - ClientConfiguration awsConf = new ClientConfiguration(); - initConnectionSettings(conf, awsConf); - initProxySupport(conf, awsConf); - initUserAgent(conf, awsConf); - return createAmazonS3Client(conf, credentials, awsConf); - } - - /** - * Initializes all AWS SDK settings related to connection management. - * - * @param conf Hadoop configuration - * @param awsConf AWS SDK configuration - */ - private static void initConnectionSettings(Configuration conf, - ClientConfiguration awsConf) { - awsConf.setMaxConnections(intOption(conf, MAXIMUM_CONNECTIONS, - DEFAULT_MAXIMUM_CONNECTIONS, 1)); - boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS, - DEFAULT_SECURE_CONNECTIONS); - awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP); - awsConf.setMaxErrorRetry(intOption(conf, MAX_ERROR_RETRIES, - DEFAULT_MAX_ERROR_RETRIES, 0)); - awsConf.setConnectionTimeout(intOption(conf, ESTABLISH_TIMEOUT, - DEFAULT_ESTABLISH_TIMEOUT, 0)); - awsConf.setSocketTimeout(intOption(conf, SOCKET_TIMEOUT, - DEFAULT_SOCKET_TIMEOUT, 0)); - int sockSendBuffer = intOption(conf, SOCKET_SEND_BUFFER, - DEFAULT_SOCKET_SEND_BUFFER, 2048); - int sockRecvBuffer = intOption(conf, SOCKET_RECV_BUFFER, - DEFAULT_SOCKET_RECV_BUFFER, 2048); - awsConf.setSocketBufferSizeHints(sockSendBuffer, sockRecvBuffer); - String signerOverride = conf.getTrimmed(SIGNING_ALGORITHM, ""); - if (!signerOverride.isEmpty()) { - LOG.debug("Signer override = {}", signerOverride); - awsConf.setSignerOverride(signerOverride); - } - } - - /** - * Initializes AWS SDK proxy support if configured. - * - * @param conf Hadoop configuration - * @param awsConf AWS SDK configuration - * @throws IllegalArgumentException if misconfigured - */ - private static void initProxySupport(Configuration conf, - ClientConfiguration awsConf) throws IllegalArgumentException { - String proxyHost = conf.getTrimmed(PROXY_HOST, ""); - int proxyPort = conf.getInt(PROXY_PORT, -1); - if (!proxyHost.isEmpty()) { - awsConf.setProxyHost(proxyHost); - if (proxyPort >= 0) { - awsConf.setProxyPort(proxyPort); - } else { - if (conf.getBoolean(SECURE_CONNECTIONS, DEFAULT_SECURE_CONNECTIONS)) { - LOG.warn("Proxy host set without port. Using HTTPS default 443"); - awsConf.setProxyPort(443); - } else { - LOG.warn("Proxy host set without port. Using HTTP default 80"); - awsConf.setProxyPort(80); - } - } - String proxyUsername = conf.getTrimmed(PROXY_USERNAME); - String proxyPassword = conf.getTrimmed(PROXY_PASSWORD); - if ((proxyUsername == null) != (proxyPassword == null)) { - String msg = "Proxy error: " + PROXY_USERNAME + " or " + - PROXY_PASSWORD + " set without the other."; - LOG.error(msg); - throw new IllegalArgumentException(msg); - } - awsConf.setProxyUsername(proxyUsername); - awsConf.setProxyPassword(proxyPassword); - awsConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN)); - awsConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION)); - if (LOG.isDebugEnabled()) { - LOG.debug("Using proxy server {}:{} as user {} with password {} on " + - "domain {} as workstation {}", awsConf.getProxyHost(), - awsConf.getProxyPort(), - String.valueOf(awsConf.getProxyUsername()), - awsConf.getProxyPassword(), awsConf.getProxyDomain(), - awsConf.getProxyWorkstation()); - } - } else if (proxyPort >= 0) { - String msg = - "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST; - LOG.error(msg); - throw new IllegalArgumentException(msg); - } - } - - /** - * Initializes the User-Agent header to send in HTTP requests to the S3 - * back-end. We always include the Hadoop version number. The user also - * may set an optional custom prefix to put in front of the Hadoop version - * number. The AWS SDK interally appends its own information, which seems - * to include the AWS SDK version, OS and JVM version. - * - * @param conf Hadoop configuration - * @param awsConf AWS SDK configuration - */ - private static void initUserAgent(Configuration conf, - ClientConfiguration awsConf) { - String userAgent = "Hadoop " + VersionInfo.getVersion(); - String userAgentPrefix = conf.getTrimmed(USER_AGENT_PREFIX, ""); - if (!userAgentPrefix.isEmpty()) { - userAgent = userAgentPrefix + ", " + userAgent; - } - LOG.debug("Using User-Agent: {}", userAgent); - awsConf.setUserAgentPrefix(userAgent); - } - - /** - * Creates an {@link AmazonS3Client} from the established configuration. - * - * @param conf Hadoop configuration - * @param credentials AWS credentials - * @param awsConf AWS SDK configuration - * @return S3 client - * @throws IllegalArgumentException if misconfigured - */ - private static AmazonS3 createAmazonS3Client(Configuration conf, - AWSCredentialsProvider credentials, ClientConfiguration awsConf) - throws IllegalArgumentException { - AmazonS3 s3 = new AmazonS3Client(credentials, awsConf); - String endPoint = conf.getTrimmed(ENDPOINT, ""); - if (!endPoint.isEmpty()) { - try { - s3.setEndpoint(endPoint); - } catch (IllegalArgumentException e) { - String msg = "Incorrect endpoint: " + e.getMessage(); - LOG.error(msg); - throw new IllegalArgumentException(msg, e); - } - } - enablePathStyleAccessIfRequired(s3, conf); - return s3; - } - - /** - * Enables path-style access to S3 buckets if configured. By default, the - * behavior is to use virtual hosted-style access with URIs of the form - * http://bucketname.s3.amazonaws.com. Enabling path-style access and a - * region-specific endpoint switches the behavior to use URIs of the form - * http://s3-eu-west-1.amazonaws.com/bucketname. - * - * @param s3 S3 client - * @param conf Hadoop configuration - */ - private static void enablePathStyleAccessIfRequired(AmazonS3 s3, - Configuration conf) { - final boolean pathStyleAccess = conf.getBoolean(PATH_STYLE_ACCESS, false); - if (pathStyleAccess) { - LOG.debug("Enabling path style access!"); - s3.setS3ClientOptions(S3ClientOptions.builder() - .setPathStyleAccess(true) - .build()); - } - } - } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c41f9f3b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java ---------------------------------------------------------------------- diff --cc hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java index ecdc3e1,0000000..0c88230 mode 100644,000000..100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java @@@ -1,131 -1,0 +1,131 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.s3guard; + +import java.io.IOException; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; +import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder; +import com.google.common.base.Preconditions; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.s3a.DefaultS3ClientFactory; + +import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_REGION_KEY; +import static org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet; + +/** + * Interface to create a DynamoDB client. + * + * Implementation should be configured for setting and getting configuration. + */ [email protected] +public interface DynamoDBClientFactory extends Configurable { + Logger LOG = LoggerFactory.getLogger(DynamoDBClientFactory.class); + + /** + * Create a DynamoDB client object from configuration. + * + * The DynamoDB client to create does not have to relate to any S3 buckets. + * All information needed to create a DynamoDB client is from the hadoop + * configuration. Specially, if the region is not configured, it will use the + * provided region parameter. If region is neither configured nor provided, + * it will indicate an error. + * + * @param defaultRegion the default region of the AmazonDynamoDB client + * @return a new DynamoDB client + * @throws IOException if any IO error happens + */ + AmazonDynamoDB createDynamoDBClient(String defaultRegion) throws IOException; + + /** + * The default implementation for creating an AmazonDynamoDB. + */ + class DefaultDynamoDBClientFactory extends Configured + implements DynamoDBClientFactory { + @Override + public AmazonDynamoDB createDynamoDBClient(String defaultRegion) + throws IOException { + assert getConf() != null : "Should have been configured before usage"; + + final Configuration conf = getConf(); + final AWSCredentialsProvider credentials = - createAWSCredentialProviderSet(null, conf, null); ++ createAWSCredentialProviderSet(null, conf); + final ClientConfiguration awsConf = + DefaultS3ClientFactory.createAwsConf(conf); + + final String region = getRegion(conf, defaultRegion); + LOG.debug("Creating DynamoDB client in region {}", region); + + return AmazonDynamoDBClientBuilder.standard() + .withCredentials(credentials) + .withClientConfiguration(awsConf) + .withRegion(region) + .build(); + } + + /** + * Helper method to get and validate the AWS region for DynamoDBClient. + * + * @param conf configuration + * @param defaultRegion the default region + * @return configured region or else the provided default region + * @throws IOException if the region is not valid + */ + static String getRegion(Configuration conf, String defaultRegion) + throws IOException { + String region = conf.getTrimmed(S3GUARD_DDB_REGION_KEY); + if (StringUtils.isEmpty(region)) { + region = defaultRegion; + } + try { + Regions.fromName(region); + } catch (IllegalArgumentException | NullPointerException e) { + throw new IOException("Invalid region specified: " + region + "; " + + "Region can be configured with " + S3GUARD_DDB_REGION_KEY + ": " + + validRegionsString()); + } + return region; + } + + private static String validRegionsString() { + final String DELIMITER = ", "; + Regions[] regions = Regions.values(); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < regions.length; i++) { + if (i > 0) { + sb.append(DELIMITER); + } + sb.append(regions[i].getName()); + } + return sb.toString(); + + } + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c41f9f3b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md ---------------------------------------------------------------------- diff --cc hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 0d8e803,a3a0bb1..d7a2b3c --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@@ -41,9 -41,9 +41,10 @@@ The specifics of using these filesystem See also: + * [Testing](testing.html) * [Troubleshooting S3a](troubleshooting_s3a.html) +* [S3Guard](s3guard.html) ### Warning #1: Object Stores are not filesystems http://git-wip-us.apache.org/repos/asf/hadoop/blob/c41f9f3b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c41f9f3b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c41f9f3b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBLocalClientFactory.java ---------------------------------------------------------------------- diff --cc hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBLocalClientFactory.java index be1642a,0000000..750cfb3 mode 100644,000000..100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBLocalClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBLocalClientFactory.java @@@ -1,148 -1,0 +1,148 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.s3guard; + +import java.io.File; +import java.io.IOException; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; +import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder; +import com.amazonaws.services.dynamodbv2.local.main.ServerRunner; +import com.amazonaws.services.dynamodbv2.local.server.DynamoDBProxyServer; +import org.apache.commons.lang3.StringUtils; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.s3a.DefaultS3ClientFactory; +import org.apache.hadoop.net.ServerSocketUtil; + +import static org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet; +import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBClientFactory.DefaultDynamoDBClientFactory.getRegion; + +/** + * A DynamoDBClientFactory implementation that creates AmazonDynamoDB clients + * against an in-memory DynamoDBLocal server instance. + * + * You won't be charged bills for issuing any DynamoDB requests. However, the + * DynamoDBLocal is considered a simulator of the DynamoDB web service, so it + * may be stale or different. For example, the throttling is not yet supported + * in DynamoDBLocal. This is for testing purpose only. + * + * To use this for creating DynamoDB client in tests: + * 1. As all DynamoDBClientFactory implementations, this should be configured. + * 2. The singleton DynamoDBLocal server instance is started automatically when + * creating the AmazonDynamoDB client for the first time. It still merits to + * launch the server before all the tests and fail fast if error happens. + * 3. The sever can be stopped explicitly, which is not actually needed in tests + * as JVM termination will do that. + * + * @see DefaultDynamoDBClientFactory + */ +public class DynamoDBLocalClientFactory extends Configured + implements DynamoDBClientFactory { + + /** The DynamoDBLocal dynamoDBLocalServer instance for testing. */ + private static DynamoDBProxyServer dynamoDBLocalServer; + private static String ddbEndpoint; + + private static final String SYSPROP_SQLITE_LIB = "sqlite4java.library.path"; + + @Override + public AmazonDynamoDB createDynamoDBClient(String defaultRegion) + throws IOException { + startSingletonServer(); + + final Configuration conf = getConf(); + final AWSCredentialsProvider credentials = - createAWSCredentialProviderSet(null, conf, null); ++ createAWSCredentialProviderSet(null, conf); + final ClientConfiguration awsConf = + DefaultS3ClientFactory.createAwsConf(conf); + // fail fast in case of service errors + awsConf.setMaxErrorRetry(3); + + final String region = getRegion(conf, defaultRegion); + LOG.info("Creating DynamoDBLocal client using endpoint {} in region {}", + ddbEndpoint, region); + + return AmazonDynamoDBClientBuilder.standard() + .withCredentials(credentials) + .withClientConfiguration(awsConf) + .withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration(ddbEndpoint, region)) + .build(); + } + + /** + * Start a singleton in-memory DynamoDBLocal server if not started yet. + * @throws IOException if any error occurs + */ + public synchronized static void startSingletonServer() throws IOException { + if (dynamoDBLocalServer != null) { + return; + } + + // Set this property if it has not been set elsewhere + if (StringUtils.isEmpty(System.getProperty(SYSPROP_SQLITE_LIB))) { + String projectBuildDir = System.getProperty("project.build.directory"); + if (StringUtils.isEmpty(projectBuildDir)) { + projectBuildDir = "target"; + } + // sqlite4java lib should have been copied to $projectBuildDir/native-libs + System.setProperty(SYSPROP_SQLITE_LIB, + projectBuildDir + File.separator + "native-libs"); + LOG.info("Setting {} -> {}", + SYSPROP_SQLITE_LIB, System.getProperty(SYSPROP_SQLITE_LIB)); + } + + try { + // Start an in-memory local DynamoDB instance + final String port = String.valueOf(ServerSocketUtil.getPort(0, 100)); + dynamoDBLocalServer = ServerRunner.createServerFromCommandLineArgs( + new String[]{"-inMemory", "-port", port}); + dynamoDBLocalServer.start(); + ddbEndpoint = "http://localhost:"; + port; + LOG.info("DynamoDBLocal singleton server was started at {}", ddbEndpoint); + } catch (Throwable t) { + String msg = "Error starting DynamoDBLocal server at " + ddbEndpoint; + LOG.error(msg, t); + throw new IOException(msg, t); + } + } + + /** + * Stop the in-memory DynamoDBLocal server if it is started. + * @throws IOException if any error occurs + */ + public synchronized static void stopSingletonServer() throws IOException { + if (dynamoDBLocalServer != null) { + LOG.info("Shutting down the in-memory DynamoDBLocal server"); + try { + dynamoDBLocalServer.stop(); + } catch (Throwable t) { + String msg = "Error stopping DynamoDBLocal server at " + ddbEndpoint; + LOG.error(msg, t); + throw new IOException(msg, t); + } + } + } + +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
