Merge branch 'trunk' into HDFS-7240
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
Added the following code in:
hadoop/ozone/container/common/impl/ContainerManagerImpl.java
@Override
public void readLockInterruptibly() throws InterruptedException {
this.lock.readLock().lockInterruptibly();
}
and Manually updated the value of version in
modified: hadoop-tools/hadoop-ozone/pom.xml
to
<version>3.2.0-SNAPSHOT</version>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/47919787
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/47919787
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/47919787
Branch: refs/heads/HDFS-7240
Commit: 479197872ba89159ec2160fbdda92a1665362b5d
Parents: fc84744 4747395
Author: Anu Engineer <aengin...@apache.org>
Authored: Thu Feb 15 15:28:08 2018 -0800
Committer: Anu Engineer <aengin...@apache.org>
Committed: Thu Feb 15 15:28:08 2018 -0800
----------------------------------------------------------------------
hadoop-assemblies/pom.xml | 4 +-
hadoop-build-tools/pom.xml | 2 +-
hadoop-client-modules/hadoop-client-api/pom.xml | 4 +-
.../hadoop-client-check-invariants/pom.xml | 4 +-
.../hadoop-client-check-test-invariants/pom.xml | 4 +-
.../hadoop-client-integration-tests/pom.xml | 4 +-
.../hadoop-client-minicluster/pom.xml | 4 +-
.../hadoop-client-runtime/pom.xml | 4 +-
hadoop-client-modules/hadoop-client/pom.xml | 4 +-
hadoop-client-modules/pom.xml | 2 +-
.../hadoop-cloud-storage/pom.xml | 4 +-
hadoop-cloud-storage-project/pom.xml | 4 +-
.../hadoop-annotations/pom.xml | 4 +-
.../hadoop-auth-examples/pom.xml | 4 +-
hadoop-common-project/hadoop-auth/pom.xml | 14 +-
.../client/AuthenticatorTestCase.java | 51 +-
.../client/TestKerberosAuthenticator.java | 41 +-
hadoop-common-project/hadoop-common/pom.xml | 4 +-
.../hadoop-common/src/main/bin/hadoop | 4 +
.../hadoop-common/src/main/bin/hadoop.cmd | 7 +-
.../org/apache/hadoop/conf/Configuration.java | 80 ++
.../org/apache/hadoop/conf/StorageSize.java | 106 +++
.../org/apache/hadoop/conf/StorageUnit.java | 530 +++++++++++
.../hadoop/fs/CommonConfigurationKeys.java | 2 +-
.../java/org/apache/hadoop/fs/FileUtil.java | 257 +++++-
.../org/apache/hadoop/fs/LocalFileSystem.java | 2 +-
.../apache/hadoop/ha/ActiveStandbyElector.java | 30 +-
.../apache/hadoop/ha/FailoverController.java | 20 +-
.../org/apache/hadoop/ha/HealthMonitor.java | 9 +-
.../org/apache/hadoop/http/HttpServer2.java | 2 +-
.../org/apache/hadoop/io/retry/RetryUtils.java | 11 +-
.../main/java/org/apache/hadoop/net/DNS.java | 39 +-
.../AbstractDelegationTokenSecretManager.java | 6 +
.../apache/hadoop/service/AbstractService.java | 27 +-
.../hadoop/service/ServiceOperations.java | 6 +-
.../org/apache/hadoop/util/CombinedIPList.java | 59 ++
.../hadoop/util/GenericOptionsParser.java | 8 +-
.../apache/hadoop/util/JsonSerialization.java | 8 +
.../java/org/apache/hadoop/util/RunJar.java | 69 ++
.../src/main/resources/core-default.xml | 13 +-
.../src/site/markdown/CommandsManual.md | 6 +
.../src/site/markdown/SecureMode.md | 32 +-
.../apache/hadoop/conf/TestConfiguration.java | 76 ++
.../org/apache/hadoop/conf/TestStorageUnit.java | 277 ++++++
.../hadoop/fs/TestDelegateToFileSystem.java | 2 +-
.../java/org/apache/hadoop/fs/TestFileUtil.java | 86 ++
.../apache/hadoop/ha/TestSshFenceByTcpPort.java | 2 +-
.../hadoop/service/TestServiceOperations.java | 3 +-
.../org/apache/hadoop/test/LambdaTestUtils.java | 40 +-
.../apache/hadoop/test/TestLambdaTestUtils.java | 36 +
.../java/org/apache/hadoop/util/TestRunJar.java | 57 ++
.../src/test/scripts/start-build-env.bats | 102 +++
hadoop-common-project/hadoop-kms/pom.xml | 4 +-
.../hadoop/crypto/key/kms/server/KMS.java | 4 +-
hadoop-common-project/hadoop-minikdc/pom.xml | 4 +-
hadoop-common-project/hadoop-nfs/pom.xml | 4 +-
hadoop-common-project/pom.xml | 4 +-
hadoop-dist/pom.xml | 4 +-
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 4 +-
.../org/apache/hadoop/hdfs/ClientContext.java | 3 +-
.../hadoop/hdfs/DFSStripedOutputStream.java | 4 +-
.../org/apache/hadoop/hdfs/DFSUtilClient.java | 6 +-
.../hdfs/client/HdfsClientConfigKeys.java | 5 +-
.../hdfs/client/impl/BlockReaderFactory.java | 56 +-
.../hadoop/hdfs/client/impl/DfsClientConf.java | 19 +-
.../sasl/SaslDataTransferClient.java | 12 +-
.../hdfs/shortcircuit/DomainSocketFactory.java | 17 +-
.../hdfs/shortcircuit/ShortCircuitCache.java | 16 +-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 11 +-
.../ha/TestRequestHedgingProxyProvider.java | 6 +-
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 4 +-
.../hadoop-hdfs-native-client/pom.xml | 4 +-
hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 4 +-
.../hadoop/hdfs/nfs/nfs3/DFSClientCache.java | 25 +-
.../hadoop/hdfs/nfs/nfs3/OpenFileCtx.java | 314 +++----
.../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 370 ++++----
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 4 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 9 +-
.../java/org/apache/hadoop/hdfs/HAUtil.java | 9 +-
.../BlackListBasedTrustedChannelResolver.java | 143 +++
.../protocol/datatransfer/package-info.java | 24 +
...uterAdminProtocolServerSideTranslatorPB.java | 60 ++
.../RouterAdminProtocolTranslatorPB.java | 60 +-
.../protocol/InterQJournalProtocol.java | 4 +-
...rQJournalProtocolServerSideTranslatorPB.java | 11 +-
.../InterQJournalProtocolTranslatorPB.java | 13 +-
.../hadoop/hdfs/qjournal/server/JNStorage.java | 9 +-
.../hdfs/qjournal/server/JournalNode.java | 129 ++-
.../qjournal/server/JournalNodeRpcServer.java | 21 +-
.../hdfs/qjournal/server/JournalNodeSyncer.java | 8 +-
.../DelegationTokenSecretManager.java | 53 +-
.../BlockReconstructionWork.java | 6 +
.../blockmanagement/ErasureCodingWork.java | 3 +-
.../blockmanagement/LowRedundancyBlocks.java | 2 +-
.../server/blockmanagement/ReplicationWork.java | 6 +-
.../server/blockmanagement/SlowDiskTracker.java | 2 +-
.../hadoop/hdfs/server/common/Storage.java | 75 +-
.../hdfs/server/datanode/BlockPoolManager.java | 15 +-
.../server/datanode/BlockPoolSliceStorage.java | 88 +-
.../hdfs/server/datanode/BlockSender.java | 56 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 213 ++---
.../hdfs/server/datanode/DataStorage.java | 112 ++-
.../hdfs/server/datanode/DataXceiver.java | 172 ++--
.../hdfs/server/datanode/FinalizedReplica.java | 74 +-
.../hdfs/server/datanode/ReplicaBuilder.java | 11 +-
.../datanode/checker/ThrottledAsyncChecker.java | 2 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 100 ++-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 21 +-
.../federation/metrics/FederationMBean.java | 6 +
.../federation/metrics/FederationMetrics.java | 71 +-
.../federation/router/FederationUtil.java | 8 +-
.../federation/router/RouterAdminServer.java | 46 +-
.../server/federation/router/RouterClient.java | 4 +
.../federation/router/RouterStateManager.java | 50 ++
.../store/protocol/EnterSafeModeRequest.java | 32 +
.../store/protocol/EnterSafeModeResponse.java | 50 ++
.../store/protocol/GetSafeModeRequest.java | 31 +
.../store/protocol/GetSafeModeResponse.java | 49 ++
.../store/protocol/LeaveSafeModeRequest.java | 32 +
.../store/protocol/LeaveSafeModeResponse.java | 50 ++
.../impl/pb/EnterSafeModeRequestPBImpl.java | 62 ++
.../impl/pb/EnterSafeModeResponsePBImpl.java | 73 ++
.../impl/pb/GetSafeModeRequestPBImpl.java | 62 ++
.../impl/pb/GetSafeModeResponsePBImpl.java | 73 ++
.../impl/pb/LeaveSafeModeRequestPBImpl.java | 62 ++
.../impl/pb/LeaveSafeModeResponsePBImpl.java | 73 ++
.../federation/store/records/RouterState.java | 8 +-
.../records/impl/pb/RouterStatePBImpl.java | 12 +-
.../hdfs/server/namenode/FSDirConcatOp.java | 4 +-
.../server/namenode/FSDirErasureCodingOp.java | 19 +-
.../hdfs/server/namenode/FSDirXAttrOp.java | 3 +-
.../hdfs/server/namenode/FSNamesystem.java | 21 +-
.../hdfs/server/namenode/FSNamesystemLock.java | 7 +
.../hdfs/server/namenode/INodeDirectory.java | 5 +-
.../hadoop/hdfs/server/namenode/INodeFile.java | 3 +-
.../hdfs/server/namenode/LeaseManager.java | 13 +-
.../hadoop/hdfs/server/namenode/NNStorage.java | 61 +-
.../server/namenode/ReencryptionHandler.java | 3 +-
.../server/namenode/ha/StandbyCheckpointer.java | 34 +-
.../snapshot/AbstractINodeDiffList.java | 27 +-
.../hdfs/server/namenode/snapshot/DiffList.java | 140 +++
.../namenode/snapshot/DiffListByArrayList.java | 80 ++
.../snapshot/DirectorySnapshottableFeature.java | 12 +-
.../snapshot/DirectoryWithSnapshotFeature.java | 10 +-
.../snapshot/FSImageFormatPBSnapshot.java | 7 +-
.../server/namenode/snapshot/FileDiffList.java | 11 +-
.../snapshot/FileWithSnapshotFeature.java | 2 +-
.../snapshot/SnapshotFSImageFormat.java | 4 +-
.../namenode/snapshot/SnapshotManager.java | 12 +-
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 192 ++--
.../hdfs/tools/federation/RouterAdmin.java | 75 +-
.../org/apache/hadoop/hdfs/util/RwLock.java | 5 +-
.../common/impl/ContainerManagerImpl.java | 5 +
.../src/main/proto/FederationProtocol.proto | 27 +-
.../src/main/proto/InterQJournalProtocol.proto | 16 +-
.../src/main/proto/RouterProtocol.proto | 15 +
.../src/main/resources/hdfs-default.xml | 33 +-
.../main/webapps/router/federationhealth.html | 41 +
.../src/main/webapps/router/federationhealth.js | 26 +
.../src/site/markdown/HDFSCommands.md | 2 +
.../markdown/HDFSHighAvailabilityWithNFS.md | 6 +-
.../markdown/HDFSHighAvailabilityWithQJM.md | 6 +-
.../src/site/markdown/HDFSRouterFederation.md | 10 +-
.../hadoop-hdfs/src/site/markdown/HdfsDesign.md | 2 +-
.../hadoop-hdfs/src/site/markdown/ViewFs.md | 32 +-
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 25 +-
.../hadoop/hdfs/TestDFSClientFailover.java | 2 +-
.../org/apache/hadoop/hdfs/TestDFSUtil.java | 28 +-
.../hdfs/TestErasureCodingExerciseAPIs.java | 546 ++++++++++++
.../hadoop/hdfs/TestErasureCodingPolicies.java | 34 +
.../TestErasureCodingPolicyWithSnapshot.java | 84 +-
.../java/org/apache/hadoop/hdfs/TestQuota.java | 2 +-
.../hadoop/hdfs/TestSnapshotCommands.java | 18 +
.../client/impl/TestBlockReaderFactory.java | 144 ++-
...estBlackListBasedTrustedChannelResolver.java | 89 ++
.../datatransfer/sasl/TestSaslDataTransfer.java | 154 +++-
.../hdfs/qjournal/server/TestJournalNode.java | 148 +++-
.../blockmanagement/TestBlockTokenWithDFS.java | 2 +-
.../TestBlockTokenWithDFSStriped.java | 2 +-
.../TestLowRedundancyBlockQueues.java | 15 +
.../server/datanode/TestBlockPoolManager.java | 12 +-
.../datanode/TestBlockPoolSliceStorage.java | 14 +-
.../fsdataset/impl/TestFsDatasetImpl.java | 103 ++-
.../server/federation/RouterConfigBuilder.java | 6 +
.../metrics/TestFederationMetrics.java | 50 ++
.../federation/metrics/TestMetricsBase.java | 45 +
.../federation/router/TestRouterAdminCLI.java | 48 +
.../store/FederationStateStoreTestUtils.java | 17 +-
.../store/TestStateStoreRouterState.java | 3 +-
.../store/records/TestRouterState.java | 6 +-
.../hadoop/hdfs/server/mover/TestMover.java | 209 +++++
.../hdfs/server/namenode/FSXAttrBaseTest.java | 63 +-
.../hdfs/server/namenode/TestAllowFormat.java | 4 +-
.../namenode/TestFSImageWithSnapshot.java | 3 +-
.../namenode/TestListCorruptFileBlocks.java | 4 +-
.../namenode/TestSecurityTokenEditLog.java | 24 +-
.../namenode/snapshot/TestNestedSnapshots.java | 8 +-
.../snapshot/TestRenameWithSnapshots.java | 40 +-
.../snapshot/TestSetQuotaWithSnapshot.java | 3 +-
.../server/namenode/snapshot/TestSnapshot.java | 4 +-
.../namenode/snapshot/TestSnapshotDeletion.java | 126 +++
.../namenode/snapshot/TestSnapshotRename.java | 3 +-
.../hadoop/hdfs/tools/TestDFSAdminWithHA.java | 464 +++++++++-
.../apache/hadoop/hdfs/tools/TestGetConf.java | 6 +-
.../hadoop/test/MiniDFSClusterManager.java | 26 +-
hadoop-hdfs-project/pom.xml | 4 +-
.../hadoop-mapreduce-client-app/pom.xml | 4 +-
.../v2/app/rm/TestRMContainerAllocator.java | 15 +-
.../hadoop-mapreduce-client-common/pom.xml | 4 +-
.../hadoop-mapreduce-client-core/pom.xml | 4 +-
.../java/org/apache/hadoop/mapred/MapTask.java | 12 +
.../java/org/apache/hadoop/mapred/Task.java | 16 +-
.../org/apache/hadoop/mapred/TestMapTask.java | 87 ++
.../java/org/apache/hadoop/mapred/TestTask.java | 89 ++
.../hadoop-mapreduce-client-hs-plugins/pom.xml | 4 +-
.../hadoop-mapreduce-client-hs/pom.xml | 4 +-
.../src/test/resources/job_0.23.9-FAILED.jhist | 2 +-
.../hadoop-mapreduce-client-jobclient/pom.xml | 4 +-
.../mapred/TestFixedLengthInputFormat.java | 2 +-
.../hadoop-mapreduce-client-nativetask/pom.xml | 4 +-
.../hadoop-mapreduce-client-shuffle/pom.xml | 4 +-
.../hadoop-mapreduce-client-uploader/pom.xml | 4 +-
.../hadoop-mapreduce-client/pom.xml | 4 +-
.../hadoop-mapreduce-examples/pom.xml | 4 +-
hadoop-mapreduce-project/pom.xml | 4 +-
hadoop-maven-plugins/pom.xml | 2 +-
hadoop-minicluster/pom.xml | 4 +-
hadoop-project-dist/pom.xml | 4 +-
hadoop-project/pom.xml | 16 +-
hadoop-tools/hadoop-aliyun/pom.xml | 2 +-
hadoop-tools/hadoop-archive-logs/pom.xml | 4 +-
hadoop-tools/hadoop-archives/pom.xml | 4 +-
hadoop-tools/hadoop-aws/pom.xml | 4 +-
.../fs/s3a/AssumedRoleCredentialProvider.java | 197 -----
.../org/apache/hadoop/fs/s3a/Constants.java | 2 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 17 +-
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 53 +-
.../s3a/auth/AssumedRoleCredentialProvider.java | 205 +++++
.../apache/hadoop/fs/s3a/auth/RoleModel.java | 314 +++++++
.../apache/hadoop/fs/s3a/auth/RolePolicies.java | 228 +++++
.../apache/hadoop/fs/s3a/auth/package-info.java | 27 +
.../hadoop/fs/s3a/commit/CommitOperations.java | 2 +-
.../markdown/tools/hadoop-aws/assumed_roles.md | 274 +++++-
.../markdown/tools/hadoop-aws/encryption.md | 21 +-
.../src/site/markdown/tools/hadoop-aws/index.md | 77 +-
.../markdown/tools/hadoop-aws/performance.md | 518 +++++++++++
.../site/markdown/tools/hadoop-aws/testing.md | 15 +-
.../tools/hadoop-aws/troubleshooting_s3a.md | 753 ++++++++++------
.../s3a/ITestS3AContractDistCpAssumedRole.java | 52 --
.../apache/hadoop/fs/s3a/ITestAssumeRole.java | 324 -------
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 46 +-
.../fs/s3a/TestS3AAWSCredentialsProvider.java | 40 +-
.../hadoop/fs/s3a/auth/ITestAssumeRole.java | 789 +++++++++++++++++
.../auth/ITestAssumedRoleCommitOperations.java | 130 +++
.../hadoop/fs/s3a/auth/RoleTestUtils.java | 171 ++++
.../fs/s3a/commit/AbstractCommitITest.java | 12 +-
.../fs/s3a/commit/ITestCommitOperations.java | 4 +-
hadoop-tools/hadoop-azure-datalake/pom.xml | 2 +-
.../org/apache/hadoop/fs/adl/AdlConfKeys.java | 5 +
.../org/apache/hadoop/fs/adl/AdlFileSystem.java | 81 +-
.../src/site/markdown/index.md | 41 +
.../src/site/markdown/troubleshooting_adl.md | 146 ++++
.../apache/hadoop/fs/adl/AdlMockWebServer.java | 102 ---
.../apache/hadoop/fs/adl/TestACLFeatures.java | 262 ------
.../org/apache/hadoop/fs/adl/TestAdlRead.java | 196 -----
.../adl/TestConcurrentDataReadOperations.java | 299 -------
.../hadoop/fs/adl/TestCustomTokenProvider.java | 140 ---
.../apache/hadoop/fs/adl/TestGetFileStatus.java | 102 ---
.../apache/hadoop/fs/adl/TestListStatus.java | 137 ---
.../fs/adl/TestValidateConfiguration.java | 152 +++-
.../hadoop/fs/adl/TestableAdlFileSystem.java | 30 -
.../hadoop/fs/adl/common/ExpectedResponse.java | 71 --
.../hadoop/fs/adl/common/Parallelized.java | 2 +-
.../hadoop/fs/adl/common/TestDataForRead.java | 122 ---
hadoop-tools/hadoop-azure/pom.xml | 2 +-
hadoop-tools/hadoop-datajoin/pom.xml | 4 +-
hadoop-tools/hadoop-distcp/pom.xml | 4 +-
.../org/apache/hadoop/tools/CopyFilter.java | 2 +-
.../src/site/markdown/DistCp.md.vm | 70 +-
.../apache/hadoop/tools/TestOptionsParser.java | 294 +++----
hadoop-tools/hadoop-extras/pom.xml | 4 +-
hadoop-tools/hadoop-fs2img/pom.xml | 4 +-
hadoop-tools/hadoop-gridmix/pom.xml | 4 +-
.../apache/hadoop/mapred/gridmix/Gridmix.java | 22 +-
hadoop-tools/hadoop-kafka/pom.xml | 4 +-
hadoop-tools/hadoop-openstack/pom.xml | 4 +-
.../swift/http/HttpInputStreamWithRelease.java | 29 +-
.../hadoop-openstack/src/site/markdown/index.md | 2 +-
hadoop-tools/hadoop-ozone/pom.xml | 2 +-
hadoop-tools/hadoop-pipes/pom.xml | 4 +-
hadoop-tools/hadoop-resourceestimator/pom.xml | 2 +-
hadoop-tools/hadoop-rumen/pom.xml | 4 +-
hadoop-tools/hadoop-sls/pom.xml | 4 +-
.../src/main/data/2jobs2min-rumen-jh.json | 12 +-
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 6 +
.../yarn/sls/scheduler/RMNodeWrapper.java | 6 +
.../sls/scheduler/SLSCapacityScheduler.java | 15 +-
.../yarn/sls/scheduler/SLSFairScheduler.java | 12 +-
hadoop-tools/hadoop-streaming/pom.xml | 4 +-
hadoop-tools/hadoop-tools-dist/pom.xml | 4 +-
hadoop-tools/pom.xml | 4 +-
.../dev-support/findbugs-exclude.xml | 8 +
.../hadoop-yarn/hadoop-yarn-api/pom.xml | 4 +-
.../yarn/ams/ApplicationMasterServiceUtils.java | 16 +
.../api/protocolrecords/AllocateRequest.java | 42 +
.../api/protocolrecords/AllocateResponse.java | 23 +
.../RegisterApplicationMasterRequest.java | 42 +-
.../hadoop/yarn/api/records/Container.java | 15 +
.../api/records/RejectedSchedulingRequest.java | 70 ++
.../yarn/api/records/RejectionReason.java | 44 +
.../hadoop/yarn/api/records/ResourceSizing.java | 91 ++
.../yarn/api/records/SchedulingRequest.java | 206 +++++
.../yarn/api/resource/PlacementConstraint.java | 744 ++++++++++++++++
.../yarn/api/resource/PlacementConstraints.java | 320 +++++++
.../hadoop/yarn/api/resource/package-info.java | 21 +
.../hadoop/yarn/conf/YarnConfiguration.java | 31 +
...SchedulerInvalidResoureRequestException.java | 47 +
.../PlacementConstraintParseException.java | 28 +
.../constraint/PlacementConstraintParser.java | 615 +++++++++++++
.../yarn/util/constraint/package-info.java | 22 +
.../src/main/proto/yarn_protos.proto | 86 ++
.../src/main/proto/yarn_service_protos.proto | 3 +
.../resource/TestPlacementConstraintParser.java | 372 ++++++++
.../api/resource/TestPlacementConstraints.java | 107 +++
.../pom.xml | 4 +-
.../distributedshell/ApplicationMaster.java | 132 ++-
.../applications/distributedshell/Client.java | 58 +-
.../distributedshell/PlacementSpec.java | 95 ++
.../pom.xml | 4 +-
.../hadoop-yarn-services-api/pom.xml | 2 +-
.../yarn/service/client/ApiServiceClient.java | 50 +-
.../hadoop/yarn/service/webapp/ApiServer.java | 13 +-
.../hadoop-yarn-services-core/pom.xml | 2 +-
.../yarn/service/client/ServiceClient.java | 4 +
.../hadoop/yarn/service/ServiceTestUtils.java | 5 +
.../yarn/service/TestYarnNativeServices.java | 128 +++
.../hadoop-yarn-services/pom.xml | 2 +-
.../hadoop-yarn-applications/pom.xml | 4 +-
.../hadoop-yarn/hadoop-yarn-client/pom.xml | 4 +-
.../hadoop/yarn/client/api/AMRMClient.java | 38 +-
.../yarn/client/api/async/AMRMClientAsync.java | 48 +
.../api/async/impl/AMRMClientAsyncImpl.java | 49 +-
.../yarn/client/api/impl/AMRMClientImpl.java | 142 ++-
.../client/api/impl/BaseAMRMClientTest.java | 212 +++++
.../yarn/client/api/impl/TestAMRMClient.java | 156 +---
.../api/impl/TestAMRMClientOnRMRestart.java | 9 +-
.../TestAMRMClientPlacementConstraints.java | 204 +++++
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 4 +-
.../PlacementConstraintFromProtoConverter.java | 116 +++
.../pb/PlacementConstraintToProtoConverter.java | 174 ++++
.../apache/hadoop/yarn/api/pb/package-info.java | 23 +
.../impl/pb/AllocateRequestPBImpl.java | 84 ++
.../impl/pb/AllocateResponsePBImpl.java | 85 ++
.../RegisterApplicationMasterRequestPBImpl.java | 106 ++-
.../api/records/impl/pb/ContainerPBImpl.java | 31 +
.../yarn/api/records/impl/pb/ProtoUtils.java | 43 +
.../pb/RejectedSchedulingRequestPBImpl.java | 148 ++++
.../records/impl/pb/ResourceSizingPBImpl.java | 128 +++
.../impl/pb/SchedulingRequestPBImpl.java | 296 +++++++
.../PlacementConstraintTransformations.java | 200 +++++
.../hadoop/yarn/api/resource/package-info.java | 23 +
.../LogAggregationFileController.java | 7 +-
.../ifile/IndexedFileAggregatedLogsBlock.java | 2 +-
.../LogAggregationIndexedFileController.java | 69 +-
.../tfile/LogAggregationTFileController.java | 5 +-
.../yarn/security/ContainerTokenIdentifier.java | 69 +-
.../DockerCredentialTokenIdentifier.java | 159 ++++
.../yarn/util/DockerClientConfigHandler.java | 183 ++++
.../org/apache/hadoop/yarn/util/FSDownload.java | 215 +++--
.../src/main/proto/yarn_security_token.proto | 6 +
...apache.hadoop.security.token.TokenIdentifier | 1 +
.../src/main/resources/yarn-default.xml | 36 +
.../hadoop/yarn/api/BasePBImplRecordsTest.java | 11 +
.../hadoop/yarn/api/TestPBImplRecords.java | 21 +
.../TestPlacementConstraintPBConversion.java | 195 +++++
.../TestPlacementConstraintTransformations.java | 166 ++++
...TestLogAggregationFileControllerFactory.java | 5 +-
.../TestLogAggregationIndexFileController.java | 21 +
.../security/TestDockerClientConfigHandler.java | 129 +++
.../apache/hadoop/yarn/util/TestFSDownload.java | 28 +-
.../hadoop-yarn/hadoop-yarn-registry/pom.xml | 4 +-
.../hadoop/registry/RegistryTestHelper.java | 4 +-
.../pom.xml | 4 +-
.../hadoop-yarn-server-common/pom.xml | 4 +-
.../api/protocolrecords/NMContainerStatus.java | 14 +
.../impl/pb/NMContainerStatusPBImpl.java | 33 +
.../server/scheduler/SchedulerRequestKey.java | 11 +
.../yarn_server_common_service_protos.proto | 1 +
.../hadoop-yarn-server-nodemanager/pom.xml | 4 +-
.../nodemanager/LinuxContainerExecutor.java | 3 +
.../containermanager/ContainerManagerImpl.java | 3 +-
.../container/ContainerImpl.java | 19 +-
.../launcher/ContainerLaunch.java | 52 +-
.../launcher/ContainerRelaunch.java | 10 +-
.../runtime/DockerLinuxContainerRuntime.java | 62 +-
.../runtime/LinuxContainerRuntimeConstants.java | 4 +
.../linux/runtime/docker/DockerCommand.java | 16 +
.../linux/runtime/docker/DockerRunCommand.java | 14 +
.../localizer/ResourceLocalizationService.java | 30 -
.../executor/ContainerStartContext.java | 26 +
.../impl/container-executor.c | 92 +-
.../container-executor/impl/utils/docker-util.c | 92 +-
.../container-executor/impl/utils/docker-util.h | 3 +-
.../test/utils/test_docker_util.cc | 256 ++++--
.../TestLinuxContainerExecutorWithMocks.java | 8 +
.../impl/pb/TestPBRecordImpl.java | 2 +-
.../launcher/TestContainerRelaunch.java | 99 +++
.../runtime/TestDockerContainerRuntime.java | 376 ++++----
.../runtime/docker/TestDockerRunCommand.java | 8 +
.../TestResourceLocalizationService.java | 144 ---
.../hadoop-yarn-server-resourcemanager/pom.xml | 4 +-
.../ApplicationMasterService.java | 15 +
.../resourcemanager/DefaultAMSProcessor.java | 13 +-
.../resourcemanager/RMActiveServiceContext.java | 30 +
.../yarn/server/resourcemanager/RMContext.java | 11 +
.../server/resourcemanager/RMContextImpl.java | 25 +
.../server/resourcemanager/ResourceManager.java | 22 +
.../rmapp/attempt/RMAppAttemptImpl.java | 5 +-
.../rmcontainer/RMContainer.java | 8 +
.../rmcontainer/RMContainerImpl.java | 37 +-
.../server/resourcemanager/rmnode/RMNode.java | 7 +
.../resourcemanager/rmnode/RMNodeImpl.java | 6 +
.../scheduler/AbstractYarnScheduler.java | 21 +-
.../scheduler/AppSchedulingInfo.java | 214 ++++-
.../ApplicationPlacementAllocatorFactory.java | 68 ++
.../scheduler/ApplicationPlacementFactory.java | 63 --
.../scheduler/ContainerUpdateContext.java | 4 +-
.../scheduler/ResourceScheduler.java | 13 +
.../scheduler/SchedulerApplicationAttempt.java | 24 +-
.../scheduler/SchedulerNode.java | 20 +-
.../scheduler/SchedulerUtils.java | 36 +-
.../scheduler/YarnScheduler.java | 15 +-
.../scheduler/capacity/CapacityScheduler.java | 158 +++-
.../CapacitySchedulerConfiguration.java | 5 +
.../allocator/RegularContainerAllocator.java | 3 +-
.../scheduler/common/ContainerRequest.java | 12 +
.../scheduler/common/PendingAsk.java | 6 +
.../common/ResourceAllocationCommitter.java | 12 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 43 +-
.../constraint/AllocationTagsManager.java | 564 ++++++++++++
.../InvalidAllocationTagsQueryException.java | 35 +
.../MemoryPlacementConstraintManager.java | 324 +++++++
.../constraint/PlacementConstraintManager.java | 164 ++++
.../PlacementConstraintManagerService.java | 93 ++
.../constraint/PlacementConstraintsUtil.java | 261 ++++++
.../constraint/algorithm/CircularIterator.java | 86 ++
.../algorithm/DefaultPlacementAlgorithm.java | 285 ++++++
.../algorithm/LocalAllocationTagsManager.java | 167 ++++
.../iterators/PopularTagsIterator.java | 71 ++
.../algorithm/iterators/SerialIterator.java | 53 ++
.../algorithm/iterators/package-info.java | 29 +
.../constraint/algorithm/package-info.java | 29 +
.../api/ConstraintPlacementAlgorithm.java | 43 +
.../api/ConstraintPlacementAlgorithmInput.java | 32 +
.../api/ConstraintPlacementAlgorithmOutput.java | 57 ++
...traintPlacementAlgorithmOutputCollector.java | 32 +
.../constraint/api/PlacedSchedulingRequest.java | 79 ++
.../SchedulingRequestWithPlacementAttempt.java | 52 ++
.../constraint/api/SchedulingResponse.java | 70 ++
.../scheduler/constraint/api/package-info.java | 28 +
.../scheduler/constraint/package-info.java | 29 +
.../constraint/processor/BatchedRequests.java | 144 +++
.../processor/NodeCandidateSelector.java | 38 +
.../processor/PlacementDispatcher.java | 145 ++++
.../processor/PlacementProcessor.java | 377 ++++++++
.../constraint/processor/package-info.java | 29 +
.../scheduler/fair/AllocationConfiguration.java | 99 +--
.../fair/AllocationFileLoaderService.java | 481 ++--------
.../scheduler/fair/FSPreemptionThread.java | 62 +-
.../scheduler/fair/FairScheduler.java | 12 +-
.../fair/allocation/AllocationFileParser.java | 258 ++++++
.../allocation/AllocationFileQueueParser.java | 268 ++++++
.../fair/allocation/QueueProperties.java | 280 ++++++
.../scheduler/fifo/FifoScheduler.java | 7 +-
.../placement/AppPlacementAllocator.java | 68 +-
.../LocalityAppPlacementAllocator.java | 35 +-
.../SingleConstraintAppPlacementAllocator.java | 533 ++++++++++++
.../security/RMContainerTokenSecretManager.java | 21 +-
.../resourcemanager/webapp/NodesPage.java | 3 +
.../webapp/dao/AllocationTagInfo.java | 56 ++
.../webapp/dao/AllocationTagsInfo.java | 59 ++
.../resourcemanager/webapp/dao/NodeInfo.java | 15 +
.../server/resourcemanager/Application.java | 9 +-
.../yarn/server/resourcemanager/MockAM.java | 77 ++
.../yarn/server/resourcemanager/MockNodes.java | 6 +
.../yarn/server/resourcemanager/MockRM.java | 14 +
.../TestApplicationMasterService.java | 190 ++++
.../attempt/TestRMAppAttemptTransitions.java | 10 +-
.../rmcontainer/TestRMContainerImpl.java | 151 +++-
.../scheduler/TestAppSchedulingInfo.java | 4 +-
.../capacity/CapacitySchedulerTestBase.java | 79 ++
.../capacity/TestCapacityScheduler.java | 91 +-
.../TestCapacitySchedulerAsyncScheduling.java | 12 +-
.../TestCapacitySchedulerAutoQueueCreation.java | 2 +-
...apacitySchedulerSchedulingRequestUpdate.java | 262 ++++++
...CapacitySchedulerWithMultiResourceTypes.java | 37 +
.../capacity/TestContainerAllocation.java | 5 +-
.../capacity/TestIncreaseAllocationExpirer.java | 2 +-
...estSchedulingRequestContainerAllocation.java | 269 ++++++
...hedulingRequestContainerAllocationAsync.java | 138 +++
.../scheduler/capacity/TestUtils.java | 19 +-
.../constraint/TestAllocationTagsManager.java | 413 +++++++++
.../TestBatchedRequestsIterators.java | 82 ++
.../TestPlacementConstraintManagerService.java | 264 ++++++
.../TestPlacementConstraintsUtil.java | 511 +++++++++++
.../constraint/TestPlacementProcessor.java | 870 +++++++++++++++++++
.../algorithm/TestCircularIterator.java | 84 ++
.../TestLocalAllocationTagsManager.java | 139 +++
.../scheduler/fair/FairSchedulerTestBase.java | 6 +-
.../fair/TestAllocationFileLoaderService.java | 187 ++--
.../fair/TestContinuousScheduling.java | 10 +-
.../scheduler/fair/TestFairScheduler.java | 30 +-
.../fair/TestFairSchedulerPreemption.java | 55 ++
.../allocationfile/AllocationFileQueue.java | 82 ++
.../AllocationFileQueueBuilder.java | 115 +++
.../AllocationFileQueueProperties.java | 202 +++++
.../AllocationFileSimpleQueueBuilder.java | 64 ++
.../AllocationFileSubQueueBuilder.java | 54 ++
.../allocationfile/AllocationFileWriter.java | 175 ++++
.../fair/allocationfile/UserSettings.java | 80 ++
.../scheduler/fifo/TestFifoScheduler.java | 33 +-
...stSingleConstraintAppPlacementAllocator.java | 408 +++++++++
.../resourcemanager/webapp/TestNodesPage.java | 4 +-
.../webapp/TestRMWebServiceAppsNodelabel.java | 13 +-
.../webapp/TestRMWebServicesNodes.java | 77 +-
.../hadoop-yarn-server-router/pom.xml | 4 +-
.../pom.xml | 4 +-
.../hadoop-yarn-server-tests/pom.xml | 4 +-
.../hadoop/yarn/server/MiniYARNCluster.java | 5 +-
.../TestMiniYarnClusterNodeUtilization.java | 20 +-
.../pom.xml | 4 +-
.../pom.xml | 4 +-
.../pom.xml | 2 +-
.../hadoop-yarn-server-timelineservice/pom.xml | 4 +-
.../hadoop-yarn-server-web-proxy/pom.xml | 4 +-
.../hadoop-yarn/hadoop-yarn-server/pom.xml | 4 +-
.../hadoop-yarn/hadoop-yarn-site/pom.xml | 4 +-
.../src/site/markdown/DockerContainers.md | 29 +-
.../site/markdown/PlacementConstraints.md.vm | 149 ++++
.../src/site/markdown/yarn-service/Examples.md | 4 +
.../hadoop-yarn/hadoop-yarn-ui/pom.xml | 4 +-
.../hadoop-yarn-ui/public/crossdomain.xml | 15 -
.../main/webapp/app/adapters/yarn-servicedef.js | 9 +-
.../webapp/app/components/deploy-service.js | 12 +-
.../main/webapp/app/components/timeline-view.js | 5 +
.../webapp/app/controllers/app-table-columns.js | 2 +-
.../webapp/app/controllers/yarn-app-attempt.js | 2 +-
.../src/main/webapp/app/controllers/yarn-app.js | 6 +-
.../app/controllers/yarn-app/components.js | 5 +
.../webapp/app/controllers/yarn-app/info.js | 4 +-
.../webapp/app/controllers/yarn-app/logs.js | 13 +-
.../webapp/app/controllers/yarn-apps/apps.js | 1 +
.../app/controllers/yarn-component-instance.js | 4 +-
.../app/controllers/yarn-component-instances.js | 2 +-
.../yarn-component-instances/info.js | 5 +
.../app/controllers/yarn-deploy-service.js | 12 +-
.../webapp/app/controllers/yarn-flowrun/info.js | 5 +
.../app/controllers/yarn-flowrun/metrics.js | 5 +
.../webapp/app/controllers/yarn-nodes/table.js | 4 +-
.../webapp/app/controllers/yarn-queue/apps.js | 1 +
.../webapp/app/controllers/yarn-services.js | 1 +
.../app/controllers/yarn-tools/yarn-conf.js | 4 +
.../src/main/webapp/app/helpers/lower.js | 3 +
.../src/main/webapp/app/initializers/loader.js | 6 +
.../src/main/webapp/app/models/yarn-app.js | 18 +-
.../src/main/webapp/app/router.js | 1 +
.../src/main/webapp/app/routes/application.js | 2 +
.../src/main/webapp/app/routes/yarn-app/logs.js | 4 +-
.../app/templates/components/deploy-service.hbs | 10 +
.../app/templates/components/timeline-view.hbs | 2 +-
.../src/main/webapp/app/templates/notauth.hbs | 20 +
.../src/main/webapp/app/templates/yarn-app.hbs | 16 +-
.../app/templates/yarn-app/components.hbs | 2 +-
.../main/webapp/app/templates/yarn-app/info.hbs | 4 +-
.../main/webapp/app/templates/yarn-app/logs.hbs | 4 +
.../templates/yarn-component-instances/info.hbs | 2 +-
.../webapp/app/templates/yarn-flowrun/info.hbs | 2 +-
.../app/templates/yarn-flowrun/metrics.hbs | 6 +-
.../app/templates/yarn-tools/yarn-conf.hbs | 6 +-
.../src/main/webapp/app/utils/error-utils.js | 3 +
.../src/main/webapp/app/utils/info-seeder.js | 3 +-
.../src/main/webapp/config/default-config.js | 2 +-
.../hadoop-yarn-ui/src/main/webapp/package.json | 6 +-
.../hadoop-yarn-ui/src/main/webapp/yarn.lock | 6 +-
hadoop-yarn-project/hadoop-yarn/pom.xml | 4 +-
hadoop-yarn-project/pom.xml | 4 +-
pom.xml | 2 +-
start-build-env.sh | 32 +-
588 files changed, 30516 insertions(+), 6345 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47919787/hadoop-client-modules/hadoop-client-minicluster/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47919787/hadoop-client-modules/hadoop-client-runtime/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47919787/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47919787/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
----------------------------------------------------------------------
diff --cc
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index 80cf5ed,8743be5..63f3376
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@@ -18,26 -18,9 +18,27 @@@
package org.apache.hadoop.fs;
+import com.google.common.base.Preconditions;
+import org.apache.commons.collections.map.CaseInsensitiveMap;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.Shell.ShellCommandExecutor;
+import org.apache.hadoop.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
+ import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
@@@ -48,12 -32,18 +50,19 @@@ import java.io.OutputStream
import java.net.InetAddress;
import java.net.URI;
import java.net.UnknownHostException;
+ import java.nio.charset.Charset;
import java.nio.file.AccessDeniedException;
+ import java.nio.file.FileSystems;
+ import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Enumeration;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
+ import java.util.concurrent.ExecutionException;
+ import java.util.concurrent.ExecutorService;
+ import java.util.concurrent.Executors;
+ import java.util.concurrent.Future;
import java.util.jar.Attributes;
import java.util.jar.JarOutputStream;
import java.util.jar.Manifest;
@@@ -62,10 -50,26 +71,11 @@@ import java.util.zip.CheckedOutputStrea
import java.util.zip.GZIPInputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
+import java.util.zip.ZipOutputStream;
+ import java.util.zip.ZipInputStream;
-import org.apache.commons.collections.map.CaseInsensitiveMap;
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.nativeio.NativeIO;
-import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.util.Shell.ShellCommandExecutor;
-import org.apache.hadoop.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
/**
- * A collection of file-processing util methods
+ * A collection of file-processing util methods.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
@@@ -587,70 -607,48 +613,108 @@@ public class FileUtil
}
/**
- * Given a stream input it will unzip the it in the unzip directory.
++<<<<<<< HEAD
+ * creates zip archieve of the source dir and writes a zip file.
+ *
+ * @param sourceDir - The directory to zip.
+ * @param archiveName - The destination file, the parent directory is
assumed
+ * to exist.
+ * @return Checksum of the Archive.
+ * @throws IOException - Throws if zipFileName already exists or if the
+ * sourceDir does not exist.
+ */
+ public static Long zip(File sourceDir, File archiveName) throws IOException
{
+ Preconditions.checkNotNull(sourceDir, "source directory cannot be null");
+ Preconditions.checkState(sourceDir.exists(), "source directory must " +
+ "exist");
+
+ Preconditions.checkNotNull(archiveName, "Destination file cannot be
null");
+ Preconditions.checkNotNull(archiveName.getParent(), "Destination " +
+ "directory cannot be null");
+ Preconditions.checkState(new File(archiveName.getParent()).exists(),
+ "Destination directory must exist");
+ Preconditions.checkState(!archiveName.exists(), "Destination file " +
+ "already exists. Refusing to overwrite existing file.");
+
+ CheckedOutputStream checksum;
+ try (FileOutputStream outputStream =
+ new FileOutputStream(archiveName)) {
+ checksum = new CheckedOutputStream(outputStream, new CRC32());
+ byte[] data = new byte[BUFFER_SIZE];
+ try (ZipOutputStream out =
+ new ZipOutputStream(new BufferedOutputStream(checksum))) {
+
+ Iterator<File> fileIter = FileUtils.iterateFiles(sourceDir, null,
true);
+ while (fileIter.hasNext()) {
+ File file = fileIter.next();
+ LOG.debug("Compressing file : " + file.getPath());
+ try (FileInputStream currentFile = new FileInputStream(file)) {
+ ZipEntry entry = new ZipEntry(file.getCanonicalPath());
+ out.putNextEntry(entry);
+ try (BufferedInputStream sourceFile
+ = new BufferedInputStream(currentFile, BUFFER_SIZE)) {
+ int bytesRead;
+ while ((bytesRead = sourceFile.read(data, 0, BUFFER_SIZE)) !=
+ -1) {
+ out.write(data, 0, bytesRead);
+ }
+ }
+ }
+ }
+ out.flush();
+ }
+ }
+ // Exit condition -- ZipFile must exist.
+ Preconditions.checkState(archiveName.exists(),
+ "Expected archive file missing: {}", archiveName.toPath());
+ long crc32 = checksum.getChecksum().getValue();
+ checksum.close();
+ return crc32;
+ }
+
+ /**
+ * Given a File input it will unzip the file in a the unzip directory
* passed as the second parameter
+ * @param inputStream The zip file as input
+ * @param toDir The unzip directory where to unzip the zip file.
+ * @throws IOException an exception occurred
+ */
+ public static void unZip(InputStream inputStream, File toDir)
+ throws IOException {
+ try (ZipInputStream zip = new ZipInputStream(inputStream)) {
+ int numOfFailedLastModifiedSet = 0;
+ for(ZipEntry entry = zip.getNextEntry();
+ entry != null;
+ entry = zip.getNextEntry()) {
+ if (!entry.isDirectory()) {
+ File file = new File(toDir, entry.getName());
+ File parent = file.getParentFile();
+ if (!parent.mkdirs() &&
+ !parent.isDirectory()) {
+ throw new IOException("Mkdirs failed to create " +
+ parent.getAbsolutePath());
+ }
+ try (OutputStream out = new FileOutputStream(file)) {
+ IOUtils.copyBytes(zip, out, BUFFER_SIZE);
+ }
+ if (!file.setLastModified(entry.getTime())) {
+ numOfFailedLastModifiedSet++;
+ }
+ }
+ }
+ if (numOfFailedLastModifiedSet > 0) {
+ LOG.warn("Could not set last modfied time for {} file(s)",
+ numOfFailedLastModifiedSet);
+ }
+ }
+ }
+
+ /**
+ * Given a File input it will unzip it in the unzip directory.
+ * passed as the second parameter
* @param inFile The zip file as input
* @param unzipDir The unzip directory where to unzip the zip file.
- * @throws IOException
+ * @throws IOException An I/O exception has occurred
*/
public static void unZip(File inFile, File unzipDir) throws IOException {
Enumeration<? extends ZipEntry> entries;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47919787/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47919787/hadoop-dist/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47919787/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47919787/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47919787/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47919787/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --cc
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 3f2939e5,30f75ba..61accc9
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@@ -2143,10 -2080,10 +2138,10 @@@ public class DataNode extends Reconfigu
try {
this.blockPoolManager.shutDownAll(bposArray);
} catch (InterruptedException ie) {
- LOG.warn("Received exception in BlockPoolManager#shutDownAll: ", ie);
+ LOG.warn("Received exception in BlockPoolManager#shutDownAll", ie);
}
}
-
+
if (storage != null) {
try {
this.storage.unlockAll();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47919787/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --cc
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index a842a98,c141293..1852192
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@@ -1689,14 -1745,9 +1745,9 @@@ class FsDatasetImpl implements FsDatase
}
assert newReplicaInfo.getState() == ReplicaState.FINALIZED
: "Replica should be finalized";
- if(volumeMap.get(bpid, replicaInfo.getBlockId()).getGenerationStamp() <=
- newReplicaInfo.getGenerationStamp()) {
- volumeMap.add(bpid, newReplicaInfo);
- return newReplicaInfo;
- } else {
- throw new IOException("Generation Stamp should be monotonically " +
- "increased. That assumption is violated here.");
- }
-
++
+ volumeMap.add(bpid, newReplicaInfo);
+ return newReplicaInfo;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
