Merge branch 'trunk' into HDFS-6581 Conflicts: hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b1000fbb Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b1000fbb Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b1000fbb Branch: refs/heads/HDFS-6581 Commit: b1000fbba43786a8d1da129bc7c7a1bf253a9e7e Parents: bfc7b7e 428a766 Author: arp <a...@apache.org> Authored: Wed Sep 24 20:08:32 2014 -0700 Committer: arp <a...@apache.org> Committed: Wed Sep 24 20:08:32 2014 -0700 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 5 + .../AbstractDelegationTokenSecretManager.java | 16 +- .../org/apache/hadoop/util/NativeCrc32.java | 33 + .../src/org/apache/hadoop/util/NativeCrc32.c | 12 + .../org/apache/hadoop/util/TestNativeCrc32.java | 229 ++++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 18 +- .../hadoop-hdfs/src/CMakeLists.txt | 17 +- .../hadoop-hdfs/src/main/bin/hdfs | 4 + .../hadoop-hdfs/src/main/bin/hdfs.cmd | 7 +- .../src/main/conf/blockStoragePolicy-site.xml | 21 - .../hadoop-hdfs/src/main/conf/hdfs-site.xml | 3 +- .../apache/hadoop/hdfs/BlockStoragePolicy.java | 419 ------- .../java/org/apache/hadoop/hdfs/DFSClient.java | 8 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 8 - .../hadoop/hdfs/DistributedFileSystem.java | 7 + .../apache/hadoop/hdfs/client/HdfsAdmin.java | 11 + .../hdfs/protocol/BlockStoragePolicy.java | 244 +++++ .../hadoop/hdfs/protocol/ClientProtocol.java | 9 +- .../protocol/SnapshottableDirectoryStatus.java | 4 +- ...tNamenodeProtocolServerSideTranslatorPB.java | 23 + .../ClientNamenodeProtocolTranslatorPB.java | 25 +- .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 65 +- .../server/blockmanagement/BlockManager.java | 12 +- .../blockmanagement/BlockPlacementPolicy.java | 2 +- .../BlockPlacementPolicyDefault.java | 2 +- .../BlockStoragePolicySuite.java | 119 ++ .../apache/hadoop/hdfs/server/mover/Mover.java | 35 +- .../hdfs/server/namenode/FSDirectory.java | 17 +- .../hdfs/server/namenode/FSEditLogLoader.java | 4 +- .../hdfs/server/namenode/FSNamesystem.java | 20 +- .../hadoop/hdfs/server/namenode/INode.java | 4 +- .../hdfs/server/namenode/INodeDirectory.java | 11 +- .../namenode/INodeDirectoryAttributes.java | 3 - .../hadoop/hdfs/server/namenode/INodeFile.java | 7 +- .../hadoop/hdfs/server/namenode/INodeMap.java | 7 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 6 + .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 20 +- .../hadoop/hdfs/tools/GetStoragePolicies.java | 65 ++ .../org/apache/hadoop/hdfs/web/JsonUtil.java | 5 +- .../hadoop-hdfs/src/main/native/libhdfs/hdfs.c | 2 +- .../main/native/libhdfs/test/test_libhdfs_ops.c | 22 +- .../src/main/native/libhdfs/test/vecsum.c | 33 +- .../main/native/libhdfs/test_libhdfs_threaded.c | 2 +- .../src/main/proto/ClientNamenodeProtocol.proto | 9 + .../hadoop-hdfs/src/main/proto/hdfs.proto | 14 + .../src/main/resources/hdfs-default.xml | 3 +- .../hadoop/hdfs/TestBlockStoragePolicy.java | 86 +- .../apache/hadoop/hdfs/TestEncryptionZones.java | 21 + .../hadoop/hdfs/TestStoragePolicyCommands.java | 11 +- .../hdfs/server/mover/TestStorageMover.java | 19 +- .../hdfs/server/namenode/TestDeleteRace.java | 2 +- hadoop-mapreduce-project/CHANGES.txt | 3 + .../mapreduce/v2/hs/TestJobHistoryParsing.java | 2 +- .../hadoop/tools/CopyListingFileStatus.java | 2 +- .../apache/hadoop/tools/DistCpOptionSwitch.java | 21 +- .../org/apache/hadoop/tools/DistCpOptions.java | 2 +- .../apache/hadoop/tools/util/DistCpUtils.java | 58 +- .../apache/hadoop/tools/TestOptionsParser.java | 2 +- .../hadoop/tools/mapred/TestCopyMapper.java | 1 + .../hadoop/tools/util/TestDistCpUtils.java | 1024 ++++++++++++++++-- hadoop-yarn-project/CHANGES.txt | 13 + .../api/records/timeline/TimelineDomain.java | 194 ++++ .../api/records/timeline/TimelineDomains.java | 86 ++ .../hadoop/yarn/client/api/TimelineClient.java | 17 + .../client/api/impl/TimelineClientImpl.java | 116 +- .../yarn/security/ContainerTokenIdentifier.java | 34 + .../records/timeline/TestTimelineRecords.java | 38 + .../client/api/impl/TestTimelineClient.java | 85 +- .../server/timeline/LeveldbTimelineStore.java | 216 +++- .../server/timeline/MemoryTimelineStore.java | 99 ++ .../server/timeline/TimelineDataManager.java | 74 ++ .../yarn/server/timeline/TimelineReader.java | 23 + .../yarn/server/timeline/TimelineWriter.java | 20 +- .../timeline/security/TimelineACLsManager.java | 26 + .../timeline/webapp/TimelineWebServices.java | 100 ++ .../timeline/TestLeveldbTimelineStore.java | 19 +- .../timeline/TestMemoryTimelineStore.java | 15 +- .../server/timeline/TimelineStoreTestUtils.java | 96 +- .../security/TestTimelineACLsManager.java | 63 +- .../webapp/TestTimelineWebServices.java | 231 +++- .../webapp/TestTimelineWebServicesWithSSL.java | 5 +- .../src/CMakeLists.txt | 3 + .../src/config.h.cmake | 2 + .../containermanager/ContainerManagerImpl.java | 26 +- .../application/ApplicationImpl.java | 18 +- .../application/ApplicationInitEvent.java | 13 + .../event/LogHandlerAppStartedEvent.java | 13 + .../container-executor/impl/configuration.c | 7 +- .../impl/container-executor.c | 83 +- .../test/test-container-executor.c | 16 +- .../yarn_server_nodemanager_recovery.proto | 1 + .../containermanager/TestContainerManager.java | 12 +- .../TestContainerManagerRecovery.java | 25 +- .../scheduler/SchedulerApplicationAttempt.java | 8 +- .../security/RMContainerTokenSecretManager.java | 23 +- .../webapp/JAXBContextResolver.java | 77 +- .../yarn/server/resourcemanager/MockRM.java | 22 +- .../capacity/TestContainerAllocation.java | 60 +- .../TestRMWebServicesAppsModification.java | 9 +- .../src/site/apt/ResourceManagerRest.apt.vm | 30 +- 100 files changed, 4006 insertions(+), 878 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1000fbb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1000fbb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java index 060a954,31feb1e..2b2d921 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java @@@ -60,9 -60,9 +60,9 @@@ public class SnapshottableDirectoryStat FsPermission permission, String owner, String group, byte[] localName, long inodeId, int childrenNum, int snapshotNumber, int snapshotQuota, byte[] parentFullPath) { - this.dirStatus = new HdfsFileStatus(0, true, 0, 0, modification_time, + this.dirStatus = new HdfsFileStatus(0, true, 0, 0, false, modification_time, access_time, permission, owner, group, null, localName, inodeId, - childrenNum, null, BlockStoragePolicy.ID_UNSPECIFIED); + childrenNum, null, BlockStoragePolicySuite.ID_UNSPECIFIED); this.snapshotNumber = snapshotNumber; this.snapshotQuota = snapshotQuota; this.parentFullPath = parentFullPath; http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1000fbb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1000fbb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1000fbb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index 63dbfb2,94d7fcf..fc47f5c --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@@ -1536,9 -1532,9 +1537,9 @@@ public class FSDirectory implements Clo private HdfsFileStatus getFileInfo4DotSnapshot(String src) throws UnresolvedLinkException { if (getINode4DotSnapshot(src) != null) { - return new HdfsFileStatus(0, true, 0, 0, 0, 0, null, null, null, null, + return new HdfsFileStatus(0, true, 0, 0, false, 0, 0, null, null, null, null, HdfsFileStatus.EMPTY_NAME, -1L, 0, null, - BlockStoragePolicy.ID_UNSPECIFIED); + BlockStoragePolicySuite.ID_UNSPECIFIED); } return null; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1000fbb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1000fbb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1000fbb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index 2d39063,583c193..d2a2de3 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@@ -78,9 -78,9 +78,10 @@@ public class INodeFile extends INodeWit */ static enum HeaderFormat { PREFERRED_BLOCK_SIZE(null, 48, 1), - REPLICATION(PREFERRED_BLOCK_SIZE.BITS, 12, 1), + REPLICATION(PREFERRED_BLOCK_SIZE.BITS, 11, 1), - STORAGE_POLICY_ID(REPLICATION.BITS, BlockStoragePolicy.ID_BIT_LENGTH, 0), + STORAGE_POLICY_ID(REPLICATION.BITS, BlockStoragePolicySuite.ID_BIT_LENGTH, - 0); ++ 0), + LAZY_PERSIST(STORAGE_POLICY_ID.BITS, 1, 0); private final LongBitFormat BITS; http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1000fbb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java index 4a79689,0eb7c61..e67cc6b --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java @@@ -266,11 -265,10 +267,11 @@@ public class JsonUtil : childrenNumLong.intValue(); final byte storagePolicy = m.containsKey("storagePolicy") ? (byte) (long) (Long) m.get("storagePolicy") : - BlockStoragePolicy.ID_UNSPECIFIED; + BlockStoragePolicySuite.ID_UNSPECIFIED; return new HdfsFileStatus(len, type == PathType.DIRECTORY, replication, - blockSize, mTime, aTime, permission, owner, group, symlink, - DFSUtil.string2Bytes(localName), fileId, childrenNum, null, storagePolicy); + blockSize, isLazyPersist, mTime, aTime, permission, owner, group, + symlink, DFSUtil.string2Bytes(localName), fileId, childrenNum, null, + storagePolicy); } /** Convert an ExtendedBlock to a Json map. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1000fbb/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1000fbb/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1000fbb/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ----------------------------------------------------------------------