Merge branch 'trunk' into HDFS-6581
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b1000fbb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b1000fbb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b1000fbb
Branch: refs/heads/HDFS-6581
Commit: b1000fbba43786a8d1da129bc7c7a1bf253a9e7e
Parents: bfc7b7e 428a766
Author: arp <a...@apache.org>
Authored: Wed Sep 24 20:08:32 2014 -0700
Committer: arp <a...@apache.org>
Committed: Wed Sep 24 20:08:32 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 5 +
.../AbstractDelegationTokenSecretManager.java | 16 +-
.../org/apache/hadoop/util/NativeCrc32.java | 33 +
.../src/org/apache/hadoop/util/NativeCrc32.c | 12 +
.../org/apache/hadoop/util/TestNativeCrc32.java | 229 ++++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 18 +-
.../hadoop-hdfs/src/CMakeLists.txt | 17 +-
.../hadoop-hdfs/src/main/bin/hdfs | 4 +
.../hadoop-hdfs/src/main/bin/hdfs.cmd | 7 +-
.../src/main/conf/blockStoragePolicy-site.xml | 21 -
.../hadoop-hdfs/src/main/conf/hdfs-site.xml | 3 +-
.../apache/hadoop/hdfs/BlockStoragePolicy.java | 419 -------
.../java/org/apache/hadoop/hdfs/DFSClient.java | 8 +
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 8 -
.../hadoop/hdfs/DistributedFileSystem.java | 7 +
.../apache/hadoop/hdfs/client/HdfsAdmin.java | 11 +
.../hdfs/protocol/BlockStoragePolicy.java | 244 +++++
.../hadoop/hdfs/protocol/ClientProtocol.java | 9 +-
.../protocol/SnapshottableDirectoryStatus.java | 4 +-
...tNamenodeProtocolServerSideTranslatorPB.java | 23 +
.../ClientNamenodeProtocolTranslatorPB.java | 25 +-
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 65 +-
.../server/blockmanagement/BlockManager.java | 12 +-
.../blockmanagement/BlockPlacementPolicy.java | 2 +-
.../BlockPlacementPolicyDefault.java | 2 +-
.../BlockStoragePolicySuite.java | 119 ++
.../apache/hadoop/hdfs/server/mover/Mover.java | 35 +-
.../hdfs/server/namenode/FSDirectory.java | 17 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 4 +-
.../hdfs/server/namenode/FSNamesystem.java | 20 +-
.../hadoop/hdfs/server/namenode/INode.java | 4 +-
.../hdfs/server/namenode/INodeDirectory.java | 11 +-
.../namenode/INodeDirectoryAttributes.java | 3 -
.../hadoop/hdfs/server/namenode/INodeFile.java | 7 +-
.../hadoop/hdfs/server/namenode/INodeMap.java | 7 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 6 +
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 20 +-
.../hadoop/hdfs/tools/GetStoragePolicies.java | 65 ++
.../org/apache/hadoop/hdfs/web/JsonUtil.java | 5 +-
.../hadoop-hdfs/src/main/native/libhdfs/hdfs.c | 2 +-
.../main/native/libhdfs/test/test_libhdfs_ops.c | 22 +-
.../src/main/native/libhdfs/test/vecsum.c | 33 +-
.../main/native/libhdfs/test_libhdfs_threaded.c | 2 +-
.../src/main/proto/ClientNamenodeProtocol.proto | 9 +
.../hadoop-hdfs/src/main/proto/hdfs.proto | 14 +
.../src/main/resources/hdfs-default.xml | 3 +-
.../hadoop/hdfs/TestBlockStoragePolicy.java | 86 +-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 21 +
.../hadoop/hdfs/TestStoragePolicyCommands.java | 11 +-
.../hdfs/server/mover/TestStorageMover.java | 19 +-
.../hdfs/server/namenode/TestDeleteRace.java | 2 +-
hadoop-mapreduce-project/CHANGES.txt | 3 +
.../mapreduce/v2/hs/TestJobHistoryParsing.java | 2 +-
.../hadoop/tools/CopyListingFileStatus.java | 2 +-
.../apache/hadoop/tools/DistCpOptionSwitch.java | 21 +-
.../org/apache/hadoop/tools/DistCpOptions.java | 2 +-
.../apache/hadoop/tools/util/DistCpUtils.java | 58 +-
.../apache/hadoop/tools/TestOptionsParser.java | 2 +-
.../hadoop/tools/mapred/TestCopyMapper.java | 1 +
.../hadoop/tools/util/TestDistCpUtils.java | 1024 ++++++++++++++++--
hadoop-yarn-project/CHANGES.txt | 13 +
.../api/records/timeline/TimelineDomain.java | 194 ++++
.../api/records/timeline/TimelineDomains.java | 86 ++
.../hadoop/yarn/client/api/TimelineClient.java | 17 +
.../client/api/impl/TimelineClientImpl.java | 116 +-
.../yarn/security/ContainerTokenIdentifier.java | 34 +
.../records/timeline/TestTimelineRecords.java | 38 +
.../client/api/impl/TestTimelineClient.java | 85 +-
.../server/timeline/LeveldbTimelineStore.java | 216 +++-
.../server/timeline/MemoryTimelineStore.java | 99 ++
.../server/timeline/TimelineDataManager.java | 74 ++
.../yarn/server/timeline/TimelineReader.java | 23 +
.../yarn/server/timeline/TimelineWriter.java | 20 +-
.../timeline/security/TimelineACLsManager.java | 26 +
.../timeline/webapp/TimelineWebServices.java | 100 ++
.../timeline/TestLeveldbTimelineStore.java | 19 +-
.../timeline/TestMemoryTimelineStore.java | 15 +-
.../server/timeline/TimelineStoreTestUtils.java | 96 +-
.../security/TestTimelineACLsManager.java | 63 +-
.../webapp/TestTimelineWebServices.java | 231 +++-
.../webapp/TestTimelineWebServicesWithSSL.java | 5 +-
.../src/CMakeLists.txt | 3 +
.../src/config.h.cmake | 2 +
.../containermanager/ContainerManagerImpl.java | 26 +-
.../application/ApplicationImpl.java | 18 +-
.../application/ApplicationInitEvent.java | 13 +
.../event/LogHandlerAppStartedEvent.java | 13 +
.../container-executor/impl/configuration.c | 7 +-
.../impl/container-executor.c | 83 +-
.../test/test-container-executor.c | 16 +-
.../yarn_server_nodemanager_recovery.proto | 1 +
.../containermanager/TestContainerManager.java | 12 +-
.../TestContainerManagerRecovery.java | 25 +-
.../scheduler/SchedulerApplicationAttempt.java | 8 +-
.../security/RMContainerTokenSecretManager.java | 23 +-
.../webapp/JAXBContextResolver.java | 77 +-
.../yarn/server/resourcemanager/MockRM.java | 22 +-
.../capacity/TestContainerAllocation.java | 60 +-
.../TestRMWebServicesAppsModification.java | 9 +-
.../src/site/apt/ResourceManagerRest.apt.vm | 30 +-
100 files changed, 4006 insertions(+), 878 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1000fbb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1000fbb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java
----------------------------------------------------------------------
diff --cc
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java
index 060a954,31feb1e..2b2d921
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java
@@@ -60,9 -60,9 +60,9 @@@ public class SnapshottableDirectoryStat
FsPermission permission, String owner, String group, byte[] localName,
long inodeId, int childrenNum,
int snapshotNumber, int snapshotQuota, byte[] parentFullPath) {
- this.dirStatus = new HdfsFileStatus(0, true, 0, 0, modification_time,
+ this.dirStatus = new HdfsFileStatus(0, true, 0, 0, false,
modification_time,
access_time, permission, owner, group, null, localName, inodeId,
- childrenNum, null, BlockStoragePolicy.ID_UNSPECIFIED);
+ childrenNum, null, BlockStoragePolicySuite.ID_UNSPECIFIED);
this.snapshotNumber = snapshotNumber;
this.snapshotQuota = snapshotQuota;
this.parentFullPath = parentFullPath;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1000fbb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1000fbb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1000fbb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --cc
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 63dbfb2,94d7fcf..fc47f5c
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@@ -1536,9 -1532,9 +1537,9 @@@ public class FSDirectory implements Clo
private HdfsFileStatus getFileInfo4DotSnapshot(String src)
throws UnresolvedLinkException {
if (getINode4DotSnapshot(src) != null) {
- return new HdfsFileStatus(0, true, 0, 0, 0, 0, null, null, null, null,
+ return new HdfsFileStatus(0, true, 0, 0, false, 0, 0, null, null, null,
null,
HdfsFileStatus.EMPTY_NAME, -1L, 0, null,
- BlockStoragePolicy.ID_UNSPECIFIED);
+ BlockStoragePolicySuite.ID_UNSPECIFIED);
}
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1000fbb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1000fbb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1000fbb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --cc
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index 2d39063,583c193..d2a2de3
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@@ -78,9 -78,9 +78,10 @@@ public class INodeFile extends INodeWit
*/
static enum HeaderFormat {
PREFERRED_BLOCK_SIZE(null, 48, 1),
- REPLICATION(PREFERRED_BLOCK_SIZE.BITS, 12, 1),
+ REPLICATION(PREFERRED_BLOCK_SIZE.BITS, 11, 1),
- STORAGE_POLICY_ID(REPLICATION.BITS, BlockStoragePolicy.ID_BIT_LENGTH, 0),
+ STORAGE_POLICY_ID(REPLICATION.BITS, BlockStoragePolicySuite.ID_BIT_LENGTH,
- 0);
++ 0),
+ LAZY_PERSIST(STORAGE_POLICY_ID.BITS, 1, 0);
private final LongBitFormat BITS;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1000fbb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
----------------------------------------------------------------------
diff --cc
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
index 4a79689,0eb7c61..e67cc6b
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
@@@ -266,11 -265,10 +267,11 @@@ public class JsonUtil
: childrenNumLong.intValue();
final byte storagePolicy = m.containsKey("storagePolicy") ?
(byte) (long) (Long) m.get("storagePolicy") :
- BlockStoragePolicy.ID_UNSPECIFIED;
+ BlockStoragePolicySuite.ID_UNSPECIFIED;
return new HdfsFileStatus(len, type == PathType.DIRECTORY, replication,
- blockSize, mTime, aTime, permission, owner, group, symlink,
- DFSUtil.string2Bytes(localName), fileId, childrenNum, null,
storagePolicy);
+ blockSize, isLazyPersist, mTime, aTime, permission, owner, group,
+ symlink, DFSUtil.string2Bytes(localName), fileId, childrenNum, null,
+ storagePolicy);
}
/** Convert an ExtendedBlock to a Json map. */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1000fbb/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1000fbb/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1000fbb/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
