[37/51] [abbrv] hadoop git commit: HDFS-7936. Erasure coding: resolving conflicts when merging with HDFS-7903, HDFS-7435 and HDFS-7930 (this commit is for HDFS-7930 only)
HDFS-7936. Erasure coding: resolving conflicts when merging with HDFS-7903, HDFS-7435 and HDFS-7930 (this commit is for HDFS-7930 only) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/37686c5d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/37686c5d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/37686c5d Branch: refs/heads/HDFS-7285 Commit: 37686c5dfbbcd7e7feaca56829eb946d132f0df2 Parents: a129dda Author: Zhe Zhang z...@apache.org Authored: Mon Mar 23 11:25:40 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:21:01 2015 -0700 -- .../hadoop/hdfs/server/blockmanagement/BlockManager.java | 7 --- .../org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 7 --- .../org/apache/hadoop/hdfs/server/namenode/INodeFile.java | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/37686c5d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 291347b..a7a7a45 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -2114,17 +2114,18 @@ public class BlockManager { * Mark block replicas as corrupt except those on the storages in * newStorages list. */ - public void markBlockReplicasAsCorrupt(BlockInfoContiguous block, + public void markBlockReplicasAsCorrupt(Block oldBlock, + BlockInfo block, long oldGenerationStamp, long oldNumBytes, DatanodeStorageInfo[] newStorages) throws IOException { assert namesystem.hasWriteLock(); BlockToMarkCorrupt b = null; if (block.getGenerationStamp() != oldGenerationStamp) { - b = new BlockToMarkCorrupt(block, oldGenerationStamp, + b = new BlockToMarkCorrupt(oldBlock, block, oldGenerationStamp, genstamp does not match + oldGenerationStamp + : + block.getGenerationStamp(), Reason.GENSTAMP_MISMATCH); } else if (block.getNumBytes() != oldNumBytes) { - b = new BlockToMarkCorrupt(block, + b = new BlockToMarkCorrupt(oldBlock, block, length does not match + oldNumBytes + : + block.getNumBytes(), Reason.SIZE_MISMATCH); } else { http://git-wip-us.apache.org/repos/asf/hadoop/blob/37686c5d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 777a084..3fec364 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -2795,7 +2795,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, /** Compute quota change for converting a complete block to a UC block */ private QuotaCounts computeQuotaDeltaForUCBlock(INodeFile file) { final QuotaCounts delta = new QuotaCounts.Builder().build(); -final BlockInfoContiguous lastBlock = file.getLastBlock(); +final BlockInfo lastBlock = file.getLastBlock(); if (lastBlock != null) { final long diff = file.getPreferredBlockSize() - lastBlock.getNumBytes(); final short repl = file.getBlockReplication(); @@ -4359,8 +4359,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, } else { iFile.convertLastBlockToUC(storedBlock, trimmedStorageInfos); if (closeFile) { -blockManager.markBlockReplicasAsCorrupt(storedBlock, -oldGenerationStamp, oldNumBytes, trimmedStorageInfos); +blockManager.markBlockReplicasAsCorrupt(oldBlock.getLocalBlock(), +storedBlock, oldGenerationStamp, oldNumBytes, +trimmedStorageInfos); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/37686c5d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java -- diff --git
[13/51] [abbrv] hadoop git commit: HADOOP-11541. Raw XOR coder
HADOOP-11541. Raw XOR coder Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f50b3df5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f50b3df5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f50b3df5 Branch: refs/heads/HDFS-7285 Commit: f50b3df51ffce1ce0d67b9601840ce0fdcb7e750 Parents: b431203 Author: Kai Zheng dran...@apache.org Authored: Sun Feb 8 01:40:27 2015 +0800 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:01:05 2015 -0700 -- .../io/erasurecode/rawcoder/XorRawDecoder.java | 81 ++ .../io/erasurecode/rawcoder/XorRawEncoder.java | 61 + .../hadoop/io/erasurecode/TestCoderBase.java| 262 +++ .../erasurecode/rawcoder/TestRawCoderBase.java | 96 +++ .../erasurecode/rawcoder/TestXorRawCoder.java | 52 5 files changed, 552 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f50b3df5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XorRawDecoder.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XorRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XorRawDecoder.java new file mode 100644 index 000..98307a7 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XorRawDecoder.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.rawcoder; + +import java.nio.ByteBuffer; + +/** + * A raw decoder in XOR code scheme in pure Java, adapted from HDFS-RAID. + */ +public class XorRawDecoder extends AbstractRawErasureDecoder { + + @Override + protected void doDecode(ByteBuffer[] inputs, int[] erasedIndexes, + ByteBuffer[] outputs) { +assert(erasedIndexes.length == outputs.length); +assert(erasedIndexes.length = 1); + +int bufSize = inputs[0].remaining(); +int erasedIdx = erasedIndexes[0]; + +// Set the output to zeros. +for (int j = 0; j bufSize; j++) { + outputs[0].put(j, (byte) 0); +} + +// Process the inputs. +for (int i = 0; i inputs.length; i++) { + // Skip the erased location. + if (i == erasedIdx) { +continue; + } + + for (int j = 0; j bufSize; j++) { +outputs[0].put(j, (byte) (outputs[0].get(j) ^ inputs[i].get(j))); + } +} + } + + @Override + protected void doDecode(byte[][] inputs, int[] erasedIndexes, + byte[][] outputs) { +assert(erasedIndexes.length == outputs.length); +assert(erasedIndexes.length = 1); + +int bufSize = inputs[0].length; +int erasedIdx = erasedIndexes[0]; + +// Set the output to zeros. +for (int j = 0; j bufSize; j++) { + outputs[0][j] = 0; +} + +// Process the inputs. +for (int i = 0; i inputs.length; i++) { + // Skip the erased location. + if (i == erasedIdx) { +continue; + } + + for (int j = 0; j bufSize; j++) { +outputs[0][j] ^= inputs[i][j]; + } +} + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f50b3df5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XorRawEncoder.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XorRawEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XorRawEncoder.java new file mode 100644 index 000..99b20b9 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XorRawEncoder.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE
[23/51] [abbrv] hadoop git commit: HDFS-7872. Erasure Coding: INodeFile.dumpTreeRecursively() supports to print striped blocks. Contributed by Takuya Fukudome.
HDFS-7872. Erasure Coding: INodeFile.dumpTreeRecursively() supports to print striped blocks. Contributed by Takuya Fukudome. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ffe990b6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ffe990b6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ffe990b6 Branch: refs/heads/HDFS-7285 Commit: ffe990b622bea084f89c94663af9c192e4fb0303 Parents: d0de9b4 Author: Jing Zhao ji...@apache.org Authored: Thu Mar 5 16:44:38 2015 -0800 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:20:57 2015 -0700 -- .../java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffe990b6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index 093ba25..22d61bc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -877,8 +877,8 @@ public class INodeFile extends INodeWithAdditionalFields out.print(, fileSize= + computeFileSize(snapshotId)); // only compare the first block out.print(, blocks=); -out.print(blocks == null || blocks.length == 0? null: blocks[0]); -// TODO print striped blocks +BlockInfo[] blks = getBlocks(); +out.print(blks == null || blks.length == 0? null: blks[0]); out.println(); }
[45/51] [abbrv] hadoop git commit: HDFS-7907. Erasure Coding: track invalid, corrupt, and under-recovery striped blocks in NameNode. Contributed by Jing Zhao.
HDFS-7907. Erasure Coding: track invalid, corrupt, and under-recovery striped blocks in NameNode. Contributed by Jing Zhao. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0767bd70 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0767bd70 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0767bd70 Branch: refs/heads/HDFS-7285 Commit: 0767bd70c1f987c27bfa184735be185ce937ffe1 Parents: 5e3f732 Author: Jing Zhao ji...@apache.org Authored: Mon Mar 30 11:25:09 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:21:03 2015 -0700 -- .../blockmanagement/BlockInfoStriped.java | 25 ++- .../server/blockmanagement/BlockManager.java| 203 ++- .../blockmanagement/DecommissionManager.java| 86 .../hdfs/server/namenode/FSNamesystem.java | 8 +- .../server/blockmanagement/TestNodeCount.java | 2 +- .../TestOverReplicatedBlocks.java | 4 +- 6 files changed, 172 insertions(+), 156 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0767bd70/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java index 30b5ee7..4a85efb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java @@ -18,11 +18,13 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; + import java.io.DataOutput; import java.io.IOException; +import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_STRIPED_CHUNK_SIZE; + /** * Subclass of {@link BlockInfo}, presenting a block group in erasure coding. * @@ -37,7 +39,6 @@ import java.io.IOException; * array to record the block index for each triplet. */ public class BlockInfoStriped extends BlockInfo { - private final int chunkSize = HdfsConstants.BLOCK_STRIPED_CHUNK_SIZE; private final short dataBlockNum; private final short parityBlockNum; /** @@ -132,6 +133,22 @@ public class BlockInfoStriped extends BlockInfo { return i == -1 ? -1 : indices[i]; } + /** + * Identify the block stored in the given datanode storage. Note that + * the returned block has the same block Id with the one seen/reported by the + * DataNode. + */ + Block getBlockOnStorage(DatanodeStorageInfo storage) { +int index = getStorageBlockIndex(storage); +if (index 0) { + return null; +} else { + Block block = new Block(this); + block.setBlockId(this.getBlockId() + index); + return block; +} + } + @Override boolean removeStorage(DatanodeStorageInfo storage) { int dnIndex = findStorageInfoFromEnd(storage); @@ -186,8 +203,8 @@ public class BlockInfoStriped extends BlockInfo { // In case striped blocks, total usage by this striped blocks should // be the total of data blocks and parity blocks because // `getNumBytes` is the total of actual data block size. -return ((getNumBytes() - 1) / (dataBlockNum * chunkSize) + 1) -* chunkSize * parityBlockNum + getNumBytes(); +return ((getNumBytes() - 1) / (dataBlockNum * BLOCK_STRIPED_CHUNK_SIZE) + 1) +* BLOCK_STRIPED_CHUNK_SIZE * parityBlockNum + getNumBytes(); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/0767bd70/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 66a02d4..e05330c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -177,7 +177,11 @@ public class BlockManager { /** Store blocks - datanodedescriptor(s) map of corrupt replicas */ final CorruptReplicasMap corruptReplicas = new CorruptReplicasMap(); -
[46/51] [abbrv] hadoop git commit: HDFS-7839. Erasure coding: implement facilities in NameNode to create and manage EC zones. Contributed by Zhe Zhang
HDFS-7839. Erasure coding: implement facilities in NameNode to create and manage EC zones. Contributed by Zhe Zhang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ea91d07e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ea91d07e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ea91d07e Branch: refs/heads/HDFS-7285 Commit: ea91d07e417e0f678954f1f694d2176b6aebe682 Parents: a4a4032 Author: Zhe Zhang z...@apache.org Authored: Thu Apr 2 22:38:29 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:21:04 2015 -0700 -- .../java/org/apache/hadoop/hdfs/DFSClient.java | 15 ++ .../hadoop/hdfs/protocol/ClientProtocol.java| 8 + .../hadoop/hdfs/protocol/HdfsConstants.java | 2 - ...tNamenodeProtocolServerSideTranslatorPB.java | 14 ++ .../ClientNamenodeProtocolTranslatorPB.java | 16 ++ .../BlockStoragePolicySuite.java| 5 - .../hdfs/server/common/HdfsServerConstants.java | 2 + .../namenode/ErasureCodingZoneManager.java | 112 ++ .../hdfs/server/namenode/FSDirRenameOp.java | 2 + .../hdfs/server/namenode/FSDirectory.java | 26 +++- .../hdfs/server/namenode/FSNamesystem.java | 40 + .../hadoop/hdfs/server/namenode/INodeFile.java | 10 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 16 ++ .../src/main/proto/ClientNamenodeProtocol.proto | 9 ++ .../hadoop/hdfs/TestBlockStoragePolicy.java | 12 +- .../hadoop/hdfs/TestErasureCodingZones.java | 151 +++ .../TestBlockInitialEncoding.java | 75 - .../server/namenode/TestAddStripedBlocks.java | 2 +- .../server/namenode/TestFSEditLogLoader.java| 6 +- .../hdfs/server/namenode/TestFSImage.java | 23 ++- .../namenode/TestRecoverStripedBlocks.java | 7 +- 21 files changed, 431 insertions(+), 122 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea91d07e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 29bb604..6a82160 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -3320,6 +3320,21 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, return new EncryptionZoneIterator(namenode, traceSampler); } + public void createErasureCodingZone(String src) + throws IOException { +checkOpen(); +TraceScope scope = getPathTraceScope(createErasureCodingZone, src); +try { + namenode.createErasureCodingZone(src); +} catch (RemoteException re) { + throw re.unwrapRemoteException(AccessControlException.class, + SafeModeException.class, + UnresolvedPathException.class); +} finally { + scope.close(); +} + } + public void setXAttr(String src, String name, byte[] value, EnumSetXAttrSetFlag flag) throws IOException { checkOpen(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea91d07e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index bafb02b..8efe344 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -1363,6 +1363,14 @@ public interface ClientProtocol { long prevId) throws IOException; /** + * Create an erasure coding zone (currently with hardcoded schema) + * TODO: Configurable and pluggable schemas (HDFS-7337) + */ + @Idempotent + public void createErasureCodingZone(String src) + throws IOException; + + /** * Set xattr of a file or directory. * The name must be prefixed with the namespace followed by .. For example, * user.attr. http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea91d07e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
[48/51] [abbrv] hadoop git commit: HDFS-7936. Erasure coding: resolving conflicts in the branch when merging (this commit is for conflicts from HDFS-6945). Contributed by Zhe Zhang.
HDFS-7936. Erasure coding: resolving conflicts in the branch when merging (this commit is for conflicts from HDFS-6945). Contributed by Zhe Zhang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a4a40322 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a4a40322 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a4a40322 Branch: refs/heads/HDFS-7285 Commit: a4a403226b8b61b063a5b18b61ab1f383f2d7ac9 Parents: 0800245 Author: Zhe Zhang z...@apache.org Authored: Thu Apr 2 11:25:58 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:21:04 2015 -0700 -- .../apache/hadoop/hdfs/server/blockmanagement/BlockManager.java| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4a40322/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index f42adcd..bce789a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -3667,7 +3667,7 @@ public class BlockManager { private void removeFromExcessReplicateMap(Block block) { for (DatanodeStorageInfo info : blocksMap.getStorages(block)) { String uuid = info.getDatanodeDescriptor().getDatanodeUuid(); - LightWeightLinkedSetBlock excessReplicas = excessReplicateMap.get(uuid); + LightWeightLinkedSetBlockInfo excessReplicas = excessReplicateMap.get(uuid); if (excessReplicas != null) { if (excessReplicas.remove(block)) { excessBlocksCount.decrementAndGet();
[19/51] [abbrv] hadoop git commit: HDFS-7749. Erasure Coding: Add striped block support in INodeFile. Contributed by Jing Zhao.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/565cef08/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileWithStripedBlocksFeature.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileWithStripedBlocksFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileWithStripedBlocksFeature.java new file mode 100644 index 000..47445be --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileWithStripedBlocksFeature.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped; + +/** + * Feature for file with striped blocks + */ +class FileWithStripedBlocksFeature implements INode.Feature { + private BlockInfoStriped[] blocks; + + FileWithStripedBlocksFeature() { +blocks = new BlockInfoStriped[0]; + } + + FileWithStripedBlocksFeature(BlockInfoStriped[] blocks) { +Preconditions.checkArgument(blocks != null); +this.blocks = blocks; + } + + BlockInfoStriped[] getBlocks() { +return this.blocks; + } + + void setBlock(int index, BlockInfoStriped blk) { +blocks[index] = blk; + } + + BlockInfoStriped getLastBlock() { +return blocks == null || blocks.length == 0 ? +null : blocks[blocks.length - 1]; + } + + int numBlocks() { +return blocks == null ? 0 : blocks.length; + } + + void updateBlockCollection(INodeFile file) { +if (blocks != null) { + for (BlockInfoStriped blk : blocks) { +blk.setBlockCollection(file); + } +} + } + + private void setBlocks(BlockInfoStriped[] blocks) { +this.blocks = blocks; + } + + void addBlock(BlockInfoStriped newBlock) { +if (this.blocks == null) { + this.setBlocks(new BlockInfoStriped[]{newBlock}); +} else { + int size = this.blocks.length; + BlockInfoStriped[] newlist = new BlockInfoStriped[size + 1]; + System.arraycopy(this.blocks, 0, newlist, 0, size); + newlist[size] = newBlock; + this.setBlocks(newlist); +} + } + + boolean removeLastBlock(Block oldblock) { +if (blocks == null || blocks.length == 0) { + return false; +} +int newSize = blocks.length - 1; +if (!blocks[newSize].equals(oldblock)) { + return false; +} + +//copy to a new list +BlockInfoStriped[] newlist = new BlockInfoStriped[newSize]; +System.arraycopy(blocks, 0, newlist, 0, newSize); +setBlocks(newlist); +return true; + } + + void truncateStripedBlocks(int n) { +final BlockInfoStriped[] newBlocks; +if (n == 0) { + newBlocks = new BlockInfoStriped[0]; +} else { + newBlocks = new BlockInfoStriped[n]; + System.arraycopy(getBlocks(), 0, newBlocks, 0, n); +} +// set new blocks +setBlocks(newBlocks); + } + + void clear() { +this.blocks = null; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/565cef08/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index b09463d..5462da7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.server.namenode; import static org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite.ID_UNSPECIFIED; +import static
[26/51] [abbrv] hadoop git commit: HADOOP-11646. Erasure Coder API for encoding and decoding of block group ( Contributed by Kai Zheng )
HADOOP-11646. Erasure Coder API for encoding and decoding of block group ( Contributed by Kai Zheng ) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c8d4c251 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c8d4c251 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c8d4c251 Branch: refs/heads/HDFS-7285 Commit: c8d4c25153b26b57e916261d63936a0de2aa8b14 Parents: ffe990b Author: Vinayakumar B vinayakum...@apache.org Authored: Mon Mar 9 12:32:26 2015 +0530 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:20:58 2015 -0700 -- .../hadoop-common/CHANGES-HDFS-EC-7285.txt | 2 + .../apache/hadoop/io/erasurecode/ECBlock.java | 80 ++ .../hadoop/io/erasurecode/ECBlockGroup.java | 82 ++ .../erasurecode/coder/AbstractErasureCoder.java | 63 + .../coder/AbstractErasureCodingStep.java| 59 .../coder/AbstractErasureDecoder.java | 152 +++ .../coder/AbstractErasureEncoder.java | 50 .../io/erasurecode/coder/ErasureCoder.java | 77 ++ .../io/erasurecode/coder/ErasureCodingStep.java | 55 .../io/erasurecode/coder/ErasureDecoder.java| 41 +++ .../erasurecode/coder/ErasureDecodingStep.java | 52 .../io/erasurecode/coder/ErasureEncoder.java| 39 +++ .../erasurecode/coder/ErasureEncodingStep.java | 49 .../io/erasurecode/coder/XorErasureDecoder.java | 78 ++ .../io/erasurecode/coder/XorErasureEncoder.java | 45 .../erasurecode/rawcoder/RawErasureCoder.java | 2 +- .../erasurecode/coder/TestErasureCoderBase.java | 266 +++ .../io/erasurecode/coder/TestXorCoder.java | 50 18 files changed, 1241 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8d4c251/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt index ee42c84..c17a1bd 100644 --- a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt @@ -15,4 +15,6 @@ HADOOP-11643. Define EC schema API for ErasureCodec. Contributed by Kai Zheng ( Kai Zheng ) +HADOOP-11646. Erasure Coder API for encoding and decoding of block group +( Kai Zheng via vinayakumarb ) http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8d4c251/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECBlock.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECBlock.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECBlock.java new file mode 100644 index 000..956954a --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECBlock.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode; + +/** + * A wrapper of block level data source/output that {@link ECChunk}s can be + * extracted from. For HDFS, it can be an HDFS block (250MB). Note it only cares + * about erasure coding specific logic thus avoids coupling with any HDFS block + * details. We can have something like HdfsBlock extend it. + */ +public class ECBlock { + + private boolean isParity; + private boolean isErased; + + /** + * A default constructor. isParity and isErased are false by default. + */ + public ECBlock() { +this(false, false); + } + + /** + * A constructor specifying isParity and isErased. + * @param isParity + * @param isErased + */ + public ECBlock(boolean isParity, boolean isErased) { +this.isParity = isParity; +this.isErased = isErased; + } + + /** + * Set true if it's for a parity block. + * @param isParity + */
[35/51] [abbrv] hadoop git commit: HDFS-7864. Erasure Coding: Update safemode calculation for striped blocks. Contributed by GAO Rui.
HDFS-7864. Erasure Coding: Update safemode calculation for striped blocks. Contributed by GAO Rui. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7c7ba714 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7c7ba714 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7c7ba714 Branch: refs/heads/HDFS-7285 Commit: 7c7ba714b047c20815ebe2b35083d5f664e7d972 Parents: 37686c5 Author: Jing Zhao ji...@apache.org Authored: Mon Mar 23 15:06:53 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:21:01 2015 -0700 -- .../server/blockmanagement/BlockIdManager.java | 6 ++ .../hdfs/server/blockmanagement/BlockManager.java | 12 +++- .../hdfs/server/blockmanagement/BlocksMap.java | 2 +- .../hadoop/hdfs/server/namenode/FSNamesystem.java | 17 - .../hadoop/hdfs/server/namenode/SafeMode.java | 5 +++-- .../java/org/apache/hadoop/hdfs/TestSafeMode.java | 15 +-- 6 files changed, 42 insertions(+), 15 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c7ba714/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java index 1d69d74..187f8c9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java @@ -233,6 +233,12 @@ public class BlockIdManager { return id 0; } + /** + * The last 4 bits of HdfsConstants.BLOCK_GROUP_INDEX_MASK(15) is , + * so the last 4 bits of (~HdfsConstants.BLOCK_GROUP_INDEX_MASK) is + * and the other 60 bits are 1. Group ID is the first 60 bits of any + * data/parity block id in the same striped block group. + */ public static long convertToStripedID(long id) { return id (~HdfsConstants.BLOCK_GROUP_INDEX_MASK); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c7ba714/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index a7a7a45..2f3c87c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -684,8 +684,10 @@ public class BlockManager { // a forced completion when a file is getting closed by an // OP_CLOSE edit on the standby). namesystem.adjustSafeModeBlockTotals(0, 1); +final int minStorage = curBlock.isStriped() ? +((BlockInfoStriped) curBlock).getDataBlockNum() : minReplication; namesystem.incrementSafeBlockCount( -Math.min(numNodes, minReplication)); +Math.min(numNodes, minStorage), curBlock); // replace block in the blocksMap return blocksMap.replaceBlock(completeBlock); @@ -2208,7 +2210,7 @@ public class BlockManager { // refer HDFS-5283 if (namesystem.isInSnapshot(storedBlock.getBlockCollection())) { int numOfReplicas = BlockInfo.getNumExpectedLocations(storedBlock); - namesystem.incrementSafeBlockCount(numOfReplicas); + namesystem.incrementSafeBlockCount(numOfReplicas, storedBlock); } //and fall through to next clause } @@ -2589,14 +2591,14 @@ public class BlockManager { // only complete blocks are counted towards that. // In the case that the block just became complete above, completeBlock() // handles the safe block count maintenance. - namesystem.incrementSafeBlockCount(numCurrentReplica); + namesystem.incrementSafeBlockCount(numCurrentReplica, storedBlock); } } /** * Modify (block--datanode) map. Remove block from set of * needed replications if this takes care of the problem. - * @return the block that is stored in blockMap. + * @return the block that is stored in blocksMap. */ private Block addStoredBlock(final BlockInfo block, final Block reportedBlock, @@ -2665,7 +2667,7 @@ public class BlockManager {
[31/51] [abbrv] hadoop git commit: Updated CHANGES-HDFS-EC-7285.txt accordingly
Updated CHANGES-HDFS-EC-7285.txt accordingly Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/08a4c9e2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/08a4c9e2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/08a4c9e2 Branch: refs/heads/HDFS-7285 Commit: 08a4c9e2c32d25942e438ba005db26b64d00b5b4 Parents: 37b917d Author: Kai Zheng kai.zh...@intel.com Authored: Wed Mar 18 19:24:24 2015 +0800 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:21:00 2015 -0700 -- hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt | 5 - 1 file changed, 4 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/08a4c9e2/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt index a97dc34..e27ff5c 100644 --- a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt @@ -19,6 +19,9 @@ ( Kai Zheng via vinayakumarb ) HADOOP-11705. Make erasure coder configurable. Contributed by Kai Zheng -( Kai Zheng ) +( Kai Zheng ) + +HADOOP-11706. Refine a little bit erasure coder API. Contributed by Kai Zheng +( Kai Zheng )
[50/51] [abbrv] hadoop git commit: HADOOP-11740. Combine erasure encoder and decoder interfaces. Contributed by Zhe Zhang.
HADOOP-11740. Combine erasure encoder and decoder interfaces. Contributed by Zhe Zhang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bdb67d96 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bdb67d96 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bdb67d96 Branch: refs/heads/HDFS-7285 Commit: bdb67d96fbb0f23679f32e419bb17a7f8684411e Parents: ea91d07 Author: Zhe Zhang z...@apache.org Authored: Fri Apr 3 15:22:50 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:21:05 2015 -0700 -- .../coder/AbstractErasureDecoder.java | 7 ++-- .../coder/AbstractErasureEncoder.java | 7 ++-- .../io/erasurecode/coder/ErasureCoder.java | 12 ++ .../io/erasurecode/coder/ErasureDecoder.java| 41 .../io/erasurecode/coder/ErasureEncoder.java| 39 --- .../erasurecode/coder/TestErasureCoderBase.java | 20 +- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 14 ++- 7 files changed, 41 insertions(+), 99 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdb67d96/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java index 54a6d1e..cd31294 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java @@ -23,13 +23,12 @@ import org.apache.hadoop.io.erasurecode.ECBlockGroup; /** * An abstract erasure decoder that's to be inherited by new decoders. * - * It implements the {@link ErasureDecoder} interface. + * It implements the {@link ErasureCoder} interface. */ -public abstract class AbstractErasureDecoder extends AbstractErasureCoder -implements ErasureDecoder { +public abstract class AbstractErasureDecoder extends AbstractErasureCoder { @Override - public ErasureCodingStep decode(ECBlockGroup blockGroup) { + public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) { // We may have more than this when considering complicate cases. HADOOP-11550 return prepareDecodingStep(blockGroup); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdb67d96/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java index 09b31e5..a836b75 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java @@ -23,13 +23,12 @@ import org.apache.hadoop.io.erasurecode.ECBlockGroup; /** * An abstract erasure encoder that's to be inherited by new encoders. * - * It implements the {@link ErasureEncoder} interface. + * It implements the {@link ErasureCoder} interface. */ -public abstract class AbstractErasureEncoder extends AbstractErasureCoder -implements ErasureEncoder { +public abstract class AbstractErasureEncoder extends AbstractErasureCoder { @Override - public ErasureCodingStep encode(ECBlockGroup blockGroup) { + public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) { // We may have more than this when considering complicate cases. HADOOP-11550 return prepareEncodingStep(blockGroup); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdb67d96/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java index c5922f3..fb90156 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java +++
[20/51] [abbrv] hadoop git commit: HDFS-7749. Erasure Coding: Add striped block support in INodeFile. Contributed by Jing Zhao.
HDFS-7749. Erasure Coding: Add striped block support in INodeFile. Contributed by Jing Zhao. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/565cef08 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/565cef08 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/565cef08 Branch: refs/heads/HDFS-7285 Commit: 565cef0818471d59d16c5540d06b279cd9dc3e1f Parents: ae79047 Author: Jing Zhao ji...@apache.org Authored: Wed Feb 25 22:10:26 2015 -0800 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:19:29 2015 -0700 -- .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 17 ++ .../server/blockmanagement/BlockCollection.java | 13 +- .../hdfs/server/blockmanagement/BlockInfo.java | 88 ++- .../BlockInfoContiguousUnderConstruction.java | 6 +- .../blockmanagement/BlockInfoStriped.java | 31 +++ .../BlockInfoStripedUnderConstruction.java | 240 ++ .../server/blockmanagement/BlockManager.java| 151 +-- .../CacheReplicationMonitor.java| 16 +- .../hdfs/server/namenode/FSDirConcatOp.java | 8 +- .../hdfs/server/namenode/FSDirectory.java | 5 +- .../hadoop/hdfs/server/namenode/FSEditLog.java | 8 +- .../hdfs/server/namenode/FSEditLogLoader.java | 16 +- .../hdfs/server/namenode/FSImageFormat.java | 7 +- .../server/namenode/FSImageFormatPBINode.java | 46 +++- .../hdfs/server/namenode/FSNamesystem.java | 110 .../namenode/FileUnderConstructionFeature.java | 15 +- .../namenode/FileWithStripedBlocksFeature.java | 112 .../hadoop/hdfs/server/namenode/INodeFile.java | 254 +-- .../hdfs/server/namenode/LeaseManager.java | 6 +- .../hdfs/server/namenode/NamenodeFsck.java | 4 +- .../hadoop/hdfs/server/namenode/Namesystem.java | 3 +- .../snapshot/FSImageFormatPBSnapshot.java | 7 +- .../server/namenode/snapshot/FileDiffList.java | 9 +- .../hadoop-hdfs/src/main/proto/fsimage.proto| 5 + .../hadoop-hdfs/src/main/proto/hdfs.proto | 10 + .../org/apache/hadoop/hdfs/DFSTestUtil.java | 3 +- .../blockmanagement/TestReplicationPolicy.java | 4 +- .../hdfs/server/namenode/TestAddBlock.java | 12 +- .../hdfs/server/namenode/TestAddBlockgroup.java | 3 +- .../namenode/TestBlockUnderConstruction.java| 6 +- .../hdfs/server/namenode/TestFSImage.java | 4 +- .../hdfs/server/namenode/TestFileTruncate.java | 4 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 4 +- .../snapshot/TestSnapshotBlocksMap.java | 24 +- .../namenode/snapshot/TestSnapshotDeletion.java | 16 +- 35 files changed, 953 insertions(+), 314 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/565cef08/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index 1942ea9..14410e6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -172,6 +172,7 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageReportProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypeProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypesProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageUuidsProto; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StripedBlockProto; import org.apache.hadoop.hdfs.protocol.proto.InotifyProtos; import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.JournalInfoProto; import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsResponseProto; @@ -184,6 +185,7 @@ import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey; import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped; import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType; @@ -430,6 +432,21 @@ public class PBHelper { return new Block(b.getBlockId(), b.getNumBytes(), b.getGenStamp()); } + public static BlockInfoStriped
[41/51] [abbrv] hadoop git commit: HDFS-7936. Erasure coding: resolving conflicts when merging with HDFS-7903, HDFS-7435, HDFS-7930, HDFS-7960 (this commit is for HDFS-7960)
HDFS-7936. Erasure coding: resolving conflicts when merging with HDFS-7903, HDFS-7435, HDFS-7930, HDFS-7960 (this commit is for HDFS-7960) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a4e62629 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a4e62629 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a4e62629 Branch: refs/heads/HDFS-7285 Commit: a4e626296989ed49d72f611fce30381298040225 Parents: f62c711 Author: Zhe Zhang z...@apache.org Authored: Tue Mar 24 11:39:36 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:21:02 2015 -0700 -- .../apache/hadoop/hdfs/server/blockmanagement/BlockManager.java | 4 ++-- .../blockmanagement/TestNameNodePrunesMissingStorages.java | 5 - .../hadoop/hdfs/server/namenode/TestAddStripedBlocks.java | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4e62629/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 2f3c87c..66a02d4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1975,10 +1975,10 @@ public class BlockManager { longer exists on the DataNode., Long.toHexString(context.getReportId()), zombie.getStorageID()); assert(namesystem.hasWriteLock()); -IteratorBlockInfoContiguous iter = zombie.getBlockIterator(); +IteratorBlockInfo iter = zombie.getBlockIterator(); int prevBlocks = zombie.numBlocks(); while (iter.hasNext()) { - BlockInfoContiguous block = iter.next(); + BlockInfo block = iter.next(); // We assume that a block can be on only one storage in a DataNode. // That's why we pass in the DatanodeDescriptor rather than the // DatanodeStorageInfo. http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4e62629/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java index 4b97d01..e9329cd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java @@ -171,9 +171,12 @@ public class TestNameNodePrunesMissingStorages { String datanodeUuid; // Find the first storage which this block is in. try { +BlockInfo storedBlock = +cluster.getNamesystem().getBlockManager(). +getStoredBlock(block.getLocalBlock()); IteratorDatanodeStorageInfo storageInfoIter = cluster.getNamesystem().getBlockManager(). -getStorages(block.getLocalBlock()).iterator(); +blocksMap.getStorages(storedBlock).iterator(); assertTrue(storageInfoIter.hasNext()); DatanodeStorageInfo info = storageInfoIter.next(); storageIdToRemove = info.getStorageID(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4e62629/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java index 05aec4b..7d7c81e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java @@ -269,7 +269,7 @@ public class TestAddStripedBlocks { StorageBlockReport[] reports = {new StorageBlockReport(storage, bll)};
[30/51] [abbrv] hadoop git commit: HDFS-7826. Erasure Coding: Update INodeFile quota computation for striped blocks. Contributed by Kai Sasaki.
HDFS-7826. Erasure Coding: Update INodeFile quota computation for striped blocks. Contributed by Kai Sasaki. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/904099c4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/904099c4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/904099c4 Branch: refs/heads/HDFS-7285 Commit: 904099c49e217980ddc537cd668dc3fb6a436dd8 Parents: cef8ce4 Author: Jing Zhao ji...@apache.org Authored: Mon Mar 16 16:37:08 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:20:59 2015 -0700 -- .../hadoop/hdfs/protocol/HdfsConstants.java | 3 + .../blockmanagement/BlockInfoStriped.java | 12 ++- .../hadoop/hdfs/server/namenode/INodeFile.java | 89 +--- 3 files changed, 90 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/904099c4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java index 245b630..07b72e6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java @@ -186,4 +186,7 @@ public class HdfsConstants { public static final byte NUM_PARITY_BLOCKS = 2; public static final long BLOCK_GROUP_INDEX_MASK = 15; public static final byte MAX_BLOCKS_IN_GROUP = 16; + + // The chunk size for striped block which is used by erasure coding + public static final int BLOCK_STRIPED_CHUNK_SIZE = 64 * 1024; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/904099c4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java index 84c3be6..cef8318 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; /** @@ -34,6 +35,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; * array to record the block index for each triplet. */ public class BlockInfoStriped extends BlockInfo { + private final int chunkSize = HdfsConstants.BLOCK_STRIPED_CHUNK_SIZE; private final short dataBlockNum; private final short parityBlockNum; /** @@ -56,7 +58,7 @@ public class BlockInfoStriped extends BlockInfo { this.setBlockCollection(b.getBlockCollection()); } - short getTotalBlockNum() { + public short getTotalBlockNum() { return (short) (dataBlockNum + parityBlockNum); } @@ -178,6 +180,14 @@ public class BlockInfoStriped extends BlockInfo { } } + public long spaceConsumed() { +// In case striped blocks, total usage by this striped blocks should +// be the total of data blocks and parity blocks because +// `getNumBytes` is the total of actual data block size. +return ((getNumBytes() - 1) / (dataBlockNum * chunkSize) + 1) +* chunkSize * parityBlockNum + getNumBytes(); + } + @Override public final boolean isStriped() { return true; http://git-wip-us.apache.org/repos/asf/hadoop/blob/904099c4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index 22d61bc..9cd7ddd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import
[40/51] [abbrv] hadoop git commit: HDFS-7936. Erasure coding: resolving conflicts in the branch when merging (this commit is for HDFS-7742)
HDFS-7936. Erasure coding: resolving conflicts in the branch when merging (this commit is for HDFS-7742) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e3f7327 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e3f7327 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e3f7327 Branch: refs/heads/HDFS-7285 Commit: 5e3f7327dac9747da63d1d49316b13fb722977ba Parents: a5097a4 Author: Zhe Zhang z...@apache.org Authored: Mon Mar 30 10:23:09 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:21:02 2015 -0700 -- .../hdfs/server/blockmanagement/TestBlockManager.java | 12 ++-- 1 file changed, 6 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e3f7327/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java index cbea3d8..43f4607 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java @@ -552,11 +552,11 @@ public class TestBlockManager { assertNotNull(Chooses decommissioning source node for a normal replication + if all available source nodes have reached their replication + limits below the hard limit., -bm.chooseSourceDatanode( -aBlock, +bm.chooseSourceDatanodes( +bm.getStoredBlock(aBlock), cntNodes, liveNodes, -new NumberReplicas(), +new NumberReplicas(), new LinkedListShort(), 1, UnderReplicatedBlocks.QUEUE_UNDER_REPLICATED)); @@ -566,11 +566,11 @@ public class TestBlockManager { assertNull(Does not choose a source decommissioning node for a normal + replication when all available nodes exceed the hard limit., -bm.chooseSourceDatanode( -aBlock, +bm.chooseSourceDatanodes( +bm.getStoredBlock(aBlock), cntNodes, liveNodes, -new NumberReplicas(), +new NumberReplicas(), new LinkedListShort(), 1, UnderReplicatedBlocks.QUEUE_UNDER_REPLICATED)); }
[29/51] [abbrv] hadoop git commit: HDFS-7912. Erasure Coding: track BlockInfo instead of Block in UnderReplicatedBlocks and PendingReplicationBlocks. Contributed by Jing Zhao.
HDFS-7912. Erasure Coding: track BlockInfo instead of Block in UnderReplicatedBlocks and PendingReplicationBlocks. Contributed by Jing Zhao. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ed82e683 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ed82e683 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ed82e683 Branch: refs/heads/HDFS-7285 Commit: ed82e6836f443b708a4af308195d821c2a3f6970 Parents: 904099c Author: Jing Zhao ji...@apache.org Authored: Tue Mar 17 10:18:50 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:20:59 2015 -0700 -- .../server/blockmanagement/BlockManager.java| 47 - .../PendingReplicationBlocks.java | 51 +-- .../blockmanagement/UnderReplicatedBlocks.java | 49 +- .../hdfs/server/namenode/FSDirAttrOp.java | 10 ++-- .../hdfs/server/namenode/FSNamesystem.java | 21 .../hadoop/hdfs/server/namenode/INode.java | 12 ++--- .../hadoop/hdfs/server/namenode/INodeFile.java | 4 +- .../hdfs/server/namenode/NamenodeFsck.java | 10 ++-- .../hadoop/hdfs/server/namenode/SafeMode.java | 3 +- .../blockmanagement/BlockManagerTestUtil.java | 5 +- .../blockmanagement/TestBlockManager.java | 8 +-- .../server/blockmanagement/TestNodeCount.java | 3 +- .../TestOverReplicatedBlocks.java | 5 +- .../blockmanagement/TestPendingReplication.java | 19 --- .../TestRBWBlockInvalidation.java | 4 +- .../blockmanagement/TestReplicationPolicy.java | 53 +++- .../TestUnderReplicatedBlockQueues.java | 16 +++--- .../datanode/TestReadOnlySharedStorage.java | 9 ++-- .../namenode/TestProcessCorruptBlocks.java | 5 +- 19 files changed, 180 insertions(+), 154 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed82e683/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index cac13b4..5cbad40 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1336,7 +1336,7 @@ public class BlockManager { * @return number of blocks scheduled for replication during this iteration. */ int computeReplicationWork(int blocksToProcess) { -ListListBlock blocksToReplicate = null; +ListListBlockInfo blocksToReplicate = null; namesystem.writeLock(); try { // Choose the blocks to be replicated @@ -1354,7 +1354,7 @@ public class BlockManager { * @return the number of blocks scheduled for replication */ @VisibleForTesting - int computeReplicationWorkForBlocks(ListListBlock blocksToReplicate) { + int computeReplicationWorkForBlocks(ListListBlockInfo blocksToReplicate) { int requiredReplication, numEffectiveReplicas; ListDatanodeDescriptor containingNodes; DatanodeDescriptor srcNode; @@ -1368,7 +1368,7 @@ public class BlockManager { try { synchronized (neededReplications) { for (int priority = 0; priority blocksToReplicate.size(); priority++) { - for (Block block : blocksToReplicate.get(priority)) { + for (BlockInfo block : blocksToReplicate.get(priority)) { // block should belong to a file bc = blocksMap.getBlockCollection(block); // abandoned block or block reopened for append @@ -1452,7 +1452,7 @@ public class BlockManager { } synchronized (neededReplications) { - Block block = rw.block; + BlockInfo block = rw.block; int priority = rw.priority; // Recheck since global lock was released // block should belong to a file @@ -1710,7 +1710,7 @@ public class BlockManager { * and put them back into the neededReplication queue */ private void processPendingReplications() { -Block[] timedOutItems = pendingReplications.getTimedOutBlocks(); +BlockInfo[] timedOutItems = pendingReplications.getTimedOutBlocks(); if (timedOutItems != null) { namesystem.writeLock(); try { @@ -2883,13 +2883,13 @@ public class BlockManager { /** Set replication for the blocks. */ public void setReplication(final short oldRepl, final short newRepl, - final String src, final Block... blocks) { + final String src,
[21/51] [abbrv] hadoop git commit: HDFS-7837. Erasure Coding: allocate and persist striped blocks in NameNode. Contributed by Jing Zhao.
HDFS-7837. Erasure Coding: allocate and persist striped blocks in NameNode. Contributed by Jing Zhao. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1af3d5d2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1af3d5d2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1af3d5d2 Branch: refs/heads/HDFS-7285 Commit: 1af3d5d2d30a03092400b0fef6be789f3c239363 Parents: 565cef0 Author: Jing Zhao ji...@apache.org Authored: Mon Mar 2 13:44:33 2015 -0800 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:20:44 2015 -0700 -- .../server/blockmanagement/BlockIdManager.java | 31 +++- .../hdfs/server/blockmanagement/BlockInfo.java | 4 +- .../blockmanagement/BlockInfoContiguous.java| 5 + .../blockmanagement/BlockInfoStriped.java | 8 +- .../server/blockmanagement/BlockManager.java| 44 -- .../hdfs/server/blockmanagement/BlocksMap.java | 20 ++- .../blockmanagement/DecommissionManager.java| 9 +- .../hdfs/server/namenode/FSDirectory.java | 27 +++- .../hdfs/server/namenode/FSEditLogLoader.java | 69 ++--- .../hdfs/server/namenode/FSImageFormat.java | 12 +- .../server/namenode/FSImageFormatPBINode.java | 5 +- .../server/namenode/FSImageFormatProtobuf.java | 9 +- .../hdfs/server/namenode/FSNamesystem.java | 27 ++-- .../hadoop/hdfs/server/namenode/INodeFile.java | 25 +++- .../server/namenode/NameNodeLayoutVersion.java | 3 +- .../hadoop-hdfs/src/main/proto/fsimage.proto| 1 + .../hdfs/server/namenode/TestAddBlockgroup.java | 85 --- .../server/namenode/TestAddStripedBlocks.java | 146 +++ 18 files changed, 351 insertions(+), 179 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1af3d5d2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java index 3ae54ce..1d69d74 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java @@ -103,21 +103,38 @@ public class BlockIdManager { } /** - * Sets the maximum allocated block ID for this filesystem. This is + * Sets the maximum allocated contiguous block ID for this filesystem. This is * the basis for allocating new block IDs. */ - public void setLastAllocatedBlockId(long blockId) { + public void setLastAllocatedContiguousBlockId(long blockId) { blockIdGenerator.skipTo(blockId); } /** - * Gets the maximum sequentially allocated block ID for this filesystem + * Gets the maximum sequentially allocated contiguous block ID for this + * filesystem */ - public long getLastAllocatedBlockId() { + public long getLastAllocatedContiguousBlockId() { return blockIdGenerator.getCurrentValue(); } /** + * Sets the maximum allocated striped block ID for this filesystem. This is + * the basis for allocating new block IDs. + */ + public void setLastAllocatedStripedBlockId(long blockId) { +blockGroupIdGenerator.skipTo(blockId); + } + + /** + * Gets the maximum sequentially allocated striped block ID for this + * filesystem + */ + public long getLastAllocatedStripedBlockId() { +return blockGroupIdGenerator.getCurrentValue(); + } + + /** * Sets the current generation stamp for legacy blocks */ public void setGenerationStampV1(long stamp) { @@ -188,11 +205,11 @@ public class BlockIdManager { /** * Increments, logs and then returns the block ID */ - public long nextBlockId() { + public long nextContiguousBlockId() { return blockIdGenerator.nextValue(); } - public long nextBlockGroupId() { + public long nextStripedBlockId() { return blockGroupIdGenerator.nextValue(); } @@ -216,7 +233,7 @@ public class BlockIdManager { return id 0; } - public static long convertToGroupID(long id) { + public static long convertToStripedID(long id) { return id (~HdfsConstants.BLOCK_GROUP_INDEX_MASK); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1af3d5d2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
[25/51] [abbrv] hadoop git commit: HDFS-7853. Erasure coding: extend LocatedBlocks to support reading from striped files. Contributed by Jing Zhao.
HDFS-7853. Erasure coding: extend LocatedBlocks to support reading from striped files. Contributed by Jing Zhao. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2e853a67 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2e853a67 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2e853a67 Branch: refs/heads/HDFS-7285 Commit: 2e853a6700dc81b68984d2af9966564e409f14af Parents: c8d4c25 Author: Jing Zhao ji...@apache.org Authored: Mon Mar 9 14:59:58 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:20:58 2015 -0700 -- .../hadoop/hdfs/protocol/LocatedBlock.java | 5 +- .../hdfs/protocol/LocatedStripedBlock.java | 68 + ...tNamenodeProtocolServerSideTranslatorPB.java | 14 +- .../ClientNamenodeProtocolTranslatorPB.java | 13 +- .../DatanodeProtocolClientSideTranslatorPB.java | 2 +- .../DatanodeProtocolServerSideTranslatorPB.java | 2 +- .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 80 +++ .../blockmanagement/BlockInfoStriped.java | 5 + .../BlockInfoStripedUnderConstruction.java | 99 +++-- .../server/blockmanagement/BlockManager.java| 51 --- .../blockmanagement/DatanodeDescriptor.java | 4 +- .../blockmanagement/DatanodeStorageInfo.java| 3 +- .../server/namenode/FSImageFormatPBINode.java | 21 +-- .../hdfs/server/namenode/FSNamesystem.java | 34 +++-- .../hadoop-hdfs/src/main/proto/hdfs.proto | 1 + .../org/apache/hadoop/hdfs/DFSTestUtil.java | 12 ++ .../hadoop/hdfs/protocolPB/TestPBHelper.java| 16 +-- .../datanode/TestIncrementalBrVariations.java | 14 +- .../server/namenode/TestAddStripedBlocks.java | 141 +++ .../hdfs/server/namenode/TestFSImage.java | 5 +- 20 files changed, 444 insertions(+), 146 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e853a67/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java index e729869..a38e8f2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java @@ -25,7 +25,6 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; -import org.apache.hadoop.hdfs.protocol.DatanodeInfoWithStorage; import org.apache.hadoop.security.token.Token; import com.google.common.collect.Lists; @@ -51,14 +50,14 @@ public class LocatedBlock { // else false. If block has few corrupt replicas, they are filtered and // their locations are not part of this object private boolean corrupt; - private TokenBlockTokenIdentifier blockToken = new TokenBlockTokenIdentifier(); + private TokenBlockTokenIdentifier blockToken = new Token(); /** * List of cached datanode locations */ private DatanodeInfo[] cachedLocs; // Used when there are no locations - private static final DatanodeInfoWithStorage[] EMPTY_LOCS = + static final DatanodeInfoWithStorage[] EMPTY_LOCS = new DatanodeInfoWithStorage[0]; public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e853a67/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedStripedBlock.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedStripedBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedStripedBlock.java new file mode 100644 index 000..97e3a69 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedStripedBlock.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by
[08/51] [abbrv] hadoop git commit: HDFS-7347. Configurable erasure coding policy for individual files and directories ( Contributed by Zhe Zhang )
HDFS-7347. Configurable erasure coding policy for individual files and directories ( Contributed by Zhe Zhang ) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d552fda8 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d552fda8 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d552fda8 Branch: refs/heads/HDFS-7285 Commit: d552fda896fd19880f902c08814e8d898f9b3109 Parents: 28bebc8 Author: Vinayakumar B vinayakum...@apache.org Authored: Thu Nov 6 10:03:26 2014 +0530 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 09:31:54 2015 -0700 -- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 4 ++ .../hadoop/hdfs/protocol/HdfsConstants.java | 2 + .../BlockStoragePolicySuite.java| 5 ++ .../hadoop/hdfs/TestBlockStoragePolicy.java | 12 +++- .../TestBlockInitialEncoding.java | 75 5 files changed, 95 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d552fda8/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt new file mode 100644 index 000..2ef8527 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -0,0 +1,4 @@ + BREAKDOWN OF HDFS-7285 SUBTASKS AND RELATED JIRAS + +HDFS-7347. Configurable erasure coding policy for individual files and +directories ( Zhe Zhang via vinayakumarb ) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/d552fda8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java index 7cf8a47..54c650b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java @@ -171,6 +171,7 @@ public class HdfsConstants { public static final String ONESSD_STORAGE_POLICY_NAME = ONE_SSD; public static final String HOT_STORAGE_POLICY_NAME = HOT; public static final String WARM_STORAGE_POLICY_NAME = WARM; + public static final String EC_STORAGE_POLICY_NAME = EC; public static final String COLD_STORAGE_POLICY_NAME = COLD; public static final byte MEMORY_STORAGE_POLICY_ID = 15; @@ -178,5 +179,6 @@ public class HdfsConstants { public static final byte ONESSD_STORAGE_POLICY_ID = 10; public static final byte HOT_STORAGE_POLICY_ID = 7; public static final byte WARM_STORAGE_POLICY_ID = 5; + public static final byte EC_STORAGE_POLICY_ID = 4; public static final byte COLD_STORAGE_POLICY_ID = 2; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d552fda8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java index 020cb5f..3d121cc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java @@ -78,6 +78,11 @@ public class BlockStoragePolicySuite { new StorageType[]{StorageType.DISK, StorageType.ARCHIVE}, new StorageType[]{StorageType.DISK, StorageType.ARCHIVE}, new StorageType[]{StorageType.DISK, StorageType.ARCHIVE}); +final byte ecId = HdfsConstants.EC_STORAGE_POLICY_ID; +policies[ecId] = new BlockStoragePolicy(ecId, +HdfsConstants.EC_STORAGE_POLICY_NAME, +new StorageType[]{StorageType.DISK}, StorageType.EMPTY_ARRAY, +new StorageType[]{StorageType.ARCHIVE}); final byte coldId = HdfsConstants.COLD_STORAGE_POLICY_ID; policies[coldId] = new BlockStoragePolicy(coldId, HdfsConstants.COLD_STORAGE_POLICY_NAME, http://git-wip-us.apache.org/repos/asf/hadoop/blob/d552fda8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java -- diff --git
[49/51] [abbrv] hadoop git commit: HDFS-7617. Add unit tests for editlog transactions for EC. Contributed by Hui Zheng.
HDFS-7617. Add unit tests for editlog transactions for EC. Contributed by Hui Zheng. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/86f579e1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/86f579e1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/86f579e1 Branch: refs/heads/HDFS-7285 Commit: 86f579e1f03d859b1fd3ed4508125a920bdda006 Parents: 18f3585 Author: Zhe Zhang z...@apache.org Authored: Tue Mar 31 10:46:04 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:21:04 2015 -0700 -- .../server/namenode/TestFSEditLogLoader.java| 157 +++ 1 file changed, 157 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/86f579e1/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java index 833ef95..d3cb749 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java @@ -39,14 +39,18 @@ import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped; +import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.test.PathUtils; import org.apache.log4j.Level; @@ -414,4 +418,157 @@ public class TestFSEditLogLoader { fromByte(code), FSEditLogOpCodes.fromByte(code)); } } + + @Test + public void testAddNewStripedBlock() throws IOException{ +// start a cluster +Configuration conf = new HdfsConfiguration(); +MiniDFSCluster cluster = null; +try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(9) + .build(); + cluster.waitActive(); + DistributedFileSystem fs = cluster.getFileSystem(); + FSNamesystem fns = cluster.getNamesystem(); + + String testDir = /ec; + String testFile = testfile_001; + String testFilePath = testDir + / + testFile; + String clientName = testUser1; + String clientMachine = testMachine1; + long blkId = 1; + long blkNumBytes = 1024; + long timestamp = 1426222918; + short blockNum = HdfsConstants.NUM_DATA_BLOCKS; + short parityNum = HdfsConstants.NUM_PARITY_BLOCKS; + + //set the storage policy of the directory + fs.mkdir(new Path(testDir), new FsPermission(755)); + fs.setStoragePolicy(new Path(testDir), + HdfsConstants.EC_STORAGE_POLICY_NAME); + + // Create a file with striped block + Path p = new Path(testFilePath); + DFSTestUtil.createFile(fs, p, 0, (short) 1, 1); + + fns.enterSafeMode(false); + fns.saveNamespace(0, 0); + fns.leaveSafeMode(); + + // Add a striped block to the file + BlockInfoStriped stripedBlk = new BlockInfoStriped( + new Block(blkId, blkNumBytes, timestamp), blockNum, parityNum); + INodeFile file = (INodeFile)fns.getFSDirectory().getINode(testFilePath); + file.toUnderConstruction(clientName, clientMachine); + file.getStripedBlocksFeature().addBlock(stripedBlk); + fns.getEditLog().logAddBlock(testFilePath, file); + file.toCompleteFile(System.currentTimeMillis()); + + //If the block by loaded is the same as above it means that + //we have successfully applied the edit log to the fsimage. + cluster.restartNameNodes(); + cluster.waitActive(); + fns = cluster.getNamesystem(); + + INodeFile inodeLoaded = (INodeFile)fns.getFSDirectory() + .getINode(testFilePath); + + assertTrue(inodeLoaded.isWithStripedBlocks()); + + BlockInfoStriped[] blks = (BlockInfoStriped[])inodeLoaded.getBlocks(); + assertEquals(1,
[11/51] [abbrv] hadoop git commit: HDFS-7652. Process block reports for erasure coded blocks. Contributed by Zhe Zhang
HDFS-7652. Process block reports for erasure coded blocks. Contributed by Zhe Zhang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7e39e00c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7e39e00c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7e39e00c Branch: refs/heads/HDFS-7285 Commit: 7e39e00c1012b877a7b8b29adf8e5a7250f542b5 Parents: 2b2859e Author: Zhe Zhang z...@apache.org Authored: Mon Feb 9 10:27:14 2015 -0800 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:01:04 2015 -0700 -- .../server/blockmanagement/BlockIdManager.java| 8 .../hdfs/server/blockmanagement/BlockManager.java | 18 +- 2 files changed, 21 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e39e00c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java index c8b9d20..e7f8a05 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java @@ -211,4 +211,12 @@ public class BlockIdManager { .LAST_RESERVED_BLOCK_ID); generationStampV1Limit = GenerationStamp.GRANDFATHER_GENERATION_STAMP; } + + public static boolean isStripedBlockID(long id) { +return id 0; + } + + public static long convertToGroupID(long id) { +return id (~(HdfsConstants.MAX_BLOCKS_IN_GROUP - 1)); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e39e00c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 524afa0..97dd2c3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1925,7 +1925,7 @@ public class BlockManager { break; } -BlockInfoContiguous bi = blocksMap.getStoredBlock(b); +BlockInfoContiguous bi = getStoredBlock(b); if (bi == null) { if (LOG.isDebugEnabled()) { LOG.debug(BLOCK* rescanPostponedMisreplicatedBlocks: + @@ -2068,7 +2068,7 @@ public class BlockManager { continue; } - BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(iblk); + BlockInfoContiguous storedBlock = getStoredBlock(iblk); // If block does not belong to any file, we are done. if (storedBlock == null) continue; @@ -2208,7 +2208,7 @@ public class BlockManager { } // find block by blockId -BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(block); +BlockInfoContiguous storedBlock = getStoredBlock(block); if(storedBlock == null) { // If blocksMap does not contain reported block id, // the replica should be removed from the data-node. @@ -2499,7 +2499,7 @@ public class BlockManager { DatanodeDescriptor node = storageInfo.getDatanodeDescriptor(); if (block instanceof BlockInfoContiguousUnderConstruction) { //refresh our copy in case the block got completed in another thread - storedBlock = blocksMap.getStoredBlock(block); + storedBlock = getStoredBlock(block); } else { storedBlock = block; } @@ -3361,7 +3361,15 @@ public class BlockManager { } public BlockInfoContiguous getStoredBlock(Block block) { -return blocksMap.getStoredBlock(block); +BlockInfoContiguous info = null; +if (BlockIdManager.isStripedBlockID(block.getBlockId())) { + info = blocksMap.getStoredBlock( + new Block(BlockIdManager.convertToGroupID(block.getBlockId(; +} +if (info == null) { + info = blocksMap.getStoredBlock(block); +} +return info; } /** updates a block in under replication queue */
[33/51] [abbrv] hadoop git commit: HADOOP-11707. Add factory to create raw erasure coder. Contributed by Kai Zheng
HADOOP-11707. Add factory to create raw erasure coder. Contributed by Kai Zheng Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c6d2f21b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c6d2f21b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c6d2f21b Branch: refs/heads/HDFS-7285 Commit: c6d2f21b97851b7b658ac8ba6dbebc726cff88dc Parents: 527ddbd Author: Kai Zheng kai.zh...@intel.com Authored: Fri Mar 20 15:07:00 2015 +0800 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:21:00 2015 -0700 -- .../hadoop-common/CHANGES-HDFS-EC-7285.txt | 3 +- .../rawcoder/JRSRawErasureCoderFactory.java | 34 ++ .../rawcoder/RawErasureCoderFactory.java| 38 .../rawcoder/XorRawErasureCoderFactory.java | 34 ++ 4 files changed, 108 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6d2f21b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt index e27ff5c..f566f0e 100644 --- a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt @@ -24,4 +24,5 @@ HADOOP-11706. Refine a little bit erasure coder API. Contributed by Kai Zheng ( Kai Zheng ) - +HADOOP-11707. Add factory to create raw erasure coder. Contributed by Kai Zheng +( Kai Zheng ) http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6d2f21b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/JRSRawErasureCoderFactory.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/JRSRawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/JRSRawErasureCoderFactory.java new file mode 100644 index 000..d6b40aa --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/JRSRawErasureCoderFactory.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.rawcoder; + +/** + * A raw coder factory for raw Reed-Solomon coder in Java. + */ +public class JRSRawErasureCoderFactory implements RawErasureCoderFactory { + + @Override + public RawErasureEncoder createEncoder() { +return new JRSRawEncoder(); + } + + @Override + public RawErasureDecoder createDecoder() { +return new JRSRawDecoder(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6d2f21b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java new file mode 100644 index 000..95a1cfe --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in
[36/51] [abbrv] hadoop git commit: HADOOP-11647. Reed-Solomon ErasureCoder. Contributed by Kai Zheng
HADOOP-11647. Reed-Solomon ErasureCoder. Contributed by Kai Zheng Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a129ddaf Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a129ddaf Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a129ddaf Branch: refs/heads/HDFS-7285 Commit: a129ddaffc26c3f3ba3a212b405ad7b6add16d10 Parents: c6d2f21 Author: Kai Zheng kai.zh...@intel.com Authored: Fri Mar 20 19:15:52 2015 +0800 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:21:01 2015 -0700 -- .../hadoop-common/CHANGES-HDFS-EC-7285.txt | 3 + .../hadoop/fs/CommonConfigurationKeys.java | 15 .../erasurecode/coder/AbstractErasureCoder.java | 65 ++ .../coder/AbstractErasureDecoder.java | 6 +- .../coder/AbstractErasureEncoder.java | 6 +- .../io/erasurecode/coder/RSErasureDecoder.java | 83 ++ .../io/erasurecode/coder/RSErasureEncoder.java | 47 ++ .../io/erasurecode/coder/XorErasureDecoder.java | 2 +- .../io/erasurecode/coder/XorErasureEncoder.java | 2 +- .../erasurecode/coder/TestRSErasureCoder.java | 92 10 files changed, 315 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a129ddaf/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt index f566f0e..b69e69a 100644 --- a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt @@ -26,3 +26,6 @@ HADOOP-11707. Add factory to create raw erasure coder. Contributed by Kai Zheng ( Kai Zheng ) + +HADOOP-11647. Reed-Solomon ErasureCoder. Contributed by Kai Zheng +( Kai Zheng ) http://git-wip-us.apache.org/repos/asf/hadoop/blob/a129ddaf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java index 7575496..70fea01 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java @@ -135,6 +135,21 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic { false; /** + * Erasure Coding configuration family + */ + + /** Supported erasure codec classes */ + public static final String IO_ERASURECODE_CODECS_KEY = io.erasurecode.codecs; + + /** Use XOR raw coder when possible for the RS codec */ + public static final String IO_ERASURECODE_CODEC_RS_USEXOR_KEY = + io.erasurecode.codec.rs.usexor; + + /** Raw coder factory for the RS codec */ + public static final String IO_ERASURECODE_CODEC_RS_RAWCODER_KEY = + io.erasurecode.codec.rs.rawcoder; + + /** * Service Authorization */ public static final String http://git-wip-us.apache.org/repos/asf/hadoop/blob/a129ddaf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java index 8d3bc34..0e4de89 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java @@ -17,7 +17,12 @@ */ package org.apache.hadoop.io.erasurecode.coder; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoder; +import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory; +import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; +import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; /** * A common class of basic facilities to be shared by encoder and decoder @@ -31,6 +36,66 @@ public abstract class AbstractErasureCoder private int numParityUnits; private int chunkSize; + /** + * Create raw decoder using the factory specified by
[43/51] [abbrv] hadoop git commit: HDFS-8005. Erasure Coding: simplify striped block recovery work computation and add tests. Contributed by Jing Zhao.
HDFS-8005. Erasure Coding: simplify striped block recovery work computation and add tests. Contributed by Jing Zhao. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/62d49a84 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/62d49a84 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/62d49a84 Branch: refs/heads/HDFS-7285 Commit: 62d49a84d13a3ff7f63281a318a1940acee64743 Parents: 0767bd7 Author: Jing Zhao ji...@apache.org Authored: Mon Mar 30 13:35:36 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:21:03 2015 -0700 -- .../server/blockmanagement/BlockManager.java| 138 +--- .../blockmanagement/DatanodeDescriptor.java | 14 +- .../hadoop/hdfs/server/namenode/INodeFile.java | 1 + .../blockmanagement/TestBlockManager.java | 33 +-- .../TestRecoverStripedBlocks.java | 107 -- .../server/namenode/TestAddStripedBlocks.java | 2 +- .../namenode/TestRecoverStripedBlocks.java | 210 +++ 7 files changed, 292 insertions(+), 213 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/62d49a84/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index e05330c..f42adcd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -538,7 +538,7 @@ public class BlockManager { // source node returned is not used chooseSourceDatanodes(getStoredBlock(block), containingNodes, containingLiveReplicasNodes, numReplicas, -new LinkedListShort(), 1, UnderReplicatedBlocks.LEVEL); +new LinkedListShort(), UnderReplicatedBlocks.LEVEL); // containingLiveReplicasNodes can include READ_ONLY_SHARED replicas which are // not included in the numReplicas.liveReplicas() count @@ -1376,7 +1376,7 @@ public class BlockManager { int computeRecoveryWorkForBlocks(ListListBlockInfo blocksToRecover) { int requiredReplication, numEffectiveReplicas; ListDatanodeDescriptor containingNodes; -BlockCollection bc = null; +BlockCollection bc; int additionalReplRequired; int scheduledWork = 0; @@ -1404,13 +1404,10 @@ public class BlockManager { containingNodes = new ArrayList(); ListDatanodeStorageInfo liveReplicaNodes = new ArrayList(); NumberReplicas numReplicas = new NumberReplicas(); -ListShort missingBlockIndices = new LinkedList(); -DatanodeDescriptor[] srcNodes; -int numSourceNodes = bc.isStriped() ? -HdfsConstants.NUM_DATA_BLOCKS : 1; -srcNodes = chooseSourceDatanodes( -block, containingNodes, liveReplicaNodes, numReplicas, -missingBlockIndices, numSourceNodes, priority); +ListShort liveBlockIndices = new ArrayList(); +final DatanodeDescriptor[] srcNodes = chooseSourceDatanodes(block, +containingNodes, liveReplicaNodes, numReplicas, +liveBlockIndices, priority); if(srcNodes == null || srcNodes.length == 0) { // block can not be replicated from any node LOG.debug(Block + block + cannot be recovered + @@ -1442,15 +1439,14 @@ public class BlockManager { } else { additionalReplRequired = 1; // Needed on a new rack } -if (bc.isStriped()) { +if (block.isStriped()) { + short[] indices = new short[liveBlockIndices.size()]; + for (int i = 0 ; i liveBlockIndices.size(); i++) { +indices[i] = liveBlockIndices.get(i); + } ErasureCodingWork ecw = new ErasureCodingWork(block, bc, srcNodes, containingNodes, liveReplicaNodes, additionalReplRequired, - priority); - short[] missingBlockArray = new short[missingBlockIndices.size()]; - for (int i = 0 ; i missingBlockIndices.size(); i++) { -missingBlockArray[i] = missingBlockIndices.get(i); - } - ecw.setMissingBlockIndices(missingBlockArray); + priority, indices); recovWork.add(ecw); } else { recovWork.add(new ReplicationWork(block, bc,
[24/51] [abbrv] hadoop git commit: HADOOP-11705. Make erasure coder configurable. Contributed by Kai Zheng
HADOOP-11705. Make erasure coder configurable. Contributed by Kai Zheng Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db8a83ac Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db8a83ac Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db8a83ac Branch: refs/heads/HDFS-7285 Commit: db8a83ac0027804d069aecb89f7460c1ff8740b4 Parents: 2e853a6 Author: drankye kai.zh...@intel.com Authored: Thu Mar 12 23:35:22 2015 +0800 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:20:58 2015 -0700 -- .../hadoop-common/CHANGES-HDFS-EC-7285.txt | 4 +++ .../erasurecode/coder/AbstractErasureCoder.java | 5 ++- .../rawcoder/AbstractRawErasureCoder.java | 5 ++- .../hadoop/io/erasurecode/TestCoderBase.java| 6 .../erasurecode/coder/TestErasureCoderBase.java | 36 +--- .../erasurecode/rawcoder/TestRawCoderBase.java | 13 +-- 6 files changed, 60 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/db8a83ac/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt index c17a1bd..a97dc34 100644 --- a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt @@ -18,3 +18,7 @@ HADOOP-11646. Erasure Coder API for encoding and decoding of block group ( Kai Zheng via vinayakumarb ) +HADOOP-11705. Make erasure coder configurable. Contributed by Kai Zheng +( Kai Zheng ) + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/db8a83ac/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java index f2cc041..8d3bc34 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java @@ -17,12 +17,15 @@ */ package org.apache.hadoop.io.erasurecode.coder; +import org.apache.hadoop.conf.Configured; + /** * A common class of basic facilities to be shared by encoder and decoder * * It implements the {@link ErasureCoder} interface. */ -public abstract class AbstractErasureCoder implements ErasureCoder { +public abstract class AbstractErasureCoder +extends Configured implements ErasureCoder { private int numDataUnits; private int numParityUnits; http://git-wip-us.apache.org/repos/asf/hadoop/blob/db8a83ac/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java index 74d2ab6..e6f3d92 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java @@ -17,12 +17,15 @@ */ package org.apache.hadoop.io.erasurecode.rawcoder; +import org.apache.hadoop.conf.Configured; + /** * A common class of basic facilities to be shared by encoder and decoder * * It implements the {@link RawErasureCoder} interface. */ -public abstract class AbstractRawErasureCoder implements RawErasureCoder { +public abstract class AbstractRawErasureCoder +extends Configured implements RawErasureCoder { private int numDataUnits; private int numParityUnits; http://git-wip-us.apache.org/repos/asf/hadoop/blob/db8a83ac/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java -- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java index 3c4288c..194413a 100644 ---
[28/51] [abbrv] hadoop git commit: Fixed a compiling issue introduced by HADOOP-11705.
Fixed a compiling issue introduced by HADOOP-11705. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aac871d3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aac871d3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aac871d3 Branch: refs/heads/HDFS-7285 Commit: aac871d3d5f3e3732573946745fbc7d576e79db0 Parents: db8a83a Author: Kai Zheng kai.zh...@intel.com Authored: Fri Mar 13 00:13:06 2015 +0800 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:20:59 2015 -0700 -- .../apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/aac871d3/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java -- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java index 36e061a..d911db9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java @@ -162,7 +162,7 @@ public abstract class TestErasureCoderBase extends TestCoderBase { } encoder.initialize(numDataUnits, numParityUnits, chunkSize); -encoder.setConf(conf); +((AbstractErasureCoder)encoder).setConf(conf); return encoder; } @@ -179,7 +179,7 @@ public abstract class TestErasureCoderBase extends TestCoderBase { } decoder.initialize(numDataUnits, numParityUnits, chunkSize); -decoder.setConf(conf); +((AbstractErasureCoder)decoder).setConf(conf); return decoder; }
[32/51] [abbrv] hadoop git commit: HDFS-7369. Erasure coding: distribute recovery work for striped blocks to DataNode. Contributed by Zhe Zhang.
HDFS-7369. Erasure coding: distribute recovery work for striped blocks to DataNode. Contributed by Zhe Zhang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/527ddbd3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/527ddbd3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/527ddbd3 Branch: refs/heads/HDFS-7285 Commit: 527ddbd3843ae4fb85eedba3ccbb6df48cc6bee0 Parents: 08a4c9e Author: Zhe Zhang z...@apache.org Authored: Wed Mar 18 15:52:36 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:21:00 2015 -0700 -- .../server/blockmanagement/BlockCollection.java | 5 + .../server/blockmanagement/BlockManager.java| 294 +-- .../blockmanagement/DatanodeDescriptor.java | 72 - .../server/blockmanagement/DatanodeManager.java | 20 +- .../hadoop/hdfs/server/namenode/INodeFile.java | 9 +- .../server/protocol/BlockECRecoveryCommand.java | 63 .../hdfs/server/protocol/DatanodeProtocol.java | 1 + .../blockmanagement/BlockManagerTestUtil.java | 2 +- .../blockmanagement/TestBlockManager.java | 22 +- .../TestRecoverStripedBlocks.java | 107 +++ 10 files changed, 486 insertions(+), 109 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/527ddbd3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java index 440a081..50dd17b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java @@ -86,4 +86,9 @@ public interface BlockCollection { * @return whether the block collection is under construction. */ public boolean isUnderConstruction(); + + /** + * @return whether the block collection is in striping format + */ + public boolean isStriped(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/527ddbd3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 5cbad40..291347b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -42,6 +42,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HAUtil; @@ -531,9 +532,9 @@ public class BlockManager { NumberReplicas numReplicas = new NumberReplicas(); // source node returned is not used -chooseSourceDatanode(block, containingNodes, +chooseSourceDatanodes(getStoredBlock(block), containingNodes, containingLiveReplicasNodes, numReplicas, -UnderReplicatedBlocks.LEVEL); +new LinkedListShort(), 1, UnderReplicatedBlocks.LEVEL); // containingLiveReplicasNodes can include READ_ONLY_SHARED replicas which are // not included in the numReplicas.liveReplicas() count @@ -1327,15 +1328,15 @@ public class BlockManager { } /** - * Scan blocks in {@link #neededReplications} and assign replication - * work to data-nodes they belong to. + * Scan blocks in {@link #neededReplications} and assign recovery + * (replication or erasure coding) work to data-nodes they belong to. * * The number of process blocks equals either twice the number of live * data-nodes or the number of under-replicated blocks whichever is less. * * @return number of blocks scheduled for replication during this iteration. */ - int computeReplicationWork(int blocksToProcess) { + int computeBlockRecoveryWork(int blocksToProcess) { ListListBlockInfo blocksToReplicate = null; namesystem.writeLock(); try { @@ -1345,30 +1346,32 @@ public class
[51/51] [abbrv] hadoop git commit: HDFS-7936. Erasure coding: resolving conflicts in the branch when merging trunk changes (this commit is for HDFS-8035). Contributed by Zhe Zhang
HDFS-7936. Erasure coding: resolving conflicts in the branch when merging trunk changes (this commit is for HDFS-8035). Contributed by Zhe Zhang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e6ecbaaf Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e6ecbaaf Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e6ecbaaf Branch: refs/heads/HDFS-7285 Commit: e6ecbaafd940824bd0c9d0f20b524d70f7455c1c Parents: bdb67d9 Author: Zhe Zhang z...@apache.org Authored: Mon Apr 6 10:37:23 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:37:23 2015 -0700 -- .../hadoop/hdfs/server/blockmanagement/BlockManager.java | 11 +-- .../apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 10 +- 2 files changed, 10 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6ecbaaf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index bce789a..d30e623 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -3529,13 +3529,12 @@ public class BlockManager { String src, BlockInfo[] blocks) { for (BlockInfo b: blocks) { if (!b.isComplete()) { -final BlockInfoContiguousUnderConstruction uc = -(BlockInfoContiguousUnderConstruction)b; final int numNodes = b.numNodes(); -LOG.info(BLOCK* + b + is not COMPLETE (ucState = - + uc.getBlockUCState() + , replication# = + numNodes - + (numNodes minReplication ? : = ) - + minimum = + minReplication + ) in file + src); +final int min = getMinStorageNum(b); +final BlockUCState state = b.getBlockUCState(); +LOG.info(BLOCK* + b + is not COMPLETE (ucState = + state ++ , replication# = + numNodes + (numNodes min ?: = ) ++ minimum = + min + ) in file + src); return false; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6ecbaaf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 6fb258a..009e5a8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -3100,7 +3100,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, // Part II. // Allocate a new block, add it to the INode and the BlocksMap. -BlockInfo newBlockInfo = null; +Block newBlock = null; long offset; checkOperation(OperationCategory.WRITE); waitForLoadingFSImage(); @@ -3133,8 +3133,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, ExtendedBlock.getLocalBlock(previous)); // allocate new block, record block locations in INode. - Block newBlock = createNewBlock(isStriped); - newBlockInfo = saveAllocatedBlock(src, fileState.iip, newBlock, targets, + newBlock = createNewBlock(isStriped); + saveAllocatedBlock(src, fileState.iip, newBlock, targets, isStriped); persistNewBlock(src, pendingFile); @@ -3145,7 +3145,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, getEditLog().logSync(); // Return located block -return makeLocatedBlock(newBlockInfo, targets, offset); +return makeLocatedBlock(getStoredBlock(newBlock), targets, offset); } /* @@ -3602,7 +3602,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, BlockInfo b = v.getPenultimateBlock(); return b == null || blockManager.checkBlocksProperlyReplicated( -src, new BlockInfoContiguous[] { b }); +src, new BlockInfo[] { b }); } } finally { readUnlock();
[47/51] [abbrv] hadoop git commit: HADOOP-11782 Correct two thrown messages in ECSchema class. Contributed by Xinwei Qin
HADOOP-11782 Correct two thrown messages in ECSchema class. Contributed by Xinwei Qin Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0800245d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0800245d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0800245d Branch: refs/heads/HDFS-7285 Commit: 0800245d16f71e919a944b0548622353a75f1a87 Parents: 86f579e Author: Kai Zheng kai.zh...@intel.com Authored: Thu Apr 2 05:12:35 2015 +0800 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:21:04 2015 -0700 -- hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt | 3 +++ .../src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0800245d/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt index b69e69a..01280db 100644 --- a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt @@ -29,3 +29,6 @@ HADOOP-11647. Reed-Solomon ErasureCoder. Contributed by Kai Zheng ( Kai Zheng ) + +HADOOP-11782 Correct two thrown messages in ECSchema class. Contributed by Xinwei Qin +( Xinwei Qin via Kai Zheng ) http://git-wip-us.apache.org/repos/asf/hadoop/blob/0800245d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java index 8dc3f45..27be00e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java @@ -64,7 +64,7 @@ public class ECSchema { } } catch (NumberFormatException e) { throw new IllegalArgumentException(Option value + - options.get(CHUNK_SIZE_KEY) + for + CHUNK_SIZE_KEY + + options.get(NUM_DATA_UNITS_KEY) + for + NUM_DATA_UNITS_KEY + is found. It should be an integer); } @@ -74,7 +74,7 @@ public class ECSchema { } } catch (NumberFormatException e) { throw new IllegalArgumentException(Option value + - options.get(CHUNK_SIZE_KEY) + for + CHUNK_SIZE_KEY + + options.get(NUM_PARITY_UNITS_KEY) + for + NUM_PARITY_UNITS_KEY + is found. It should be an integer); }
[44/51] [abbrv] hadoop git commit: HDFS-8027. Erasure Coding: Update CHANGES-HDFS-7285.txt with branch commits (Vinayakumar B)
HDFS-8027. Erasure Coding: Update CHANGES-HDFS-7285.txt with branch commits (Vinayakumar B) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/18f35854 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/18f35854 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/18f35854 Branch: refs/heads/HDFS-7285 Commit: 18f35854a5196c8c1676a9b4eb5cd268e65c4fb4 Parents: 62d49a8 Author: Vinayakumar B vinayakuma...@intel.com Authored: Tue Mar 31 15:12:09 2015 +0530 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:21:03 2015 -0700 -- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 40 +++- 1 file changed, 39 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/18f35854/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 21e4c03..a686315 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -3,6 +3,44 @@ HDFS-7347. Configurable erasure coding policy for individual files and directories ( Zhe Zhang via vinayakumarb ) -HDFS-7716. Add a test for BlockGroup support in FSImage. +HDFS-7339. Representing striped block groups in NameNode with hierarchical +naming protocol ( Zhe Zhang ) + +HDFS-7652. Process block reports for erasure coded blocks (Zhe Zhang) + +HDFS-7716. Erasure Coding: extend BlockInfo to handle EC info (Jing Zhao) + +HDFS-7749. Erasure Coding: Add striped block support in INodeFile (Jing Zhao) + +HDFS-7837. Erasure Coding: allocate and persist striped blocks in NameNode +(Jing Zhao via Zhe Zhang) + +HDFS-7872. Erasure Coding: INodeFile.dumpTreeRecursively() supports to print +striped blocks (Takuya Fukudome via jing9) + +HDFS-7853. Erasure coding: extend LocatedBlocks to support reading from +striped files (Jing Zhao) + +HDFS-7826. Erasure Coding: Update INodeFile quota computation for striped +blocks ( Kai Sasaki via jing9 ) + +HDFS-7912. Erasure Coding: track BlockInfo instead of Block in +UnderReplicatedBlocks and PendingReplicationBlocks (Jing Zhao) + +HDFS-7369. Erasure coding: distribute recovery work for striped blocks to +DataNode (Zhe Zhang) + +HDFS-7864. Erasure Coding: Update safemode calculation for striped blocks +(GAO Rui via jing9) + +HDFS-7827. Erasure Coding: support striped blocks in non-protobuf fsimage +( Hui Zheng via jing9 ) + +HDFS-7616. Add a test for BlockGroup support in FSImage. (Takuya Fukudome via szetszwo) +HDFS-7907. Erasure Coding: track invalid, corrupt, and under-recovery striped +blocks in NameNode (Jing Zhao) + +HDFS-8005. Erasure Coding: simplify striped block recovery work computation +and add tests (Jing Zhao) \ No newline at end of file
[02/51] [abbrv] hadoop git commit: HADOOP-11776. Fixed the broken JDiff support in Hadoop 2. Contributed by Li Lu.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b3948ea/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.6.0.xml -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.6.0.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.6.0.xml new file mode 100644 index 000..5514700 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.6.0.xml @@ -0,0 +1,19520 @@ +?xml version=1.0 encoding=iso-8859-1 standalone=no? +!-- Generated by the JDiff Javadoc doclet -- +!-- (http://www.jdiff.org) -- +!-- on Mon Mar 30 15:30:43 PDT 2015 -- + +api + xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' + xsi:noNamespaceSchemaLocation='api.xsd' + name=hadoop-hdfs 2.6.0 + jdversion=1.0.9 + +!-- Command line arguments = -doclet org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet -docletpath /Users/llu/hadoop-common/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-annotations.jar:/Users/llu/hadoop-common/hadoop-hdfs-project/hadoop-hdfs/target/jdiff.jar -verbose -classpath /Users/llu/hadoop-common/hadoop-hdfs-project/hadoop-hdfs/target/classes:/Users/llu/hadoop-common/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-2.6.0.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_67.jdk/Contents/Home/lib/tools.jar:/Users/llu/hadoop-common/hadoop-common-project/hadoop-auth/target/hadoop-auth-2.6.0.jar:/Users/llu/.m2/repository/org/slf4j/slf4j-api/1.7.5/slf4j-api-1.7.5.jar:/Users/llu/.m2/repository/org/apache/httpcomponents/httpclient/4.2.5/httpclient-4.2.5.jar:/Users/llu/.m2/repository/org/apache/httpcomponents/httpcore/4.2.5/httpcore-4.2.5.jar:/Users/llu/.m2/repository/org/apache/directory/server/apacheds-kerberos-codec/2.0.0-M15/apacheds-ker beros-codec-2.0.0-M15.jar:/Users/llu/.m2/repository/org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar:/Users/llu/.m2/repository/org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar:/Users/llu/.m2/repository/org/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar:/Users/llu/.m2/repository/org/apache/zookeeper/zookeeper/3.4.6/zookeeper-3.4.6.jar:/Users/llu/.m2/repository/org/apache/curator/curator-framework/2.6.0/curator-framework-2.6.0.jar:/Users/llu/hadoop-common/hadoop-common-project/hadoop-common/target/hadoop-common-2.6.0.jar:/Users/llu/.m2/repository/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/Users/llu/.m2/repository/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar:/Users/llu/.m2/repository/commons-net/commons-net/3.1/commons-net-3.1.jar:/Users/llu/.m2/repository/commons-collections/commons-collections/3.2.1/commons-collections-3.2.1.jar:/Users/llu/.m2/repository/com/sun/jerse y/jersey-json/1.9/jersey-json-1.9.jar:/Users/llu/.m2/repository/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/Users/llu/.m2/repository/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/Users/llu/.m2/repository/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar:/Users/llu/.m2/repository/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar:/Users/llu/.m2/repository/javax/activation/activation/1.1/activation-1.1.jar:/Users/llu/.m2/repository/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/Users/llu/.m2/repository/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/Users/llu/.m2/repository/tomcat/jasper-compiler/5.5.23/jasper-compiler-5.5.23.jar:/Users/llu/.m2/repository/commons-el/commons-el/1.0/commons-el-1.0.jar:/Users/llu/.m2/repository/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar:/Users/llu/.m2/repository/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.jar:/Users/llu/.m2/repository/commons-configuration/commons-configuratio n/1.6/commons-configuration-1.6.jar:/Users/llu/.m2/repository/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/Users/llu/.m2/repository/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar:/Users/llu/.m2/repository/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar:/Users/llu/.m2/repository/org/apache/avro/avro/1.7.4/avro-1.7.4.jar:/Users/llu/.m2/repository/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/Users/llu/.m2/repository/org/xerial/snappy/snappy-java/1.0.4.1/snappy-java-1.0.4.1.jar:/Users/llu/.m2/repository/com/google/code/gson/gson/2.2.4/gson-2.2.4.jar:/Users/llu/.m2/repository/com/jcraft/jsch/0.1.42/jsch-0.1.42.jar:/Users/llu/.m2/repository/org/apache/curator/curator-client/2.6.0/curator-client-2.6.0.jar:/Users/llu/.m2/repository/org/apache/curator/curator-recipes/2.6.0/curator-recipes-2.6.0.jar:/Users/llu/.m2/repository/com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar:/Users/llu/.m2/repository/
[05/51] [abbrv] hadoop git commit: YARN-3435. AM container to be allocated Appattempt AM container shown as null. Contributed by Bibin A Chundatt
YARN-3435. AM container to be allocated Appattempt AM container shown as null. Contributed by Bibin A Chundatt Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96d72118 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96d72118 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96d72118 Branch: refs/heads/HDFS-7285 Commit: 96d72118f5c81aa4e0dca0dc0241fbf1a3fff4d2 Parents: 4b3948e Author: Xuan xg...@apache.org Authored: Sun Apr 5 00:40:57 2015 -0700 Committer: Xuan xg...@apache.org Committed: Sun Apr 5 00:40:57 2015 -0700 -- hadoop-yarn-project/CHANGES.txt | 3 +++ .../yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/96d72118/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 594833d..b1f8913 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -142,6 +142,9 @@ Release 2.8.0 - UNRELEASED YARN-3415. Non-AM containers can be counted towards amResourceUsage of a Fair Scheduler queue (Zhihai Xu via Sandy Ryza) +YARN-3435. AM container to be allocated Appattempt AM container shown as null. +(Bibin A Chundatt via xgong) + Release 2.7.0 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/96d72118/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java index 1831920..506e31f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java @@ -209,7 +209,7 @@ public class RMAppAttemptBlock extends AppAttemptBlock{ AM Container:, appAttempt.getAmContainerId() == null || containers == null || !hasAMContainer(appAttemptReport.getAMContainerId(), containers) -? null : root_url(container, appAttempt.getAmContainerId()), +? N/A : root_url(container, appAttempt.getAmContainerId()), String.valueOf(appAttempt.getAmContainerId())) ._(Node:, node) ._(
[10/51] [abbrv] hadoop git commit: Fix Compilation Error in TestAddBlockgroup.java after the merge
Fix Compilation Error in TestAddBlockgroup.java after the merge Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dd5fd7b6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dd5fd7b6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dd5fd7b6 Branch: refs/heads/HDFS-7285 Commit: dd5fd7b612241a371d573c60714f0f11de97dbf7 Parents: 7e39e00 Author: Jing Zhao ji...@apache.org Authored: Sun Feb 8 16:01:03 2015 -0800 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:01:04 2015 -0700 -- .../apache/hadoop/hdfs/server/namenode/TestAddBlockgroup.java| 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd5fd7b6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockgroup.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockgroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockgroup.java index 95133ce..06dfade 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockgroup.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockgroup.java @@ -26,7 +26,7 @@ import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -75,7 +75,7 @@ public class TestAddBlockgroup { final Path file1 = new Path(/file1); DFSTestUtil.createFile(fs, file1, BLOCKSIZE * 2, REPLICATION, 0L); INodeFile file1Node = fsdir.getINode4Write(file1.toString()).asFile(); -BlockInfo[] file1Blocks = file1Node.getBlocks(); +BlockInfoContiguous[] file1Blocks = file1Node.getBlocks(); assertEquals(2, file1Blocks.length); assertEquals(GROUP_SIZE, file1Blocks[0].numNodes()); assertEquals(HdfsConstants.MAX_BLOCKS_IN_GROUP,
[01/51] [abbrv] hadoop git commit: HADOOP-11776. Fixed the broken JDiff support in Hadoop 2. Contributed by Li Lu.
Repository: hadoop Updated Branches: refs/heads/HDFS-7285 8525fdc5a - e6ecbaafd (forced update) http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b3948ea/hadoop-project-dist/pom.xml -- diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml index 33f5a04..d206593 100644 --- a/hadoop-project-dist/pom.xml +++ b/hadoop-project-dist/pom.xml @@ -165,8 +165,10 @@ activeByDefaultfalse/activeByDefault /activation properties -jdiff.stable.api0.20.2/jdiff.stable.api +jdiff.stable.api2.6.0/jdiff.stable.api jdiff.stability-unstable/jdiff.stability +!-- Commented out for HADOOP-11776 -- +!-- Uncomment param name=${jdiff.compatibility} in javadoc doclet if compatibility is not empty -- jdiff.compatibility/jdiff.compatibility jdiff.javadoc.maxmemory512m/jdiff.javadoc.maxmemory /properties @@ -227,6 +229,14 @@ outputDirectory${project.build.directory}/outputDirectory destFileNamehadoop-annotations.jar/destFileName /artifactItem +artifactItem + groupIdxerces/groupId + artifactIdxercesImpl/artifactId + version${xerces.jdiff.version}/version + overWritefalse/overWrite + outputDirectory${project.build.directory}/outputDirectory + destFileNamexerces.jar/destFileName +/artifactItem /artifactItems /configuration /execution @@ -276,7 +286,7 @@ doclet name=org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet path=${project.build.directory}/hadoop-annotations.jar:${project.build.directory}/jdiff.jar param name=-apidir value=${project.build.directory}/site/jdiff/xml/ -param name=-apiname value=hadoop-core ${project.version}/ +param name=-apiname value=${project.name} ${project.version}/ param name=${jdiff.stability}/ /doclet packageset dir=${basedir}/src/main/java/ @@ -289,17 +299,17 @@ sourceFiles=${basedir}/dev-support/jdiff/Null.java maxmemory=${jdiff.javadoc.maxmemory} doclet name=org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet - path=${project.build.directory}/hadoop-annotations.jar:${project.build.directory}/jdiff.jar -param name=-oldapi value=hadoop-core ${jdiff.stable.api}/ -param name=-newapi value=hadoop-core ${project.version}/ + path=${project.build.directory}/hadoop-annotations.jar:${project.build.directory}/jdiff.jar:${project.build.directory}/xerces.jar +param name=-oldapi value=${project.name} ${jdiff.stable.api}/ +param name=-newapi value=${project.name} ${project.version}/ param name=-oldapidir value=${basedir}/dev-support/jdiff/ param name=-newapidir value=${project.build.directory}/site/jdiff/xml/ param name=-javadocold - value=http://hadoop.apache.org/docs/${jdiff.stable.api}/api// + value=http://hadoop.apache.org/docs/r${jdiff.stable.api}/api// param name=-javadocnew value=${project.build.directory}/site/api/ param name=-stats/ param name=${jdiff.stability}/ -param name=${jdiff.compatibility}/ +!--param name=${jdiff.compatibility}/-- /doclet classpath path refid=maven.compile.classpath/ http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b3948ea/hadoop-project/pom.xml -- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 739cae0..a59ec06 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -42,6 +42,8 @@ !-- These 2 versions are defined here becuase they are used -- !-- JDIFF generation from embedded ant in the antrun plugin -- jdiff.version1.0.9/jdiff.version +!-- Version number for xerces used by JDiff -- +xerces.jdiff.version2.11.0/xerces.jdiff.version hadoop.assemblies.version${project.version}/hadoop.assemblies.version
[42/51] [abbrv] hadoop git commit: HADOOP-11664. Loading predefined EC schemas from configuration. Contributed by Kai Zheng.
HADOOP-11664. Loading predefined EC schemas from configuration. Contributed by Kai Zheng. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a5097a48 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a5097a48 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a5097a48 Branch: refs/heads/HDFS-7285 Commit: a5097a481b0b8e1cc9f293882b2a5fa60075b6e1 Parents: d3ccfa9 Author: Zhe Zhang z...@apache.org Authored: Fri Mar 27 14:52:50 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:21:02 2015 -0700 -- .../src/main/conf/ecschema-def.xml | 40 + .../hadoop/fs/CommonConfigurationKeys.java | 5 + .../hadoop/io/erasurecode/SchemaLoader.java | 147 +++ .../hadoop/io/erasurecode/TestSchemaLoader.java | 80 ++ 4 files changed, 272 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5097a48/hadoop-common-project/hadoop-common/src/main/conf/ecschema-def.xml -- diff --git a/hadoop-common-project/hadoop-common/src/main/conf/ecschema-def.xml b/hadoop-common-project/hadoop-common/src/main/conf/ecschema-def.xml new file mode 100644 index 000..e619485 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/conf/ecschema-def.xml @@ -0,0 +1,40 @@ +?xml version=1.0? + +!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + License); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an AS IS BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-- + +!-- +Please define your EC schemas here. Note, once these schemas are loaded +and referenced by EC storage policies, any change to them will be ignored. +You can modify and remove those not used yet, or add new ones. +-- + +schemas + schema name=RS-6-3 +k6/k +m3/m +codecRS/codec + /schema + schema name=RS-10-4 +k10/k +m4/m +codecRS/codec + /schema +/schemas \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5097a48/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java index 70fea01..af32674 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java @@ -141,6 +141,11 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic { /** Supported erasure codec classes */ public static final String IO_ERASURECODE_CODECS_KEY = io.erasurecode.codecs; + public static final String IO_ERASURECODE_SCHEMA_FILE_KEY = + io.erasurecode.schema.file; + public static final String IO_ERASURECODE_SCHEMA_FILE_DEFAULT = + ecschema-def.xml; + /** Use XOR raw coder when possible for the RS codec */ public static final String IO_ERASURECODE_CODEC_RS_USEXOR_KEY = io.erasurecode.codec.rs.usexor; http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5097a48/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/SchemaLoader.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/SchemaLoader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/SchemaLoader.java new file mode 100644 index 000..c51ed37 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/SchemaLoader.java @@ -0,0 +1,147 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + *
[09/51] [abbrv] hadoop git commit: HDFS-7339. Allocating and persisting block groups in NameNode. Contributed by Zhe Zhang
HDFS-7339. Allocating and persisting block groups in NameNode. Contributed by Zhe Zhang Conflicts: hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Conflicts: hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2b2859e6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2b2859e6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2b2859e6 Branch: refs/heads/HDFS-7285 Commit: 2b2859e68314c626d1cac785c92ad7577425d3c4 Parents: d552fda Author: Zhe Zhang z...@apache.org Authored: Fri Jan 30 16:16:26 2015 -0800 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:00:42 2015 -0700 -- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 + .../hadoop/hdfs/protocol/HdfsConstants.java | 4 + .../server/blockmanagement/BlockIdManager.java | 8 +- .../SequentialBlockGroupIdGenerator.java| 82 +++ .../SequentialBlockIdGenerator.java | 6 +- .../hdfs/server/namenode/FSDirectory.java | 8 +- .../hdfs/server/namenode/FSNamesystem.java | 34 +--- .../hadoop/hdfs/server/namenode/INodeFile.java | 11 +++ .../hdfs/server/namenode/TestAddBlockgroup.java | 84 9 files changed, 223 insertions(+), 16 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2859e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 610932a..eff457c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -221,6 +221,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final int DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT = 3; public static final String DFS_NAMENODE_REPLICATION_MIN_KEY = dfs.namenode.replication.min; public static final int DFS_NAMENODE_REPLICATION_MIN_DEFAULT = 1; + public static final String DFS_NAMENODE_STRIPE_MIN_KEY = dfs.namenode.stripe.min; + public static final int DFS_NAMENODE_STRIPE_MIN_DEFAULT = 1; public static final String DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_KEY = dfs.namenode.replication.pending.timeout-sec; public static final int DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_DEFAULT = -1; public static final String DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY = dfs.namenode.replication.max-streams; http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2859e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java index 54c650b..de60b6e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java @@ -181,4 +181,8 @@ public class HdfsConstants { public static final byte WARM_STORAGE_POLICY_ID = 5; public static final byte EC_STORAGE_POLICY_ID = 4; public static final byte COLD_STORAGE_POLICY_ID = 2; + + public static final byte NUM_DATA_BLOCKS = 3; + public static final byte NUM_PARITY_BLOCKS = 2; + public static final byte MAX_BLOCKS_IN_GROUP = 16; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2859e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java index 1c69203..c8b9d20 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java @@ -53,10 +53,12 @@ public class
[17/51] [abbrv] hadoop git commit: HDFS-7716. Erasure Coding: extend BlockInfo to handle EC info. Contributed by Jing Zhao.
HDFS-7716. Erasure Coding: extend BlockInfo to handle EC info. Contributed by Jing Zhao. Conflicts: hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e3a48212 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e3a48212 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e3a48212 Branch: refs/heads/HDFS-7285 Commit: e3a48212e2ac6e1638eb06732645b809e32d1d0d Parents: 958e8c1 Author: Jing Zhao ji...@apache.org Authored: Tue Feb 10 17:54:10 2015 -0800 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:05:11 2015 -0700 -- .../hadoop/hdfs/protocol/HdfsConstants.java | 1 + .../server/blockmanagement/BlockCollection.java | 13 +- .../server/blockmanagement/BlockIdManager.java | 7 +- .../hdfs/server/blockmanagement/BlockInfo.java | 339 + .../blockmanagement/BlockInfoContiguous.java| 363 +++ .../BlockInfoContiguousUnderConstruction.java | 137 +-- .../blockmanagement/BlockInfoStriped.java | 179 + .../server/blockmanagement/BlockManager.java| 188 +- .../hdfs/server/blockmanagement/BlocksMap.java | 46 +-- .../CacheReplicationMonitor.java| 10 +- .../blockmanagement/DatanodeDescriptor.java | 22 +- .../blockmanagement/DatanodeStorageInfo.java| 38 +- .../ReplicaUnderConstruction.java | 119 ++ .../hdfs/server/namenode/FSDirectory.java | 4 +- .../hdfs/server/namenode/FSNamesystem.java | 20 +- .../hdfs/server/namenode/NamenodeFsck.java | 3 +- .../snapshot/FSImageFormatPBSnapshot.java | 4 +- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 4 +- .../server/blockmanagement/TestBlockInfo.java | 6 +- .../blockmanagement/TestBlockInfoStriped.java | 219 +++ .../blockmanagement/TestBlockManager.java | 4 +- .../blockmanagement/TestReplicationPolicy.java | 2 +- 22 files changed, 1122 insertions(+), 606 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3a48212/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java index de60b6e..245b630 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java @@ -184,5 +184,6 @@ public class HdfsConstants { public static final byte NUM_DATA_BLOCKS = 3; public static final byte NUM_PARITY_BLOCKS = 2; + public static final long BLOCK_GROUP_INDEX_MASK = 15; public static final byte MAX_BLOCKS_IN_GROUP = 16; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3a48212/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java index e9baf85..b14efb4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java @@ -39,12 +39,12 @@ public interface BlockCollection { public ContentSummary computeContentSummary(BlockStoragePolicySuite bsps); /** - * @return the number of blocks + * @return the number of blocks or block groups */ public int numBlocks(); /** - * Get the blocks. + * Get the blocks or block groups. */ public BlockInfoContiguous[] getBlocks(); @@ -55,8 +55,8 @@ public interface BlockCollection { public long getPreferredBlockSize(); /** - * Get block replication for the collection - * @return block replication value + * Get block replication for the collection. + * @return block replication value. Return 0 if the file is erasure coded. */ public short getBlockReplication(); @@ -71,7 +71,7 @@ public interface BlockCollection { public String getName(); /** - * Set the block at the given index. + * Set the block/block-group at the given index. */ public void setBlock(int index, BlockInfoContiguous blk); @@ -79,7
[34/51] [abbrv] hadoop git commit: HADOOP-11706 Refine a little bit erasure coder API
HADOOP-11706 Refine a little bit erasure coder API Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/37b917d1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/37b917d1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/37b917d1 Branch: refs/heads/HDFS-7285 Commit: 37b917d14968792aff4f6cead24c7d2ed4e97a91 Parents: ed82e68 Author: Kai Zheng kai.zh...@intel.com Authored: Wed Mar 18 19:21:37 2015 +0800 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:21:00 2015 -0700 -- .../io/erasurecode/coder/ErasureCoder.java | 4 +++- .../erasurecode/rawcoder/RawErasureCoder.java | 4 +++- .../hadoop/io/erasurecode/TestCoderBase.java| 17 +--- .../erasurecode/coder/TestErasureCoderBase.java | 21 +++- .../erasurecode/rawcoder/TestJRSRawCoder.java | 12 +-- .../erasurecode/rawcoder/TestRawCoderBase.java | 2 ++ 6 files changed, 31 insertions(+), 29 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/37b917d1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java index 68875c0..c5922f3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.io.erasurecode.coder; +import org.apache.hadoop.conf.Configurable; + /** * An erasure coder to perform encoding or decoding given a group. Generally it * involves calculating necessary internal steps according to codec logic. For @@ -31,7 +33,7 @@ package org.apache.hadoop.io.erasurecode.coder; * of multiple coding steps. * */ -public interface ErasureCoder { +public interface ErasureCoder extends Configurable { /** * Initialize with the important parameters for the code. http://git-wip-us.apache.org/repos/asf/hadoop/blob/37b917d1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java index 91a9abf..9af5b6c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.io.erasurecode.rawcoder; +import org.apache.hadoop.conf.Configurable; + /** * RawErasureCoder is a common interface for {@link RawErasureEncoder} and * {@link RawErasureDecoder} as both encoder and decoder share some properties. @@ -31,7 +33,7 @@ package org.apache.hadoop.io.erasurecode.rawcoder; * low level constructs, since it only takes care of the math calculation with * a group of byte buffers. */ -public interface RawErasureCoder { +public interface RawErasureCoder extends Configurable { /** * Initialize with the important parameters for the code. http://git-wip-us.apache.org/repos/asf/hadoop/blob/37b917d1/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java -- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java index 194413a..22fd98d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java @@ -17,11 +17,12 @@ */ package org.apache.hadoop.io.erasurecode; +import org.apache.hadoop.conf.Configuration; + import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Random; -import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertTrue; /** @@ -31,6 +32,7 @@ import static org.junit.Assert.assertTrue; public abstract class TestCoderBase { protected static Random RAND = new Random(); + private
[39/51] [abbrv] hadoop git commit: HDFS-7716. Add a test for BlockGroup support in FSImage. Contributed by Takuya Fukudome
HDFS-7716. Add a test for BlockGroup support in FSImage. Contributed by Takuya Fukudome Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d3ccfa98 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d3ccfa98 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d3ccfa98 Branch: refs/heads/HDFS-7285 Commit: d3ccfa987a3b089a7128b6c7ed1dd66e83662398 Parents: a4e6262 Author: Tsz-Wo Nicholas Sze szets...@hortonworks.com Authored: Wed Mar 25 19:01:03 2015 +0900 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:21:02 2015 -0700 -- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 6 ++- .../hdfs/server/namenode/TestFSImage.java | 53 2 files changed, 58 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3ccfa98/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 2ef8527..21e4c03 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -1,4 +1,8 @@ BREAKDOWN OF HDFS-7285 SUBTASKS AND RELATED JIRAS HDFS-7347. Configurable erasure coding policy for individual files and -directories ( Zhe Zhang via vinayakumarb ) \ No newline at end of file +directories ( Zhe Zhang via vinayakumarb ) + +HDFS-7716. Add a test for BlockGroup support in FSImage. +(Takuya Fukudome via szetszwo) + http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3ccfa98/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java index 71dc978..440f5cd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.namenode; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import java.io.File; @@ -31,7 +32,12 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.EnumSet; +import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.junit.Assert; @@ -46,6 +52,7 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSOutputStream; +import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; @@ -378,4 +385,50 @@ public class TestFSImage { FileUtil.fullyDelete(dfsDir); } } + + /** + * Ensure that FSImage supports BlockGroup. + */ + @Test + public void testSupportBlockGroup() throws IOException { +final short GROUP_SIZE = HdfsConstants.NUM_DATA_BLOCKS + +HdfsConstants.NUM_PARITY_BLOCKS; +final int BLOCK_SIZE = 8 * 1024 * 1024; +Configuration conf = new HdfsConfiguration(); +conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); +MiniDFSCluster cluster = null; +try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(GROUP_SIZE) + .build(); + cluster.waitActive(); + DistributedFileSystem fs = cluster.getFileSystem(); + fs.setStoragePolicy(new Path(/), HdfsConstants.EC_STORAGE_POLICY_NAME); + Path file = new Path(/striped); + FSDataOutputStream out = fs.create(file); + byte[] bytes = DFSTestUtil.generateSequentialBytes(0, BLOCK_SIZE); + out.write(bytes); + out.close(); + + fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + fs.saveNamespace(); + fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + + cluster.restartNameNodes(); + fs = cluster.getFileSystem(); +
[22/51] [abbrv] hadoop git commit: HADOOP-11643. Define EC schema API for ErasureCodec. Contributed by Kai Zheng
HADOOP-11643. Define EC schema API for ErasureCodec. Contributed by Kai Zheng Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d0de9b48 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d0de9b48 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d0de9b48 Branch: refs/heads/HDFS-7285 Commit: d0de9b48f001cfae3fd0b12b3ed77eacf6f63179 Parents: 1af3d5d Author: drankye kai.zh...@intel.com Authored: Thu Mar 5 22:51:52 2015 +0800 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:20:57 2015 -0700 -- .../hadoop-common/CHANGES-HDFS-EC-7285.txt | 4 + .../apache/hadoop/io/erasurecode/ECSchema.java | 203 +++ .../hadoop/io/erasurecode/TestECSchema.java | 54 + 3 files changed, 261 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0de9b48/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt index 7bbacf7..ee42c84 100644 --- a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt @@ -12,3 +12,7 @@ HADOOP-11542. Raw Reed-Solomon coder in pure Java. Contributed by Kai Zheng ( Kai Zheng ) +HADOOP-11643. Define EC schema API for ErasureCodec. Contributed by Kai Zheng +( Kai Zheng ) + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0de9b48/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java new file mode 100644 index 000..8dc3f45 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java @@ -0,0 +1,203 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode; + +import java.util.Collections; +import java.util.Map; + +/** + * Erasure coding schema to housekeeper relevant information. + */ +public class ECSchema { + public static final String NUM_DATA_UNITS_KEY = k; + public static final String NUM_PARITY_UNITS_KEY = m; + public static final String CODEC_NAME_KEY = codec; + public static final String CHUNK_SIZE_KEY = chunkSize; + public static final int DEFAULT_CHUNK_SIZE = 64 * 1024; // 64K + + private String schemaName; + private String codecName; + private MapString, String options; + private int numDataUnits; + private int numParityUnits; + private int chunkSize; + + /** + * Constructor with schema name and provided options. Note the options may + * contain additional information for the erasure codec to interpret further. + * @param schemaName schema name + * @param options schema options + */ + public ECSchema(String schemaName, MapString, String options) { +assert (schemaName != null ! schemaName.isEmpty()); + +this.schemaName = schemaName; + +if (options == null || options.isEmpty()) { + throw new IllegalArgumentException(No schema options are provided); +} + +String codecName = options.get(CODEC_NAME_KEY); +if (codecName == null || codecName.isEmpty()) { + throw new IllegalArgumentException(No codec option is provided); +} + +int dataUnits = 0, parityUnits = 0; +try { + if (options.containsKey(NUM_DATA_UNITS_KEY)) { +dataUnits = Integer.parseInt(options.get(NUM_DATA_UNITS_KEY)); + } +} catch (NumberFormatException e) { + throw new IllegalArgumentException(Option value + + options.get(CHUNK_SIZE_KEY) + for + CHUNK_SIZE_KEY + + is found. It should be an integer); +} + +try { + if
[18/51] [abbrv] hadoop git commit: HADOOP-11542. Raw Reed-Solomon coder in pure Java. Contributed by Kai Zheng
HADOOP-11542. Raw Reed-Solomon coder in pure Java. Contributed by Kai Zheng Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ae790470 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ae790470 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ae790470 Branch: refs/heads/HDFS-7285 Commit: ae7904705a255b96353471bf7f0e0a9586f936aa Parents: e3a4821 Author: drankye dran...@gmail.com Authored: Thu Feb 12 21:12:44 2015 +0800 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:05:28 2015 -0700 -- .../hadoop-common/CHANGES-HDFS-EC-7285.txt | 4 + .../io/erasurecode/rawcoder/JRSRawDecoder.java | 69 +++ .../io/erasurecode/rawcoder/JRSRawEncoder.java | 78 +++ .../erasurecode/rawcoder/RawErasureCoder.java | 2 +- .../erasurecode/rawcoder/util/GaloisField.java | 497 +++ .../io/erasurecode/rawcoder/util/RSUtil.java| 22 + .../hadoop/io/erasurecode/TestCoderBase.java| 28 +- .../erasurecode/rawcoder/TestJRSRawCoder.java | 93 .../erasurecode/rawcoder/TestRawCoderBase.java | 5 +- .../erasurecode/rawcoder/TestXorRawCoder.java | 1 - 10 files changed, 786 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae790470/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt index 9728f97..7bbacf7 100644 --- a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt @@ -8,3 +8,7 @@ HADOOP-11541. Raw XOR coder ( Kai Zheng ) + +HADOOP-11542. Raw Reed-Solomon coder in pure Java. Contributed by Kai Zheng +( Kai Zheng ) + http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae790470/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/JRSRawDecoder.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/JRSRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/JRSRawDecoder.java new file mode 100644 index 000..dbb689e --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/JRSRawDecoder.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.rawcoder; + +import org.apache.hadoop.io.erasurecode.rawcoder.util.RSUtil; + +import java.nio.ByteBuffer; + +/** + * A raw erasure decoder in RS code scheme in pure Java in case native one + * isn't available in some environment. Please always use native implementations + * when possible. + */ +public class JRSRawDecoder extends AbstractRawErasureDecoder { + // To describe and calculate the needed Vandermonde matrix + private int[] errSignature; + private int[] primitivePower; + + @Override + public void initialize(int numDataUnits, int numParityUnits, int chunkSize) { +super.initialize(numDataUnits, numParityUnits, chunkSize); +assert (getNumDataUnits() + getNumParityUnits() RSUtil.GF.getFieldSize()); + +this.errSignature = new int[getNumParityUnits()]; +this.primitivePower = RSUtil.getPrimitivePower(getNumDataUnits(), +getNumParityUnits()); + } + + @Override + protected void doDecode(ByteBuffer[] inputs, int[] erasedIndexes, + ByteBuffer[] outputs) { +for (int i = 0; i erasedIndexes.length; i++) { + errSignature[i] = primitivePower[erasedIndexes[i]]; + RSUtil.GF.substitute(inputs, outputs[i], primitivePower[i]); +} + +int dataLen = inputs[0].remaining(); +RSUtil.GF.solveVandermondeSystem(errSignature, outputs, +erasedIndexes.length, dataLen); + } + + @Override + protected void
[38/51] [abbrv] hadoop git commit: HDFS-7827. Erasure Coding: support striped blocks in non-protobuf fsimage. Contributed by Hui Zheng.
HDFS-7827. Erasure Coding: support striped blocks in non-protobuf fsimage. Contributed by Hui Zheng. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f62c7110 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f62c7110 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f62c7110 Branch: refs/heads/HDFS-7285 Commit: f62c7110dbae620c45466b4897632d78b3bd37f6 Parents: 7c7ba71 Author: Jing Zhao ji...@apache.org Authored: Mon Mar 23 15:10:10 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:21:01 2015 -0700 -- .../blockmanagement/BlockInfoStriped.java | 11 +- .../hdfs/server/namenode/FSImageFormat.java | 62 ++-- .../server/namenode/FSImageSerialization.java | 78 +++--- .../blockmanagement/TestBlockInfoStriped.java | 34 + .../hdfs/server/namenode/TestFSImage.java | 148 ++- 5 files changed, 300 insertions(+), 33 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f62c7110/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java index cef8318..30b5ee7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java @@ -20,6 +20,8 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; +import java.io.DataOutput; +import java.io.IOException; /** * Subclass of {@link BlockInfo}, presenting a block group in erasure coding. @@ -206,6 +208,13 @@ public class BlockInfoStriped extends BlockInfo { return num; } + @Override + public void write(DataOutput out) throws IOException { +out.writeShort(dataBlockNum); +out.writeShort(parityBlockNum); +super.write(out); + } + /** * Convert a complete block to an under construction block. * @return BlockInfoUnderConstruction - an under construction block. @@ -215,7 +224,7 @@ public class BlockInfoStriped extends BlockInfo { final BlockInfoStripedUnderConstruction ucBlock; if(isComplete()) { ucBlock = new BlockInfoStripedUnderConstruction(this, getDataBlockNum(), - getParityBlockNum(), s, targets); + getParityBlockNum(), s, targets); ucBlock.setBlockCollection(getBlockCollection()); } else { // the block is already under construction http://git-wip-us.apache.org/repos/asf/hadoop/blob/f62c7110/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java index 2e6e741..ad96863 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java @@ -47,13 +47,16 @@ import org.apache.hadoop.fs.PathIsNotDirectoryException; import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.LayoutFlags; import org.apache.hadoop.hdfs.protocol.LayoutVersion; import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStripedUnderConstruction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException; @@
[27/51] [abbrv] hadoop git commit: HDFS-7936. Erasure coding: resolving conflicts when merging with HDFS-7903 and HDFS-7435. Contributed by Zhe Zhang.
HDFS-7936. Erasure coding: resolving conflicts when merging with HDFS-7903 and HDFS-7435. Contributed by Zhe Zhang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cef8ce48 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cef8ce48 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cef8ce48 Branch: refs/heads/HDFS-7285 Commit: cef8ce48f199c8342e93193737d720a973b2fca9 Parents: aac871d Author: Jing Zhao ji...@apache.org Authored: Mon Mar 16 14:27:21 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:20:59 2015 -0700 -- .../hadoop/hdfs/server/blockmanagement/DecommissionManager.java | 2 +- .../org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java| 2 +- .../hadoop/hdfs/server/namenode/snapshot/FileDiffList.java | 3 ++- .../src/test/java/org/apache/hadoop/hdfs/TestDecommission.java | 5 ++--- .../hadoop/hdfs/server/namenode/TestAddStripedBlocks.java | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef8ce48/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java index 0faf3ad..df31d6e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java @@ -536,7 +536,7 @@ public class DecommissionManager { */ private void processBlocksForDecomInternal( final DatanodeDescriptor datanode, -final IteratorBlockInfoContiguous it, +final Iterator? extends BlockInfo it, final ListBlockInfoContiguous insufficientlyReplicated, boolean pruneSufficientlyReplicated) { boolean firstReplicationLog = true; http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef8ce48/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 0e2441d..69bdc4d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -2001,7 +2001,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, } // Check if the file is already being truncated with the same length -final BlockInfoContiguous last = file.getLastBlock(); +final BlockInfo last = file.getLastBlock(); if (last != null last.getBlockUCState() == BlockUCState.UNDER_RECOVERY) { final Block truncateBlock = ((BlockInfoContiguousUnderConstruction)last).getTruncateBlock(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef8ce48/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java index a1263c5..d0248eb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java @@ -21,6 +21,7 @@ import java.util.Collections; import java.util.List; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; @@ -132,7 +133,7 @@ public class FileDiffList extends break; } // Check if last block is part of truncate recovery -BlockInfoContiguous lastBlock = file.getLastBlock(); +BlockInfo lastBlock = file.getLastBlock(); Block
[16/51] [abbrv] hadoop git commit: HDFS-7716. Erasure Coding: extend BlockInfo to handle EC info. Contributed by Jing Zhao.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3a48212/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java index be16a87..fa7f263 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java @@ -24,6 +24,7 @@ import java.util.List; import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State; @@ -80,10 +81,10 @@ public class DatanodeStorageInfo { /** * Iterates over the list of blocks belonging to the data-node. */ - class BlockIterator implements IteratorBlockInfoContiguous { -private BlockInfoContiguous current; + class BlockIterator implements IteratorBlockInfo { +private BlockInfo current; -BlockIterator(BlockInfoContiguous head) { +BlockIterator(BlockInfo head) { this.current = head; } @@ -91,8 +92,8 @@ public class DatanodeStorageInfo { return current != null; } -public BlockInfoContiguous next() { - BlockInfoContiguous res = current; +public BlockInfo next() { + BlockInfo res = current; current = current.getNext(current.findStorageInfo(DatanodeStorageInfo.this)); return res; } @@ -112,7 +113,7 @@ public class DatanodeStorageInfo { private volatile long remaining; private long blockPoolUsed; - private volatile BlockInfoContiguous blockList = null; + private volatile BlockInfo blockList = null; private int numBlocks = 0; // The ID of the last full block report which updated this storage. @@ -226,7 +227,7 @@ public class DatanodeStorageInfo { return blockPoolUsed; } - public AddBlockResult addBlock(BlockInfoContiguous b) { + public AddBlockResult addBlock(BlockInfo b, Block reportedBlock) { // First check whether the block belongs to a different storage // on the same DN. AddBlockResult result = AddBlockResult.ADDED; @@ -245,13 +246,21 @@ public class DatanodeStorageInfo { } // add to the head of the data-node list -b.addStorage(this); +b.addStorage(this, reportedBlock); +insertToList(b); +return result; + } + + AddBlockResult addBlock(BlockInfoContiguous b) { +return addBlock(b, b); + } + + public void insertToList(BlockInfo b) { blockList = b.listInsert(blockList, this); numBlocks++; -return result; } - public boolean removeBlock(BlockInfoContiguous b) { + public boolean removeBlock(BlockInfo b) { blockList = b.listRemove(blockList, this); if (b.removeStorage(this)) { numBlocks--; @@ -265,16 +274,15 @@ public class DatanodeStorageInfo { return numBlocks; } - IteratorBlockInfoContiguous getBlockIterator() { + IteratorBlockInfo getBlockIterator() { return new BlockIterator(blockList); - } /** * Move block to the head of the list of blocks belonging to the data-node. * @return the index of the head of the blockList */ - int moveBlockToHead(BlockInfoContiguous b, int curIndex, int headIndex) { + int moveBlockToHead(BlockInfo b, int curIndex, int headIndex) { blockList = b.moveBlockToHead(blockList, this, curIndex, headIndex); return curIndex; } @@ -284,7 +292,7 @@ public class DatanodeStorageInfo { * @return the head of the blockList */ @VisibleForTesting - BlockInfoContiguous getBlockListHeadForTesting(){ + BlockInfo getBlockListHeadForTesting(){ return blockList; } @@ -371,6 +379,6 @@ public class DatanodeStorageInfo { } static enum AddBlockResult { -ADDED, REPLACED, ALREADY_EXIST; +ADDED, REPLACED, ALREADY_EXIST } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3a48212/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicaUnderConstruction.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicaUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicaUnderConstruction.java new file mode 100644 index 000..f4600cb7 --- /dev/null +++
[03/51] [abbrv] hadoop git commit: HADOOP-11776. Fixed the broken JDiff support in Hadoop 2. Contributed by Li Lu.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b3948ea/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.6.0.xml -- diff --git a/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.6.0.xml b/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.6.0.xml new file mode 100644 index 000..b105253 --- /dev/null +++ b/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.6.0.xml @@ -0,0 +1,45596 @@ +?xml version=1.0 encoding=iso-8859-1 standalone=no? +!-- Generated by the JDiff Javadoc doclet -- +!-- (http://www.jdiff.org) -- +!-- on Mon Mar 30 15:27:52 PDT 2015 -- + +api + xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' + xsi:noNamespaceSchemaLocation='api.xsd' + name=hadoop-core 2.6.0 + jdversion=1.0.9 + +!-- Command line arguments = -doclet org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet -docletpath /Users/llu/hadoop-common/hadoop-common-project/hadoop-common/target/hadoop-annotations.jar:/Users/llu/hadoop-common/hadoop-common-project/hadoop-common/target/jdiff.jar -verbose -classpath /Users/llu/hadoop-common/hadoop-common-project/hadoop-common/target/classes:/Users/llu/hadoop-common/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-2.6.0.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_67.jdk/Contents/Home/lib/tools.jar:/Users/llu/.m2/repository/com/google/guava/guava/11.0.2/guava-11.0.2.jar:/Users/llu/.m2/repository/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/Users/llu/.m2/repository/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/Users/llu/.m2/repository/xmlenc/xmlenc/0.52/xmlenc-0.52.jar:/Users/llu/.m2/repository/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar:/Users/llu/.m2/repository/common s-codec/commons-codec/1.4/commons-codec-1.4.jar:/Users/llu/.m2/repository/commons-io/commons-io/2.4/commons-io-2.4.jar:/Users/llu/.m2/repository/commons-net/commons-net/3.1/commons-net-3.1.jar:/Users/llu/.m2/repository/commons-collections/commons-collections/3.2.1/commons-collections-3.2.1.jar:/Users/llu/.m2/repository/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar:/Users/llu/.m2/repository/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar:/Users/llu/.m2/repository/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/Users/llu/.m2/repository/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar:/Users/llu/.m2/repository/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar:/Users/llu/.m2/repository/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/Users/llu/.m2/repository/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/Users/llu/.m2/repository/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar:/Users/llu/.m2/repository/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar:/ Users/llu/.m2/repository/javax/activation/activation/1.1/activation-1.1.jar:/Users/llu/.m2/repository/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/Users/llu/.m2/repository/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/Users/llu/.m2/repository/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar:/Users/llu/.m2/repository/asm/asm/3.2/asm-3.2.jar:/Users/llu/.m2/repository/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/Users/llu/.m2/repository/log4j/log4j/1.2.17/log4j-1.2.17.jar:/Users/llu/.m2/repository/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar:/Users/llu/.m2/repository/org/apache/httpcomponents/httpclient/4.2.5/httpclient-4.2.5.jar:/Users/llu/.m2/repository/org/apache/httpcomponents/httpcore/4.2.5/httpcore-4.2.5.jar:/Users/llu/.m2/repository/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.jar:/Users/llu/.m2/repository/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/Users/llu/.m2/repository/commons-conf iguration/commons-configuration/1.6/commons-configuration-1.6.jar:/Users/llu/.m2/repository/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/Users/llu/.m2/repository/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar:/Users/llu/.m2/repository/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar:/Users/llu/.m2/repository/org/slf4j/slf4j-api/1.7.5/slf4j-api-1.7.5.jar:/Users/llu/.m2/repository/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/Users/llu/.m2/repository/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/Users/llu/.m2/repository/org/apache/avro/avro/1.7.4/avro-1.7.4.jar:/Users/llu/.m2/repository/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/Users/llu/.m2/repository/org/xerial/snappy/snappy-java/1.0.4.1/snappy-java-1.0.4.1.jar:/Users/llu/.m2/repository/org/apache/ant/ant/1.8.1/ant-1.8.1.jar:/Users/llu/.m2/repository/org/apache/ant/ant-launcher/1.8.1/
[07/51] [abbrv] hadoop git commit: HDFS-7999. FsDatasetImpl#createTemporary sometimes holds the FSDatasetImpl lock for a very long time (sinago via cmccabe)
HDFS-7999. FsDatasetImpl#createTemporary sometimes holds the FSDatasetImpl lock for a very long time (sinago via cmccabe) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/28bebc81 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/28bebc81 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/28bebc81 Branch: refs/heads/HDFS-7285 Commit: 28bebc81db8bb6d1bc2574de7564fe4c595cfe09 Parents: 53959e6 Author: Colin Patrick Mccabe cmcc...@cloudera.com Authored: Mon Apr 6 08:54:46 2015 -0700 Committer: Colin Patrick Mccabe cmcc...@cloudera.com Committed: Mon Apr 6 08:56:52 2015 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../datanode/fsdataset/impl/FsDatasetImpl.java | 77 +--- 2 files changed, 52 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/28bebc81/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 6fafec8..52325a2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1379,6 +1379,9 @@ Release 2.7.0 - UNRELEASED HDFS-8051. FsVolumeList#addVolume should release volume reference if not put it into BlockScanner. (Lei (Eddy) Xu via Colin P. McCabe) +HDFS-7999. FsDatasetImpl#createTemporary sometimes holds the FSDatasetImpl +lock for a very long time (sinago via cmccabe) + BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS HDFS-7720. Quota by Storage Type API, tools and ClientNameNode http://git-wip-us.apache.org/repos/asf/hadoop/blob/28bebc81/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index f15f649..6bcbe5a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -1412,38 +1412,59 @@ class FsDatasetImpl implements FsDatasetSpiFsVolumeImpl { } @Override // FsDatasetSpi - public synchronized ReplicaHandler createTemporary( + public ReplicaHandler createTemporary( StorageType storageType, ExtendedBlock b) throws IOException { -ReplicaInfo replicaInfo = volumeMap.get(b.getBlockPoolId(), b.getBlockId()); -if (replicaInfo != null) { - if (replicaInfo.getGenerationStamp() b.getGenerationStamp() - replicaInfo instanceof ReplicaInPipeline) { -// Stop the previous writer -((ReplicaInPipeline)replicaInfo) - .stopWriter(datanode.getDnConf().getXceiverStopTimeout()); -invalidate(b.getBlockPoolId(), new Block[]{replicaInfo}); - } else { -throw new ReplicaAlreadyExistsException(Block + b + - already exists in state + replicaInfo.getState() + - and thus cannot be created.); +long startTimeMs = Time.monotonicNow(); +long writerStopTimeoutMs = datanode.getDnConf().getXceiverStopTimeout(); +ReplicaInfo lastFoundReplicaInfo = null; +do { + synchronized (this) { +ReplicaInfo currentReplicaInfo = +volumeMap.get(b.getBlockPoolId(), b.getBlockId()); +if (currentReplicaInfo == lastFoundReplicaInfo) { + if (lastFoundReplicaInfo != null) { +invalidate(b.getBlockPoolId(), new Block[] { lastFoundReplicaInfo }); + } + FsVolumeReference ref = + volumes.getNextVolume(storageType, b.getNumBytes()); + FsVolumeImpl v = (FsVolumeImpl) ref.getVolume(); + // create a temporary file to hold block in the designated volume + File f; + try { +f = v.createTmpFile(b.getBlockPoolId(), b.getLocalBlock()); + } catch (IOException e) { +IOUtils.cleanup(null, ref); +throw e; + } + ReplicaInPipeline newReplicaInfo = + new ReplicaInPipeline(b.getBlockId(), b.getGenerationStamp(), v, + f.getParentFile(), 0); + volumeMap.add(b.getBlockPoolId(), newReplicaInfo); + return new ReplicaHandler(newReplicaInfo, ref); +} else { + if (!(currentReplicaInfo.getGenerationStamp() b + .getGenerationStamp() currentReplicaInfo
[15/51] [abbrv] hadoop git commit: HADOOP-11514. Raw Erasure Coder API for concrete encoding and decoding (Kai Zheng via umamahesh)
HADOOP-11514. Raw Erasure Coder API for concrete encoding and decoding (Kai Zheng via umamahesh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7175f1cc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7175f1cc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7175f1cc Branch: refs/heads/HDFS-7285 Commit: 7175f1cc0b9c8eb88cf7b0285c127e8c11045245 Parents: dd5fd7b Author: Uma Maheswara Rao G umamah...@apache.org Authored: Thu Jan 29 14:15:13 2015 +0530 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:01:05 2015 -0700 -- .../hadoop-common/CHANGES-HDFS-EC-7285.txt | 4 + .../apache/hadoop/io/erasurecode/ECChunk.java | 82 + .../rawcoder/AbstractRawErasureCoder.java | 63 + .../rawcoder/AbstractRawErasureDecoder.java | 93 .../rawcoder/AbstractRawErasureEncoder.java | 93 .../erasurecode/rawcoder/RawErasureCoder.java | 78 .../erasurecode/rawcoder/RawErasureDecoder.java | 55 .../erasurecode/rawcoder/RawErasureEncoder.java | 54 8 files changed, 522 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7175f1cc/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt new file mode 100644 index 000..8ce5a89 --- /dev/null +++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt @@ -0,0 +1,4 @@ + BREAKDOWN OF HADOOP-11264 SUBTASKS AND RELATED JIRAS (Common part of HDFS-7285) + +HADOOP-11514. Raw Erasure Coder API for concrete encoding and decoding +(Kai Zheng via umamahesh) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/7175f1cc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java new file mode 100644 index 000..f84eb11 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode; + +import java.nio.ByteBuffer; + +/** + * A wrapper for ByteBuffer or bytes array for an erasure code chunk. + */ +public class ECChunk { + + private ByteBuffer chunkBuffer; + + /** + * Wrapping a ByteBuffer + * @param buffer + */ + public ECChunk(ByteBuffer buffer) { +this.chunkBuffer = buffer; + } + + /** + * Wrapping a bytes array + * @param buffer + */ + public ECChunk(byte[] buffer) { +this.chunkBuffer = ByteBuffer.wrap(buffer); + } + + /** + * Convert to ByteBuffer + * @return ByteBuffer + */ + public ByteBuffer getBuffer() { +return chunkBuffer; + } + + /** + * Convert an array of this chunks to an array of ByteBuffers + * @param chunks + * @return an array of ByteBuffers + */ + public static ByteBuffer[] toBuffers(ECChunk[] chunks) { +ByteBuffer[] buffers = new ByteBuffer[chunks.length]; + +for (int i = 0; i chunks.length; i++) { + buffers[i] = chunks[i].getBuffer(); +} + +return buffers; + } + + /** + * Convert an array of this chunks to an array of byte array + * @param chunks + * @return an array of byte array + */ + public static byte[][] toArray(ECChunk[] chunks) { +byte[][] bytesArr = new byte[chunks.length][]; + +for (int i = 0; i chunks.length; i++) { + bytesArr[i] = chunks[i].getBuffer().array(); +} + +return bytesArr; + } +}
[06/51] [abbrv] hadoop git commit: TestFairScheduler.testContinuousScheduling fails Intermittently. Contributed by Zhihai Xu.
TestFairScheduler.testContinuousScheduling fails Intermittently. Contributed by Zhihai Xu. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/53959e69 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/53959e69 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/53959e69 Branch: refs/heads/HDFS-7285 Commit: 53959e69f7ef4442324627fbfb159012968a2092 Parents: 96d7211 Author: Tsuyoshi Ozawa oz...@apache.org Authored: Mon Apr 6 20:13:21 2015 +0900 Committer: Tsuyoshi Ozawa oz...@apache.org Committed: Mon Apr 6 20:19:13 2015 +0900 -- hadoop-yarn-project/CHANGES.txt | 3 +++ .../resourcemanager/scheduler/fair/TestFairScheduler.java | 7 ++- 2 files changed, 5 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/53959e69/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b1f8913..695c4a6 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -145,6 +145,9 @@ Release 2.8.0 - UNRELEASED YARN-3435. AM container to be allocated Appattempt AM container shown as null. (Bibin A Chundatt via xgong) +YARN-2666. TestFairScheduler.testContinuousScheduling fails Intermittently. +(Zhihai Xu via ozawa) + Release 2.7.0 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/53959e69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index b5bfb8c..98877e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -3922,12 +3922,9 @@ public class TestFairScheduler extends FairSchedulerTestBase { createResourceRequest(1024, 1, ResourceRequest.ANY, 2, 1, true); ask.clear(); ask.add(request); +scheduler.stop(); scheduler.allocate(appAttemptId, ask, new ArrayListContainerId(), null, null); - -// Wait until app gets resources -while (app.getCurrentConsumption() -.equals(Resources.createResource(1024, 1))) { } - +scheduler.continuousSchedulingAttempt(); Assert.assertEquals(2048, app.getCurrentConsumption().getMemory()); Assert.assertEquals(2, app.getCurrentConsumption().getVirtualCores());
[12/51] [abbrv] hadoop git commit: Added the missed entry for commit of HADOOP-11541
Added the missed entry for commit of HADOOP-11541 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/958e8c1e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/958e8c1e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/958e8c1e Branch: refs/heads/HDFS-7285 Commit: 958e8c1eabbbf05da16dc037077791777924dc3d Parents: f50b3df Author: drankye dran...@gmail.com Authored: Mon Feb 9 22:04:08 2015 +0800 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:01:05 2015 -0700 -- hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt | 5 - 1 file changed, 4 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/958e8c1e/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt index 2124800..9728f97 100644 --- a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt @@ -4,4 +4,7 @@ (Kai Zheng via umamahesh) HADOOP-11534. Minor improvements for raw erasure coders -( Kai Zheng via vinayakumarb ) \ No newline at end of file +( Kai Zheng via vinayakumarb ) + +HADOOP-11541. Raw XOR coder +( Kai Zheng )
[14/51] [abbrv] hadoop git commit: HADOOP-11534. Minor improvements for raw erasure coders ( Contributed by Kai Zheng )
HADOOP-11534. Minor improvements for raw erasure coders ( Contributed by Kai Zheng ) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b431203a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b431203a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b431203a Branch: refs/heads/HDFS-7285 Commit: b431203a8b2bb6d480347d51a28b9e3a4f23e322 Parents: 7175f1c Author: Vinayakumar B vinayakuma...@intel.com Authored: Mon Feb 2 14:39:53 2015 +0530 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 10:01:05 2015 -0700 -- .../hadoop-common/CHANGES-HDFS-EC-7285.txt | 5 - .../org/apache/hadoop/io/erasurecode/ECChunk.java| 15 +-- .../rawcoder/AbstractRawErasureCoder.java| 12 ++-- 3 files changed, 23 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b431203a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt index 8ce5a89..2124800 100644 --- a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt @@ -1,4 +1,7 @@ BREAKDOWN OF HADOOP-11264 SUBTASKS AND RELATED JIRAS (Common part of HDFS-7285) HADOOP-11514. Raw Erasure Coder API for concrete encoding and decoding -(Kai Zheng via umamahesh) \ No newline at end of file +(Kai Zheng via umamahesh) + +HADOOP-11534. Minor improvements for raw erasure coders +( Kai Zheng via vinayakumarb ) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/b431203a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java index f84eb11..01e8f35 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java @@ -66,15 +66,26 @@ public class ECChunk { } /** - * Convert an array of this chunks to an array of byte array + * Convert an array of this chunks to an array of byte array. + * Note the chunk buffers are not affected. * @param chunks * @return an array of byte array */ public static byte[][] toArray(ECChunk[] chunks) { byte[][] bytesArr = new byte[chunks.length][]; +ByteBuffer buffer; for (int i = 0; i chunks.length; i++) { - bytesArr[i] = chunks[i].getBuffer().array(); + buffer = chunks[i].getBuffer(); + if (buffer.hasArray()) { +bytesArr[i] = buffer.array(); + } else { +bytesArr[i] = new byte[buffer.remaining()]; +// Avoid affecting the original one +buffer.mark(); +buffer.get(bytesArr[i]); +buffer.reset(); + } } return bytesArr; http://git-wip-us.apache.org/repos/asf/hadoop/blob/b431203a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java index 474542b..74d2ab6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java @@ -24,26 +24,26 @@ package org.apache.hadoop.io.erasurecode.rawcoder; */ public abstract class AbstractRawErasureCoder implements RawErasureCoder { - private int dataSize; - private int paritySize; + private int numDataUnits; + private int numParityUnits; private int chunkSize; @Override public void initialize(int numDataUnits, int numParityUnits, int chunkSize) { -this.dataSize = numDataUnits; -this.paritySize = numParityUnits; +this.numDataUnits = numDataUnits; +this.numParityUnits = numParityUnits; this.chunkSize = chunkSize; } @Override public int getNumDataUnits() { -return dataSize; +return numDataUnits; } @Override public int
[04/51] [abbrv] hadoop git commit: HADOOP-11776. Fixed the broken JDiff support in Hadoop 2. Contributed by Li Lu.
HADOOP-11776. Fixed the broken JDiff support in Hadoop 2. Contributed by Li Lu. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4b3948ea Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4b3948ea Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4b3948ea Branch: refs/heads/HDFS-7285 Commit: 4b3948ea365db07df7a9369a271009fafd1ba8f5 Parents: 5370e71 Author: Vinod Kumar Vavilapalli vino...@apache.org Authored: Sat Apr 4 13:52:01 2015 -0700 Committer: Vinod Kumar Vavilapalli vino...@apache.org Committed: Sat Apr 4 13:52:01 2015 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 2 + .../jdiff/Apache_Hadoop_Common_2.6.0.xml| 45596 + .../jdiff/Apache_Hadoop_HDFS_2.6.0.xml | 19520 +++ hadoop-project-dist/pom.xml |24 +- hadoop-project/pom.xml | 2 + 5 files changed, 65137 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b3948ea/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index fd926aa..f52e09f 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -1205,6 +1205,8 @@ Release 2.7.0 - UNRELEASED HADOOP-11377. Added Null.java without which jdiff completely flops. (Tsuyoshi Ozawa via vinodkv) + +HADOOP-11776. Fixed the broken JDiff support in Hadoop 2. (Li Lu via vinodkv) Release 2.6.1 - UNRELEASED
[40/50] [abbrv] hadoop git commit: HDFS-8051. FsVolumeList#addVolume should release volume reference if not put it into BlockScanner. (Lei (Eddy) Xu via Colin P. McCabe)
HDFS-8051. FsVolumeList#addVolume should release volume reference if not put it into BlockScanner. (Lei (Eddy) Xu via Colin P. McCabe) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e1ffb3ea Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e1ffb3ea Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e1ffb3ea Branch: refs/heads/YARN-2928 Commit: e1ffb3ea615e830b4215fafd0b2c1d05a50d4b0c Parents: 5864e88 Author: Colin Patrick Mccabe cmcc...@cloudera.com Authored: Fri Apr 3 16:34:23 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:16 2015 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../datanode/fsdataset/impl/FsVolumeList.java| 5 + .../fsdataset/impl/TestFsVolumeList.java | 19 +++ 3 files changed, 27 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e1ffb3ea/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 2d399a4..6fafec8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1376,6 +1376,9 @@ Release 2.7.0 - UNRELEASED HDFS-7996. After swapping a volume, BlockReceiver reports ReplicaNotFoundException (Lei (Eddy) Xu via Colin P. McCabe) +HDFS-8051. FsVolumeList#addVolume should release volume reference if not +put it into BlockScanner. (Lei (Eddy) Xu via Colin P. McCabe) + BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS HDFS-7720. Quota by Storage Type API, tools and ClientNameNode http://git-wip-us.apache.org/repos/asf/hadoop/blob/e1ffb3ea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java index 4fddfb9..d87595c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.VolumeChoosingPolicy; import org.apache.hadoop.hdfs.server.datanode.BlockScanner; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.DiskChecker.DiskErrorException; import org.apache.hadoop.util.Time; @@ -292,6 +293,10 @@ class FsVolumeList { } if (blockScanner != null) { blockScanner.addVolumeScanner(ref); +} else { + // If the volume is not put into a volume scanner, it does not need to + // hold the reference. + IOUtils.cleanup(FsDatasetImpl.LOG, ref); } // If the volume is used to replace a failed volume, it needs to reset the // volume failure info for this volume. http://git-wip-us.apache.org/repos/asf/hadoop/blob/e1ffb3ea/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java index 46189ba..eccff89 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java @@ -35,6 +35,7 @@ import java.util.Collections; import java.util.List; import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; public class TestFsVolumeList { @@ -101,4 +102,22 @@ public class TestFsVolumeList { // checkDirs() should ignore the 2nd volume since it is closed. volumeList.checkDirs(); } + + @Test + public void testReleaseVolumeRefIfNoBlockScanner() throws IOException { +FsVolumeList volumeList = new FsVolumeList( +Collections.VolumeFailureInfoemptyList(), null, blockChooser); +File volDir = new File(baseDir,
[36/50] [abbrv] hadoop git commit: HADOOP-11800. Clean up some test methods in TestCodec.java. Contributed by Brahma Reddy Battula.
HADOOP-11800. Clean up some test methods in TestCodec.java. Contributed by Brahma Reddy Battula. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ff35b525 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ff35b525 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ff35b525 Branch: refs/heads/YARN-2928 Commit: ff35b525ac2a2c92d4d2e366deee80a9fb48f576 Parents: aae48de Author: Akira Ajisaka aajis...@apache.org Authored: Sat Apr 4 00:32:18 2015 +0900 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:15 2015 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../src/test/java/org/apache/hadoop/io/compress/TestCodec.java | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff35b525/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 495c42d..7dcf8c1 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -494,6 +494,9 @@ Release 2.8.0 - UNRELEASED HADOOP-11760. Fix typo of javadoc in DistCp. (Brahma Reddy Battula via ozawa). +HADOOP-11800. Clean up some test methods in TestCodec.java. +(Brahma Reddy Battula via aajisaka) + Release 2.7.0 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff35b525/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java -- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java index 7246bf5..5d4af91 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java @@ -827,7 +827,7 @@ public class TestCodec { br.close(); } - public void testGzipCodecWrite(boolean useNative) throws IOException { + private void testGzipCodecWrite(boolean useNative) throws IOException { // Create a gzipped file using a compressor from the CodecPool, // and try to read it back via the regular GZIPInputStream. @@ -894,7 +894,7 @@ public class TestCodec { public void testGzipNativeCodecWrite() throws IOException { testGzipCodecWrite(true); } - + @Test public void testCodecPoolAndGzipDecompressor() { // BuiltInZlibInflater should not be used as the GzipCodec decompressor. // Assert that this is the case.
[26/50] [abbrv] hadoop git commit: HDFS-7888. Change DFSOutputStream and DataStreamer for convenience of subclassing. Contributed by Li Bo
HDFS-7888. Change DFSOutputStream and DataStreamer for convenience of subclassing. Contributed by Li Bo Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b98f637c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b98f637c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b98f637c Branch: refs/heads/YARN-2928 Commit: b98f637c1843b1f79016fd86244ecc68039ba54a Parents: b4c3ce1 Author: Tsz-Wo Nicholas Sze szets...@hortonworks.com Authored: Thu Apr 2 10:59:26 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:13 2015 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../org/apache/hadoop/hdfs/DFSOutputStream.java | 116 ++- .../org/apache/hadoop/hdfs/DataStreamer.java| 13 +-- 3 files changed, 69 insertions(+), 63 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b98f637c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 80d958d..0c66309 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -376,6 +376,9 @@ Release 2.8.0 - UNRELEASED HDFS-7978. Add LOG.isDebugEnabled() guard for some LOG.debug(..). (Walter Su via wang) +HDFS-7888. Change DFSOutputStream and DataStreamer for convenience of +subclassing. (Li Bo via szetszwo) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/b98f637c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index c88639d..f6733e3 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InterruptedIOException; -import java.net.Socket; import java.nio.channels.ClosedChannelException; import java.util.EnumSet; import java.util.concurrent.atomic.AtomicReference; @@ -95,29 +94,29 @@ public class DFSOutputStream extends FSOutputSummer static CryptoProtocolVersion[] SUPPORTED_CRYPTO_VERSIONS = CryptoProtocolVersion.supported(); - private final DFSClient dfsClient; - private final ByteArrayManager byteArrayManager; + protected final DFSClient dfsClient; + protected final ByteArrayManager byteArrayManager; // closed is accessed by different threads under different locks. - private volatile boolean closed = false; - - private final String src; - private final long fileId; - private final long blockSize; - private final int bytesPerChecksum; - - private DFSPacket currentPacket = null; - private DataStreamer streamer; - private int packetSize = 0; // write packet size, not including the header. - private int chunksPerPacket = 0; - private long lastFlushOffset = 0; // offset when flush was invoked + protected volatile boolean closed = false; + + protected final String src; + protected final long fileId; + protected final long blockSize; + protected final int bytesPerChecksum; + + protected DFSPacket currentPacket = null; + protected DataStreamer streamer; + protected int packetSize = 0; // write packet size, not including the header. + protected int chunksPerPacket = 0; + protected long lastFlushOffset = 0; // offset when flush was invoked private long initialFileSize = 0; // at time of file open private final short blockReplication; // replication factor of file - private boolean shouldSyncBlock = false; // force blocks to disk upon close - private final AtomicReferenceCachingStrategy cachingStrategy; + protected boolean shouldSyncBlock = false; // force blocks to disk upon close + protected final AtomicReferenceCachingStrategy cachingStrategy; private FileEncryptionInfo fileEncryptionInfo; /** Use {@link ByteArrayManager} to create buffer for non-heartbeat packets.*/ - private DFSPacket createPacket(int packetSize, int chunksPerPkt, long offsetInBlock, + protected DFSPacket createPacket(int packetSize, int chunksPerPkt, long offsetInBlock, long seqno, boolean lastPacketInBlock) throws InterruptedIOException { final byte[] buf; final int bufferSize = PacketHeader.PKT_MAX_HEADER_LEN + packetSize; @@
[05/50] [abbrv] hadoop git commit: HDFS-7671. hdfs user guide should point to the common rack awareness doc. Contributed by Kai Sasaki.
HDFS-7671. hdfs user guide should point to the common rack awareness doc. Contributed by Kai Sasaki. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8af286f4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8af286f4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8af286f4 Branch: refs/heads/YARN-2928 Commit: 8af286f4262328228c801cf0b1e8e6e49e255d6d Parents: 6916c77 Author: Akira Ajisaka aajis...@apache.org Authored: Wed Apr 1 00:26:16 2015 +0900 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:09 2015 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop-hdfs/src/site/markdown/HdfsUserGuide.md| 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8af286f4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index e8075e6..f3537b0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -368,6 +368,9 @@ Release 2.8.0 - UNRELEASED HDFS-7944. Minor cleanup of BlockPoolManager#getAllNamenodeThreads. (Arpit Agarwal) +HDFS-7671. hdfs user guide should point to the common rack awareness doc. +(Kai Sasaki via aajisaka) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/8af286f4/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md index 37fa4be..ffd8532 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md @@ -264,7 +264,7 @@ For command usage, see [balancer](./HDFSCommands.html#balancer). Rack Awareness -- -Typically large Hadoop clusters are arranged in racks and network traffic between different nodes with in the same rack is much more desirable than network traffic across the racks. In addition NameNode tries to place replicas of block on multiple racks for improved fault tolerance. Hadoop lets the cluster administrators decide which rack a node belongs to through configuration variable `net.topology.script.file.name`. When this script is configured, each node runs the script to determine its rack id. A default installation assumes all the nodes belong to the same rack. This feature and configuration is further described in PDF attached to [HADOOP-692](https://issues.apache.org/jira/browse/HADOOP-692). +A HDFS cluster can recognize the topology of racks where each nodes are put. It is important to configure this topology in order to optimize the data capacity and usage. For more detail, please check the [rack awareness](../hadoop-common/RackAwareness.html) in common document. Safemode
[35/50] [abbrv] hadoop git commit: HDFS-7811. Avoid recursive call getStoragePolicyID in INodeFile#computeQuotaUsage. Contributed by Xiaoyu Yao and Jing Zhao.
HDFS-7811. Avoid recursive call getStoragePolicyID in INodeFile#computeQuotaUsage. Contributed by Xiaoyu Yao and Jing Zhao. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aae48dea Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aae48dea Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aae48dea Branch: refs/heads/YARN-2928 Commit: aae48deac10c1ef26504997ae57d298bd6d6fd8d Parents: c134f99 Author: Jing Zhao ji...@apache.org Authored: Thu Apr 2 21:20:30 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:15 2015 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hadoop/hdfs/server/namenode/FSImage.java| 14 +--- .../hadoop/hdfs/server/namenode/INode.java | 29 +++-- .../hdfs/server/namenode/INodeDirectory.java| 34 +--- .../hadoop/hdfs/server/namenode/INodeFile.java | 13 .../hadoop/hdfs/server/namenode/INodeMap.java | 4 +-- .../hdfs/server/namenode/INodeReference.java| 15 + .../hdfs/server/namenode/INodeSymlink.java | 2 +- .../snapshot/DirectoryWithSnapshotFeature.java | 7 ++-- 9 files changed, 84 insertions(+), 37 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/aae48dea/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 85c013e..04eac75 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -877,6 +877,9 @@ Release 2.7.0 - UNRELEASED HDFS-8008. Support client-side back off when the datanodes are congested. (wheat9) +HDFS-7811. Avoid recursive call getStoragePolicyID in +INodeFile#computeQuotaUsage. (Xiaoyu Yao and jing9) + OPTIMIZATIONS HDFS-7454. Reduce memory footprint for AclEntries in NameNode. http://git-wip-us.apache.org/repos/asf/hadoop/blob/aae48dea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index 7454850..d62b952 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -856,23 +856,27 @@ public class FSImage implements Closeable { */ static void updateCountForQuota(BlockStoragePolicySuite bsps, INodeDirectory root) { -updateCountForQuotaRecursively(bsps, root, new QuotaCounts.Builder().build()); +updateCountForQuotaRecursively(bsps, root.getStoragePolicyID(), root, +new QuotaCounts.Builder().build()); } private static void updateCountForQuotaRecursively(BlockStoragePolicySuite bsps, - INodeDirectory dir, QuotaCounts counts) { + byte blockStoragePolicyId, INodeDirectory dir, QuotaCounts counts) { final long parentNamespace = counts.getNameSpace(); final long parentStoragespace = counts.getStorageSpace(); final EnumCountersStorageType parentTypeSpaces = counts.getTypeSpaces(); -dir.computeQuotaUsage4CurrentDirectory(bsps, counts); +dir.computeQuotaUsage4CurrentDirectory(bsps, blockStoragePolicyId, counts); for (INode child : dir.getChildrenList(Snapshot.CURRENT_STATE_ID)) { + final byte childPolicyId = child.getStoragePolicyIDForQuota(blockStoragePolicyId); if (child.isDirectory()) { -updateCountForQuotaRecursively(bsps, child.asDirectory(), counts); +updateCountForQuotaRecursively(bsps, childPolicyId, +child.asDirectory(), counts); } else { // file or symlink: count here to reduce recursive calls. -child.computeQuotaUsage(bsps, counts, false); +child.computeQuotaUsage(bsps, childPolicyId, counts, false, +Snapshot.CURRENT_STATE_ID); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/aae48dea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java index 586cce4..e629441 100644 ---
[20/50] [abbrv] hadoop git commit: HDFS-8036. Use snapshot path as source when using snapshot diff report in DistCp. Contributed by Jing Zhao.
HDFS-8036. Use snapshot path as source when using snapshot diff report in DistCp. Contributed by Jing Zhao. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/61487d36 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/61487d36 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/61487d36 Branch: refs/heads/YARN-2928 Commit: 61487d3642b69381c99b1463fce176063de15726 Parents: 093e092 Author: Haohui Mai whe...@apache.org Authored: Wed Apr 1 16:50:59 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:12 2015 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../org/apache/hadoop/tools/DistCpSync.java | 21 +- .../hadoop/tools/mapred/CopyCommitter.java | 3 +- .../org/apache/hadoop/tools/TestDistCpSync.java | 40 +++- 4 files changed, 63 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/61487d36/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index f265ead..1d9e200 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1345,6 +1345,9 @@ Release 2.7.0 - UNRELEASED HDFS-7748. Separate ECN flags from the Status in the DataTransferPipelineAck. (Anu Engineer and Haohui Mai via wheat9) +HDFS-8036. Use snapshot path as source when using snapshot diff report in +DistCp. (Jing Zhao via wheat9) + BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS HDFS-7720. Quota by Storage Type API, tools and ClientNameNode http://git-wip-us.apache.org/repos/asf/hadoop/blob/61487d36/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java -- diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java index 26d7eb4..8e71b6f 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java @@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import java.io.IOException; @@ -86,6 +87,22 @@ class DistCpSync { } finally { deleteTargetTmpDir(targetFs, tmpDir); // TODO: since we have tmp directory, we can support undo with failures + // set the source path using the snapshot path + inputOptions.setSourcePaths(Arrays.asList(getSourceSnapshotPath(sourceDir, + inputOptions.getToSnapshot(; +} + } + + private static String getSnapshotName(String name) { +return Path.CUR_DIR.equals(name) ? : name; + } + + private static Path getSourceSnapshotPath(Path sourceDir, String snapshotName) { +if (Path.CUR_DIR.equals(snapshotName)) { + return sourceDir; +} else { + return new Path(sourceDir, + HdfsConstants.DOT_SNAPSHOT_DIR + Path.SEPARATOR + snapshotName); } } @@ -136,8 +153,10 @@ class DistCpSync { static DiffInfo[] getDiffs(DistCpOptions inputOptions, DistributedFileSystem fs, Path sourceDir, Path targetDir) { try { + final String from = getSnapshotName(inputOptions.getFromSnapshot()); + final String to = getSnapshotName(inputOptions.getToSnapshot()); SnapshotDiffReport sourceDiff = fs.getSnapshotDiffReport(sourceDir, - inputOptions.getFromSnapshot(), inputOptions.getToSnapshot()); + from, to); return DiffInfo.getDiffs(sourceDiff, targetDir); } catch (IOException e) { DistCp.LOG.warn(Failed to compute snapshot diff on + sourceDir, e); http://git-wip-us.apache.org/repos/asf/hadoop/blob/61487d36/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java -- diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java index 9ec57f4..2b1e510 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java @@ -90,8 +90,7 @@ public class CopyCommitter extends
[48/50] [abbrv] hadoop git commit: TestFairScheduler.testContinuousScheduling fails Intermittently. Contributed by Zhihai Xu.
TestFairScheduler.testContinuousScheduling fails Intermittently. Contributed by Zhihai Xu. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/37725e71 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/37725e71 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/37725e71 Branch: refs/heads/YARN-2928 Commit: 37725e71643f9dd6f68796736cc9c407ea437cb3 Parents: 0b09b40 Author: Tsuyoshi Ozawa oz...@apache.org Authored: Mon Apr 6 20:13:21 2015 +0900 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:17 2015 -0700 -- hadoop-yarn-project/CHANGES.txt | 3 +++ .../resourcemanager/scheduler/fair/TestFairScheduler.java | 7 ++- 2 files changed, 5 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/37725e71/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 0c5aac8..5f20ca3 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -201,6 +201,9 @@ Release 2.8.0 - UNRELEASED YARN-3435. AM container to be allocated Appattempt AM container shown as null. (Bibin A Chundatt via xgong) +YARN-2666. TestFairScheduler.testContinuousScheduling fails Intermittently. +(Zhihai Xu via ozawa) + Release 2.7.0 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/37725e71/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index b5bfb8c..98877e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -3922,12 +3922,9 @@ public class TestFairScheduler extends FairSchedulerTestBase { createResourceRequest(1024, 1, ResourceRequest.ANY, 2, 1, true); ask.clear(); ask.add(request); +scheduler.stop(); scheduler.allocate(appAttemptId, ask, new ArrayListContainerId(), null, null); - -// Wait until app gets resources -while (app.getCurrentConsumption() -.equals(Resources.createResource(1024, 1))) { } - +scheduler.continuousSchedulingAttempt(); Assert.assertEquals(2048, app.getCurrentConsumption().getMemory()); Assert.assertEquals(2, app.getCurrentConsumption().getVirtualCores());
[46/50] [abbrv] hadoop git commit: HADOOP-11776. Fixed the broken JDiff support in Hadoop 2. Contributed by Li Lu.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1069dcee/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.6.0.xml -- diff --git a/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.6.0.xml b/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.6.0.xml new file mode 100644 index 000..b105253 --- /dev/null +++ b/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.6.0.xml @@ -0,0 +1,45596 @@ +?xml version=1.0 encoding=iso-8859-1 standalone=no? +!-- Generated by the JDiff Javadoc doclet -- +!-- (http://www.jdiff.org) -- +!-- on Mon Mar 30 15:27:52 PDT 2015 -- + +api + xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' + xsi:noNamespaceSchemaLocation='api.xsd' + name=hadoop-core 2.6.0 + jdversion=1.0.9 + +!-- Command line arguments = -doclet org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet -docletpath /Users/llu/hadoop-common/hadoop-common-project/hadoop-common/target/hadoop-annotations.jar:/Users/llu/hadoop-common/hadoop-common-project/hadoop-common/target/jdiff.jar -verbose -classpath /Users/llu/hadoop-common/hadoop-common-project/hadoop-common/target/classes:/Users/llu/hadoop-common/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-2.6.0.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_67.jdk/Contents/Home/lib/tools.jar:/Users/llu/.m2/repository/com/google/guava/guava/11.0.2/guava-11.0.2.jar:/Users/llu/.m2/repository/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/Users/llu/.m2/repository/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/Users/llu/.m2/repository/xmlenc/xmlenc/0.52/xmlenc-0.52.jar:/Users/llu/.m2/repository/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar:/Users/llu/.m2/repository/common s-codec/commons-codec/1.4/commons-codec-1.4.jar:/Users/llu/.m2/repository/commons-io/commons-io/2.4/commons-io-2.4.jar:/Users/llu/.m2/repository/commons-net/commons-net/3.1/commons-net-3.1.jar:/Users/llu/.m2/repository/commons-collections/commons-collections/3.2.1/commons-collections-3.2.1.jar:/Users/llu/.m2/repository/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar:/Users/llu/.m2/repository/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar:/Users/llu/.m2/repository/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/Users/llu/.m2/repository/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar:/Users/llu/.m2/repository/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar:/Users/llu/.m2/repository/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/Users/llu/.m2/repository/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/Users/llu/.m2/repository/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar:/Users/llu/.m2/repository/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar:/ Users/llu/.m2/repository/javax/activation/activation/1.1/activation-1.1.jar:/Users/llu/.m2/repository/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/Users/llu/.m2/repository/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/Users/llu/.m2/repository/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar:/Users/llu/.m2/repository/asm/asm/3.2/asm-3.2.jar:/Users/llu/.m2/repository/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/Users/llu/.m2/repository/log4j/log4j/1.2.17/log4j-1.2.17.jar:/Users/llu/.m2/repository/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar:/Users/llu/.m2/repository/org/apache/httpcomponents/httpclient/4.2.5/httpclient-4.2.5.jar:/Users/llu/.m2/repository/org/apache/httpcomponents/httpcore/4.2.5/httpcore-4.2.5.jar:/Users/llu/.m2/repository/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.jar:/Users/llu/.m2/repository/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/Users/llu/.m2/repository/commons-conf iguration/commons-configuration/1.6/commons-configuration-1.6.jar:/Users/llu/.m2/repository/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/Users/llu/.m2/repository/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar:/Users/llu/.m2/repository/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar:/Users/llu/.m2/repository/org/slf4j/slf4j-api/1.7.5/slf4j-api-1.7.5.jar:/Users/llu/.m2/repository/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/Users/llu/.m2/repository/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/Users/llu/.m2/repository/org/apache/avro/avro/1.7.4/avro-1.7.4.jar:/Users/llu/.m2/repository/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/Users/llu/.m2/repository/org/xerial/snappy/snappy-java/1.0.4.1/snappy-java-1.0.4.1.jar:/Users/llu/.m2/repository/org/apache/ant/ant/1.8.1/ant-1.8.1.jar:/Users/llu/.m2/repository/org/apache/ant/ant-launcher/1.8.1/
[45/50] [abbrv] hadoop git commit: HADOOP-11776. Fixed the broken JDiff support in Hadoop 2. Contributed by Li Lu.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1069dcee/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.6.0.xml -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.6.0.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.6.0.xml new file mode 100644 index 000..5514700 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.6.0.xml @@ -0,0 +1,19520 @@ +?xml version=1.0 encoding=iso-8859-1 standalone=no? +!-- Generated by the JDiff Javadoc doclet -- +!-- (http://www.jdiff.org) -- +!-- on Mon Mar 30 15:30:43 PDT 2015 -- + +api + xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' + xsi:noNamespaceSchemaLocation='api.xsd' + name=hadoop-hdfs 2.6.0 + jdversion=1.0.9 + +!-- Command line arguments = -doclet org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet -docletpath /Users/llu/hadoop-common/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-annotations.jar:/Users/llu/hadoop-common/hadoop-hdfs-project/hadoop-hdfs/target/jdiff.jar -verbose -classpath /Users/llu/hadoop-common/hadoop-hdfs-project/hadoop-hdfs/target/classes:/Users/llu/hadoop-common/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-2.6.0.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_67.jdk/Contents/Home/lib/tools.jar:/Users/llu/hadoop-common/hadoop-common-project/hadoop-auth/target/hadoop-auth-2.6.0.jar:/Users/llu/.m2/repository/org/slf4j/slf4j-api/1.7.5/slf4j-api-1.7.5.jar:/Users/llu/.m2/repository/org/apache/httpcomponents/httpclient/4.2.5/httpclient-4.2.5.jar:/Users/llu/.m2/repository/org/apache/httpcomponents/httpcore/4.2.5/httpcore-4.2.5.jar:/Users/llu/.m2/repository/org/apache/directory/server/apacheds-kerberos-codec/2.0.0-M15/apacheds-ker beros-codec-2.0.0-M15.jar:/Users/llu/.m2/repository/org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar:/Users/llu/.m2/repository/org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar:/Users/llu/.m2/repository/org/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar:/Users/llu/.m2/repository/org/apache/zookeeper/zookeeper/3.4.6/zookeeper-3.4.6.jar:/Users/llu/.m2/repository/org/apache/curator/curator-framework/2.6.0/curator-framework-2.6.0.jar:/Users/llu/hadoop-common/hadoop-common-project/hadoop-common/target/hadoop-common-2.6.0.jar:/Users/llu/.m2/repository/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/Users/llu/.m2/repository/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar:/Users/llu/.m2/repository/commons-net/commons-net/3.1/commons-net-3.1.jar:/Users/llu/.m2/repository/commons-collections/commons-collections/3.2.1/commons-collections-3.2.1.jar:/Users/llu/.m2/repository/com/sun/jerse y/jersey-json/1.9/jersey-json-1.9.jar:/Users/llu/.m2/repository/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/Users/llu/.m2/repository/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/Users/llu/.m2/repository/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar:/Users/llu/.m2/repository/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar:/Users/llu/.m2/repository/javax/activation/activation/1.1/activation-1.1.jar:/Users/llu/.m2/repository/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/Users/llu/.m2/repository/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/Users/llu/.m2/repository/tomcat/jasper-compiler/5.5.23/jasper-compiler-5.5.23.jar:/Users/llu/.m2/repository/commons-el/commons-el/1.0/commons-el-1.0.jar:/Users/llu/.m2/repository/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar:/Users/llu/.m2/repository/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.jar:/Users/llu/.m2/repository/commons-configuration/commons-configuratio n/1.6/commons-configuration-1.6.jar:/Users/llu/.m2/repository/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/Users/llu/.m2/repository/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar:/Users/llu/.m2/repository/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar:/Users/llu/.m2/repository/org/apache/avro/avro/1.7.4/avro-1.7.4.jar:/Users/llu/.m2/repository/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/Users/llu/.m2/repository/org/xerial/snappy/snappy-java/1.0.4.1/snappy-java-1.0.4.1.jar:/Users/llu/.m2/repository/com/google/code/gson/gson/2.2.4/gson-2.2.4.jar:/Users/llu/.m2/repository/com/jcraft/jsch/0.1.42/jsch-0.1.42.jar:/Users/llu/.m2/repository/org/apache/curator/curator-client/2.6.0/curator-client-2.6.0.jar:/Users/llu/.m2/repository/org/apache/curator/curator-recipes/2.6.0/curator-recipes-2.6.0.jar:/Users/llu/.m2/repository/com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar:/Users/llu/.m2/repository/
[31/50] [abbrv] hadoop git commit: YARN-3415. Non-AM containers can be counted towards amResourceUsage of a fairscheduler queue (Zhihai Xu via Sandy Ryza)
YARN-3415. Non-AM containers can be counted towards amResourceUsage of a fairscheduler queue (Zhihai Xu via Sandy Ryza) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e75307b2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e75307b2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e75307b2 Branch: refs/heads/YARN-2928 Commit: e75307b282014f73b435c0d4f6e5399af4ba128c Parents: 97092e4 Author: Sandy Ryza sa...@cloudera.com Authored: Thu Apr 2 13:56:08 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:14 2015 -0700 -- hadoop-yarn-project/CHANGES.txt | 3 ++ .../scheduler/fair/FSAppAttempt.java| 31 +++- .../scheduler/fair/FSLeafQueue.java | 5 +- .../scheduler/fair/FairScheduler.java | 6 --- .../scheduler/fair/TestFairScheduler.java | 50 5 files changed, 64 insertions(+), 31 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e75307b2/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index bc8d06a..18a3c37 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -188,6 +188,9 @@ Release 2.8.0 - UNRELEASED YARN-3425. NPE from RMNodeLabelsManager.serviceStop when NodeLabelsManager.serviceInit failed. (Bibin A Chundatt via wangda) +YARN-3415. Non-AM containers can be counted towards amResourceUsage of a +Fair Scheduler queue (Zhihai Xu via Sandy Ryza) + Release 2.7.0 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/e75307b2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index 46617ff..f0d1ed1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -523,8 +523,11 @@ public class FSAppAttempt extends SchedulerApplicationAttempt // Inform the node node.allocateContainer(allocatedContainer); - // If this container is used to run AM, update the leaf queue's AM usage - if (getLiveContainers().size() == 1 !getUnmanagedAM()) { + // If not running unmanaged, the first container we allocate is always + // the AM. Set the amResource for this app and update the leaf queue's AM + // usage + if (!isAmRunning() !getUnmanagedAM()) { +setAMResource(container.getResource()); getQueue().addAMResourceUsage(container.getResource()); setAmRunning(true); } @@ -551,6 +554,19 @@ public class FSAppAttempt extends SchedulerApplicationAttempt LOG.debug(Node offered to app: + getName() + reserved: + reserved); } +// Check the AM resource usage for the leaf queue +if (!isAmRunning() !getUnmanagedAM()) { + ListResourceRequest ask = appSchedulingInfo.getAllResourceRequests(); + if (ask.isEmpty() || !getQueue().canRunAppAM( + ask.get(0).getCapability())) { +if (LOG.isDebugEnabled()) { + LOG.debug(Skipping allocation because maxAMShare limit would + + be exceeded); +} +return Resources.none(); + } +} + CollectionPriority prioritiesToTry = (reserved) ? Arrays.asList(node.getReservedContainer().getReservedPriority()) : getPriorities(); @@ -567,17 +583,6 @@ public class FSAppAttempt extends SchedulerApplicationAttempt addSchedulingOpportunity(priority); -// Check the AM resource usage for the leaf queue -if (getLiveContainers().size() == 0 !getUnmanagedAM()) { - if (!getQueue().canRunAppAM(getAMResource())) { -if (LOG.isDebugEnabled()) { - LOG.debug(Skipping allocation because maxAMShare limit would + - be exceeded); -} -return Resources.none(); - } -}
[07/50] [abbrv] hadoop git commit: YARN-3425. NPE from RMNodeLabelsManager.serviceStop when NodeLabelsManager.serviceInit failed. (Bibin A Chundatt via wangda)
YARN-3425. NPE from RMNodeLabelsManager.serviceStop when NodeLabelsManager.serviceInit failed. (Bibin A Chundatt via wangda) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a920ab07 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a920ab07 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a920ab07 Branch: refs/heads/YARN-2928 Commit: a920ab0779e9de764ba80d4d2c53b7cb6f6d6d36 Parents: f95a2de Author: Wangda Tan wan...@apache.org Authored: Wed Apr 1 10:14:48 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:10 2015 -0700 -- hadoop-yarn-project/CHANGES.txt | 3 +++ .../apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a920ab07/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index c48de2c..6fbc26e 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -185,6 +185,9 @@ Release 2.8.0 - UNRELEASED YARN-3412. RM tests should use MockRM where possible. (kasha) +YARN-3425. NPE from RMNodeLabelsManager.serviceStop when +NodeLabelsManager.serviceInit failed. (Bibin A Chundatt via wangda) + Release 2.7.0 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/a920ab07/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java index a5e2756..fe38164 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java @@ -258,7 +258,9 @@ public class CommonNodeLabelsManager extends AbstractService { // for UT purpose protected void stopDispatcher() { AsyncDispatcher asyncDispatcher = (AsyncDispatcher) dispatcher; -asyncDispatcher.stop(); +if (null != asyncDispatcher) { + asyncDispatcher.stop(); +} } @Override
[13/50] [abbrv] hadoop git commit: HDFS-7978. Add LOG.isDebugEnabled() guard for some LOG.debug(..). Contributed by Walter Su.
HDFS-7978. Add LOG.isDebugEnabled() guard for some LOG.debug(..). Contributed by Walter Su. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b9fbc73e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b9fbc73e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b9fbc73e Branch: refs/heads/YARN-2928 Commit: b9fbc73e57f2d64cbf7109f232d9c670ca6d7df6 Parents: 188b7d0 Author: Andrew Wang w...@apache.org Authored: Wed Apr 1 12:53:25 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:11 2015 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../apache/hadoop/hdfs/BlockReaderFactory.java | 24 +--- .../java/org/apache/hadoop/hdfs/HAUtil.java | 12 ++ .../hdfs/server/datanode/BPServiceActor.java| 8 --- .../datanode/fsdataset/impl/FsDatasetCache.java | 8 --- .../server/namenode/FileJournalManager.java | 22 +++--- .../hadoop/hdfs/server/namenode/NameNode.java | 4 +++- .../hdfs/shortcircuit/ShortCircuitCache.java| 4 +++- .../tools/offlineImageViewer/FSImageLoader.java | 6 +++-- .../hadoop/hdfs/util/LightWeightHashSet.java| 6 +++-- .../org/apache/hadoop/hdfs/web/TokenAspect.java | 8 +-- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 16 + 12 files changed, 78 insertions(+), 43 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9fbc73e/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index cba53a3..435fdd7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -373,6 +373,9 @@ Release 2.8.0 - UNRELEASED HDFS-8009. Signal congestion on the DataNode. (wheat9) +HDFS-7978. Add LOG.isDebugEnabled() guard for some LOG.debug(..). +(Walter Su via wang) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9fbc73e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java index 1e915b2..8f33899 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java @@ -369,9 +369,9 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator { return null; } if (clientContext.getDisableLegacyBlockReaderLocal()) { - PerformanceAdvisory.LOG.debug(this + : can't construct + - BlockReaderLocalLegacy because + - disableLegacyBlockReaderLocal is set.); +PerformanceAdvisory.LOG.debug({}: can't construct + +BlockReaderLocalLegacy because + +disableLegacyBlockReaderLocal is set., this); return null; } IOException ioe = null; @@ -410,8 +410,8 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator { getPathInfo(inetSocketAddress, conf); } if (!pathInfo.getPathState().getUsableForShortCircuit()) { - PerformanceAdvisory.LOG.debug(this + : + pathInfo + is not + - usable for short circuit; giving up on BlockReaderLocal.); + PerformanceAdvisory.LOG.debug({}: {} is not usable for short circuit; + + giving up on BlockReaderLocal., this, pathInfo); return null; } ShortCircuitCache cache = clientContext.getShortCircuitCache(); @@ -426,11 +426,9 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator { throw exc; } if (info.getReplica() == null) { - if (LOG.isTraceEnabled()) { -PerformanceAdvisory.LOG.debug(this + : failed to get + -ShortCircuitReplica. Cannot construct + -BlockReaderLocal via + pathInfo.getPath()); - } + PerformanceAdvisory.LOG.debug({}: failed to get + + ShortCircuitReplica. Cannot construct + + BlockReaderLocal via {}, this, pathInfo.getPath()); return null; } return new BlockReaderLocal.Builder(conf). @@ -610,9 +608,9 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator { getPathInfo(inetSocketAddress, conf); } if (!pathInfo.getPathState().getUsableForDataTransfer()) { - PerformanceAdvisory.LOG.debug(this + : not trying to create a + - remote
[23/50] [abbrv] hadoop git commit: HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor goes for infinite loop (Contributed by Vinayakumar B)
HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor goes for infinite loop (Contributed by Vinayakumar B) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b4c3ce1a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b4c3ce1a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b4c3ce1a Branch: refs/heads/YARN-2928 Commit: b4c3ce1ac62e734cc0f725ffda1e1319d2a05aca Parents: 0e6b61e Author: Vinayakumar B vinayakum...@apache.org Authored: Thu Apr 2 08:12:00 2015 +0530 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:13 2015 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop/hdfs/server/datanode/ReportBadBlockAction.java| 8 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4c3ce1a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 518df9f..80d958d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -412,6 +412,9 @@ Release 2.8.0 - UNRELEASED HDFS-7922. ShortCircuitCache#close is not releasing ScheduledThreadPoolExecutors (Rakesh R via Colin P. McCabe) +HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor +goes for infinite loop (vinayakumarb) + Release 2.7.0 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4c3ce1a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java index fd01a01..991b56d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; +import org.apache.hadoop.ipc.StandbyException; /** * ReportBadBlockAction is an instruction issued by {{BPOfferService}} to @@ -58,8 +59,11 @@ public class ReportBadBlockAction implements BPServiceActorAction { dnArr, uuids, types) }; try { - bpNamenode.reportBadBlocks(locatedBlock); -} catch (IOException e){ + bpNamenode.reportBadBlocks(locatedBlock); +} catch (StandbyException e) { + DataNode.LOG.warn(Failed to report bad block + block + + to standby namenode); +} catch (IOException e) { throw new BPServiceActorActionException(Failed to report bad block + block + to namenode: ); }
[29/50] [abbrv] hadoop git commit: YARN-3365. Enhanced NodeManager to support using the 'tc' tool via container-executor for outbound network traffic control. Contributed by Sidharta Seethana.
YARN-3365. Enhanced NodeManager to support using the 'tc' tool via container-executor for outbound network traffic control. Contributed by Sidharta Seethana. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4f66d408 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4f66d408 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4f66d408 Branch: refs/heads/YARN-2928 Commit: 4f66d40802a81de710ad8b1909abee2e20e1007d Parents: 503e490 Author: Vinod Kumar Vavilapalli vino...@apache.org Authored: Thu Apr 2 16:53:59 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:14 2015 -0700 -- hadoop-yarn-project/CHANGES.txt | 4 + .../impl/container-executor.c | 100 +++- .../impl/container-executor.h | 34 +- .../main/native/container-executor/impl/main.c | 465 +-- .../nodemanager/TestLinuxContainerExecutor.java | 421 +++-- 5 files changed, 722 insertions(+), 302 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f66d408/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 18a3c37..bcd2286 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -112,6 +112,10 @@ Release 2.8.0 - UNRELEASED YARN-3345. Add non-exclusive node label API. (Wangda Tan via jianhe) +YARN-3365. Enhanced NodeManager to support using the 'tc' tool via +container-executor for outbound network traffic control. (Sidharta Seethana +via vinodkv) + IMPROVEMENTS YARN-1880. Cleanup TestApplicationClientProtocolOnHA http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f66d408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c index edfd25f..485399a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c @@ -40,6 +40,12 @@ static const int DEFAULT_MIN_USERID = 1000; static const char* DEFAULT_BANNED_USERS[] = {yarn, mapred, hdfs, bin, 0}; +//location of traffic control binary +static const char* TC_BIN = /sbin/tc; +static const char* TC_MODIFY_STATE_OPTS [] = { -b , NULL}; +static const char* TC_READ_STATE_OPTS [] = { -b, NULL}; +static const char* TC_READ_STATS_OPTS [] = { -s, -b, NULL}; + //struct to store the user details struct passwd *user_detail = NULL; @@ -291,27 +297,20 @@ static int write_exit_code_file(const char* exit_code_file, int exit_code) { return 0; } -/** - * Wait for the container process to exit and write the exit code to - * the exit code file. - * Returns the exit code of the container process. - */ -static int wait_and_write_exit_code(pid_t pid, const char* exit_code_file) { +static int wait_and_get_exit_code(pid_t pid) { int child_status = -1; int exit_code = -1; int waitpid_result; - if (change_effective_user(nm_uid, nm_gid) != 0) { -return -1; - } do { -waitpid_result = waitpid(pid, child_status, 0); + waitpid_result = waitpid(pid, child_status, 0); } while (waitpid_result == -1 errno == EINTR); + if (waitpid_result 0) { -fprintf(LOGFILE, Error waiting for container process %d - %s\n, -pid, strerror(errno)); +fprintf(LOGFILE, error waiting for process %d - %s\n, pid, strerror(errno)); return -1; } + if (WIFEXITED(child_status)) { exit_code = WEXITSTATUS(child_status); } else if (WIFSIGNALED(child_status)) { @@ -319,9 +318,26 @@ static int wait_and_write_exit_code(pid_t pid, const char* exit_code_file) { } else { fprintf(LOGFILE, Unable to determine exit status for pid %d\n, pid); } + + return exit_code; +} + +/** + * Wait for the container process to exit and write the exit code to + * the exit code file. + * Returns the exit code of the container process. + */ +static int wait_and_write_exit_code(pid_t pid, const char* exit_code_file) { + int exit_code = -1; + + if (change_effective_user(nm_uid, nm_gid) != 0) { +return -1; + } + exit_code = wait_and_get_exit_code(pid); if
[18/50] [abbrv] hadoop git commit: HDFS-7922. ShortCircuitCache#close is not releasing ScheduledThreadPoolExecutors (Rakesh R via Colin P. McCabe)
HDFS-7922. ShortCircuitCache#close is not releasing ScheduledThreadPoolExecutors (Rakesh R via Colin P. McCabe) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/093e0925 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/093e0925 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/093e0925 Branch: refs/heads/YARN-2928 Commit: 093e0925a4459bb07e4093a30455016a815734d5 Parents: f4afce0 Author: Colin Patrick Mccabe cmcc...@cloudera.com Authored: Wed Apr 1 16:02:39 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:12 2015 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hdfs/shortcircuit/ShortCircuitCache.java| 28 .../shortcircuit/TestShortCircuitCache.java | 2 +- 3 files changed, 32 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/093e0925/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index b5591e0..f265ead 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -409,6 +409,9 @@ Release 2.8.0 - UNRELEASED HDFS-6945. BlockManager should remove a block from excessReplicateMap and decrement ExcessBlocks metric when the block is removed. (aajisaka) +HDFS-7922. ShortCircuitCache#close is not releasing +ScheduledThreadPoolExecutors (Rakesh R via Colin P. McCabe) + Release 2.7.0 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/093e0925/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java index 73c52d5..d1ec3b8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java @@ -916,6 +916,34 @@ public class ShortCircuitCache implements Closeable { } finally { lock.unlock(); } + +releaserExecutor.shutdown(); +cleanerExecutor.shutdown(); +// wait for existing tasks to terminate +try { + if (!releaserExecutor.awaitTermination(30, TimeUnit.SECONDS)) { +LOG.error(Forcing SlotReleaserThreadPool to shutdown!); +releaserExecutor.shutdownNow(); + } +} catch (InterruptedException e) { + releaserExecutor.shutdownNow(); + Thread.currentThread().interrupt(); + LOG.error(Interrupted while waiting for SlotReleaserThreadPool + + to terminate, e); +} + +// wait for existing tasks to terminate +try { + if (!cleanerExecutor.awaitTermination(30, TimeUnit.SECONDS)) { +LOG.error(Forcing CleanerThreadPool to shutdown!); +cleanerExecutor.shutdownNow(); + } +} catch (InterruptedException e) { + cleanerExecutor.shutdownNow(); + Thread.currentThread().interrupt(); + LOG.error(Interrupted while waiting for CleanerThreadPool + + to terminate, e); +} IOUtils.cleanup(LOG, shmManager); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/093e0925/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java index 7daabd0..7d26dee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java @@ -203,7 +203,7 @@ public class TestShortCircuitCache { cache.close(); } - @Test(timeout=6) + @Test(timeout=10) public void testExpiry() throws Exception { final ShortCircuitCache cache = new ShortCircuitCache(2, 1, 1, 1000, 1, 1000, 0);
[47/50] [abbrv] hadoop git commit: HADOOP-11776. Fixed the broken JDiff support in Hadoop 2. Contributed by Li Lu.
HADOOP-11776. Fixed the broken JDiff support in Hadoop 2. Contributed by Li Lu. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1069dcee Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1069dcee Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1069dcee Branch: refs/heads/YARN-2928 Commit: 1069dceea24fe019e77619a545909e52430b4275 Parents: 3485a8d Author: Vinod Kumar Vavilapalli vino...@apache.org Authored: Sat Apr 4 13:52:01 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:17 2015 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 2 + .../jdiff/Apache_Hadoop_Common_2.6.0.xml| 45596 + .../jdiff/Apache_Hadoop_HDFS_2.6.0.xml | 19520 +++ hadoop-project-dist/pom.xml |24 +- hadoop-project/pom.xml | 2 + 5 files changed, 65137 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1069dcee/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index fd926aa..f52e09f 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -1205,6 +1205,8 @@ Release 2.7.0 - UNRELEASED HADOOP-11377. Added Null.java without which jdiff completely flops. (Tsuyoshi Ozawa via vinodkv) + +HADOOP-11776. Fixed the broken JDiff support in Hadoop 2. (Li Lu via vinodkv) Release 2.6.1 - UNRELEASED
[17/50] [abbrv] hadoop git commit: HDFS-8008. Support client-side back off when the datanodes are congested. Contributed by Haohui Mai.
HDFS-8008. Support client-side back off when the datanodes are congested. Contributed by Haohui Mai. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f6d4143f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f6d4143f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f6d4143f Branch: refs/heads/YARN-2928 Commit: f6d4143f32845a4230919c998269f59b9f3be11b Parents: 61487d3 Author: Haohui Mai whe...@apache.org Authored: Wed Apr 1 16:54:46 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:12 2015 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../org/apache/hadoop/hdfs/DataStreamer.java| 63 .../hdfs/protocol/datatransfer/PipelineAck.java | 4 ++ .../apache/hadoop/hdfs/TestDFSOutputStream.java | 42 + 4 files changed, 112 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6d4143f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 1d9e200..34c0556 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -868,6 +868,9 @@ Release 2.7.0 - UNRELEASED HDFS-7742. Favoring decommissioning node for replication can cause a block to stay underreplicated for long periods (Nathan Roberts via kihwal) +HDFS-8008. Support client-side back off when the datanodes are congested. +(wheat9) + OPTIMIZATIONS HDFS-7454. Reduce memory footprint for AclEntries in NameNode. http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6d4143f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java index 9c437ba..6ff4c24 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java @@ -218,6 +218,13 @@ class DataStreamer extends Daemon { private boolean failPacket = false; private final long dfsclientSlowLogThresholdMs; private long artificialSlowdown = 0; + // List of congested data nodes. The stream will back off if the DataNodes + // are congested + private final ArrayListDatanodeInfo congestedNodes = new ArrayList(); + private static final int CONGESTION_BACKOFF_MEAN_TIME_IN_MS = 5000; + private static final int CONGESTION_BACK_OFF_MAX_TIME_IN_MS = + CONGESTION_BACKOFF_MEAN_TIME_IN_MS * 10; + private int lastCongestionBackoffTime; private final LoadingCacheDatanodeInfo, DatanodeInfo excludedNodes; @@ -386,6 +393,11 @@ class DataStreamer extends Daemon { one = createHeartbeatPacket(); assert one != null; } else { +try { + backOffIfNecessary(); +} catch (InterruptedException e) { + DFSClient.LOG.warn(Caught exception , e); +} one = dataQueue.getFirst(); // regular data packet long parents[] = one.getTraceParents(); if (parents.length 0) { @@ -815,9 +827,14 @@ class DataStreamer extends Daemon { long seqno = ack.getSeqno(); // processes response status from datanodes. + ArrayListDatanodeInfo congestedNodesFromAck = new ArrayList(); for (int i = ack.getNumOfReplies()-1; i =0 dfsClient.clientRunning; i--) { final Status reply = PipelineAck.getStatusFromHeader(ack .getHeaderFlag(i)); +if (PipelineAck.getECNFromHeader(ack.getHeaderFlag(i)) == +PipelineAck.ECN.CONGESTED) { + congestedNodesFromAck.add(targets[i]); +} // Restart will not be treated differently unless it is // the local node or the only one in the pipeline. if (PipelineAck.isRestartOOBStatus(reply) @@ -839,6 +856,18 @@ class DataStreamer extends Daemon { } } + if (!congestedNodesFromAck.isEmpty()) { +synchronized (congestedNodes) { + congestedNodes.clear(); + congestedNodes.addAll(congestedNodesFromAck); +} + } else { +synchronized (congestedNodes) { + congestedNodes.clear(); + lastCongestionBackoffTime = 0; +} + } + assert seqno != PipelineAck.UNKOWN_SEQNO : Ack for unknown seqno
[01/50] [abbrv] hadoop git commit: YARN-3304. Addendum patch. Cleaning up ResourceCalculatorProcessTree APIs for public use and removing inconsistencies in the default values. (Junping Du and Karthik
Repository: hadoop Updated Branches: refs/heads/YARN-2928 ae0a11167 - 8898d14c0 YARN-3304. Addendum patch. Cleaning up ResourceCalculatorProcessTree APIs for public use and removing inconsistencies in the default values. (Junping Du and Karthik Kambatla via vinodkv) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0834fe37 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0834fe37 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0834fe37 Branch: refs/heads/YARN-2928 Commit: 0834fe372ce55a4fd2b8566cbae95b7b908be71d Parents: 71b4374 Author: Vinod Kumar Vavilapalli vino...@apache.org Authored: Tue Mar 31 17:27:46 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:09 2015 -0700 -- .../yarn/util/ProcfsBasedProcessTree.java | 82 +++-- .../util/ResourceCalculatorProcessTree.java | 55 ++- .../yarn/util/WindowsBasedProcessTree.java | 24 - .../yarn/util/TestProcfsBasedProcessTree.java | 96 ++-- .../util/TestResourceCalculatorProcessTree.java | 10 ++ .../yarn/util/TestWindowsBasedProcessTree.java | 15 ++- 6 files changed, 236 insertions(+), 46 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0834fe37/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java index 9996a79..df9d28a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java @@ -344,15 +344,23 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree { public long getVirtualMemorySize(int olderThanAge) { long total = UNAVAILABLE; for (ProcessInfo p : processTree.values()) { - if ((p != null) (p.getAge() olderThanAge)) { + if (p != null) { if (total == UNAVAILABLE ) { total = 0; } -total += p.getVmem(); +if (p.getAge() olderThanAge) { + total += p.getVmem(); +} } } return total; } + + @Override + @SuppressWarnings(deprecation) + public long getCumulativeVmem(int olderThanAge) { +return getVirtualMemorySize(olderThanAge); + } @Override public long getRssMemorySize(int olderThanAge) { @@ -365,13 +373,21 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree { boolean isAvailable = false; long totalPages = 0; for (ProcessInfo p : processTree.values()) { - if ((p != null) (p.getAge() olderThanAge)) { -totalPages += p.getRssmemPage(); + if ((p != null) ) { +if (p.getAge() olderThanAge) { + totalPages += p.getRssmemPage(); +} isAvailable = true; } } return isAvailable ? totalPages * PAGE_SIZE : UNAVAILABLE; // convert # pages to byte } + + @Override + @SuppressWarnings(deprecation) + public long getCumulativeRssmem(int olderThanAge) { +return getRssMemorySize(olderThanAge); + } /** * Get the resident set size (RSS) memory used by all the processes @@ -388,36 +404,42 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree { private long getSmapBasedRssMemorySize(int olderThanAge) { long total = UNAVAILABLE; for (ProcessInfo p : processTree.values()) { - if ((p != null) (p.getAge() olderThanAge)) { -ProcessTreeSmapMemInfo procMemInfo = processSMAPTree.get(p.getPid()); -if (procMemInfo != null) { - for (ProcessSmapMemoryInfo info : procMemInfo.getMemoryInfoList()) { -// Do not account for r--s or r-xs mappings -if (info.getPermission().trim() - .equalsIgnoreCase(READ_ONLY_WITH_SHARED_PERMISSION) -|| info.getPermission().trim() - .equalsIgnoreCase(READ_EXECUTE_WITH_SHARED_PERMISSION)) { - continue; -} -if (total == UNAVAILABLE){ - total = 0; -} -total += -Math.min(info.sharedDirty, info.pss) + info.privateDirty -+ info.privateClean; -if (LOG.isDebugEnabled()) { - LOG.debug( total( + olderThanAge + ): PID : + p.getPid() - + , SharedDirty : + info.sharedDirty + , PSS : -
[11/50] [abbrv] hadoop git commit: YARN-3248. Display count of nodes blacklisted by apps in the web UI. Contributed by Varun Vasudev
YARN-3248. Display count of nodes blacklisted by apps in the web UI. Contributed by Varun Vasudev Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/40f362b1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/40f362b1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/40f362b1 Branch: refs/heads/YARN-2928 Commit: 40f362b1cbbae7d86d478e781d099e4de5b6ac92 Parents: db2998e Author: Xuan xg...@apache.org Authored: Wed Apr 1 04:19:18 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:10 2015 -0700 -- hadoop-yarn-project/CHANGES.txt | 3 + .../yarn/server/webapp/AppAttemptBlock.java | 66 + .../hadoop/yarn/server/webapp/AppBlock.java | 37 +++-- .../hadoop/yarn/server/webapp/AppsBlock.java| 85 ++- .../scheduler/AppSchedulingInfo.java| 4 + .../scheduler/SchedulerApplicationAttempt.java | 4 + .../webapp/AppsBlockWithMetrics.java| 2 +- .../webapp/CapacitySchedulerPage.java | 2 +- .../webapp/RMAppAttemptBlock.java | 67 + .../resourcemanager/webapp/RMAppBlock.java | 110 ++ .../resourcemanager/webapp/RMAppsBlock.java | 146 +++ .../resourcemanager/webapp/RMWebServices.java | 3 +- .../webapp/dao/AppAttemptInfo.java | 19 ++- .../webapp/TestRMWebServicesApps.java | 2 +- 14 files changed, 468 insertions(+), 82 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/40f362b1/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index f18cf30..461f07d 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -920,6 +920,9 @@ Release 2.7.0 - UNRELEASED removing inconsistencies in the default values. (Junping Du and Karthik Kambatla via vinodkv) +YARN-3248. Display count of nodes blacklisted by apps in the web UI. +(Varun Vasudev via xgong) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/40f362b1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java index dca39d6..8df94e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java @@ -72,7 +72,7 @@ public class AppAttemptBlock extends HtmlBlock { } UserGroupInformation callerUGI = getCallerUGI(); -ApplicationAttemptReport appAttemptReport = null; +ApplicationAttemptReport appAttemptReport; try { final GetApplicationAttemptReportRequest request = GetApplicationAttemptReportRequest.newInstance(appAttemptId); @@ -135,34 +135,7 @@ public class AppAttemptBlock extends HtmlBlock { appAttempt.getRpcPort() 65536) { node = appAttempt.getHost() + : + appAttempt.getRpcPort(); } -info(Application Attempt Overview) - ._( -Application Attempt State:, -appAttempt.getAppAttemptState() == null ? UNAVAILABLE : appAttempt - .getAppAttemptState()) - ._( -AM Container:, -appAttempt.getAmContainerId() == null || containers == null -|| !hasAMContainer(appAttemptReport.getAMContainerId(), containers) -? null : root_url(container, appAttempt.getAmContainerId()), -String.valueOf(appAttempt.getAmContainerId())) - ._(Node:, node) - ._( -Tracking URL:, -appAttempt.getTrackingUrl() == null -|| appAttempt.getTrackingUrl() == UNAVAILABLE ? null -: root_url(appAttempt.getTrackingUrl()), -appAttempt.getTrackingUrl() == null -|| appAttempt.getTrackingUrl() == UNAVAILABLE -? Unassigned -: appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FINISHED -|| appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FAILED -|| appAttempt.getAppAttemptState() == YarnApplicationAttemptState.KILLED -?
[06/50] [abbrv] hadoop git commit: HDFS-7997. The first non-existing xattr should also throw IOException. (zhouyingchao via yliu)
HDFS-7997. The first non-existing xattr should also throw IOException. (zhouyingchao via yliu) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6916c77b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6916c77b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6916c77b Branch: refs/heads/YARN-2928 Commit: 6916c77b9dae46d7e2ce2d0a3a233f33f044a469 Parents: 49db780 Author: yliu y...@apache.org Authored: Tue Mar 31 21:17:44 2015 +0800 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:09 2015 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java | 3 ++- .../hadoop-hdfs/src/test/resources/testXAttrConf.xml | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6916c77b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 0bea916..e8075e6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -392,6 +392,9 @@ Release 2.8.0 - UNRELEASED DatanodeDescriptor#updateHeartbeatState() (Brahma Reddy Battula via Colin P. McCabe) +HDFS-7997. The first non-existing xattr should also throw IOException. +(zhouyingchao via yliu) + Release 2.7.0 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/6916c77b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java index 45e63f2..d5c9124 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java @@ -108,7 +108,8 @@ class FSDirXAttrOp { return filteredAll; } if (filteredAll == null || filteredAll.isEmpty()) { - return null; + throw new IOException( + At least one of the attributes provided was not found.); } ListXAttr toGet = Lists.newArrayListWithCapacity(xAttrs.size()); for (XAttr xAttr : xAttrs) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/6916c77b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testXAttrConf.xml -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testXAttrConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testXAttrConf.xml index 9c66cba..c2e836c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testXAttrConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testXAttrConf.xml @@ -420,8 +420,8 @@ /cleanup-commands comparators comparator - typeExactComparator/type - expected-output# file: /file1#LF#/expected-output + typeSubstringComparator/type + expected-outputAt least one of the attributes provided was not found/expected-output /comparator /comparators /test
[42/50] [abbrv] hadoop git commit: HDFS-8039. Fix TestDebugAdmin#testRecoverLease and testVerfiyBlockChecksumCommand on Windows. Contributed by Xiaoyu Yao.
HDFS-8039. Fix TestDebugAdmin#testRecoverLease and testVerfiyBlockChecksumCommand on Windows. Contributed by Xiaoyu Yao. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4aff1e7b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4aff1e7b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4aff1e7b Branch: refs/heads/YARN-2928 Commit: 4aff1e7bf7866f547163a09b750a14a70fa0602e Parents: def2277 Author: cnauroth cnaur...@apache.org Authored: Fri Apr 3 10:44:02 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:16 2015 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../org/apache/hadoop/hdfs/tools/TestDebugAdmin.java | 15 --- 2 files changed, 11 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4aff1e7b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 366d8fb..ae56f1b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1370,6 +1370,9 @@ Release 2.7.0 - UNRELEASED TestDFSClientRetries#testDFSClientConfigurationLocateFollowingBlockInitialDelay for Windows. (Xiaoyu Yao via cnauroth) +HDFS-8039. Fix TestDebugAdmin#testRecoverLease and +testVerifyBlockChecksumCommand on Windows. (Xiaoyu Yao via cnauroth) + BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS HDFS-7720. Quota by Storage Type API, tools and ClientNameNode http://git-wip-us.apache.org/repos/asf/hadoop/blob/4aff1e7b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java index 44b6ba9..52b194d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java @@ -77,17 +77,18 @@ public class TestDebugAdmin { System.setOut(oldOut); IOUtils.closeStream(out); } -return ret: + ret + , + bytes.toString(); +return ret: + ret + , + +bytes.toString().replaceAll(System.getProperty(line.separator), ); } @Test(timeout = 6) public void testRecoverLease() throws Exception { -assertEquals(ret: 1, You must supply a -path argument to recoverLease.\n, +assertEquals(ret: 1, You must supply a -path argument to recoverLease., runCmd(new String[]{recoverLease, -retries, 1})); FSDataOutputStream out = fs.create(new Path(/foo)); out.write(123); out.close(); -assertEquals(ret: 0, recoverLease SUCCEEDED on /foo\n, +assertEquals(ret: 0, recoverLease SUCCEEDED on /foo, runCmd(new String[]{recoverLease, -path, /foo})); } @@ -98,18 +99,18 @@ public class TestDebugAdmin { ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, new Path(/bar)); File blockFile = getBlockFile(fsd, block.getBlockPoolId(), block.getLocalBlock()); -assertEquals(ret: 1, You must specify a meta file with -meta\n, +assertEquals(ret: 1, You must specify a meta file with -meta, runCmd(new String[]{verify, -block, blockFile.getAbsolutePath()})); File metaFile = getMetaFile(fsd, block.getBlockPoolId(), block.getLocalBlock()); assertEquals(ret: 0, Checksum type: + - DataChecksum(type=CRC32C, chunkSize=512)\n, + DataChecksum(type=CRC32C, chunkSize=512), runCmd(new String[]{verify, -meta, metaFile.getAbsolutePath()})); assertEquals(ret: 0, Checksum type: + - DataChecksum(type=CRC32C, chunkSize=512)\n + + DataChecksum(type=CRC32C, chunkSize=512) + Checksum verification succeeded on block file + - blockFile.getAbsolutePath() + \n, + blockFile.getAbsolutePath(), runCmd(new String[]{verify, -meta, metaFile.getAbsolutePath(), -block, blockFile.getAbsolutePath()})
[24/50] [abbrv] hadoop git commit: HADOOP-11731. Rework the changelog and releasenotes (aw)
HADOOP-11731. Rework the changelog and releasenotes (aw) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0e6b61eb Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0e6b61eb Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0e6b61eb Branch: refs/heads/YARN-2928 Commit: 0e6b61ebe68350d7c4f1d5eaeb97e8977eab0b2f Parents: ccf3fd1 Author: Allen Wittenauer a...@apache.org Authored: Wed Apr 1 17:52:22 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:13 2015 -0700 -- BUILDING.txt| 7 +- dev-support/releasedocmaker.py | 460 +++ dev-support/relnotes.py | 274 --- hadoop-common-project/hadoop-common/CHANGES.txt | 2 + hadoop-common-project/hadoop-common/pom.xml | 51 ++ hadoop-project/src/site/site.xml| 6 +- 6 files changed, 518 insertions(+), 282 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e6b61eb/BUILDING.txt -- diff --git a/BUILDING.txt b/BUILDING.txt index 02b8610..f3b6853 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -73,7 +73,7 @@ Where to run Maven from? -- Maven build goals: - * Clean : mvn clean + * Clean : mvn clean [-Preleasedocs] * Compile : mvn compile [-Pnative] * Run tests : mvn test [-Pnative] * Create JAR: mvn package @@ -84,7 +84,7 @@ Maven build goals: * Run clover: mvn test -Pclover [-DcloverLicenseLocation=${user.name}/.clover.license] * Run Rat : mvn apache-rat:check * Build javadocs: mvn javadoc:javadoc - * Build distribution: mvn package [-Pdist][-Pdocs][-Psrc][-Pnative][-Dtar] + * Build distribution: mvn package [-Pdist][-Pdocs][-Psrc][-Pnative][-Dtar][-Preleasedocs] * Change Hadoop version : mvn versions:set -DnewVersion=NEWVERSION Build options: @@ -93,6 +93,7 @@ Maven build goals: * Use -Pdocs to generate bundle the documentation in the distribution (using -Pdist) * Use -Psrc to create a project source TAR.GZ * Use -Dtar to create a TAR with the distribution (using -Pdist) + * Use -Preleasedocs to include the changelog and release docs (requires Internet connectivity) Snappy build options: @@ -203,7 +204,7 @@ Create source and binary distributions with native code and documentation: Create a local staging version of the website (in /tmp/hadoop-site) - $ mvn clean site; mvn site:stage -DstagingDirectory=/tmp/hadoop-site + $ mvn clean site -Preleasedocs; mvn site:stage -DstagingDirectory=/tmp/hadoop-site -- Installing Hadoop http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e6b61eb/dev-support/releasedocmaker.py -- diff --git a/dev-support/releasedocmaker.py b/dev-support/releasedocmaker.py new file mode 100755 index 000..b00c1a7 --- /dev/null +++ b/dev-support/releasedocmaker.py @@ -0,0 +1,460 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# License); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an AS IS BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from glob import glob +from optparse import OptionParser +import os +import re +import sys +import urllib +try: + import json +except ImportError: + import simplejson as json + +releaseVersion={} +namePattern = re.compile(r' \([0-9]+\)') + +def clean(str): + return tableclean(re.sub(namePattern, , str)) + +def formatComponents(str): + str = re.sub(namePattern, '', str).replace(', ) + if str != : +ret = str + else: +# some markdown parsers don't like empty tables +ret = . + return clean(ret) + +# convert to utf-8 +# protect some known md metachars +# or chars that screw up doxia +def tableclean(str): + str=str.encode('utf-8') + str=str.replace(_,\_) +
[37/50] [abbrv] hadoop git commit: HDFS-8034. Fix TestDFSClientRetries#testDFSClientConfigurationLocateFollowingBlockInitialDelay for Windows. Contributed by Xiaoyu Yao.
HDFS-8034. Fix TestDFSClientRetries#testDFSClientConfigurationLocateFollowingBlockInitialDelay for Windows. Contributed by Xiaoyu Yao. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/310c997b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/310c997b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/310c997b Branch: refs/heads/YARN-2928 Commit: 310c997b56c53e660be31809cb4da74114824666 Parents: ff35b52 Author: cnauroth cnaur...@apache.org Authored: Fri Apr 3 10:10:11 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:15 2015 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4 +++ .../hadoop/hdfs/TestDFSClientRetries.java | 33 +++- 2 files changed, 22 insertions(+), 15 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/310c997b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 04eac75..366d8fb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1366,6 +1366,10 @@ Release 2.7.0 - UNRELEASED HDFS-7954. TestBalancer#testBalancerWithPinnedBlocks should not be executed on Windows. (Xiaoyu Yao via szetszwo) +HDFS-8034. Fix + TestDFSClientRetries#testDFSClientConfigurationLocateFollowingBlockInitialDelay +for Windows. (Xiaoyu Yao via cnauroth) + BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS HDFS-7720. Quota by Storage Type API, tools and ClientNameNode http://git-wip-us.apache.org/repos/asf/hadoop/blob/310c997b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java index 3912774..c4258eb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java @@ -1136,20 +1136,23 @@ public class TestDFSClientRetries { throws Exception { // test if DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_KEY // is not configured, verify DFSClient uses the default value 400. -Configuration dfsConf = new HdfsConfiguration(); -MiniDFSCluster cluster = new MiniDFSCluster.Builder(dfsConf).build(); -cluster.waitActive(); -NamenodeProtocols nn = cluster.getNameNodeRpc(); -DFSClient client = new DFSClient(null, nn, dfsConf, null); -assertEquals(client.getConf(). -getBlockWriteLocateFollowingInitialDelayMs(), 400); - -// change DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_KEY, -// verify DFSClient uses the configured value 1000. -dfsConf.setInt(DFSConfigKeys. -DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_KEY, 1000); -client = new DFSClient(null, nn, dfsConf, null); -assertEquals(client.getConf(). -getBlockWriteLocateFollowingInitialDelayMs(), 1000); +MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); +try { + cluster.waitActive(); + NamenodeProtocols nn = cluster.getNameNodeRpc(); + DFSClient client = new DFSClient(null, nn, conf, null); + assertEquals(client.getConf(). + getBlockWriteLocateFollowingInitialDelayMs(), 400); + + // change DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_KEY, + // verify DFSClient uses the configured value 1000. + conf.setInt(DFSConfigKeys. + DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_KEY, 1000); + client = new DFSClient(null, nn, conf, null); + assertEquals(client.getConf(). + getBlockWriteLocateFollowingInitialDelayMs(), 1000); +} finally { + cluster.shutdown(); +} } }
[10/50] [abbrv] hadoop git commit: YARN-3428. Debug log resources to be localized for a container. (kasha)
YARN-3428. Debug log resources to be localized for a container. (kasha) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b62eb19e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b62eb19e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b62eb19e Branch: refs/heads/YARN-2928 Commit: b62eb19ef02a23a766e4d8fa5172d57a36aec849 Parents: 0834fe3 Author: Karthik Kambatla ka...@apache.org Authored: Tue Mar 31 17:34:40 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:10 2015 -0700 -- hadoop-yarn-project/CHANGES.txt | 2 ++ .../localizer/ResourceLocalizationService.java | 12 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b62eb19e/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 3cf9847..dab8116 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -145,6 +145,8 @@ Release 2.8.0 - UNRELEASED YARN-3258. FairScheduler: Need to add more logging to investigate allocations. (Anubhav Dhoot via ozawa) +YARN-3428. Debug log resources to be localized for a container. (kasha) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not http://git-wip-us.apache.org/repos/asf/hadoop/blob/b62eb19e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java index dd50ead..4236392 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java @@ -448,6 +448,10 @@ public class ResourceLocalizationService extends CompositeService .getApplicationId()); for (LocalResourceRequest req : e.getValue()) { tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt)); +if (LOG.isDebugEnabled()) { + LOG.debug(Localizing + req.getPath() + + for container + c.getContainerId()); +} } } } @@ -456,10 +460,14 @@ public class ResourceLocalizationService extends CompositeService ResourceRetentionSet retain = new ResourceRetentionSet(delService, cacheTargetSize); retain.addResources(publicRsrc); -LOG.debug(Resource cleanup (public) + retain); +if (LOG.isDebugEnabled()) { + LOG.debug(Resource cleanup (public) + retain); +} for (LocalResourcesTracker t : privateRsrc.values()) { retain.addResources(t); - LOG.debug(Resource cleanup + t.getUser() + : + retain); + if (LOG.isDebugEnabled()) { +LOG.debug(Resource cleanup + t.getUser() + : + retain); + } } //TODO Check if appRsrcs should also be added to the retention set. }
[08/50] [abbrv] hadoop git commit: YARN-3248. Correct fix version from branch-2.7 to branch-2.8 in the change log.
YARN-3248. Correct fix version from branch-2.7 to branch-2.8 in the change log. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f95a2de4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f95a2de4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f95a2de4 Branch: refs/heads/YARN-2928 Commit: f95a2de4a3a1b3846a82e7cd34b1140b7154c0f0 Parents: 40f362b Author: Xuan xg...@apache.org Authored: Wed Apr 1 04:32:11 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:10 2015 -0700 -- hadoop-yarn-project/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f95a2de4/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 461f07d..c48de2c 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -150,6 +150,9 @@ Release 2.8.0 - UNRELEASED YARN-3424. Change logs for ContainerMonitorImpl's resourse monitoring from info to debug. (Anubhav Dhoot via ozawa) +YARN-3248. Display count of nodes blacklisted by apps in the web UI. +(Varun Vasudev via xgong) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not @@ -920,9 +923,6 @@ Release 2.7.0 - UNRELEASED removing inconsistencies in the default values. (Junping Du and Karthik Kambatla via vinodkv) -YARN-3248. Display count of nodes blacklisted by apps in the web UI. -(Varun Vasudev via xgong) - Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES
[43/50] [abbrv] hadoop git commit: HADOOP-11377. Added Null.java without which jdiff completely flops. Contributed by Tsuyoshi Ozawa.
HADOOP-11377. Added Null.java without which jdiff completely flops. Contributed by Tsuyoshi Ozawa. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3485a8d7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3485a8d7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3485a8d7 Branch: refs/heads/YARN-2928 Commit: 3485a8d7b42817ddbf4b13c68bf9c195806c1afc Parents: e1ffb3e Author: Vinod Kumar Vavilapalli vino...@apache.org Authored: Sat Apr 4 13:47:08 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:16 2015 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../hadoop-common/dev-support/jdiff/Null.java | 20 .../hadoop-hdfs/dev-support/jdiff/Null.java | 20 3 files changed, 43 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3485a8d7/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index d2d1181..fd926aa 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -1202,6 +1202,9 @@ Release 2.7.0 - UNRELEASED HADOOP-11757. NFS gateway should shutdown when it can't start UDP or TCP server (brandonli) + +HADOOP-11377. Added Null.java without which jdiff completely flops. (Tsuyoshi +Ozawa via vinodkv) Release 2.6.1 - UNRELEASED http://git-wip-us.apache.org/repos/asf/hadoop/blob/3485a8d7/hadoop-common-project/hadoop-common/dev-support/jdiff/Null.java -- diff --git a/hadoop-common-project/hadoop-common/dev-support/jdiff/Null.java b/hadoop-common-project/hadoop-common/dev-support/jdiff/Null.java new file mode 100644 index 000..7b00145 --- /dev/null +++ b/hadoop-common-project/hadoop-common/dev-support/jdiff/Null.java @@ -0,0 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +public class Null { + public Null() { } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3485a8d7/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Null.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Null.java b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Null.java new file mode 100644 index 000..7b00145 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Null.java @@ -0,0 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +public class Null { + public Null() { } +}
[39/50] [abbrv] hadoop git commit: MAPREDUCE-4844. Counters / AbstractCounters have constant references not declared final. (Brahma Reddy Battula via gera)
MAPREDUCE-4844. Counters / AbstractCounters have constant references not declared final. (Brahma Reddy Battula via gera) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/def22772 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/def22772 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/def22772 Branch: refs/heads/YARN-2928 Commit: def2277201a7221c8937f25da13801fd56434c7b Parents: 310c997 Author: Gera Shegalov g...@apache.org Authored: Fri Apr 3 10:24:02 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:16 2015 -0700 -- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../src/main/java/org/apache/hadoop/mapred/Counters.java | 6 +++--- .../org/apache/hadoop/mapreduce/counters/AbstractCounters.java | 6 +++--- 3 files changed, 9 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/def22772/hadoop-mapreduce-project/CHANGES.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 160c49c..c1eb6c3 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -321,6 +321,9 @@ Release 2.8.0 - UNRELEASED causes counter limits are not reset correctly. (Zhihai Xu via harsh) +MAPREDUCE-4844. Counters / AbstractCounters have constant references not +declared final. (Brahma Reddy Battula via gera) + Release 2.7.0 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/def22772/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java index b9cb210..c9e0b9f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java @@ -62,9 +62,9 @@ import com.google.common.collect.Iterators; public class Counters extends AbstractCountersCounters.Counter, Counters.Group { - public static int MAX_COUNTER_LIMIT = Limits.getCountersMax(); - public static int MAX_GROUP_LIMIT = Limits.getGroupsMax(); - private static HashMapString, String depricatedCounterMap = + public static final int MAX_COUNTER_LIMIT = Limits.getCountersMax(); + public static final int MAX_GROUP_LIMIT = Limits.getGroupsMax(); + private static final HashMapString, String depricatedCounterMap = new HashMapString, String(); static { http://git-wip-us.apache.org/repos/asf/hadoop/blob/def22772/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java index dd81ebb..e6e74da 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java @@ -64,11 +64,11 @@ public abstract class AbstractCountersC extends Counter, /** * A cache from enum values to the associated counter. */ - private MapEnum?, C cache = Maps.newIdentityHashMap(); + private final MapEnum?, C cache = Maps.newIdentityHashMap(); //framework fs groups - private MapString, G fgroups = new ConcurrentSkipListMapString, G(); + private final MapString, G fgroups = new ConcurrentSkipListMapString, G(); // other groups - private MapString, G groups = new ConcurrentSkipListMapString, G(); + private final MapString, G groups = new ConcurrentSkipListMapString, G(); private final CounterGroupFactoryC, G groupFactory; // For framework counter serialization without strings
[49/50] [abbrv] hadoop git commit: YARN-3435. AM container to be allocated Appattempt AM container shown as null. Contributed by Bibin A Chundatt
YARN-3435. AM container to be allocated Appattempt AM container shown as null. Contributed by Bibin A Chundatt Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0b09b406 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0b09b406 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0b09b406 Branch: refs/heads/YARN-2928 Commit: 0b09b406c6d09e031357ad84a810c9ddfe4a922d Parents: 1069dce Author: Xuan xg...@apache.org Authored: Sun Apr 5 00:40:57 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:17 2015 -0700 -- hadoop-yarn-project/CHANGES.txt | 3 +++ .../yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b09b406/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 004044a..0c5aac8 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -198,6 +198,9 @@ Release 2.8.0 - UNRELEASED YARN-3415. Non-AM containers can be counted towards amResourceUsage of a Fair Scheduler queue (Zhihai Xu via Sandy Ryza) +YARN-3435. AM container to be allocated Appattempt AM container shown as null. +(Bibin A Chundatt via xgong) + Release 2.7.0 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b09b406/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java index 1831920..506e31f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java @@ -209,7 +209,7 @@ public class RMAppAttemptBlock extends AppAttemptBlock{ AM Container:, appAttempt.getAmContainerId() == null || containers == null || !hasAMContainer(appAttemptReport.getAMContainerId(), containers) -? null : root_url(container, appAttempt.getAmContainerId()), +? N/A : root_url(container, appAttempt.getAmContainerId()), String.valueOf(appAttempt.getAmContainerId())) ._(Node:, node) ._(
[19/50] [abbrv] hadoop git commit: HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than DFSOutputStream#writeChunk (cmccabe)
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than DFSOutputStream#writeChunk (cmccabe) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4afce08 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4afce08 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4afce08 Branch: refs/heads/YARN-2928 Commit: f4afce0801bc488e301aee820a8091252bbc2748 Parents: f066e73 Author: Colin Patrick Mccabe cmcc...@cloudera.com Authored: Wed Apr 1 13:55:40 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:12 2015 -0700 -- .../org/apache/hadoop/fs/FSOutputSummer.java| 20 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../org/apache/hadoop/hdfs/DFSOutputStream.java | 15 --- .../org/apache/hadoop/tracing/TestTracing.java | 4 ++-- 4 files changed, 25 insertions(+), 17 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4afce08/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java index 13a5e26..d2998b6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java @@ -21,6 +21,8 @@ package org.apache.hadoop.fs; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.util.DataChecksum; +import org.apache.htrace.NullScope; +import org.apache.htrace.TraceScope; import java.io.IOException; import java.io.OutputStream; @@ -194,16 +196,26 @@ abstract public class FSOutputSummer extends OutputStream { return sum.getChecksumSize(); } + protected TraceScope createWriteTraceScope() { +return NullScope.INSTANCE; + } + /** Generate checksums for the given data chunks and output chunks checksums * to the underlying output stream. */ private void writeChecksumChunks(byte b[], int off, int len) throws IOException { sum.calculateChunkedSums(b, off, len, checksum, 0); -for (int i = 0; i len; i += sum.getBytesPerChecksum()) { - int chunkLen = Math.min(sum.getBytesPerChecksum(), len - i); - int ckOffset = i / sum.getBytesPerChecksum() * getChecksumSize(); - writeChunk(b, off + i, chunkLen, checksum, ckOffset, getChecksumSize()); +TraceScope scope = createWriteTraceScope(); +try { + for (int i = 0; i len; i += sum.getBytesPerChecksum()) { +int chunkLen = Math.min(sum.getBytesPerChecksum(), len - i); +int ckOffset = i / sum.getBytesPerChecksum() * getChecksumSize(); +writeChunk(b, off + i, chunkLen, checksum, ckOffset, +getChecksumSize()); + } +} finally { + scope.close(); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4afce08/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 435fdd7..b5591e0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -378,6 +378,9 @@ Release 2.8.0 - UNRELEASED OPTIMIZATIONS +HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than +DFSOutputStream#writeChunk (cmccabe) + BUG FIXES HDFS-7501. TransactionsSinceLastCheckpoint can be negative on SBNs. http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4afce08/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index 933d8e6..c88639d 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -372,21 +372,14 @@ public class DFSOutputStream extends FSOutputSummer } } + protected TraceScope createWriteTraceScope() { +return dfsClient.getPathTraceScope(DFSOutputStream#write, src); + } + // @see FSOutputSummer#writeChunk() @Override protected synchronized void writeChunk(byte[] b, int offset, int len, byte[] checksum, int ckoff, int cklen)
[41/50] [abbrv] hadoop git commit: HADOOP-11785. Reduce the number of listStatus operation in distcp buildListing (Zoran Dimitrijevic via Colin P. McCabe)
HADOOP-11785. Reduce the number of listStatus operation in distcp buildListing (Zoran Dimitrijevic via Colin P. McCabe) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fb13303d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fb13303d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fb13303d Branch: refs/heads/YARN-2928 Commit: fb13303de639de8b45c78e91d22c85774ad4252e Parents: 4aff1e7 Author: Colin Patrick Mccabe cmcc...@cloudera.com Authored: Fri Apr 3 14:08:25 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:16 2015 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++ .../apache/hadoop/tools/SimpleCopyListing.java | 41 +--- 2 files changed, 21 insertions(+), 23 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb13303d/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 7dcf8c1..d2d1181 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -483,6 +483,9 @@ Release 2.8.0 - UNRELEASED OPTIMIZATIONS +HADOOP-11785. Reduce the number of listStatus operation in distcp +buildListing (Zoran Dimitrijevic via Colin P. McCabe) + BUG FIXES HADOOP-10027. *Compressor_deflateBytesDirect passes instance instead of http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb13303d/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java -- diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java index 6dc827a..e8a23aa 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java @@ -193,12 +193,12 @@ public class SimpleCopyListing extends CopyListing { writeToFileListing(fileListWriter, sourceCopyListingStatus, sourcePathRoot, options); -if (isDirectoryAndNotEmpty(sourceFS, sourceStatus)) { +if (sourceStatus.isDirectory()) { if (LOG.isDebugEnabled()) { -LOG.debug(Traversing non-empty source dir: + sourceStatus.getPath()); +LOG.debug(Traversing source dir: + sourceStatus.getPath()); } - traverseNonEmptyDirectory(fileListWriter, sourceStatus, sourcePathRoot, - options); + traverseDirectory(fileListWriter, sourceFS, sourceStatus, +sourcePathRoot, options); } } } @@ -275,22 +275,17 @@ public class SimpleCopyListing extends CopyListing { SequenceFile.Writer.compression(SequenceFile.CompressionType.NONE)); } - private static boolean isDirectoryAndNotEmpty(FileSystem fileSystem, -FileStatus fileStatus) throws IOException { -return fileStatus.isDirectory() getChildren(fileSystem, fileStatus).length 0; - } - private static FileStatus[] getChildren(FileSystem fileSystem, FileStatus parent) throws IOException { return fileSystem.listStatus(parent.getPath()); } - private void traverseNonEmptyDirectory(SequenceFile.Writer fileListWriter, - FileStatus sourceStatus, - Path sourcePathRoot, - DistCpOptions options) - throws IOException { -FileSystem sourceFS = sourcePathRoot.getFileSystem(getConf()); + private void traverseDirectory(SequenceFile.Writer fileListWriter, + FileSystem sourceFS, + FileStatus sourceStatus, + Path sourcePathRoot, + DistCpOptions options) + throws IOException { final boolean preserveAcls = options.shouldPreserve(FileAttribute.ACL); final boolean preserveXAttrs = options.shouldPreserve(FileAttribute.XATTR); final boolean preserveRawXattrs = options.shouldPreserveRawXattrs(); @@ -299,9 +294,9 @@ public class SimpleCopyListing extends CopyListing { while (!pathStack.isEmpty()) { for (FileStatus child: getChildren(sourceFS, pathStack.pop())) { -if (LOG.isDebugEnabled()) -
[21/50] [abbrv] hadoop git commit: HADOOP-11757. NFS gateway should shutdown when it can't start UDP or TCP server. Contributed by Brandon Li
HADOOP-11757. NFS gateway should shutdown when it can't start UDP or TCP server. Contributed by Brandon Li Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09686762 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09686762 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09686762 Branch: refs/heads/YARN-2928 Commit: 096867629333d4b2de4e7b971b758fbabc11d1d3 Parents: f6d4143 Author: Brandon Li brando...@apache.org Authored: Wed Apr 1 17:04:44 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:12 2015 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 5 ++- .../org/apache/hadoop/mount/MountdBase.java | 26 +-- .../org/apache/hadoop/nfs/nfs3/Nfs3Base.java| 15 +++-- .../apache/hadoop/oncrpc/SimpleTcpServer.java | 31 -- .../apache/hadoop/oncrpc/SimpleUdpServer.java | 33 +--- 5 files changed, 82 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/09686762/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 111fb5e..fa98a0c 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -1186,7 +1186,10 @@ Release 2.7.0 - UNRELEASED HADOOP-11787. OpensslSecureRandom.c pthread_threadid_np usage signature is wrong on 32-bit Mac. (Kiran Kumar M R via cnauroth) - + +HADOOP-11757. NFS gateway should shutdown when it can't start UDP or TCP +server (brandonli) + Release 2.6.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/09686762/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountdBase.java -- diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountdBase.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountdBase.java index 8d7d6dc..92ca7ec 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountdBase.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountdBase.java @@ -60,7 +60,17 @@ abstract public class MountdBase { SimpleUdpServer udpServer = new SimpleUdpServer(rpcProgram.getPort(), rpcProgram, 1); rpcProgram.startDaemons(); -udpServer.run(); +try { + udpServer.run(); +} catch (Throwable e) { + LOG.fatal(Failed to start the UDP server., e); + if (udpServer.getBoundPort() 0) { +rpcProgram.unregister(PortmapMapping.TRANSPORT_UDP, +udpServer.getBoundPort()); + } + udpServer.shutdown(); + terminate(1, e); +} udpBoundPort = udpServer.getBoundPort(); } @@ -69,7 +79,17 @@ abstract public class MountdBase { SimpleTcpServer tcpServer = new SimpleTcpServer(rpcProgram.getPort(), rpcProgram, 1); rpcProgram.startDaemons(); -tcpServer.run(); +try { + tcpServer.run(); +} catch (Throwable e) { + LOG.fatal(Failed to start the TCP server., e); + if (tcpServer.getBoundPort() 0) { +rpcProgram.unregister(PortmapMapping.TRANSPORT_TCP, +tcpServer.getBoundPort()); + } + tcpServer.shutdown(); + terminate(1, e); +} tcpBoundPort = tcpServer.getBoundPort(); } @@ -83,7 +103,7 @@ abstract public class MountdBase { rpcProgram.register(PortmapMapping.TRANSPORT_UDP, udpBoundPort); rpcProgram.register(PortmapMapping.TRANSPORT_TCP, tcpBoundPort); } catch (Throwable e) { -LOG.fatal(Failed to start the server. Cause:, e); +LOG.fatal(Failed to register the MOUNT service., e); terminate(1, e); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/09686762/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Base.java -- diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Base.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Base.java index 40744bc..80faca5 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Base.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Base.java @@ -29,7 +29,6 @@ import static org.apache.hadoop.util.ExitUtil.terminate; /** * Nfs server. Supports NFS v3 using {@link RpcProgram}. - * Currently Mountd program is also started inside this
[04/50] [abbrv] hadoop git commit: YARN-3412. RM tests should use MockRM where possible. (kasha)
YARN-3412. RM tests should use MockRM where possible. (kasha) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5b9d546c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5b9d546c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5b9d546c Branch: refs/heads/YARN-2928 Commit: 5b9d546c479dc41356aad99038003349d810f84c Parents: 8af286f Author: Karthik Kambatla ka...@apache.org Authored: Tue Mar 31 09:14:15 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:09 2015 -0700 -- hadoop-yarn-project/CHANGES.txt | 2 ++ .../yarn/server/resourcemanager/TestMoveApplication.java | 3 +-- .../yarn/server/resourcemanager/TestResourceManager.java | 6 ++ .../server/resourcemanager/monitor/TestSchedulingMonitor.java | 3 ++- .../server/resourcemanager/recovery/TestZKRMStateStore.java | 7 +++ .../resourcemanager/scheduler/fair/TestFairScheduler.java | 3 +-- .../scheduler/fair/TestFairSchedulerEventLog.java | 4 ++-- .../resourcemanager/scheduler/fifo/TestFifoScheduler.java | 5 ++--- 8 files changed, 15 insertions(+), 18 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b9d546c/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 2466642..3cf9847 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -175,6 +175,8 @@ Release 2.8.0 - UNRELEASED YARN-3400. [JDK 8] Build Failure due to unreported exceptions in RPCUtil (rkanter) +YARN-3412. RM tests should use MockRM where possible. (kasha) + Release 2.7.0 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b9d546c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestMoveApplication.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestMoveApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestMoveApplication.java index 36153de..d2bde80 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestMoveApplication.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestMoveApplication.java @@ -52,8 +52,7 @@ public class TestMoveApplication { FifoSchedulerWithMove.class); conf.set(YarnConfiguration.YARN_ADMIN_ACL, ); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); -resourceManager = new ResourceManager(); -resourceManager.init(conf); +resourceManager = new MockRM(conf); resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey(); resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey(); resourceManager.start(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b9d546c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java index 6735575..fbf54fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java @@ -212,9 +212,8 @@ public class TestResourceManager { public void testResourceManagerInitConfigValidation() throws Exception { Configuration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, -1); -resourceManager = new ResourceManager(); try { - resourceManager.init(conf); + resourceManager = new MockRM(conf); fail(Exception is
[15/50] [abbrv] hadoop git commit: HDFS-8009. Signal congestion on the DataNode. Contributed by Haohui Mai.
HDFS-8009. Signal congestion on the DataNode. Contributed by Haohui Mai. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0d7c0758 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0d7c0758 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0d7c0758 Branch: refs/heads/YARN-2928 Commit: 0d7c0758b7079ed1660f124d2adcc91ccc9e7289 Parents: a920ab0 Author: Haohui Mai whe...@apache.org Authored: Wed Apr 1 10:56:53 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:11 2015 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../apache/hadoop/hdfs/server/datanode/DataNode.java| 12 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d7c0758/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 4247ea6..cba53a3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -371,6 +371,8 @@ Release 2.8.0 - UNRELEASED HDFS-7671. hdfs user guide should point to the common rack awareness doc. (Kai Sasaki via aajisaka) +HDFS-8009. Signal congestion on the DataNode. (wheat9) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d7c0758/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 071aba1..50dccb8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -354,6 +354,9 @@ public class DataNode extends ReconfigurableBase private String dnUserName = null; private SpanReceiverHost spanReceiverHost; + private static final int NUM_CORES = Runtime.getRuntime() + .availableProcessors(); + private static final double CONGESTION_RATIO = 1.5; /** * Creates a dummy DataNode for testing purpose. @@ -484,8 +487,13 @@ public class DataNode extends ReconfigurableBase * /ul */ public PipelineAck.ECN getECN() { -return pipelineSupportECN ? PipelineAck.ECN.SUPPORTED : PipelineAck.ECN - .DISABLED; +if (!pipelineSupportECN) { + return PipelineAck.ECN.DISABLED; +} +double load = ManagementFactory.getOperatingSystemMXBean() +.getSystemLoadAverage(); +return load NUM_CORES * CONGESTION_RATIO ? PipelineAck.ECN.CONGESTED : +PipelineAck.ECN.SUPPORTED; } /**
[16/50] [abbrv] hadoop git commit: Add the missing files for HDFS-8009.
Add the missing files for HDFS-8009. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/777f01b5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/777f01b5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/777f01b5 Branch: refs/heads/YARN-2928 Commit: 777f01b5defa1b53cde21042f5b56a22175e5a87 Parents: 0d7c075 Author: Haohui Mai whe...@apache.org Authored: Wed Apr 1 10:58:32 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:11 2015 -0700 -- .../hdfs/server/datanode/TestDataNodeECN.java | 45 1 file changed, 45 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/777f01b5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeECN.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeECN.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeECN.java new file mode 100644 index 000..b994386 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeECN.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.datanode; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; + +public class TestDataNodeECN { + @Test + public void testECNFlag() throws IOException { +Configuration conf = new Configuration(); +conf.setBoolean(DFSConfigKeys.DFS_PIPELINE_ECN_ENABLED, true); +MiniDFSCluster cluster = null; +try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + PipelineAck.ECN ecn = cluster.getDataNodes().get(0).getECN(); + Assert.assertNotEquals(PipelineAck.ECN.DISABLED, ecn); +} finally { + if (cluster != null) { +cluster.shutdown(); + } +} + } +}
[50/50] [abbrv] hadoop git commit: HDFS-7999. FsDatasetImpl#createTemporary sometimes holds the FSDatasetImpl lock for a very long time (sinago via cmccabe)
HDFS-7999. FsDatasetImpl#createTemporary sometimes holds the FSDatasetImpl lock for a very long time (sinago via cmccabe) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8898d14c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8898d14c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8898d14c Branch: refs/heads/YARN-2928 Commit: 8898d14c00981c48bdf6c575d6604e19adaca7f7 Parents: 37725e7 Author: Colin Patrick Mccabe cmcc...@cloudera.com Authored: Mon Apr 6 08:54:46 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:17 2015 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../datanode/fsdataset/impl/FsDatasetImpl.java | 77 +--- 2 files changed, 52 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8898d14c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 6fafec8..52325a2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1379,6 +1379,9 @@ Release 2.7.0 - UNRELEASED HDFS-8051. FsVolumeList#addVolume should release volume reference if not put it into BlockScanner. (Lei (Eddy) Xu via Colin P. McCabe) +HDFS-7999. FsDatasetImpl#createTemporary sometimes holds the FSDatasetImpl +lock for a very long time (sinago via cmccabe) + BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS HDFS-7720. Quota by Storage Type API, tools and ClientNameNode http://git-wip-us.apache.org/repos/asf/hadoop/blob/8898d14c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index f15f649..6bcbe5a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -1412,38 +1412,59 @@ class FsDatasetImpl implements FsDatasetSpiFsVolumeImpl { } @Override // FsDatasetSpi - public synchronized ReplicaHandler createTemporary( + public ReplicaHandler createTemporary( StorageType storageType, ExtendedBlock b) throws IOException { -ReplicaInfo replicaInfo = volumeMap.get(b.getBlockPoolId(), b.getBlockId()); -if (replicaInfo != null) { - if (replicaInfo.getGenerationStamp() b.getGenerationStamp() - replicaInfo instanceof ReplicaInPipeline) { -// Stop the previous writer -((ReplicaInPipeline)replicaInfo) - .stopWriter(datanode.getDnConf().getXceiverStopTimeout()); -invalidate(b.getBlockPoolId(), new Block[]{replicaInfo}); - } else { -throw new ReplicaAlreadyExistsException(Block + b + - already exists in state + replicaInfo.getState() + - and thus cannot be created.); +long startTimeMs = Time.monotonicNow(); +long writerStopTimeoutMs = datanode.getDnConf().getXceiverStopTimeout(); +ReplicaInfo lastFoundReplicaInfo = null; +do { + synchronized (this) { +ReplicaInfo currentReplicaInfo = +volumeMap.get(b.getBlockPoolId(), b.getBlockId()); +if (currentReplicaInfo == lastFoundReplicaInfo) { + if (lastFoundReplicaInfo != null) { +invalidate(b.getBlockPoolId(), new Block[] { lastFoundReplicaInfo }); + } + FsVolumeReference ref = + volumes.getNextVolume(storageType, b.getNumBytes()); + FsVolumeImpl v = (FsVolumeImpl) ref.getVolume(); + // create a temporary file to hold block in the designated volume + File f; + try { +f = v.createTmpFile(b.getBlockPoolId(), b.getLocalBlock()); + } catch (IOException e) { +IOUtils.cleanup(null, ref); +throw e; + } + ReplicaInPipeline newReplicaInfo = + new ReplicaInPipeline(b.getBlockId(), b.getGenerationStamp(), v, + f.getParentFile(), 0); + volumeMap.add(b.getBlockPoolId(), newReplicaInfo); + return new ReplicaHandler(newReplicaInfo, ref); +} else { + if (!(currentReplicaInfo.getGenerationStamp() b + .getGenerationStamp() currentReplicaInfo instanceof
[02/50] [abbrv] hadoop git commit: HDFS-6945. BlockManager should remove a block from excessReplicateMap and decrement ExcessBlocks metric when the block is removed. (aajisaka)
HDFS-6945. BlockManager should remove a block from excessReplicateMap and decrement ExcessBlocks metric when the block is removed. (aajisaka) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71b43741 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71b43741 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71b43741 Branch: refs/heads/YARN-2928 Commit: 71b43741dde686725ac188ae91b14185989944a1 Parents: 40a1282 Author: Akira Ajisaka aajis...@apache.org Authored: Wed Apr 1 09:07:28 2015 +0900 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:09 2015 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../server/blockmanagement/BlockManager.java| 22 ++-- .../namenode/metrics/TestNameNodeMetrics.java | 9 ++-- 3 files changed, 30 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/71b43741/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index f3537b0..4247ea6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -398,6 +398,9 @@ Release 2.8.0 - UNRELEASED HDFS-7997. The first non-existing xattr should also throw IOException. (zhouyingchao via yliu) +HDFS-6945. BlockManager should remove a block from excessReplicateMap and +decrement ExcessBlocks metric when the block is removed. (aajisaka) + Release 2.7.0 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/71b43741/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index f6e15a3..d9aee62 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -3351,8 +3351,7 @@ public class BlockManager { // file already removes them from the block map below. block.setNumBytes(BlockCommand.NO_ACK); addToInvalidates(block); -corruptReplicas.removeFromCorruptReplicasMap(block); -blocksMap.removeBlock(block); +removeBlockFromMap(block); // Remove the block from pendingReplications and neededReplications pendingReplications.remove(block); neededReplications.remove(block, UnderReplicatedBlocks.LEVEL); @@ -3528,11 +3527,30 @@ public class BlockManager { } public void removeBlockFromMap(Block block) { +removeFromExcessReplicateMap(block); blocksMap.removeBlock(block); // If block is removed from blocksMap remove it from corruptReplicasMap corruptReplicas.removeFromCorruptReplicasMap(block); } + /** + * If a block is removed from blocksMap, remove it from excessReplicateMap. + */ + private void removeFromExcessReplicateMap(Block block) { +for (DatanodeStorageInfo info : blocksMap.getStorages(block)) { + String uuid = info.getDatanodeDescriptor().getDatanodeUuid(); + LightWeightLinkedSetBlock excessReplicas = excessReplicateMap.get(uuid); + if (excessReplicas != null) { +if (excessReplicas.remove(block)) { + excessBlocksCount.decrementAndGet(); + if (excessReplicas.isEmpty()) { +excessReplicateMap.remove(uuid); + } +} + } +} + } + public int getCapacity() { return blocksMap.getCapacity(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/71b43741/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java index 64ea1e4..b390391 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java @@ -280,11 +280,16 @@ public class TestNameNodeMetrics { public void testExcessBlocks() throws Exception {
[34/50] [abbrv] hadoop git commit: HADOOP-11797. releasedocmaker.py needs to put ASF headers on output (aw)
HADOOP-11797. releasedocmaker.py needs to put ASF headers on output (aw) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c134f991 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c134f991 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c134f991 Branch: refs/heads/YARN-2928 Commit: c134f991272f40c87ddeeafb07fb96e801ec53df Parents: d5d8402 Author: Allen Wittenauer a...@apache.org Authored: Thu Apr 2 20:50:36 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:15 2015 -0700 -- dev-support/releasedocmaker.py | 23 hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ 2 files changed, 25 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c134f991/dev-support/releasedocmaker.py -- diff --git a/dev-support/releasedocmaker.py b/dev-support/releasedocmaker.py index b00c1a7..78c8e6b 100755 --- a/dev-support/releasedocmaker.py +++ b/dev-support/releasedocmaker.py @@ -30,6 +30,26 @@ except ImportError: releaseVersion={} namePattern = re.compile(r' \([0-9]+\)') +asflicense=''' +!--- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# License); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an AS IS BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +-- +''' + def clean(str): return tableclean(re.sub(namePattern, , str)) @@ -357,6 +377,9 @@ def main(): %(ver)s/CHANGES.%(key)s.%(ver)s.md, [HADOOP,HDFS,MAPREDUCE,YARN], {ver:maxVersion, date:reldate}) + reloutputs.writeAll(asflicense) + choutputs.writeAll(asflicense) + relhead = '# Hadoop %(key)s %(ver)s Release Notes\n\n' \ 'These release notes cover new developer and user-facing incompatibilities, features, and major improvements.\n\n' http://git-wip-us.apache.org/repos/asf/hadoop/blob/c134f991/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 260cc83..495c42d 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -431,6 +431,8 @@ Trunk (Unreleased) HADOOP-11703. git should ignore .DS_Store files on Mac OS X (Abin Shahab via aw) +HADOOP-11797. releasedocmaker.py needs to put ASF headers on output (aw) + OPTIMIZATIONS HADOOP-7761. Improve the performance of raw comparisons. (todd)
[27/50] [abbrv] hadoop git commit: HDFS-7954. TestBalancer#testBalancerWithPinnedBlocks should not be executed on Windows. Contributed by Xiaoyu Yao
HDFS-7954. TestBalancer#testBalancerWithPinnedBlocks should not be executed on Windows. Contributed by Xiaoyu Yao Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0a1dcd17 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0a1dcd17 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0a1dcd17 Branch: refs/heads/YARN-2928 Commit: 0a1dcd17cc585d145ad4ee1e84ac3d3267dd105c Parents: b98f637 Author: Tsz-Wo Nicholas Sze szets...@hortonworks.com Authored: Thu Apr 2 11:11:34 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:13 2015 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt| 3 +++ .../org/apache/hadoop/hdfs/server/balancer/TestBalancer.java | 6 ++ 2 files changed, 9 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a1dcd17/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 0c66309..85c013e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1360,6 +1360,9 @@ Release 2.7.0 - UNRELEASED HDFS-8001 RpcProgramNfs3 : wrong parsing of dfs.blocksize (Remi Catherinot via brandonli) +HDFS-7954. TestBalancer#testBalancerWithPinnedBlocks should not be executed +on Windows. (Xiaoyu Yao via szetszwo) + BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS HDFS-7720. Quota by Storage Type API, tools and ClientNameNode http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a1dcd17/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java index 8b2d11e..9aea283 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java @@ -23,6 +23,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.*; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.junit.Assume.assumeTrue; import java.io.File; import java.io.IOException; @@ -309,6 +310,11 @@ public class TestBalancer { */ @Test(timeout=10) public void testBalancerWithPinnedBlocks() throws Exception { +// This test assumes stick-bit based block pin mechanism available only +// in Linux/Unix. It can be unblocked on Windows when HDFS-7759 is ready to +// provide a different mechanism for Windows. +assumeTrue(!Path.WINDOWS); + final Configuration conf = new HdfsConfiguration(); initConf(conf); conf.setBoolean(DFS_DATANODE_BLOCK_PINNING_ENABLED, true);
[32/50] [abbrv] hadoop git commit: HDFS-7893. Update the POM to create a separate hdfs-client jar. Contributed by Haohui Mai.
HDFS-7893. Update the POM to create a separate hdfs-client jar. Contributed by Haohui Mai. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88c14681 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88c14681 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88c14681 Branch: refs/heads/YARN-2928 Commit: 88c146814b3431a8eb98c03a0ed706a675058b4e Parents: e75307b Author: Haohui Mai whe...@apache.org Authored: Thu Apr 2 15:29:18 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:14 2015 -0700 -- .../dev-support/findbugsExcludeFile.xml | 2 + hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 49 hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 5 ++ hadoop-hdfs-project/hadoop-hdfs/pom.xml | 5 ++ .../hadoop-hdfs/src/contrib/bkjournal/pom.xml | 5 ++ hadoop-hdfs-project/pom.xml | 1 + hadoop-project/pom.xml | 5 ++ 7 files changed, 72 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/88c14681/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml new file mode 100644 index 000..43bc332 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml @@ -0,0 +1,2 @@ +FindBugsFilter +/FindBugsFilter http://git-wip-us.apache.org/repos/asf/hadoop/blob/88c14681/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml new file mode 100644 index 000..900f345 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml @@ -0,0 +1,49 @@ +?xml version=1.0 encoding=UTF-8? +!-- + Licensed under the Apache License, Version 2.0 (the License); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an AS IS BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +-- +project xmlns=http://maven.apache.org/POM/4.0.0; +xmlns:xsi=http://www.w3.org/2001/XMLSchema-instance; +xsi:schemaLocation=http://maven.apache.org/POM/4.0.0 +http://maven.apache.org/xsd/maven-4.0.0.xsd; + modelVersion4.0.0/modelVersion + parent +groupIdorg.apache.hadoop/groupId +artifactIdhadoop-project-dist/artifactId +version3.0.0-SNAPSHOT/version +relativePath../../hadoop-project-dist/relativePath + /parent + groupIdorg.apache.hadoop/groupId + artifactIdhadoop-hdfs-client/artifactId + version3.0.0-SNAPSHOT/version + descriptionApache Hadoop HDFS Client/description + nameApache Hadoop HDFS Client/name + packagingjar/packaging + build +plugins + plugin +groupIdorg.apache.maven.plugins/groupId +artifactIdmaven-surefire-plugin/artifactId + /plugin + plugin +groupIdorg.apache.rat/groupId +artifactIdapache-rat-plugin/artifactId +configuration + excludes +excludedev-support/findbugsExcludeFile.xml/exclude + /excludes +/configuration + /plugin +/plugins + /build +/project http://git-wip-us.apache.org/repos/asf/hadoop/blob/88c14681/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml index 9a9d29c..ac8930c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml @@ -58,6 +58,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd; /dependency dependency groupIdorg.apache.hadoop/groupId + artifactIdhadoop-hdfs-client/artifactId + scopecompile/scope +/dependency +dependency + groupIdorg.apache.hadoop/groupId artifactIdhadoop-hdfs/artifactId scopetest/scope typetest-jar/type http://git-wip-us.apache.org/repos/asf/hadoop/blob/88c14681/hadoop-hdfs-project/hadoop-hdfs/pom.xml -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index 2d402a9..c11b963
[03/50] [abbrv] hadoop git commit: Reverted MAPREDUCE-6286, MAPREDUCE-6199, and MAPREDUCE-5875 from branch-2.7. Editing CHANGES.txt to reflect this.
Reverted MAPREDUCE-6286, MAPREDUCE-6199, and MAPREDUCE-5875 from branch-2.7. Editing CHANGES.txt to reflect this. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/40a12828 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/40a12828 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/40a12828 Branch: refs/heads/YARN-2928 Commit: 40a1282880788e2dd2a82ad54838f357348117b3 Parents: 5b9d546 Author: Vinod Kumar Vavilapalli vino...@apache.org Authored: Tue Mar 31 13:29:20 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:09 2015 -0700 -- hadoop-mapreduce-project/CHANGES.txt | 20 ++-- 1 file changed, 10 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/40a12828/hadoop-mapreduce-project/CHANGES.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index b0367a7..f5d2d1a 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -311,6 +311,16 @@ Release 2.8.0 - UNRELEASED MAPREDUCE-6294. Remove an extra parameter described in Javadoc of TockenCache. (Brahma Reddy Battula via ozawa) +MAPREDUCE-5875. Make Counter limits consistent across JobClient, +MRAppMaster, and YarnChild. (Gera Shegalov via kasha) + +MAPREDUCE-6199. AbstractCounters are not reset completely on +deserialization (adhoot via rkanter) + +MAPREDUCE-6286. A typo in HistoryViewer makes some code useless, which +causes counter limits are not reset correctly. +(Zhihai Xu via harsh) + Release 2.7.0 - UNRELEASED INCOMPATIBLE CHANGES @@ -411,10 +421,6 @@ Release 2.7.0 - UNRELEASED BUG FIXES -MAPREDUCE-6286. A typo in HistoryViewer makes some code useless, which -causes counter limits are not reset correctly. -(Zhihai Xu via harsh) - MAPREDUCE-6210. Use getApplicationAttemptId() instead of getApplicationId() for logging AttemptId in RMContainerAllocator.java (Leitao Guo via aajisaka) @@ -448,9 +454,6 @@ Release 2.7.0 - UNRELEASED MAPREDUCE-6045. need close the DataInputStream after open it in TestMapReduce.java (zxu via rkanter) -MAPREDUCE-6199. AbstractCounters are not reset completely on -deserialization (adhoot via rkanter) - MAPREDUCE-6206. TestAggregatedTransferRate fails on non-US systems (Jens Rabe via jlowe) @@ -721,9 +724,6 @@ Release 2.6.0 - 2014-11-18 MAPREDUCE-6123. TestCombineFileInputFormat incorrectly starts 2 MiniDFSCluster instances. (cnauroth) -MAPREDUCE-5875. Make Counter limits consistent across JobClient, -MRAppMaster, and YarnChild. (Gera Shegalov via kasha) - MAPREDUCE-6125. TestContainerLauncherImpl sometimes fails (Mit Desai via jlowe)
[09/50] [abbrv] hadoop git commit: YARN-3424. Change logs for ContainerMonitorImpl's resourse monitoring from info to debug. Contributed by Anubhav Dhoot.
YARN-3424. Change logs for ContainerMonitorImpl's resourse monitoring from info to debug. Contributed by Anubhav Dhoot. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db2998ed Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db2998ed Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db2998ed Branch: refs/heads/YARN-2928 Commit: db2998ed25c2331a8a42ef50c2ee4c8afdc2a65e Parents: b62eb19 Author: Tsuyoshi Ozawa oz...@apache.org Authored: Wed Apr 1 17:44:25 2015 +0900 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:10 2015 -0700 -- hadoop-yarn-project/CHANGES.txt| 3 +++ .../monitor/ContainersMonitorImpl.java | 13 - 2 files changed, 11 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/db2998ed/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index dab8116..f18cf30 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -147,6 +147,9 @@ Release 2.8.0 - UNRELEASED YARN-3428. Debug log resources to be localized for a container. (kasha) +YARN-3424. Change logs for ContainerMonitorImpl's resourse monitoring +from info to debug. (Anubhav Dhoot via ozawa) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not http://git-wip-us.apache.org/repos/asf/hadoop/blob/db2998ed/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index 6200efa..bfd6b48 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -519,11 +519,14 @@ public class ContainersMonitorImpl extends AbstractService implements long curRssMemUsageOfAgedProcesses = pTree.getRssMemorySize(1); long vmemLimit = ptInfo.getVmemLimit(); long pmemLimit = ptInfo.getPmemLimit(); -LOG.info(String.format( -Memory usage of ProcessTree %s for container-id %s: , - pId, containerId.toString()) + -formatUsageString( -currentVmemUsage, vmemLimit, currentPmemUsage, pmemLimit)); +if (LOG.isDebugEnabled()) { + LOG.debug(String.format( + Memory usage of ProcessTree %s for container-id %s: , + pId, containerId.toString()) + + formatUsageString( + currentVmemUsage, vmemLimit, + currentPmemUsage, pmemLimit)); +} // Add usage to container metrics if (containerMetricsEnabled) {
[28/50] [abbrv] hadoop git commit: HADOOP-9805. Refactor RawLocalFileSystem#rename for improved testability. Contributed by Jean-Pierre Matsumoto.
HADOOP-9805. Refactor RawLocalFileSystem#rename for improved testability. Contributed by Jean-Pierre Matsumoto. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/503e4902 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/503e4902 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/503e4902 Branch: refs/heads/YARN-2928 Commit: 503e4902c88cbb6ddcfb423e8530c431dcdf0a8f Parents: 88c1468 Author: cnauroth cnaur...@apache.org Authored: Thu Apr 2 16:13:00 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:14 2015 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++ .../apache/hadoop/fs/RawLocalFileSystem.java| 36 ++- .../rawlocal/TestRawlocalContractRename.java| 47 3 files changed, 74 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/503e4902/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index c92e378..260cc83 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -476,6 +476,9 @@ Release 2.8.0 - UNRELEASED HADOOP-11660. Add support for hardware crc of HDFS checksums on ARM aarch64 architecture (Edward Nevill via Colin P. McCabe) +HADOOP-9805. Refactor RawLocalFileSystem#rename for improved testability. +(Jean-Pierre Matsumoto via cnauroth) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/503e4902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java index d7866b8..52623b8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java @@ -43,7 +43,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.nativeio.NativeIO; -import org.apache.hadoop.io.nativeio.NativeIOException; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.StringUtils; @@ -347,11 +346,29 @@ public class RawLocalFileSystem extends FileSystem { return true; } -// Enforce POSIX rename behavior that a source directory replaces an existing -// destination if the destination is an empty directory. On most platforms, -// this is already handled by the Java API call above. Some platforms -// (notably Windows) do not provide this behavior, so the Java API call above -// fails. Delete destination and attempt rename again. +// Else try POSIX style rename on Windows only +if (Shell.WINDOWS +handleEmptyDstDirectoryOnWindows(src, srcFile, dst, dstFile)) { + return true; +} + +// The fallback behavior accomplishes the rename by a full copy. +if (LOG.isDebugEnabled()) { + LOG.debug(Falling through to a copy of + src + to + dst); +} +return FileUtil.copy(this, src, this, dst, true, getConf()); + } + + @VisibleForTesting + public final boolean handleEmptyDstDirectoryOnWindows(Path src, File srcFile, + Path dst, File dstFile) throws IOException { + +// Enforce POSIX rename behavior that a source directory replaces an +// existing destination if the destination is an empty directory. On most +// platforms, this is already handled by the Java API call above. Some +// platforms (notably Windows) do not provide this behavior, so the Java API +// call renameTo(dstFile) fails. Delete destination and attempt rename +// again. if (this.exists(dst)) { FileStatus sdst = this.getFileStatus(dst); if (sdst.isDirectory() dstFile.list().length == 0) { @@ -364,12 +381,7 @@ public class RawLocalFileSystem extends FileSystem { } } } - -// The fallback behavior accomplishes the rename by a full copy. -if (LOG.isDebugEnabled()) { - LOG.debug(Falling through to a copy of + src + to + dst); -} -return FileUtil.copy(this, src, this, dst, true, getConf()); +return false; } @Override
[33/50] [abbrv] hadoop git commit: YARN-2901. Add errors and warning metrics page to RM, NM web UI. (Varun Vasudev via wangda)
YARN-2901. Add errors and warning metrics page to RM, NM web UI. (Varun Vasudev via wangda) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d5d8402d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d5d8402d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d5d8402d Branch: refs/heads/YARN-2928 Commit: d5d8402d943a141ab1497d2d0cb8371cd6d3dd78 Parents: 4f66d40 Author: Wangda Tan wan...@apache.org Authored: Thu Apr 2 17:23:20 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 12:08:15 2015 -0700 -- .../src/main/conf/log4j.properties | 9 + hadoop-yarn-project/CHANGES.txt | 3 + .../util/Log4jWarningErrorMetricsAppender.java | 447 +++ .../TestLog4jWarningErrorMetricsAppender.java | 260 +++ .../server/webapp/ErrorsAndWarningsBlock.java | 233 ++ .../server/nodemanager/webapp/NMController.java | 4 + .../webapp/NMErrorsAndWarningsPage.java | 55 +++ .../server/nodemanager/webapp/NavBlock.java | 24 +- .../server/nodemanager/webapp/WebServer.java| 1 + .../server/resourcemanager/webapp/NavBlock.java | 27 +- .../webapp/RMErrorsAndWarningsPage.java | 54 +++ .../server/resourcemanager/webapp/RMWebApp.java | 1 + .../resourcemanager/webapp/RmController.java| 4 + 13 files changed, 1114 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5d8402d/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties -- diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties index 316c48e..3a0a3ad 100644 --- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties +++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties @@ -266,3 +266,12 @@ log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n #log4j.appender.nodemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender #log4j.appender.nodemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-nodemanager-_mm_dd.log #log4j.appender.nodemanagerrequestlog.RetainDays=3 + +# Appender for viewing information for errors and warnings +yarn.ewma.cleanupInterval=300 +yarn.ewma.messageAgeLimitSeconds=86400 +yarn.ewma.maxUniqueMessages=250 +log4j.appender.EWMA=org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender +log4j.appender.EWMA.cleanupInterval=${yarn.ewma.cleanupInterval} +log4j.appender.EWMA.messageAgeLimitSeconds=${yarn.ewma.messageAgeLimitSeconds} +log4j.appender.EWMA.maxUniqueMessages=${yarn.ewma.maxUniqueMessages} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5d8402d/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index bcd2286..004044a 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -157,6 +157,9 @@ Release 2.8.0 - UNRELEASED YARN-3248. Display count of nodes blacklisted by apps in the web UI. (Varun Vasudev via xgong) +YARN-2901. Add errors and warning metrics page to RM, NM web UI. +(Varun Vasudev via wangda) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5d8402d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Log4jWarningErrorMetricsAppender.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Log4jWarningErrorMetricsAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Log4jWarningErrorMetricsAppender.java new file mode 100644 index 000..0366ae0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Log4jWarningErrorMetricsAppender.java @@ -0,0 +1,447 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on
hadoop git commit: HDFS-7969. Erasure coding: NameNode support for lease recovery of striped block groups. Contributed by Zhe Zhang.
Repository: hadoop Updated Branches: refs/heads/HDFS-7285 e6ecbaafd - a84274f90 HDFS-7969. Erasure coding: NameNode support for lease recovery of striped block groups. Contributed by Zhe Zhang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a84274f9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a84274f9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a84274f9 Branch: refs/heads/HDFS-7285 Commit: a84274f90b5630b5869e03f6bb1d8b0b7f03ae49 Parents: e6ecbaa Author: Zhe Zhang z...@apache.org Authored: Mon Apr 6 12:52:44 2015 -0700 Committer: Zhe Zhang z...@apache.org Committed: Mon Apr 6 12:52:44 2015 -0700 -- .../BlockInfoContiguousUnderConstruction.java | 33 .../BlockInfoStripedUnderConstruction.java | 80 .../BlockInfoUnderConstruction.java | 57 ++ .../blockmanagement/DatanodeDescriptor.java | 12 +-- .../server/blockmanagement/DatanodeManager.java | 10 +-- .../hdfs/server/namenode/FSNamesystem.java | 24 +++--- .../TestBlockInfoUnderConstruction.java | 2 +- 7 files changed, 163 insertions(+), 55 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a84274f9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java index 7a052fd..9ba2978 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java @@ -31,7 +31,8 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode; * Represents a block that is currently being constructed.br * This is usually the last block of a file opened for write or append. */ -public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous { +public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous +implements BlockInfoUnderConstruction{ /** Block state. See {@link BlockUCState} */ private BlockUCState blockUCState; @@ -94,7 +95,7 @@ public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous { return new BlockInfoContiguous(this); } - /** Set expected locations */ + @Override public void setExpectedLocations(DatanodeStorageInfo[] targets) { int numLocations = targets == null ? 0 : targets.length; this.replicas = new ArrayList(numLocations); @@ -104,10 +105,7 @@ public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous { } } - /** - * Create array of expected replica locations - * (as has been assigned by chooseTargets()). - */ + @Override public DatanodeStorageInfo[] getExpectedStorageLocations() { int numLocations = replicas == null ? 0 : replicas.size(); DatanodeStorageInfo[] storages = new DatanodeStorageInfo[numLocations]; @@ -117,7 +115,7 @@ public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous { return storages; } - /** Get the number of expected locations */ + @Override public int getNumExpectedLocations() { return replicas == null ? 0 : replicas.size(); } @@ -135,25 +133,26 @@ public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous { blockUCState = s; } - /** Get block recovery ID */ + @Override public long getBlockRecoveryId() { return blockRecoveryId; } - /** Get recover block */ + @Override public Block getTruncateBlock() { return truncateBlock; } + @Override + public Block toBlock(){ +return this; + } + public void setTruncateBlock(Block recoveryBlock) { this.truncateBlock = recoveryBlock; } - /** - * Process the recorded replicas. When about to commit or finish the - * pipeline recovery sort out bad replicas. - * @param genStamp The final generation stamp for the block. - */ + @Override public void setGenerationStampAndVerifyReplicas(long genStamp) { // Set the generation stamp for the block. setGenerationStamp(genStamp); @@ -187,11 +186,7 @@ public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous { setGenerationStampAndVerifyReplicas(block.getGenerationStamp()); } - /** - * Initialize lease recovery for this block. - * Find the first alive data-node
[Hadoop Wiki] Update of ContributorsGroup by SteveLoughran
Dear Wiki user, You have subscribed to a wiki page or wiki category on Hadoop Wiki for change notification. The ContributorsGroup page has been changed by SteveLoughran: https://wiki.apache.org/hadoop/ContributorsGroup?action=diffrev1=100rev2=101 Comment: +JoshBaer * JonathanHsieh * JonathanSmith * joshdsullivan + * JoshBaer * JoydeepSensarma * JunpingDu * karthikkambatla
hadoop git commit: YARN-2429. TestAMRMTokens.testTokenExpiry fails Intermittently with error message:Invalid AMRMToken (zxu via rkanter)
Repository: hadoop Updated Branches: refs/heads/trunk 28bebc81d - 99b08a748 YARN-2429. TestAMRMTokens.testTokenExpiry fails Intermittently with error message:Invalid AMRMToken (zxu via rkanter) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/99b08a74 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/99b08a74 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/99b08a74 Branch: refs/heads/trunk Commit: 99b08a748e7b00a58b63330b353902a6da6aae27 Parents: 28bebc8 Author: Robert Kanter rkan...@apache.org Authored: Mon Apr 6 14:11:20 2015 -0700 Committer: Robert Kanter rkan...@apache.org Committed: Mon Apr 6 14:11:20 2015 -0700 -- hadoop-yarn-project/CHANGES.txt| 3 +++ .../yarn/server/resourcemanager/security/TestAMRMTokens.java | 6 ++ 2 files changed, 9 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/99b08a74/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 695c4a6..f836f04 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -148,6 +148,9 @@ Release 2.8.0 - UNRELEASED YARN-2666. TestFairScheduler.testContinuousScheduling fails Intermittently. (Zhihai Xu via ozawa) +YARN-2429. TestAMRMTokens.testTokenExpiry fails Intermittently with +error message:Invalid AMRMToken (zxu via rkanter) + Release 2.7.0 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/99b08a74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java index 0be72e3..5dfd092 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java @@ -107,6 +107,12 @@ public class TestAMRMTokens { @SuppressWarnings(unchecked) @Test public void testTokenExpiry() throws Exception { +conf.setLong( +YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, +YarnConfiguration. +DEFAULT_RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS); +conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, +YarnConfiguration.DEFAULT_RM_AM_EXPIRY_INTERVAL_MS); MyContainerManager containerManager = new MyContainerManager(); final MockRMWithAMS rm =
hadoop git commit: Move YARN-3273 from 2.8 to 2.7.
Repository: hadoop Updated Branches: refs/heads/branch-2 354d2c75c - 735e4f15a Move YARN-3273 from 2.8 to 2.7. (cherry picked from commit 5e9f4099de4b949c838a6224442f335c67d80936) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/735e4f15 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/735e4f15 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/735e4f15 Branch: refs/heads/branch-2 Commit: 735e4f15ab0dcd1eadc160bc8c04905a55d94ca8 Parents: 354d2c7 Author: Zhijie Shen zjs...@apache.org Authored: Mon Apr 6 12:28:31 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 14:15:23 2015 -0700 -- hadoop-yarn-project/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/735e4f15/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 7e6e7a4..1e6190b 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -20,9 +20,6 @@ Release 2.8.0 - UNRELEASED YARN-3243. CapacityScheduler should pass headroom from parent to children to make sure ParentQueue obey its capacity limits. (Wangda Tan via jianhe) -YARN-3273. Improve scheduler UI to facilitate scheduling analysis and -debugging. (Rohith Sharmaks via jianhe) - YARN-3357. Move TestFifoScheduler to FIFO package. (Rohith Sharmaks via devaraj) @@ -437,6 +434,9 @@ Release 2.7.0 - UNRELEASED YARN-2777. Mark the end of individual log in aggregated log. (Varun Saxena via xgong) +YARN-3273. Improve scheduler UI to facilitate scheduling analysis and +debugging. (Rohith Sharmaks via jianhe) + OPTIMIZATIONS YARN-2990. FairScheduler's delay-scheduling always waits for node-local and
hadoop git commit: YARN-2429. TestAMRMTokens.testTokenExpiry fails Intermittently with error message:Invalid AMRMToken (zxu via rkanter)
Repository: hadoop Updated Branches: refs/heads/branch-2 a82708990 - 354d2c75c YARN-2429. TestAMRMTokens.testTokenExpiry fails Intermittently with error message:Invalid AMRMToken (zxu via rkanter) (cherry picked from commit 99b08a748e7b00a58b63330b353902a6da6aae27) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/354d2c75 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/354d2c75 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/354d2c75 Branch: refs/heads/branch-2 Commit: 354d2c75cf8f48f47c5e1226ae4879952b0e3485 Parents: a827089 Author: Robert Kanter rkan...@apache.org Authored: Mon Apr 6 14:11:20 2015 -0700 Committer: Robert Kanter rkan...@apache.org Committed: Mon Apr 6 14:11:50 2015 -0700 -- hadoop-yarn-project/CHANGES.txt| 3 +++ .../yarn/server/resourcemanager/security/TestAMRMTokens.java | 6 ++ 2 files changed, 9 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/354d2c75/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index e9d9a37..7e6e7a4 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -100,6 +100,9 @@ Release 2.8.0 - UNRELEASED YARN-2666. TestFairScheduler.testContinuousScheduling fails Intermittently. (Zhihai Xu via ozawa) +YARN-2429. TestAMRMTokens.testTokenExpiry fails Intermittently with +error message:Invalid AMRMToken (zxu via rkanter) + Release 2.7.0 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/354d2c75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java index 0be72e3..5dfd092 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java @@ -107,6 +107,12 @@ public class TestAMRMTokens { @SuppressWarnings(unchecked) @Test public void testTokenExpiry() throws Exception { +conf.setLong( +YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, +YarnConfiguration. +DEFAULT_RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS); +conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, +YarnConfiguration.DEFAULT_RM_AM_EXPIRY_INTERVAL_MS); MyContainerManager containerManager = new MyContainerManager(); final MockRMWithAMS rm =
hadoop git commit: Move YARN-3273 from 2.8 to 2.7.
Repository: hadoop Updated Branches: refs/heads/trunk 99b08a748 - 3fb5abfc8 Move YARN-3273 from 2.8 to 2.7. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3fb5abfc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3fb5abfc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3fb5abfc Branch: refs/heads/trunk Commit: 3fb5abfc87953377f86e06578518801a181d7697 Parents: 99b08a7 Author: Zhijie Shen zjs...@apache.org Authored: Mon Apr 6 12:28:31 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Mon Apr 6 14:14:53 2015 -0700 -- hadoop-yarn-project/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fb5abfc/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index f836f04..efb8153 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -68,9 +68,6 @@ Release 2.8.0 - UNRELEASED YARN-3243. CapacityScheduler should pass headroom from parent to children to make sure ParentQueue obey its capacity limits. (Wangda Tan via jianhe) -YARN-3273. Improve scheduler UI to facilitate scheduling analysis and -debugging. (Rohith Sharmaks via jianhe) - YARN-3357. Move TestFifoScheduler to FIFO package. (Rohith Sharmaks via devaraj) @@ -482,6 +479,9 @@ Release 2.7.0 - UNRELEASED YARN-2777. Mark the end of individual log in aggregated log. (Varun Saxena via xgong) +YARN-3273. Improve scheduler UI to facilitate scheduling analysis and +debugging. (Rohith Sharmaks via jianhe) + OPTIMIZATIONS YARN-2990. FairScheduler's delay-scheduling always waits for node-local and