[48/50] [abbrv] hadoop git commit: Merge remote-tracking branch 'apache/trunk' into HDFS-7285
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fd55202/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java -- diff --cc hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 000,92d117c..be346a4 mode 00,100644..100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@@ -1,0 -1,3135 +1,3240 @@@ + /** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.hdfs; + + import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX; + import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_CACHE_DROP_BEHIND_READS; + import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_CACHE_DROP_BEHIND_WRITES; + import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_CACHE_READAHEAD; + import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_CONTEXT; + import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_CONTEXT_DEFAULT; + + import java.io.BufferedOutputStream; + import java.io.DataInputStream; + import java.io.DataOutputStream; + import java.io.FileNotFoundException; + import java.io.IOException; + import java.io.InputStream; + import java.io.OutputStream; + import java.net.InetAddress; + import java.net.InetSocketAddress; + import java.net.Socket; + import java.net.SocketAddress; + import java.net.URI; + import java.net.UnknownHostException; + import java.security.GeneralSecurityException; + import java.util.ArrayList; + import java.util.EnumSet; + import java.util.HashMap; + import java.util.List; + import java.util.Map; + import java.util.Random; + import java.util.concurrent.SynchronousQueue; + import java.util.concurrent.ThreadLocalRandom; + import java.util.concurrent.ThreadPoolExecutor; + import java.util.concurrent.TimeUnit; + import java.util.concurrent.atomic.AtomicBoolean; + import java.util.concurrent.atomic.AtomicInteger; + + import javax.net.SocketFactory; + + import org.apache.hadoop.HadoopIllegalArgumentException; + import org.apache.hadoop.classification.InterfaceAudience; + import org.apache.hadoop.conf.Configuration; + import org.apache.hadoop.crypto.CipherSuite; + import org.apache.hadoop.crypto.CryptoCodec; + import org.apache.hadoop.crypto.CryptoInputStream; + import org.apache.hadoop.crypto.CryptoOutputStream; + import org.apache.hadoop.crypto.CryptoProtocolVersion; + import org.apache.hadoop.crypto.key.KeyProvider; + import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion; + import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; + import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion; + import org.apache.hadoop.fs.BlockLocation; + import org.apache.hadoop.fs.CacheFlag; + import org.apache.hadoop.fs.ContentSummary; + import org.apache.hadoop.fs.CreateFlag; + import org.apache.hadoop.fs.FileAlreadyExistsException; + import org.apache.hadoop.fs.FileEncryptionInfo; + import org.apache.hadoop.fs.FileSystem; + import org.apache.hadoop.fs.FsServerDefaults; + import org.apache.hadoop.fs.FsStatus; + import org.apache.hadoop.fs.FsTracer; + import org.apache.hadoop.fs.HdfsBlockLocation; + import org.apache.hadoop.fs.InvalidPathException; + import org.apache.hadoop.fs.MD5MD5CRC32CastagnoliFileChecksum; + import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum; + import org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum; + import org.apache.hadoop.fs.Options; + import org.apache.hadoop.fs.Options.ChecksumOpt; + import org.apache.hadoop.fs.ParentNotDirectoryException; + import org.apache.hadoop.fs.Path; + import org.apache.hadoop.fs.RemoteIterator; + import org.apache.hadoop.fs.StorageType; + import org.apache.hadoop.fs.UnresolvedLinkException; + import org.apache.hadoop.fs.XAttr; + import org.apache.hadoop.fs.XAttrSetFlag; + import org.apache.hadoop.fs.permission.AclEntry; + import
[48/50] [abbrv] hadoop git commit: Merge remote-tracking branch 'apache/trunk' into HDFS-7285
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1080c373/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java -- diff --cc hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index d921507,ae0a3f6..a7cacc7 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@@ -17,25 -17,150 +17,157 @@@ */ package org.apache.hadoop.hdfs.protocolPB; + import java.io.EOFException; + import java.io.IOException; + import java.io.InputStream; + import java.util.ArrayList; + import java.util.Arrays; + import java.util.EnumSet; ++import java.util.HashMap; + import java.util.List; ++import java.util.Map; ++import java.util.Set; + + import com.google.common.base.Preconditions; import com.google.common.collect.Lists; + import com.google.common.primitives.Shorts; import com.google.protobuf.ByteString; import com.google.protobuf.CodedInputStream; + -import static com.google.common.base.Preconditions.checkNotNull; - + import org.apache.hadoop.crypto.CipherOption; + import org.apache.hadoop.crypto.CipherSuite; + import org.apache.hadoop.crypto.CryptoProtocolVersion; + import org.apache.hadoop.fs.CacheFlag; + import org.apache.hadoop.fs.ContentSummary; + import org.apache.hadoop.fs.CreateFlag; + import org.apache.hadoop.fs.FileEncryptionInfo; + import org.apache.hadoop.fs.FsServerDefaults; + import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.StorageType; + import org.apache.hadoop.fs.XAttr; + import org.apache.hadoop.fs.XAttrSetFlag; + import org.apache.hadoop.fs.permission.AclEntry; + import org.apache.hadoop.fs.permission.AclEntryScope; + import org.apache.hadoop.fs.permission.AclEntryType; + import org.apache.hadoop.fs.permission.AclStatus; + import org.apache.hadoop.fs.permission.FsAction; + import org.apache.hadoop.fs.permission.FsPermission; + import org.apache.hadoop.hdfs.DFSUtilClient; + import org.apache.hadoop.hdfs.inotify.Event; + import org.apache.hadoop.hdfs.inotify.EventBatch; + import org.apache.hadoop.hdfs.inotify.EventBatchList; + import org.apache.hadoop.hdfs.protocol.Block; + import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; + import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry; + import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; + import org.apache.hadoop.hdfs.protocol.CacheDirectiveStats; + import org.apache.hadoop.hdfs.protocol.CachePoolEntry; + import org.apache.hadoop.hdfs.protocol.CachePoolInfo; + import org.apache.hadoop.hdfs.protocol.CachePoolStats; + import org.apache.hadoop.hdfs.protocol.ClientProtocol; + import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; + import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates; import org.apache.hadoop.hdfs.protocol.DatanodeLocalInfo; + import org.apache.hadoop.hdfs.protocol.DirectoryListing; + import org.apache.hadoop.hdfs.protocol.EncryptionZone; ++import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; + import org.apache.hadoop.hdfs.protocol.FsPermissionExtension; + import org.apache.hadoop.hdfs.protocol.HdfsConstants; + import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; + import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction; + import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; + import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; + import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; + import org.apache.hadoop.hdfs.protocol.LocatedBlock; + import org.apache.hadoop.hdfs.protocol.LocatedBlocks; ++import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock; + import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo; + import org.apache.hadoop.hdfs.protocol.RollingUpgradeStatus; + import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; + import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; + import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType; + import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; + import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto; + import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.AclEntryScopeProto; + import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.AclEntryTypeProto; + import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.FsActionProto; + import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclStatusProto; + import org.apache.hadoop.hdfs.protocol.proto.AclProtos.GetAclStatusResponseProto; + import
[48/50] [abbrv] hadoop git commit: Merge remote-tracking branch 'apache/trunk' into HDFS-7285
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53358fe6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
--
diff --cc
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 6c6d758,1346ab3..8232ab9
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@@ -674,8 -648,8 +674,8 @@@ public class BlockManager implements Bl
return false; // already completed (e.g. by syncBlock)
final boolean b = commitBlock(lastBlock, commitBlock);
- if (hasMinStorage(lastBlock)) {
- completeBlock(bc, bc.numBlocks() - 1, false);
-if (countNodes(lastBlock).liveReplicas() >= minReplication) {
++ if (hasMinStorage(lastBlock)) {
+ completeBlock(lastBlock, false);
}
return b;
}
@@@ -698,9 -666,9 +692,9 @@@
}
int numNodes = curBlock.numNodes();
-if (!force && numNodes < minReplication) {
+if (!force && !hasMinStorage(curBlock, numNodes)) {
- throw new IOException("Cannot complete block: " +
- "block does not satisfy minimal replication requirement.");
+ throw new IOException("Cannot complete block: "
+ + "block does not satisfy minimal replication requirement.");
}
if (!force && curBlock.getBlockUCState() != BlockUCState.COMMITTED) {
throw new IOException(
@@@ -718,26 -683,10 +709,12 @@@
// a "forced" completion when a file is getting closed by an
// OP_CLOSE edit on the standby).
namesystem.adjustSafeModeBlockTotals(0, 1);
+final int minStorage = curBlock.isStriped() ?
+((BlockInfoStriped) curBlock).getRealDataBlockNum() : minReplication;
namesystem.incrementSafeBlockCount(
-Math.min(numNodes, minReplication));
+Math.min(numNodes, minStorage), curBlock);
-
- // replace block in the blocksMap
- return blocksMap.replaceBlock(completeBlock);
}
- private BlockInfo completeBlock(final BlockCollection bc,
- final BlockInfo block, boolean force) throws IOException {
- BlockInfo[] fileBlocks = bc.getBlocks();
- for (int idx = 0; idx < fileBlocks.length; idx++) {
- if (fileBlocks[idx] == block) {
- return completeBlock(bc, idx, force);
- }
- }
- return block;
- }
-
/**
* Force the given block in the given file to be marked as complete,
* regardless of whether enough replicas are present. This is necessary
@@@ -1270,37 -1162,29 +1245,36 @@@
private void markBlockAsCorrupt(BlockToMarkCorrupt b,
DatanodeStorageInfo storageInfo,
DatanodeDescriptor node) throws IOException {
--
- if (b.stored.isDeleted()) {
-if (b.getCorrupted().isDeleted()) {
++if (b.getStored().isDeleted()) {
blockLog.debug("BLOCK markBlockAsCorrupt: {} cannot be marked as" +
" corrupt as it does not belong to any file", b);
- addToInvalidates(b.corrupted, node);
+ addToInvalidates(b.getCorrupted(), node);
return;
-}
-short expectedReplicas = b.getCorrupted().getReplication();
+}
+short expectedReplicas =
- getExpectedReplicaNum(b.stored);
++getExpectedReplicaNum(b.getStored());
// Add replica to the data-node if it is not already there
if (storageInfo != null) {
- storageInfo.addBlock(b.stored, b.corrupted);
- storageInfo.addBlock(b.getStored());
++ storageInfo.addBlock(b.getStored(), b.getCorrupted());
}
-// Add this replica to corruptReplicas Map
-corruptReplicas.addToCorruptReplicasMap(b.getCorrupted(), node,
-b.getReason(), b.getReasonCode());
+// Add this replica to corruptReplicas Map. For striped blocks, we always
+// use the id of whole striped block group when adding to corruptReplicas
- Block corrupted = new Block(b.corrupted);
- if (b.stored.isStriped()) {
- corrupted.setBlockId(b.stored.getBlockId());
++Block corrupted = new Block(b.getCorrupted());
++if (b.getStored().isStriped()) {
++ corrupted.setBlockId(b.getStored().getBlockId());
+}
- corruptReplicas.addToCorruptReplicasMap(corrupted, node, b.reason,
- b.reasonCode);
++corruptReplicas.addToCorruptReplicasMap(corrupted, node, b.getReason(),
++b.getReasonCode());
- NumberReplicas numberOfReplicas = countNodes(b.stored);
+ NumberReplicas numberOfReplicas = countNodes(b.getStored());
boolean hasEnoughLiveReplicas = numberOfReplicas.liveReplicas() >=
expectedReplicas;
-boolean minReplicationSatisfied =
-numberOfReplicas.liveReplicas() >= minReplication;
+
- boolean minReplicationSatisfied =
