HDFS-8169. Move LocatedBlocks and related classes to hdfs-client. Contributed by Haohui Mai.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/63d0082c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/63d0082c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/63d0082c Branch: refs/heads/YARN-2928 Commit: 63d0082c0c1832c87e34d5cc123b697586300f04 Parents: 26a97e2 Author: Haohui Mai <[email protected]> Authored: Thu Apr 16 23:13:15 2015 -0700 Committer: Zhijie Shen <[email protected]> Committed: Tue Apr 21 16:16:53 2015 -0700 ---------------------------------------------------------------------- .../dev-support/findbugsExcludeFile.xml | 8 + .../org/apache/hadoop/hdfs/DFSUtilClient.java | 65 +++ .../org/apache/hadoop/hdfs/protocol/Block.java | 243 +++++++++ .../apache/hadoop/hdfs/protocol/DatanodeID.java | 279 ++++++++++ .../hadoop/hdfs/protocol/DatanodeInfo.java | 512 ++++++++++++++++++ .../hdfs/protocol/DatanodeInfoWithStorage.java | 67 +++ .../hadoop/hdfs/protocol/ExtendedBlock.java | 123 +++++ .../hdfs/protocol/FsPermissionExtension.java | 89 ++++ .../hdfs/protocol/HdfsConstantsClient.java | 1 + .../hadoop/hdfs/protocol/HdfsFileStatus.java | 271 ++++++++++ .../hadoop/hdfs/protocol/LocatedBlock.java | 206 ++++++++ .../hadoop/hdfs/protocol/LocatedBlocks.java | 189 +++++++ .../token/block/BlockTokenIdentifier.java | 189 +++++++ .../delegation/DelegationTokenIdentifier.java | 100 ++++ .../hadoop/hdfs/web/WebHdfsConstants.java | 37 ++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../java/org/apache/hadoop/hdfs/DFSUtil.java | 15 - .../org/apache/hadoop/hdfs/protocol/Block.java | 243 --------- .../apache/hadoop/hdfs/protocol/DatanodeID.java | 279 ---------- .../hadoop/hdfs/protocol/DatanodeInfo.java | 514 ------------------- .../hdfs/protocol/DatanodeInfoWithStorage.java | 67 --- .../hadoop/hdfs/protocol/ExtendedBlock.java | 123 ----- .../hdfs/protocol/FsPermissionExtension.java | 89 ---- .../hadoop/hdfs/protocol/HdfsFileStatus.java | 271 ---------- .../hadoop/hdfs/protocol/LocatedBlock.java | 217 -------- .../hadoop/hdfs/protocol/LocatedBlocks.java | 189 ------- .../protocol/SnapshottableDirectoryStatus.java | 3 +- .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 3 +- .../token/block/BlockTokenIdentifier.java | 189 ------- .../delegation/DelegationTokenIdentifier.java | 101 ---- .../server/blockmanagement/BlockManager.java | 19 +- .../BlockStoragePolicySuite.java | 1 - .../blockmanagement/DatanodeStorageInfo.java | 3 + .../blockmanagement/HeartbeatManager.java | 11 +- .../apache/hadoop/hdfs/server/mover/Mover.java | 3 +- .../server/namenode/FSDirStatAndListingOp.java | 18 +- .../hdfs/server/namenode/FSDirectory.java | 4 +- .../hdfs/server/namenode/FSEditLogLoader.java | 7 +- .../hdfs/server/namenode/FSEditLogOp.java | 5 +- .../hdfs/server/namenode/FSNamesystem.java | 5 +- .../hadoop/hdfs/server/namenode/INode.java | 13 +- .../hdfs/server/namenode/INodeDirectory.java | 9 +- .../hadoop/hdfs/server/namenode/INodeFile.java | 8 +- .../hadoop/hdfs/server/namenode/INodeMap.java | 5 +- .../snapshot/FileWithSnapshotFeature.java | 3 +- .../web/resources/NamenodeWebHdfsMethods.java | 6 +- .../hadoop/hdfs/tools/StoragePolicyAdmin.java | 4 +- .../apache/hadoop/hdfs/web/JsonUtilClient.java | 3 +- .../hadoop/hdfs/web/SWebHdfsFileSystem.java | 3 +- .../org/apache/hadoop/hdfs/web/TokenAspect.java | 10 +- .../hadoop/hdfs/web/WebHdfsConstants.java | 30 -- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 5 +- .../hadoop/hdfs/TestBlockStoragePolicy.java | 8 +- .../namenode/TestNamenodeCapacityReport.java | 19 +- .../apache/hadoop/hdfs/web/TestWebHdfsUrl.java | 2 +- 55 files changed, 2476 insertions(+), 2412 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/63d0082c/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml index 43bc332..478a931 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml @@ -1,2 +1,10 @@ <FindBugsFilter> + <Match> + <Or> + <Class name="org.apache.hadoop.hdfs.protocol.HdfsFileStatus"/> + <Class name="org.apache.hadoop.hdfs.protocol.LocatedBlock"/> + <Class name="org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier"/> + </Or> + <Bug pattern="EI_EXPOSE_REP,EI_EXPOSE_REP2" /> + </Match> </FindBugsFilter> http://git-wip-us.apache.org/repos/asf/hadoop/blob/63d0082c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java new file mode 100644 index 0000000..2817b66 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import org.apache.hadoop.util.StringUtils; + +import java.io.UnsupportedEncodingException; + +public class DFSUtilClient { + /** + * Converts a byte array to a string using UTF8 encoding. + */ + public static String bytes2String(byte[] bytes) { + return bytes2String(bytes, 0, bytes.length); + } + + /** Return used as percentage of capacity */ + public static float getPercentUsed(long used, long capacity) { + return capacity <= 0 ? 100 : (used * 100.0f)/capacity; + } + + /** Return remaining as percentage of capacity */ + public static float getPercentRemaining(long remaining, long capacity) { + return capacity <= 0 ? 0 : (remaining * 100.0f)/capacity; + } + + /** Convert percentage to a string. */ + public static String percent2String(double percentage) { + return StringUtils.format("%.2f%%", percentage); + } + + /** + * Decode a specific range of bytes of the given byte array to a string + * using UTF8. + * + * @param bytes The bytes to be decoded into characters + * @param offset The index of the first byte to decode + * @param length The number of bytes to decode + * @return The decoded string + */ + private static String bytes2String(byte[] bytes, int offset, int length) { + try { + return new String(bytes, offset, length, "UTF8"); + } catch(UnsupportedEncodingException e) { + assert false : "UTF8 encoding is not supported "; + } + return null; + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/63d0082c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/Block.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/Block.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/Block.java new file mode 100644 index 0000000..11daccc --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/Block.java @@ -0,0 +1,243 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.protocol; + +import java.io.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.io.*; + +/************************************************** + * A Block is a Hadoop FS primitive, identified by a + * long. + * + **************************************************/ [email protected] [email protected] +public class Block implements Writable, Comparable<Block> { + public static final String BLOCK_FILE_PREFIX = "blk_"; + public static final String METADATA_EXTENSION = ".meta"; + static { // register a ctor + WritableFactories.setFactory + (Block.class, + new WritableFactory() { + @Override + public Writable newInstance() { return new Block(); } + }); + } + + public static final Pattern blockFilePattern = Pattern + .compile(BLOCK_FILE_PREFIX + "(-??\\d++)$"); + public static final Pattern metaFilePattern = Pattern + .compile(BLOCK_FILE_PREFIX + "(-??\\d++)_(\\d++)\\" + METADATA_EXTENSION + + "$"); + public static final Pattern metaOrBlockFilePattern = Pattern + .compile(BLOCK_FILE_PREFIX + "(-??\\d++)(_(\\d++)\\" + METADATA_EXTENSION + + ")?$"); + + public static boolean isBlockFilename(File f) { + String name = f.getName(); + return blockFilePattern.matcher(name).matches(); + } + + public static long filename2id(String name) { + Matcher m = blockFilePattern.matcher(name); + return m.matches() ? Long.parseLong(m.group(1)) : 0; + } + + public static boolean isMetaFilename(String name) { + return metaFilePattern.matcher(name).matches(); + } + + public static File metaToBlockFile(File metaFile) { + return new File(metaFile.getParent(), metaFile.getName().substring( + 0, metaFile.getName().lastIndexOf('_'))); + } + + /** + * Get generation stamp from the name of the metafile name + */ + public static long getGenerationStamp(String metaFile) { + Matcher m = metaFilePattern.matcher(metaFile); + return m.matches() ? Long.parseLong(m.group(2)) + : HdfsConstantsClient.GRANDFATHER_GENERATION_STAMP; + } + + /** + * Get the blockId from the name of the meta or block file + */ + public static long getBlockId(String metaOrBlockFile) { + Matcher m = metaOrBlockFilePattern.matcher(metaOrBlockFile); + return m.matches() ? Long.parseLong(m.group(1)) : 0; + } + + private long blockId; + private long numBytes; + private long generationStamp; + + public Block() {this(0, 0, 0);} + + public Block(final long blkid, final long len, final long generationStamp) { + set(blkid, len, generationStamp); + } + + public Block(final long blkid) { + this(blkid, 0, HdfsConstantsClient.GRANDFATHER_GENERATION_STAMP); + } + + public Block(Block blk) { + this(blk.blockId, blk.numBytes, blk.generationStamp); + } + + /** + * Find the blockid from the given filename + */ + public Block(File f, long len, long genstamp) { + this(filename2id(f.getName()), len, genstamp); + } + + public void set(long blkid, long len, long genStamp) { + this.blockId = blkid; + this.numBytes = len; + this.generationStamp = genStamp; + } + /** + */ + public long getBlockId() { + return blockId; + } + + public void setBlockId(long bid) { + blockId = bid; + } + + /** + */ + public String getBlockName() { + return BLOCK_FILE_PREFIX + String.valueOf(blockId); + } + + /** + */ + public long getNumBytes() { + return numBytes; + } + public void setNumBytes(long len) { + this.numBytes = len; + } + + public long getGenerationStamp() { + return generationStamp; + } + + public void setGenerationStamp(long stamp) { + generationStamp = stamp; + } + + /** + */ + @Override + public String toString() { + return getBlockName() + "_" + getGenerationStamp(); + } + + public void appendStringTo(StringBuilder sb) { + sb.append(BLOCK_FILE_PREFIX) + .append(blockId) + .append("_") + .append(getGenerationStamp()); + } + + + ///////////////////////////////////// + // Writable + ///////////////////////////////////// + @Override // Writable + public void write(DataOutput out) throws IOException { + writeHelper(out); + } + + @Override // Writable + public void readFields(DataInput in) throws IOException { + readHelper(in); + } + + final void writeHelper(DataOutput out) throws IOException { + out.writeLong(blockId); + out.writeLong(numBytes); + out.writeLong(generationStamp); + } + + final void readHelper(DataInput in) throws IOException { + this.blockId = in.readLong(); + this.numBytes = in.readLong(); + this.generationStamp = in.readLong(); + if (numBytes < 0) { + throw new IOException("Unexpected block size: " + numBytes); + } + } + + // write only the identifier part of the block + public void writeId(DataOutput out) throws IOException { + out.writeLong(blockId); + out.writeLong(generationStamp); + } + + // Read only the identifier part of the block + public void readId(DataInput in) throws IOException { + this.blockId = in.readLong(); + this.generationStamp = in.readLong(); + } + + @Override // Comparable + public int compareTo(Block b) { + return blockId < b.blockId ? -1 : + blockId > b.blockId ? 1 : 0; + } + + @Override // Object + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Block)) { + return false; + } + return compareTo((Block)o) == 0; + } + + /** + * @return true if the two blocks have the same block ID and the same + * generation stamp, or if both blocks are null. + */ + public static boolean matchingIdAndGenStamp(Block a, Block b) { + if (a == b) return true; // same block, or both null + if (a == null || b == null) return false; // only one null + return a.blockId == b.blockId && + a.generationStamp == b.generationStamp; + } + + @Override // Object + public int hashCode() { + //GenerationStamp is IRRELEVANT and should not be used here + return (int)(blockId^(blockId>>>32)); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/63d0082c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java new file mode 100644 index 0000000..6d72285 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java @@ -0,0 +1,279 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.protocol; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +import com.google.common.annotations.VisibleForTesting; + +/** + * This class represents the primary identifier for a Datanode. + * Datanodes are identified by how they can be contacted (hostname + * and ports) and their storage ID, a unique number that associates + * the Datanodes blocks with a particular Datanode. + * + * {@link DatanodeInfo#getName()} should be used to get the network + * location (for topology) of a datanode, instead of using + * {@link DatanodeID#getXferAddr()} here. Helpers are defined below + * for each context in which a DatanodeID is used. + */ [email protected] [email protected] +public class DatanodeID implements Comparable<DatanodeID> { + public static final DatanodeID[] EMPTY_ARRAY = {}; + + private String ipAddr; // IP address + private String hostName; // hostname claimed by datanode + private String peerHostName; // hostname from the actual connection + private int xferPort; // data streaming port + private int infoPort; // info server port + private int infoSecurePort; // info server port + private int ipcPort; // IPC server port + private String xferAddr; + + /** + * UUID identifying a given datanode. For upgraded Datanodes this is the + * same as the StorageID that was previously used by this Datanode. + * For newly formatted Datanodes it is a UUID. + */ + private final String datanodeUuid; + + public DatanodeID(DatanodeID from) { + this(from.getDatanodeUuid(), from); + } + + @VisibleForTesting + public DatanodeID(String datanodeUuid, DatanodeID from) { + this(from.getIpAddr(), + from.getHostName(), + datanodeUuid, + from.getXferPort(), + from.getInfoPort(), + from.getInfoSecurePort(), + from.getIpcPort()); + this.peerHostName = from.getPeerHostName(); + } + + /** + * Create a DatanodeID + * @param ipAddr IP + * @param hostName hostname + * @param datanodeUuid data node ID, UUID for new Datanodes, may be the + * storage ID for pre-UUID datanodes. NULL if unknown + * e.g. if this is a new datanode. A new UUID will + * be assigned by the namenode. + * @param xferPort data transfer port + * @param infoPort info server port + * @param ipcPort ipc server port + */ + public DatanodeID(String ipAddr, String hostName, String datanodeUuid, + int xferPort, int infoPort, int infoSecurePort, int ipcPort) { + setIpAndXferPort(ipAddr, xferPort); + this.hostName = hostName; + this.datanodeUuid = checkDatanodeUuid(datanodeUuid); + this.infoPort = infoPort; + this.infoSecurePort = infoSecurePort; + this.ipcPort = ipcPort; + } + + public void setIpAddr(String ipAddr) { + //updated during registration, preserve former xferPort + setIpAndXferPort(ipAddr, xferPort); + } + + private void setIpAndXferPort(String ipAddr, int xferPort) { + // build xferAddr string to reduce cost of frequent use + this.ipAddr = ipAddr; + this.xferPort = xferPort; + this.xferAddr = ipAddr + ":" + xferPort; + } + + public void setPeerHostName(String peerHostName) { + this.peerHostName = peerHostName; + } + + /** + * @return data node ID. + */ + public String getDatanodeUuid() { + return datanodeUuid; + } + + private String checkDatanodeUuid(String uuid) { + if (uuid == null || uuid.isEmpty()) { + return null; + } else { + return uuid; + } + } + + /** + * @return ipAddr; + */ + public String getIpAddr() { + return ipAddr; + } + + /** + * @return hostname + */ + public String getHostName() { + return hostName; + } + + /** + * @return hostname from the actual connection + */ + public String getPeerHostName() { + return peerHostName; + } + + /** + * @return IP:xferPort string + */ + public String getXferAddr() { + return xferAddr; + } + + /** + * @return IP:ipcPort string + */ + private String getIpcAddr() { + return ipAddr + ":" + ipcPort; + } + + /** + * @return IP:infoPort string + */ + public String getInfoAddr() { + return ipAddr + ":" + infoPort; + } + + /** + * @return IP:infoPort string + */ + public String getInfoSecureAddr() { + return ipAddr + ":" + infoSecurePort; + } + + /** + * @return hostname:xferPort + */ + public String getXferAddrWithHostname() { + return hostName + ":" + xferPort; + } + + /** + * @return hostname:ipcPort + */ + private String getIpcAddrWithHostname() { + return hostName + ":" + ipcPort; + } + + /** + * @param useHostname true to use the DN hostname, use the IP otherwise + * @return name:xferPort + */ + public String getXferAddr(boolean useHostname) { + return useHostname ? getXferAddrWithHostname() : getXferAddr(); + } + + /** + * @param useHostname true to use the DN hostname, use the IP otherwise + * @return name:ipcPort + */ + public String getIpcAddr(boolean useHostname) { + return useHostname ? getIpcAddrWithHostname() : getIpcAddr(); + } + + /** + * @return xferPort (the port for data streaming) + */ + public int getXferPort() { + return xferPort; + } + + /** + * @return infoPort (the port at which the HTTP server bound to) + */ + public int getInfoPort() { + return infoPort; + } + + /** + * @return infoSecurePort (the port at which the HTTPS server bound to) + */ + public int getInfoSecurePort() { + return infoSecurePort; + } + + /** + * @return ipcPort (the port at which the IPC server bound to) + */ + public int getIpcPort() { + return ipcPort; + } + + @Override + public boolean equals(Object to) { + if (this == to) { + return true; + } + if (!(to instanceof DatanodeID)) { + return false; + } + return (getXferAddr().equals(((DatanodeID)to).getXferAddr()) && + datanodeUuid.equals(((DatanodeID)to).getDatanodeUuid())); + } + + @Override + public int hashCode() { + return datanodeUuid.hashCode(); + } + + @Override + public String toString() { + return getXferAddr(); + } + + /** + * Update fields when a new registration request comes in. + * Note that this does not update storageID. + */ + public void updateRegInfo(DatanodeID nodeReg) { + setIpAndXferPort(nodeReg.getIpAddr(), nodeReg.getXferPort()); + hostName = nodeReg.getHostName(); + peerHostName = nodeReg.getPeerHostName(); + infoPort = nodeReg.getInfoPort(); + infoSecurePort = nodeReg.getInfoSecurePort(); + ipcPort = nodeReg.getIpcPort(); + } + + /** + * Compare based on data transfer address. + * + * @param that datanode to compare with + * @return as specified by Comparable + */ + @Override + public int compareTo(DatanodeID that) { + return getXferAddr().compareTo(that.getXferAddr()); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/63d0082c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java new file mode 100644 index 0000000..3555add --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java @@ -0,0 +1,512 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.protocol; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hdfs.DFSUtilClient; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.net.NetworkTopology; +import org.apache.hadoop.net.Node; +import org.apache.hadoop.net.NodeBase; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.Time; + +import java.util.Date; +import java.util.LinkedList; +import java.util.List; + +import static org.apache.hadoop.hdfs.DFSUtilClient.percent2String; + +/** + * This class extends the primary identifier of a Datanode with ephemeral + * state, eg usage information, current administrative state, and the + * network location that is communicated to clients. + */ [email protected] [email protected] +public class DatanodeInfo extends DatanodeID implements Node { + private long capacity; + private long dfsUsed; + private long remaining; + private long blockPoolUsed; + private long cacheCapacity; + private long cacheUsed; + private long lastUpdate; + private long lastUpdateMonotonic; + private int xceiverCount; + private String location = NetworkTopology.DEFAULT_RACK; + private String softwareVersion; + private List<String> dependentHostNames = new LinkedList<String>(); + + + // Datanode administrative states + public enum AdminStates { + NORMAL("In Service"), + DECOMMISSION_INPROGRESS("Decommission In Progress"), + DECOMMISSIONED("Decommissioned"); + + final String value; + + AdminStates(final String v) { + this.value = v; + } + + @Override + public String toString() { + return value; + } + + public static AdminStates fromValue(final String value) { + for (AdminStates as : AdminStates.values()) { + if (as.value.equals(value)) return as; + } + return NORMAL; + } + } + + protected AdminStates adminState; + + public DatanodeInfo(DatanodeInfo from) { + super(from); + this.capacity = from.getCapacity(); + this.dfsUsed = from.getDfsUsed(); + this.remaining = from.getRemaining(); + this.blockPoolUsed = from.getBlockPoolUsed(); + this.cacheCapacity = from.getCacheCapacity(); + this.cacheUsed = from.getCacheUsed(); + this.lastUpdate = from.getLastUpdate(); + this.lastUpdateMonotonic = from.getLastUpdateMonotonic(); + this.xceiverCount = from.getXceiverCount(); + this.location = from.getNetworkLocation(); + this.adminState = from.getAdminState(); + } + + public DatanodeInfo(DatanodeID nodeID) { + super(nodeID); + this.capacity = 0L; + this.dfsUsed = 0L; + this.remaining = 0L; + this.blockPoolUsed = 0L; + this.cacheCapacity = 0L; + this.cacheUsed = 0L; + this.lastUpdate = 0L; + this.lastUpdateMonotonic = 0L; + this.xceiverCount = 0; + this.adminState = null; + } + + public DatanodeInfo(DatanodeID nodeID, String location) { + this(nodeID); + this.location = location; + } + + public DatanodeInfo(DatanodeID nodeID, String location, + final long capacity, final long dfsUsed, final long remaining, + final long blockPoolUsed, final long cacheCapacity, final long cacheUsed, + final long lastUpdate, final long lastUpdateMonotonic, + final int xceiverCount, final AdminStates adminState) { + this(nodeID.getIpAddr(), nodeID.getHostName(), nodeID.getDatanodeUuid(), + nodeID.getXferPort(), nodeID.getInfoPort(), nodeID.getInfoSecurePort(), + nodeID.getIpcPort(), capacity, dfsUsed, remaining, blockPoolUsed, + cacheCapacity, cacheUsed, lastUpdate, lastUpdateMonotonic, + xceiverCount, location, adminState); + } + + /** Constructor */ + public DatanodeInfo(final String ipAddr, final String hostName, + final String datanodeUuid, final int xferPort, final int infoPort, + final int infoSecurePort, final int ipcPort, + final long capacity, final long dfsUsed, final long remaining, + final long blockPoolUsed, final long cacheCapacity, final long cacheUsed, + final long lastUpdate, final long lastUpdateMonotonic, + final int xceiverCount, final String networkLocation, + final AdminStates adminState) { + super(ipAddr, hostName, datanodeUuid, xferPort, infoPort, + infoSecurePort, ipcPort); + this.capacity = capacity; + this.dfsUsed = dfsUsed; + this.remaining = remaining; + this.blockPoolUsed = blockPoolUsed; + this.cacheCapacity = cacheCapacity; + this.cacheUsed = cacheUsed; + this.lastUpdate = lastUpdate; + this.lastUpdateMonotonic = lastUpdateMonotonic; + this.xceiverCount = xceiverCount; + this.location = networkLocation; + this.adminState = adminState; + } + + /** Network location name */ + @Override + public String getName() { + return getXferAddr(); + } + + /** The raw capacity. */ + public long getCapacity() { return capacity; } + + /** The used space by the data node. */ + public long getDfsUsed() { return dfsUsed; } + + /** The used space by the block pool on data node. */ + public long getBlockPoolUsed() { return blockPoolUsed; } + + /** The used space by the data node. */ + public long getNonDfsUsed() { + long nonDFSUsed = capacity - dfsUsed - remaining; + return nonDFSUsed < 0 ? 0 : nonDFSUsed; + } + + /** The used space by the data node as percentage of present capacity */ + public float getDfsUsedPercent() { + return DFSUtilClient.getPercentUsed(dfsUsed, capacity); + } + + /** The raw free space. */ + public long getRemaining() { return remaining; } + + /** Used space by the block pool as percentage of present capacity */ + public float getBlockPoolUsedPercent() { + return DFSUtilClient.getPercentUsed(blockPoolUsed, capacity); + } + + /** The remaining space as percentage of configured capacity. */ + public float getRemainingPercent() { + return DFSUtilClient.getPercentRemaining(remaining, capacity); + } + + /** + * @return Amount of cache capacity in bytes + */ + public long getCacheCapacity() { + return cacheCapacity; + } + + /** + * @return Amount of cache used in bytes + */ + public long getCacheUsed() { + return cacheUsed; + } + + /** + * @return Cache used as a percentage of the datanode's total cache capacity + */ + public float getCacheUsedPercent() { + return DFSUtilClient.getPercentUsed(cacheUsed, cacheCapacity); + } + + /** + * @return Amount of cache remaining in bytes + */ + public long getCacheRemaining() { + return cacheCapacity - cacheUsed; + } + + /** + * @return Cache remaining as a percentage of the datanode's total cache + * capacity + */ + public float getCacheRemainingPercent() { + return DFSUtilClient.getPercentRemaining(getCacheRemaining(), cacheCapacity); + } + + /** + * Get the last update timestamp. + * Return value is suitable for Date conversion. + */ + public long getLastUpdate() { return lastUpdate; } + + /** + * The time when this information was accurate. <br> + * Ps: So return value is ideal for calculation of time differences. + * Should not be used to convert to Date. + */ + public long getLastUpdateMonotonic() { return lastUpdateMonotonic;} + + /** + * Set lastUpdate monotonic time + */ + public void setLastUpdateMonotonic(long lastUpdateMonotonic) { + this.lastUpdateMonotonic = lastUpdateMonotonic; + } + + /** number of active connections */ + public int getXceiverCount() { return xceiverCount; } + + /** Sets raw capacity. */ + public void setCapacity(long capacity) { + this.capacity = capacity; + } + + /** Sets the used space for the datanode. */ + public void setDfsUsed(long dfsUsed) { + this.dfsUsed = dfsUsed; + } + + /** Sets raw free space. */ + public void setRemaining(long remaining) { + this.remaining = remaining; + } + + /** Sets block pool used space */ + public void setBlockPoolUsed(long bpUsed) { + this.blockPoolUsed = bpUsed; + } + + /** Sets cache capacity. */ + public void setCacheCapacity(long cacheCapacity) { + this.cacheCapacity = cacheCapacity; + } + + /** Sets cache used. */ + public void setCacheUsed(long cacheUsed) { + this.cacheUsed = cacheUsed; + } + + /** Sets time when this information was accurate. */ + public void setLastUpdate(long lastUpdate) { + this.lastUpdate = lastUpdate; + } + + /** Sets number of active connections */ + public void setXceiverCount(int xceiverCount) { + this.xceiverCount = xceiverCount; + } + + /** network location */ + @Override + public synchronized String getNetworkLocation() {return location;} + + /** Sets the network location */ + @Override + public synchronized void setNetworkLocation(String location) { + this.location = NodeBase.normalize(location); + } + + /** Add a hostname to a list of network dependencies */ + public void addDependentHostName(String hostname) { + dependentHostNames.add(hostname); + } + + /** List of Network dependencies */ + public List<String> getDependentHostNames() { + return dependentHostNames; + } + + /** Sets the network dependencies */ + public void setDependentHostNames(List<String> dependencyList) { + dependentHostNames = dependencyList; + } + + /** A formatted string for reporting the status of the DataNode. */ + public String getDatanodeReport() { + StringBuilder buffer = new StringBuilder(); + long c = getCapacity(); + long r = getRemaining(); + long u = getDfsUsed(); + long nonDFSUsed = getNonDfsUsed(); + float usedPercent = getDfsUsedPercent(); + float remainingPercent = getRemainingPercent(); + long cc = getCacheCapacity(); + long cr = getCacheRemaining(); + long cu = getCacheUsed(); + float cacheUsedPercent = getCacheUsedPercent(); + float cacheRemainingPercent = getCacheRemainingPercent(); + String lookupName = NetUtils.getHostNameOfIP(getName()); + + buffer.append("Name: "+ getName()); + if (lookupName != null) { + buffer.append(" (" + lookupName + ")"); + } + buffer.append("\n"); + buffer.append("Hostname: " + getHostName() + "\n"); + + if (!NetworkTopology.DEFAULT_RACK.equals(location)) { + buffer.append("Rack: "+location+"\n"); + } + buffer.append("Decommission Status : "); + if (isDecommissioned()) { + buffer.append("Decommissioned\n"); + } else if (isDecommissionInProgress()) { + buffer.append("Decommission in progress\n"); + } else { + buffer.append("Normal\n"); + } + buffer.append("Configured Capacity: "+c+" ("+StringUtils.byteDesc(c)+")"+"\n"); + buffer.append("DFS Used: "+u+" ("+StringUtils.byteDesc(u)+")"+"\n"); + buffer.append("Non DFS Used: "+nonDFSUsed+" ("+StringUtils.byteDesc(nonDFSUsed)+")"+"\n"); + buffer.append("DFS Remaining: " +r+ " ("+StringUtils.byteDesc(r)+")"+"\n"); + buffer.append("DFS Used%: "+percent2String(usedPercent) + "\n"); + buffer.append("DFS Remaining%: "+percent2String(remainingPercent) + "\n"); + buffer.append("Configured Cache Capacity: "+cc+" ("+StringUtils.byteDesc(cc)+")"+"\n"); + buffer.append("Cache Used: "+cu+" ("+StringUtils.byteDesc(cu)+")"+"\n"); + buffer.append("Cache Remaining: " +cr+ " ("+StringUtils.byteDesc(cr)+")"+"\n"); + buffer.append("Cache Used%: "+percent2String(cacheUsedPercent) + "\n"); + buffer.append("Cache Remaining%: "+percent2String(cacheRemainingPercent) + "\n"); + buffer.append("Xceivers: "+getXceiverCount()+"\n"); + buffer.append("Last contact: "+new Date(lastUpdate)+"\n"); + return buffer.toString(); + } + + /** A formatted string for printing the status of the DataNode. */ + public String dumpDatanode() { + StringBuilder buffer = new StringBuilder(); + long c = getCapacity(); + long r = getRemaining(); + long u = getDfsUsed(); + long cc = getCacheCapacity(); + long cr = getCacheRemaining(); + long cu = getCacheUsed(); + buffer.append(getName()); + if (!NetworkTopology.DEFAULT_RACK.equals(location)) { + buffer.append(" "+location); + } + if (isDecommissioned()) { + buffer.append(" DD"); + } else if (isDecommissionInProgress()) { + buffer.append(" DP"); + } else { + buffer.append(" IN"); + } + buffer.append(" " + c + "(" + StringUtils.byteDesc(c)+")"); + buffer.append(" " + u + "(" + StringUtils.byteDesc(u)+")"); + buffer.append(" " + percent2String(u/(double)c)); + buffer.append(" " + r + "(" + StringUtils.byteDesc(r)+")"); + buffer.append(" " + cc + "(" + StringUtils.byteDesc(cc)+")"); + buffer.append(" " + cu + "(" + StringUtils.byteDesc(cu)+")"); + buffer.append(" " + percent2String(cu/(double)cc)); + buffer.append(" " + cr + "(" + StringUtils.byteDesc(cr)+")"); + buffer.append(" " + new Date(lastUpdate)); + return buffer.toString(); + } + + /** + * Start decommissioning a node. + * old state. + */ + public void startDecommission() { + adminState = AdminStates.DECOMMISSION_INPROGRESS; + } + + /** + * Stop decommissioning a node. + * old state. + */ + public void stopDecommission() { + adminState = null; + } + + /** + * Returns true if the node is in the process of being decommissioned + */ + public boolean isDecommissionInProgress() { + return adminState == AdminStates.DECOMMISSION_INPROGRESS; + } + + /** + * Returns true if the node has been decommissioned. + */ + public boolean isDecommissioned() { + return adminState == AdminStates.DECOMMISSIONED; + } + + /** + * Sets the admin state to indicate that decommission is complete. + */ + public void setDecommissioned() { + adminState = AdminStates.DECOMMISSIONED; + } + + /** + * Retrieves the admin state of this node. + */ + public AdminStates getAdminState() { + if (adminState == null) { + return AdminStates.NORMAL; + } + return adminState; + } + + /** + * Check if the datanode is in stale state. Here if + * the namenode has not received heartbeat msg from a + * datanode for more than staleInterval, + * the datanode will be treated as stale node. + * + * @param staleInterval + * the time interval for marking the node as stale. If the last + * update time is beyond the given time interval, the node will be + * marked as stale. + * @return true if the node is stale + */ + public boolean isStale(long staleInterval) { + return (Time.monotonicNow() - lastUpdateMonotonic) >= staleInterval; + } + + /** + * Sets the admin state of this node. + */ + protected void setAdminState(AdminStates newState) { + if (newState == AdminStates.NORMAL) { + adminState = null; + } + else { + adminState = newState; + } + } + + private transient int level; //which level of the tree the node resides + private transient Node parent; //its parent + + /** Return this node's parent */ + @Override + public Node getParent() { return parent; } + @Override + public void setParent(Node parent) {this.parent = parent;} + + /** Return this node's level in the tree. + * E.g. the root of a tree returns 0 and its children return 1 + */ + @Override + public int getLevel() { return level; } + @Override + public void setLevel(int level) {this.level = level;} + + @Override + public int hashCode() { + // Super implementation is sufficient + return super.hashCode(); + } + + @Override + public boolean equals(Object obj) { + // Sufficient to use super equality as datanodes are uniquely identified + // by DatanodeID + return (this == obj) || super.equals(obj); + } + + public String getSoftwareVersion() { + return softwareVersion; + } + + public void setSoftwareVersion(String softwareVersion) { + this.softwareVersion = softwareVersion; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/63d0082c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfoWithStorage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfoWithStorage.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfoWithStorage.java new file mode 100644 index 0000000..18c940b --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfoWithStorage.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.protocol; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; + [email protected] [email protected] +public class DatanodeInfoWithStorage extends DatanodeInfo { + private final String storageID; + private final StorageType storageType; + + public DatanodeInfoWithStorage(DatanodeInfo from, String storageID, + StorageType storageType) { + super(from); + this.storageID = storageID; + this.storageType = storageType; + setSoftwareVersion(from.getSoftwareVersion()); + setDependentHostNames(from.getDependentHostNames()); + setLevel(from.getLevel()); + setParent(from.getParent()); + } + + public String getStorageID() { + return storageID; + } + + public StorageType getStorageType() { + return storageType; + } + + @Override + public boolean equals(Object o) { + // allows this class to be used interchangeably with DatanodeInfo + return super.equals(o); + } + + @Override + public int hashCode() { + // allows this class to be used interchangeably with DatanodeInfo + return super.hashCode(); + } + + @Override + public String toString() { + return "DatanodeInfoWithStorage[" + super.toString() + "," + storageID + + "," + storageType + "]"; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/63d0082c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ExtendedBlock.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ExtendedBlock.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ExtendedBlock.java new file mode 100644 index 0000000..af24909 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ExtendedBlock.java @@ -0,0 +1,123 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.protocol; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * Identifies a Block uniquely across the block pools + */ [email protected] [email protected] +public class ExtendedBlock { + private String poolId; + private Block block; + + public ExtendedBlock() { + this(null, 0, 0, 0); + } + + public ExtendedBlock(final ExtendedBlock b) { + this(b.poolId, new Block(b.block)); + } + + public ExtendedBlock(final String poolId, final long blockId) { + this(poolId, blockId, 0, 0); + } + + public ExtendedBlock(String poolId, Block b) { + this.poolId = poolId; + this.block = b; + } + + public ExtendedBlock(final String poolId, final long blkid, final long len, + final long genstamp) { + this.poolId = poolId; + block = new Block(blkid, len, genstamp); + } + + public String getBlockPoolId() { + return poolId; + } + + /** Returns the block file name for the block */ + public String getBlockName() { + return block.getBlockName(); + } + + public long getNumBytes() { + return block.getNumBytes(); + } + + public long getBlockId() { + return block.getBlockId(); + } + + public long getGenerationStamp() { + return block.getGenerationStamp(); + } + + public void setBlockId(final long bid) { + block.setBlockId(bid); + } + + public void setGenerationStamp(final long genStamp) { + block.setGenerationStamp(genStamp); + } + + public void setNumBytes(final long len) { + block.setNumBytes(len); + } + + public void set(String poolId, Block blk) { + this.poolId = poolId; + this.block = blk; + } + + public static Block getLocalBlock(final ExtendedBlock b) { + return b == null ? null : b.getLocalBlock(); + } + + public Block getLocalBlock() { + return block; + } + + @Override // Object + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ExtendedBlock)) { + return false; + } + ExtendedBlock b = (ExtendedBlock)o; + return b.block.equals(block) && b.poolId.equals(poolId); + } + + @Override // Object + public int hashCode() { + int result = 31 + poolId.hashCode(); + return (31 * result + block.hashCode()); + } + + @Override // Object + public String toString() { + return poolId + ":" + block; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/63d0082c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java new file mode 100644 index 0000000..f74472d --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.protocol; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.fs.permission.FsPermission; + +/** + * HDFS permission subclass used to indicate an ACL is present and/or that the + * underlying file/dir is encrypted. The ACL/encrypted bits are not visible + * directly to users of {@link FsPermission} serialization. This is + * done for backwards compatibility in case any existing clients assume the + * value of FsPermission is in a particular range. + */ [email protected] +public class FsPermissionExtension extends FsPermission { + private final static short ACL_BIT = 1 << 12; + private final static short ENCRYPTED_BIT = 1 << 13; + private final boolean aclBit; + private final boolean encryptedBit; + + /** + * Constructs a new FsPermissionExtension based on the given FsPermission. + * + * @param perm FsPermission containing permission bits + */ + public FsPermissionExtension(FsPermission perm, boolean hasAcl, + boolean isEncrypted) { + super(perm.toShort()); + aclBit = hasAcl; + encryptedBit = isEncrypted; + } + + /** + * Creates a new FsPermissionExtension by calling the base class constructor. + * + * @param perm short containing permission bits + */ + public FsPermissionExtension(short perm) { + super(perm); + aclBit = (perm & ACL_BIT) != 0; + encryptedBit = (perm & ENCRYPTED_BIT) != 0; + } + + @Override + public short toExtendedShort() { + return (short)(toShort() | + (aclBit ? ACL_BIT : 0) | (encryptedBit ? ENCRYPTED_BIT : 0)); + } + + @Override + public boolean getAclBit() { + return aclBit; + } + + @Override + public boolean getEncryptedBit() { + return encryptedBit; + } + + @Override + public boolean equals(Object o) { + // This intentionally delegates to the base class. This is only overridden + // to suppress a FindBugs warning. + return super.equals(o); + } + + @Override + public int hashCode() { + // This intentionally delegates to the base class. This is only overridden + // to suppress a FindBugs warning. + return super.hashCode(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/63d0082c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstantsClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstantsClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstantsClient.java index aa5f1f9..40c3203 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstantsClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstantsClient.java @@ -31,4 +31,5 @@ public interface HdfsConstantsClient { * uses GRANDFATHER_INODE_ID for backward compatibility. */ long GRANDFATHER_INODE_ID = 0; + byte BLOCK_STORAGE_POLICY_ID_UNSPECIFIED = 0; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/63d0082c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java new file mode 100644 index 0000000..34f429a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java @@ -0,0 +1,271 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.protocol; + +import java.net.URI; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.FileEncryptionInfo; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSUtilClient; + +/** Interface that represents the over the wire information for a file. + */ [email protected] [email protected] +public class HdfsFileStatus { + + private final byte[] path; // local name of the inode that's encoded in java UTF8 + private final byte[] symlink; // symlink target encoded in java UTF8 or null + private final long length; + private final boolean isdir; + private final short block_replication; + private final long blocksize; + private final long modification_time; + private final long access_time; + private final FsPermission permission; + private final String owner; + private final String group; + private final long fileId; + + private final FileEncryptionInfo feInfo; + + // Used by dir, not including dot and dotdot. Always zero for a regular file. + private final int childrenNum; + private final byte storagePolicy; + + public static final byte[] EMPTY_NAME = new byte[0]; + + /** + * Constructor + * @param length the number of bytes the file has + * @param isdir if the path is a directory + * @param block_replication the replication factor + * @param blocksize the block size + * @param modification_time modification time + * @param access_time access time + * @param permission permission + * @param owner the owner of the path + * @param group the group of the path + * @param path the local name in java UTF8 encoding the same as that in-memory + * @param fileId the file id + * @param feInfo the file's encryption info + */ + public HdfsFileStatus(long length, boolean isdir, int block_replication, + long blocksize, long modification_time, long access_time, + FsPermission permission, String owner, String group, byte[] symlink, + byte[] path, long fileId, int childrenNum, FileEncryptionInfo feInfo, + byte storagePolicy) { + this.length = length; + this.isdir = isdir; + this.block_replication = (short)block_replication; + this.blocksize = blocksize; + this.modification_time = modification_time; + this.access_time = access_time; + this.permission = (permission == null) ? + ((isdir || symlink!=null) ? + FsPermission.getDefault() : + FsPermission.getFileDefault()) : + permission; + this.owner = (owner == null) ? "" : owner; + this.group = (group == null) ? "" : group; + this.symlink = symlink; + this.path = path; + this.fileId = fileId; + this.childrenNum = childrenNum; + this.feInfo = feInfo; + this.storagePolicy = storagePolicy; + } + + /** + * Get the length of this file, in bytes. + * @return the length of this file, in bytes. + */ + public final long getLen() { + return length; + } + + /** + * Is this a directory? + * @return true if this is a directory + */ + public final boolean isDir() { + return isdir; + } + + /** + * Is this a symbolic link? + * @return true if this is a symbolic link + */ + public boolean isSymlink() { + return symlink != null; + } + + /** + * Get the block size of the file. + * @return the number of bytes + */ + public final long getBlockSize() { + return blocksize; + } + + /** + * Get the replication factor of a file. + * @return the replication factor of a file. + */ + public final short getReplication() { + return block_replication; + } + + /** + * Get the modification time of the file. + * @return the modification time of file in milliseconds since January 1, 1970 UTC. + */ + public final long getModificationTime() { + return modification_time; + } + + /** + * Get the access time of the file. + * @return the access time of file in milliseconds since January 1, 1970 UTC. + */ + public final long getAccessTime() { + return access_time; + } + + /** + * Get FsPermission associated with the file. + * @return permssion + */ + public final FsPermission getPermission() { + return permission; + } + + /** + * Get the owner of the file. + * @return owner of the file + */ + public final String getOwner() { + return owner; + } + + /** + * Get the group associated with the file. + * @return group for the file. + */ + public final String getGroup() { + return group; + } + + /** + * Check if the local name is empty + * @return true if the name is empty + */ + public final boolean isEmptyLocalName() { + return path.length == 0; + } + + /** + * Get the string representation of the local name + * @return the local name in string + */ + public final String getLocalName() { + return DFSUtilClient.bytes2String(path); + } + + /** + * Get the Java UTF8 representation of the local name + * @return the local name in java UTF8 + */ + public final byte[] getLocalNameInBytes() { + return path; + } + + /** + * Get the string representation of the full path name + * @param parent the parent path + * @return the full path in string + */ + public final String getFullName(final String parent) { + if (isEmptyLocalName()) { + return parent; + } + + StringBuilder fullName = new StringBuilder(parent); + if (!parent.endsWith(Path.SEPARATOR)) { + fullName.append(Path.SEPARATOR); + } + fullName.append(getLocalName()); + return fullName.toString(); + } + + /** + * Get the full path + * @param parent the parent path + * @return the full path + */ + public final Path getFullPath(final Path parent) { + if (isEmptyLocalName()) { + return parent; + } + + return new Path(parent, getLocalName()); + } + + /** + * Get the string representation of the symlink. + * @return the symlink as a string. + */ + public final String getSymlink() { + return DFSUtilClient.bytes2String(symlink); + } + + public final byte[] getSymlinkInBytes() { + return symlink; + } + + public final long getFileId() { + return fileId; + } + + public final FileEncryptionInfo getFileEncryptionInfo() { + return feInfo; + } + + public final int getChildrenNum() { + return childrenNum; + } + + /** @return the storage policy id */ + public final byte getStoragePolicy() { + return storagePolicy; + } + + public final FileStatus makeQualified(URI defaultUri, Path path) { + return new FileStatus(getLen(), isDir(), getReplication(), + getBlockSize(), getModificationTime(), + getAccessTime(), + getPermission(), getOwner(), getGroup(), + isSymlink() ? new Path(getSymlink()) : null, + (getFullPath(path)).makeQualified( + defaultUri, null)); // fully-qualify path + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/63d0082c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java new file mode 100644 index 0000000..cc13f10 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java @@ -0,0 +1,206 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.protocol; + +import java.util.Arrays; +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; +import org.apache.hadoop.security.token.Token; + +import com.google.common.collect.Lists; + +/** + * Associates a block with the Datanodes that contain its replicas + * and other block metadata (E.g. the file offset associated with this + * block, whether it is corrupt, a location is cached in memory, + * security token, etc). + */ [email protected] [email protected] +public class LocatedBlock { + + private final ExtendedBlock b; + private long offset; // offset of the first byte of the block in the file + private final DatanodeInfoWithStorage[] locs; + /** Cached storage ID for each replica */ + private final String[] storageIDs; + /** Cached storage type for each replica, if reported. */ + private final StorageType[] storageTypes; + // corrupt flag is true if all of the replicas of a block are corrupt. + // else false. If block has few corrupt replicas, they are filtered and + // their locations are not part of this object + private boolean corrupt; + private Token<BlockTokenIdentifier> blockToken = new Token<BlockTokenIdentifier>(); + /** + * List of cached datanode locations + */ + private DatanodeInfo[] cachedLocs; + + // Used when there are no locations + private static final DatanodeInfoWithStorage[] EMPTY_LOCS = + new DatanodeInfoWithStorage[0]; + + public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs) { + // By default, startOffset is unknown(-1) and corrupt is false. + this(b, locs, null, null, -1, false, EMPTY_LOCS); + } + + public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs, + String[] storageIDs, StorageType[] storageTypes) { + this(b, locs, storageIDs, storageTypes, -1, false, EMPTY_LOCS); + } + + public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs, String[] storageIDs, + StorageType[] storageTypes, long startOffset, + boolean corrupt, DatanodeInfo[] cachedLocs) { + this.b = b; + this.offset = startOffset; + this.corrupt = corrupt; + if (locs==null) { + this.locs = EMPTY_LOCS; + } else { + this.locs = new DatanodeInfoWithStorage[locs.length]; + for(int i = 0; i < locs.length; i++) { + DatanodeInfo di = locs[i]; + DatanodeInfoWithStorage storage = new DatanodeInfoWithStorage(di, + storageIDs != null ? storageIDs[i] : null, + storageTypes != null ? storageTypes[i] : null); + this.locs[i] = storage; + } + } + this.storageIDs = storageIDs; + this.storageTypes = storageTypes; + + if (cachedLocs == null || cachedLocs.length == 0) { + this.cachedLocs = EMPTY_LOCS; + } else { + this.cachedLocs = cachedLocs; + } + } + + public Token<BlockTokenIdentifier> getBlockToken() { + return blockToken; + } + + public void setBlockToken(Token<BlockTokenIdentifier> token) { + this.blockToken = token; + } + + public ExtendedBlock getBlock() { + return b; + } + + /** + * Returns the locations associated with this block. The returned array is not + * expected to be modified. If it is, caller must immediately invoke + * {@link org.apache.hadoop.hdfs.protocol.LocatedBlock#updateCachedStorageInfo} + * to update the cached Storage ID/Type arrays. + */ + public DatanodeInfo[] getLocations() { + return locs; + } + + public StorageType[] getStorageTypes() { + return storageTypes; + } + + public String[] getStorageIDs() { + return storageIDs; + } + + /** + * Updates the cached StorageID and StorageType information. Must be + * called when the locations array is modified. + */ + public void updateCachedStorageInfo() { + if (storageIDs != null) { + for(int i = 0; i < locs.length; i++) { + storageIDs[i] = locs[i].getStorageID(); + } + } + if (storageTypes != null) { + for(int i = 0; i < locs.length; i++) { + storageTypes[i] = locs[i].getStorageType(); + } + } + } + + public long getStartOffset() { + return offset; + } + + public long getBlockSize() { + return b.getNumBytes(); + } + + public void setStartOffset(long value) { + this.offset = value; + } + + public void setCorrupt(boolean corrupt) { + this.corrupt = corrupt; + } + + public boolean isCorrupt() { + return this.corrupt; + } + + /** + * Add a the location of a cached replica of the block. + * + * @param loc of datanode with the cached replica + */ + public void addCachedLoc(DatanodeInfo loc) { + List<DatanodeInfo> cachedList = Lists.newArrayList(cachedLocs); + if (cachedList.contains(loc)) { + return; + } + // Try to re-use a DatanodeInfo already in loc + for (DatanodeInfoWithStorage di : locs) { + if (loc.equals(di)) { + cachedList.add(di); + cachedLocs = cachedList.toArray(cachedLocs); + return; + } + } + // Not present in loc, add it and go + cachedList.add(loc); + cachedLocs = cachedList.toArray(cachedLocs); + } + + /** + * @return Datanodes with a cached block replica + */ + public DatanodeInfo[] getCachedLocations() { + return cachedLocs; + } + + @Override + public String toString() { + return getClass().getSimpleName() + "{" + b + + "; getBlockSize()=" + getBlockSize() + + "; corrupt=" + corrupt + + "; offset=" + offset + + "; locs=" + Arrays.asList(locs) + + "}"; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/63d0082c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java new file mode 100644 index 0000000..e4896977 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java @@ -0,0 +1,189 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.protocol; + +import java.util.List; +import java.util.Collections; +import java.util.Comparator; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.FileEncryptionInfo; + +/** + * Collection of blocks with their locations and the file length. + */ [email protected] [email protected] +public class LocatedBlocks { + private final long fileLength; + private final List<LocatedBlock> blocks; // array of blocks with prioritized locations + private final boolean underConstruction; + private final LocatedBlock lastLocatedBlock; + private final boolean isLastBlockComplete; + private final FileEncryptionInfo fileEncryptionInfo; + + public LocatedBlocks() { + fileLength = 0; + blocks = null; + underConstruction = false; + lastLocatedBlock = null; + isLastBlockComplete = false; + fileEncryptionInfo = null; + } + + public LocatedBlocks(long flength, boolean isUnderConstuction, + List<LocatedBlock> blks, LocatedBlock lastBlock, + boolean isLastBlockCompleted, FileEncryptionInfo feInfo) { + fileLength = flength; + blocks = blks; + underConstruction = isUnderConstuction; + this.lastLocatedBlock = lastBlock; + this.isLastBlockComplete = isLastBlockCompleted; + this.fileEncryptionInfo = feInfo; + } + + /** + * Get located blocks. + */ + public List<LocatedBlock> getLocatedBlocks() { + return blocks; + } + + /** Get the last located block. */ + public LocatedBlock getLastLocatedBlock() { + return lastLocatedBlock; + } + + /** Is the last block completed? */ + public boolean isLastBlockComplete() { + return isLastBlockComplete; + } + + /** + * Get located block. + */ + public LocatedBlock get(int index) { + return blocks.get(index); + } + + /** + * Get number of located blocks. + */ + public int locatedBlockCount() { + return blocks == null ? 0 : blocks.size(); + } + + /** + * + */ + public long getFileLength() { + return this.fileLength; + } + + /** + * Return true if file was under construction when this LocatedBlocks was + * constructed, false otherwise. + */ + public boolean isUnderConstruction() { + return underConstruction; + } + + /** + * @return the FileEncryptionInfo for the LocatedBlocks + */ + public FileEncryptionInfo getFileEncryptionInfo() { + return fileEncryptionInfo; + } + + /** + * Find block containing specified offset. + * + * @return block if found, or null otherwise. + */ + public int findBlock(long offset) { + // create fake block of size 0 as a key + LocatedBlock key = new LocatedBlock( + new ExtendedBlock(), new DatanodeInfo[0]); + key.setStartOffset(offset); + key.getBlock().setNumBytes(1); + Comparator<LocatedBlock> comp = + new Comparator<LocatedBlock>() { + // Returns 0 iff a is inside b or b is inside a + @Override + public int compare(LocatedBlock a, LocatedBlock b) { + long aBeg = a.getStartOffset(); + long bBeg = b.getStartOffset(); + long aEnd = aBeg + a.getBlockSize(); + long bEnd = bBeg + b.getBlockSize(); + if(aBeg <= bBeg && bEnd <= aEnd + || bBeg <= aBeg && aEnd <= bEnd) + return 0; // one of the blocks is inside the other + if(aBeg < bBeg) + return -1; // a's left bound is to the left of the b's + return 1; + } + }; + return Collections.binarySearch(blocks, key, comp); + } + + public void insertRange(int blockIdx, List<LocatedBlock> newBlocks) { + int oldIdx = blockIdx; + int insStart = 0, insEnd = 0; + for(int newIdx = 0; newIdx < newBlocks.size() && oldIdx < blocks.size(); + newIdx++) { + long newOff = newBlocks.get(newIdx).getStartOffset(); + long oldOff = blocks.get(oldIdx).getStartOffset(); + if(newOff < oldOff) { + insEnd++; + } else if(newOff == oldOff) { + // replace old cached block by the new one + blocks.set(oldIdx, newBlocks.get(newIdx)); + if(insStart < insEnd) { // insert new blocks + blocks.addAll(oldIdx, newBlocks.subList(insStart, insEnd)); + oldIdx += insEnd - insStart; + } + insStart = insEnd = newIdx+1; + oldIdx++; + } else { // newOff > oldOff + assert false : "List of LocatedBlock must be sorted by startOffset"; + } + } + insEnd = newBlocks.size(); + if(insStart < insEnd) { // insert new blocks + blocks.addAll(oldIdx, newBlocks.subList(insStart, insEnd)); + } + } + + public static int getInsertIndex(int binSearchResult) { + return binSearchResult >= 0 ? binSearchResult : -(binSearchResult+1); + } + + @Override + public String toString() { + final StringBuilder b = new StringBuilder(getClass().getSimpleName()); + b.append("{") + .append("\n fileLength=").append(fileLength) + .append("\n underConstruction=").append(underConstruction) + .append("\n blocks=").append(blocks) + .append("\n lastLocatedBlock=").append(lastLocatedBlock) + .append("\n isLastBlockComplete=").append(isLastBlockComplete) + .append("}"); + return b.toString(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/63d0082c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java new file mode 100644 index 0000000..3f2c9ca --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java @@ -0,0 +1,189 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.security.token.block; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.EnumSet; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.WritableUtils; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; + [email protected] +public class BlockTokenIdentifier extends TokenIdentifier { + static final Text KIND_NAME = new Text("HDFS_BLOCK_TOKEN"); + + public enum AccessMode { + READ, WRITE, COPY, REPLACE + } + + private long expiryDate; + private int keyId; + private String userId; + private String blockPoolId; + private long blockId; + private final EnumSet<AccessMode> modes; + + private byte [] cache; + + public BlockTokenIdentifier() { + this(null, null, 0, EnumSet.noneOf(AccessMode.class)); + } + + public BlockTokenIdentifier(String userId, String bpid, long blockId, + EnumSet<AccessMode> modes) { + this.cache = null; + this.userId = userId; + this.blockPoolId = bpid; + this.blockId = blockId; + this.modes = modes == null ? EnumSet.noneOf(AccessMode.class) : modes; + } + + @Override + public Text getKind() { + return KIND_NAME; + } + + @Override + public UserGroupInformation getUser() { + if (userId == null || "".equals(userId)) { + String user = blockPoolId + ":" + Long.toString(blockId); + return UserGroupInformation.createRemoteUser(user); + } + return UserGroupInformation.createRemoteUser(userId); + } + + public long getExpiryDate() { + return expiryDate; + } + + public void setExpiryDate(long expiryDate) { + this.cache = null; + this.expiryDate = expiryDate; + } + + public int getKeyId() { + return this.keyId; + } + + public void setKeyId(int keyId) { + this.cache = null; + this.keyId = keyId; + } + + public String getUserId() { + return userId; + } + + public String getBlockPoolId() { + return blockPoolId; + } + + public long getBlockId() { + return blockId; + } + + public EnumSet<AccessMode> getAccessModes() { + return modes; + } + + @Override + public String toString() { + return "block_token_identifier (expiryDate=" + this.getExpiryDate() + + ", keyId=" + this.getKeyId() + ", userId=" + this.getUserId() + + ", blockPoolId=" + this.getBlockPoolId() + + ", blockId=" + this.getBlockId() + ", access modes=" + + this.getAccessModes() + ")"; + } + + static boolean isEqual(Object a, Object b) { + return a == null ? b == null : a.equals(b); + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (obj instanceof BlockTokenIdentifier) { + BlockTokenIdentifier that = (BlockTokenIdentifier) obj; + return this.expiryDate == that.expiryDate && this.keyId == that.keyId + && isEqual(this.userId, that.userId) + && isEqual(this.blockPoolId, that.blockPoolId) + && this.blockId == that.blockId + && isEqual(this.modes, that.modes); + } + return false; + } + + @Override + public int hashCode() { + return (int) expiryDate ^ keyId ^ (int) blockId ^ modes.hashCode() + ^ (userId == null ? 0 : userId.hashCode()) + ^ (blockPoolId == null ? 0 : blockPoolId.hashCode()); + } + + @Override + public void readFields(DataInput in) throws IOException { + this.cache = null; + expiryDate = WritableUtils.readVLong(in); + keyId = WritableUtils.readVInt(in); + userId = WritableUtils.readString(in); + blockPoolId = WritableUtils.readString(in); + blockId = WritableUtils.readVLong(in); + int length = WritableUtils.readVIntInRange(in, 0, + AccessMode.class.getEnumConstants().length); + for (int i = 0; i < length; i++) { + modes.add(WritableUtils.readEnum(in, AccessMode.class)); + } + } + + @Override + public void write(DataOutput out) throws IOException { + WritableUtils.writeVLong(out, expiryDate); + WritableUtils.writeVInt(out, keyId); + WritableUtils.writeString(out, userId); + WritableUtils.writeString(out, blockPoolId); + WritableUtils.writeVLong(out, blockId); + WritableUtils.writeVInt(out, modes.size()); + for (AccessMode aMode : modes) { + WritableUtils.writeEnum(out, aMode); + } + } + + @Override + public byte[] getBytes() { + if(cache == null) cache = super.getBytes(); + + return cache; + } + + @InterfaceAudience.Private + public static class Renewer extends Token.TrivialRenewer { + @Override + protected Text getKind() { + return KIND_NAME; + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/63d0082c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java new file mode 100644 index 0000000..2a5d5fe --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.security.token.delegation; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hdfs.web.WebHdfsConstants; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; + +/** + * A delegation token identifier that is specific to HDFS. + */ [email protected] +public class DelegationTokenIdentifier + extends AbstractDelegationTokenIdentifier { + public static final Text HDFS_DELEGATION_KIND = new Text("HDFS_DELEGATION_TOKEN"); + + /** + * Create an empty delegation token identifier for reading into. + */ + public DelegationTokenIdentifier() { + } + + /** + * Create a new delegation token identifier + * @param owner the effective username of the token owner + * @param renewer the username of the renewer + * @param realUser the real username of the token owner + */ + public DelegationTokenIdentifier(Text owner, Text renewer, Text realUser) { + super(owner, renewer, realUser); + } + + @Override + public Text getKind() { + return HDFS_DELEGATION_KIND; + } + + @Override + public String toString() { + return getKind() + " token " + getSequenceNumber() + + " for " + getUser().getShortUserName(); + } + + /** @return a string representation of the token */ + public static String stringifyToken(final Token<?> token) throws IOException { + DelegationTokenIdentifier ident = new DelegationTokenIdentifier(); + ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier()); + DataInputStream in = new DataInputStream(buf); + ident.readFields(in); + + if (token.getService().getLength() > 0) { + return ident + " on " + token.getService(); + } else { + return ident.toString(); + } + } + + public static class WebHdfsDelegationTokenIdentifier + extends DelegationTokenIdentifier { + public WebHdfsDelegationTokenIdentifier() { + super(); + } + @Override + public Text getKind() { + return WebHdfsConstants.WEBHDFS_TOKEN_KIND; + } + } + + public static class SWebHdfsDelegationTokenIdentifier extends WebHdfsDelegationTokenIdentifier { + public SWebHdfsDelegationTokenIdentifier() { + super(); + } + @Override + public Text getKind() { + return WebHdfsConstants.SWEBHDFS_TOKEN_KIND; + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/63d0082c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConstants.java new file mode 100644 index 0000000..25f3cfd --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConstants.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.web; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.io.Text; + [email protected] +public class WebHdfsConstants { + /** Delegation token kind */ + public static final Text WEBHDFS_TOKEN_KIND = new Text("WEBHDFS delegation"); + public static final Text SWEBHDFS_TOKEN_KIND = new Text("SWEBHDFS delegation"); + + enum PathType { + FILE, DIRECTORY, SYMLINK; + + static PathType valueOf(HdfsFileStatus status) { + return status.isDir()? DIRECTORY: status.isSymlink()? SYMLINK: FILE; + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/63d0082c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 39ee33e..9bb1fd4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -450,6 +450,8 @@ Release 2.8.0 - UNRELEASED HDFS-8152. Refactoring of lazy persist storage cases. (Arpit Agarwal) + HDFS-8169. Move LocatedBlocks and related classes to hdfs-client. (wheat9) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/63d0082c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 8b3f512..a2c16b7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -1234,21 +1234,6 @@ public class DFSUtil { } } } - - /** Return used as percentage of capacity */ - public static float getPercentUsed(long used, long capacity) { - return capacity <= 0 ? 100 : (used * 100.0f)/capacity; - } - - /** Return remaining as percentage of capacity */ - public static float getPercentRemaining(long remaining, long capacity) { - return capacity <= 0 ? 0 : (remaining * 100.0f)/capacity; - } - - /** Convert percentage to a string. */ - public static String percent2String(double percentage) { - return StringUtils.format("%.2f%%", percentage); - } /** * Round bytes to GiB (gibibyte)
