Author: umamahesh Date: Tue Apr 29 13:08:03 2014 New Revision: 1590974 URL: http://svn.apache.org/r1590974 Log: HDFS-6299. Protobuf for XAttr and client-side implementation. Contributed by Yi Liu.
Added: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/xattr.proto Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-2006.txt hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/pom.xml hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-2006.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-2006.txt?rev=1590974&r1=1590973&r2=1590974&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-2006.txt (original) +++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-2006.txt Tue Apr 29 13:08:03 2014 @@ -8,6 +8,8 @@ HDFS-2006 (Unreleased) IMPROVEMENTS + HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh) + OPTIMIZATIONS BUG FIXES Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/pom.xml?rev=1590974&r1=1590973&r2=1590974&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/pom.xml (original) +++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/pom.xml Tue Apr 29 13:08:03 2014 @@ -439,6 +439,7 @@ http://maven.apache.org/xsd/maven-4.0.0. <include>NamenodeProtocol.proto</include> <include>QJournalProtocol.proto</include> <include>acl.proto</include> + <include>xattr.proto</include> <include>datatransfer.proto</include> <include>fsimage.proto</include> <include>hdfs.proto</include> Added: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java?rev=1590974&view=auto ============================================================================== --- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java (added) +++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java Tue Apr 29 13:08:03 2014 @@ -0,0 +1,144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs; + +import java.util.Arrays; + +/** + * XAttr is POSIX Extended Attribute model, similar to the one in traditional + * Operating Systems. Extended Attribute consists of a name and associated + * data, and 4 namespaces are defined: user, trusted, security and system. + * <br> + * 1). USER namespace extended attribute may be assigned for storing + * arbitrary additional information, and its access permissions are + * defined by file/directory permission bits. + * <br> + * 2). TRUSTED namespace extended attribute are visible and accessible + * only to privilege user (file/directory owner or fs admin), and it is + * available from both user space (filesystem API) and fs kernel. + * <br> + * 3). SYSTEM namespace extended attribute is used by fs kernel to store + * system objects, and only available in fs kernel. It's not visible to users. + * <br> + * 4). SECURITY namespace extended attribute is used by fs kernel for + * security features, and it's not visible to users. + * <p/> + * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes"> + * http://en.wikipedia.org/wiki/Extended_file_attributes</a> + * + */ +public class XAttr { + + public static enum NameSpace { + USER, + TRUSTED, + SECURITY, + SYSTEM; + } + + private final NameSpace ns; + private final String name; + private final byte[] value; + + public static class Builder { + private NameSpace ns = NameSpace.USER; + private String name; + private byte[] value; + + public Builder setNameSpace(NameSpace ns) { + this.ns = ns; + return this; + } + + public Builder setName(String name) { + this.name = name; + return this; + } + + public Builder setValue(byte[] value) { + this.value = value; + return this; + } + + public XAttr build() { + return new XAttr(ns, name, value); + } + } + + private XAttr(NameSpace ns, String name, byte[] value) { + this.ns = ns; + this.name = name; + this.value = value; + } + + public NameSpace getNameSpace() { + return ns; + } + + public String getName() { + return name; + } + + public byte[] getValue() { + return value; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((name == null) ? 0 : name.hashCode()); + result = prime * result + ((ns == null) ? 0 : ns.hashCode()); + result = prime * result + Arrays.hashCode(value); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + XAttr other = (XAttr) obj; + if (name == null) { + if (other.name != null) { + return false; + } + } else if (!name.equals(other.name)) { + return false; + } + if (ns != other.ns) { + return false; + } + if (!Arrays.equals(value, other.value)) { + return false; + } + return true; + } + + @Override + public String toString() { + return "XAttr [ns=" + ns + ", name=" + name + ", value=" + + Arrays.toString(value) + "]"; + } +} Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java?rev=1590974&r1=1590973&r2=1590974&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java (original) +++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java Tue Apr 29 13:08:03 2014 @@ -91,6 +91,7 @@ import javax.net.SocketFactory; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; @@ -109,6 +110,8 @@ import org.apache.hadoop.fs.MD5MD5CRC32C import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum; import org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum; import org.apache.hadoop.fs.Options; +import org.apache.hadoop.fs.XAttr; +import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Path; @@ -191,6 +194,8 @@ import org.apache.hadoop.util.Time; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.net.InetAddresses; /******************************************************** @@ -2757,6 +2762,149 @@ public class DFSClient implements java.i UnresolvedPathException.class); } } + + XAttr buildXAttr(String name, byte[] value) { + if (name == null) { + throw new NullPointerException("XAttr name can not be null."); + } + + int prefixIndex = name.indexOf("."); + if (prefixIndex == -1) { + throw new HadoopIllegalArgumentException("XAttr name must be prefixed with" + + " user/trusted/security/system which followed by '.'"); + } else if (prefixIndex == name.length() -1) { + throw new HadoopIllegalArgumentException("XAttr name can not be empty."); + } + + XAttr.NameSpace ns; + String prefix = name.substring(0, prefixIndex).toUpperCase(); + if (prefix.equals(XAttr.NameSpace.USER.toString())) { + ns = XAttr.NameSpace.USER; + } else if (prefix.equals(XAttr.NameSpace.TRUSTED.toString())) { + ns = XAttr.NameSpace.TRUSTED; + } else if (prefix.equals(XAttr.NameSpace.SECURITY.toString())) { + ns = XAttr.NameSpace.SECURITY; + } else if (prefix.equals(XAttr.NameSpace.SYSTEM.toString())) { + ns = XAttr.NameSpace.SYSTEM; + } else { + throw new HadoopIllegalArgumentException("XAttr name must be prefixed with" + + " user/trusted/security/system which followed by '.'"); + } + XAttr xAttr = (new XAttr.Builder()).setNameSpace(ns).setName(name. + substring(prefixIndex + 1)).setValue(value).build(); + + return xAttr; + } + + public void setXAttr(String src, String name, byte[] value, + EnumSet<XAttrSetFlag> flag) throws IOException { + checkOpen(); + try { + namenode.setXAttr(src, buildXAttr(name, value), flag); + } catch (RemoteException re) { + throw re.unwrapRemoteException(AccessControlException.class, + AclException.class, + FileNotFoundException.class, + NSQuotaExceededException.class, + SafeModeException.class, + SnapshotAccessControlException.class, + UnresolvedPathException.class); + } + } + + public byte[] getXAttr(String src, String name) throws IOException { + checkOpen(); + try { + XAttr xAttr = buildXAttr(name, null); + List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1); + xAttrs.add(xAttr); + List<XAttr> result = namenode.getXAttrs(src, xAttrs); + byte[] value = null; + if (result != null && result.size() > 0) { + XAttr a = result.get(0); + value = a.getValue(); + if (value == null) { + value = new byte[0]; //xattr exists, but no value. + } + } + return value; + } catch(RemoteException re) { + throw re.unwrapRemoteException(AccessControlException.class, + AclException.class, + FileNotFoundException.class, + UnresolvedPathException.class); + } + } + + Map<String, byte[]> buildXAttrMap(List<XAttr> xAttrs) { + if (xAttrs == null) { + return null; + } + Map<String, byte[]> xAttrMap = Maps.newHashMap(); + for (XAttr xAttr : xAttrs) { + String namespace = xAttr.getNameSpace().toString(); + String name = namespace.toLowerCase() + "." + xAttr.getName(); + byte[] value = xAttr.getValue(); + if (value == null) { + value = new byte[0]; + } + xAttrMap.put(name, value); + } + + return xAttrMap; + } + + public Map<String, byte[]> getXAttrs(String src) throws IOException { + checkOpen(); + try { + return buildXAttrMap(namenode.getXAttrs(src, null)); + } catch(RemoteException re) { + throw re.unwrapRemoteException(AccessControlException.class, + AclException.class, + FileNotFoundException.class, + UnresolvedPathException.class); + } + } + + List<XAttr> buildXAttrs(List<String> names) { + if (names == null || names.isEmpty()) { + throw new HadoopIllegalArgumentException("XAttr names can not be null or empty."); + } + + List<XAttr> xAttrs = Lists.newArrayListWithCapacity(names.size()); + for (String name : names) { + xAttrs.add(buildXAttr(name, null)); + } + return xAttrs; + } + + public Map<String, byte[]> getXAttrs(String src, List<String> names) + throws IOException { + checkOpen(); + try { + return buildXAttrMap(namenode.getXAttrs(src, buildXAttrs(names))); + } catch(RemoteException re) { + throw re.unwrapRemoteException(AccessControlException.class, + AclException.class, + FileNotFoundException.class, + UnresolvedPathException.class); + } + } + + public void removeXAttr(String src, String name) throws IOException { + checkOpen(); + try { + namenode.removeXAttr(src, buildXAttr(name, null)); + } catch(RemoteException re) { + throw re.unwrapRemoteException(AccessControlException.class, + AclException.class, + FileNotFoundException.class, + NSQuotaExceededException.class, + SafeModeException.class, + SnapshotAccessControlException.class, + UnresolvedPathException.class); + } + } @Override // RemotePeerFactory public Peer newConnectedPeer(InetSocketAddress addr) throws IOException { Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java?rev=1590974&r1=1590973&r2=1590974&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java (original) +++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java Tue Apr 29 13:08:03 2014 @@ -25,6 +25,7 @@ import java.net.URI; import java.util.ArrayList; import java.util.EnumSet; import java.util.List; +import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -46,6 +47,7 @@ import org.apache.hadoop.fs.FsServerDefa import org.apache.hadoop.fs.FsStatus; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Options; +import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Path; @@ -1769,4 +1771,91 @@ public class DistributedFileSystem exten } }.resolve(this, absF); } + + @Override + public void setXAttr(Path path, final String name, final byte[] value, + final EnumSet<XAttrSetFlag> flag) throws IOException { + Path absF = fixRelativePart(path); + new FileSystemLinkResolver<Void>() { + + @Override + public Void doCall(final Path p) throws IOException { + dfs.setXAttr(getPathName(p), name, value, flag); + return null; + } + + @Override + public Void next(final FileSystem fs, final Path p) throws IOException { + fs.setXAttr(p, name, value, flag); + return null; + } + }.resolve(this, absF); + } + + @Override + public byte[] getXAttr(Path path, final String name) throws IOException { + final Path absF = fixRelativePart(path); + return new FileSystemLinkResolver<byte[]>() { + @Override + public byte[] doCall(final Path p) throws IOException { + return dfs.getXAttr(getPathName(p), name); + } + @Override + public byte[] next(final FileSystem fs, final Path p) + throws IOException, UnresolvedLinkException { + return fs.getXAttr(p, name); + } + }.resolve(this, absF); + } + + @Override + public Map<String, byte[]> getXAttrs(Path path) throws IOException { + final Path absF = fixRelativePart(path); + return new FileSystemLinkResolver<Map<String, byte[]>>() { + @Override + public Map<String, byte[]> doCall(final Path p) throws IOException { + return dfs.getXAttrs(getPathName(p)); + } + @Override + public Map<String, byte[]> next(final FileSystem fs, final Path p) + throws IOException, UnresolvedLinkException { + return fs.getXAttrs(p); + } + }.resolve(this, absF); + } + + @Override + public Map<String, byte[]> getXAttrs(Path path, final List<String> names) + throws IOException { + final Path absF = fixRelativePart(path); + return new FileSystemLinkResolver<Map<String, byte[]>>() { + @Override + public Map<String, byte[]> doCall(final Path p) throws IOException { + return dfs.getXAttrs(getPathName(p), names); + } + @Override + public Map<String, byte[]> next(final FileSystem fs, final Path p) + throws IOException, UnresolvedLinkException { + return fs.getXAttrs(p, names); + } + }.resolve(this, absF); + } + + @Override + public void removeXAttr(Path path, final String name) throws IOException { + Path absF = fixRelativePart(path); + new FileSystemLinkResolver<Void>() { + @Override + public Void doCall(final Path p) throws IOException { + dfs.removeXAttr(getPathName(p), name); + return null; + } + + @Override + public Void next(final FileSystem fs, final Path p) throws IOException { + fs.removeXAttr(p, name); + return null; + } + }.resolve(this, absF); + } } Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java?rev=1590974&r1=1590973&r2=1590974&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java (original) +++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java Tue Apr 29 13:08:03 2014 @@ -31,10 +31,12 @@ import org.apache.hadoop.fs.FileAlreadyE import org.apache.hadoop.fs.FsServerDefaults; import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.Options; +import org.apache.hadoop.fs.XAttr; import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries; import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.UnresolvedLinkException; +import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsPermission; @@ -1241,4 +1243,66 @@ public interface ClientProtocol { */ @Idempotent public AclStatus getAclStatus(String src) throws IOException; + + /** + * Set xattr of a file or directory. + * A regular user only can set xattr of "user" namespace. + * A super user can set xattr of "user" and "trusted" namespace. + * XAttr of "security" and "system" namespace is only used/exposed + * internally to the FS impl. + * <p/> + * For xattr of "user" namespace, its access permissions are + * defined by the file or directory permission bits. + * XAttr will be set only when login user has correct permissions. + * <p/> + * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes"> + * http://en.wikipedia.org/wiki/Extended_file_attributes</a> + * @param src file or directory + * @param xAttr <code>XAttr</code> to set + * @param flag set flag + * @throws IOException + */ + @Idempotent + public void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag) + throws IOException; + + /** + * Get xattrs of file or directory. Values in xAttrs parameter are ignored. + * If xattrs is null or empty, equals getting all xattrs of the file or + * directory. + * Only xattrs which login user has correct permissions will be returned. + * <p/> + * A regular user only can get xattr of "user" namespace. + * A super user can get xattr of "user" and "trusted" namespace. + * XAttr of "security" and "system" namespace is only used/exposed + * internally to the FS impl. + * <p/> + * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes"> + * http://en.wikipedia.org/wiki/Extended_file_attributes</a> + * @param src file or directory + * @param xAttrs xAttrs to get + * @return List<XAttr> <code>XAttr</code> list + * @throws IOException + */ + @Idempotent + public List<XAttr> getXAttrs(String src, List<XAttr> xAttrs) + throws IOException; + + /** + * Remove xattr of a file or directory.Value in xAttr parameter is ignored. + * Name must be prefixed with user/trusted/security/system. + * <p/> + * A regular user only can remove xattr of "user" namespace. + * A super user can remove xattr of "user" and "trusted" namespace. + * XAttr of "security" and "system" namespace is only used/exposed + * internally to the FS impl. + * <p/> + * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes"> + * http://en.wikipedia.org/wiki/Extended_file_attributes</a> + * @param src file or directory + * @param xAttr <code>XAttr</code> to remove + * @throws IOException + */ + @Idempotent + public void removeXAttr(String src, XAttr xAttr) throws IOException; } Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java?rev=1590974&r1=1590973&r2=1590974&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java (original) +++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java Tue Apr 29 13:08:03 2014 @@ -174,6 +174,12 @@ import org.apache.hadoop.hdfs.protocol.p import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeIDProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfoProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto; +import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.RemoveXAttrRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.RemoveXAttrResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.SetXAttrRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.SetXAttrResponseProto; import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.server.namenode.INodeId; @@ -302,6 +308,12 @@ public class ClientNamenodeProtocolServe private static final RemoveAclResponseProto VOID_REMOVEACL_RESPONSE = RemoveAclResponseProto.getDefaultInstance(); + + private static final SetXAttrResponseProto + VOID_SETXATTR_RESPONSE = SetXAttrResponseProto.getDefaultInstance(); + + private static final RemoveXAttrResponseProto + VOID_REMOVEXATTR_RESPONSE = RemoveXAttrResponseProto.getDefaultInstance(); /** * Constructor @@ -1261,4 +1273,38 @@ public class ClientNamenodeProtocolServe throw new ServiceException(e); } } + + @Override + public SetXAttrResponseProto setXAttr(RpcController controller, + SetXAttrRequestProto req) throws ServiceException { + try { + server.setXAttr(req.getSrc(), PBHelper.convertXAttr(req.getXAttr()), + PBHelper.convert(req.getFlag())); + } catch (IOException e) { + throw new ServiceException(e); + } + return VOID_SETXATTR_RESPONSE; + } + + @Override + public GetXAttrsResponseProto getXAttrs(RpcController controller, + GetXAttrsRequestProto req) throws ServiceException { + try { + return PBHelper.convertXAttrsResponse(server.getXAttrs(req.getSrc(), + PBHelper.convertXAttrs(req.getXAttrsList()))); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override + public RemoveXAttrResponseProto removeXAttr(RpcController controller, + RemoveXAttrRequestProto req) throws ServiceException { + try { + server.removeXAttr(req.getSrc(), PBHelper.convertXAttr(req.getXAttr())); + } catch (IOException e) { + throw new ServiceException(e); + } + return VOID_REMOVEXATTR_RESPONSE; + } } Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java?rev=1590974&r1=1590973&r2=1590974&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java (original) +++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java Tue Apr 29 13:08:03 2014 @@ -35,6 +35,8 @@ import org.apache.hadoop.fs.FsServerDefa import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.UnresolvedLinkException; +import org.apache.hadoop.fs.XAttr; +import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsPermission; @@ -141,6 +143,9 @@ import org.apache.hadoop.hdfs.protocol.p import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetTimesRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdateBlockForPipelineRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdatePipelineRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.RemoveXAttrRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.SetXAttrRequestProto; import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException; @@ -1264,4 +1269,47 @@ public class ClientNamenodeProtocolTrans throw ProtobufHelper.getRemoteException(e); } } + + @Override + public void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag) + throws IOException { + SetXAttrRequestProto req = SetXAttrRequestProto.newBuilder() + .setSrc(src) + .setXAttr(PBHelper.convertXAttrProto(xAttr)) + .setFlag(PBHelper.convert(flag)) + .build(); + try { + rpcProxy.setXAttr(null, req); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + } + + @Override + public List<XAttr> getXAttrs(String src, List<XAttr> xAttrs) + throws IOException { + GetXAttrsRequestProto.Builder builder = GetXAttrsRequestProto.newBuilder(); + builder.setSrc(src); + if (xAttrs != null) { + builder.addAllXAttrs(PBHelper.convertXAttrProto(xAttrs)); + } + GetXAttrsRequestProto req = builder.build(); + try { + return PBHelper.convert(rpcProxy.getXAttrs(null, req)); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + } + + @Override + public void removeXAttr(String src, XAttr xAttr) throws IOException { + RemoveXAttrRequestProto req = RemoveXAttrRequestProto + .newBuilder().setSrc(src) + .setXAttr(PBHelper.convertXAttrProto(xAttr)).build(); + try { + rpcProxy.removeXAttr(null, req); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + } } Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java?rev=1590974&r1=1590973&r2=1590974&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java (original) +++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java Tue Apr 29 13:08:03 2014 @@ -32,6 +32,8 @@ import org.apache.hadoop.fs.ContentSumma import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FsServerDefaults; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.XAttr; +import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclEntryScope; import org.apache.hadoop.fs.permission.AclEntryType; @@ -149,6 +151,10 @@ import org.apache.hadoop.hdfs.protocol.p import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypeProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageUuidsProto; import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.JournalInfoProto; +import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.XAttrProto; +import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.XAttrProto.XAttrNamespaceProto; +import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.XAttrSetFlagProto; import org.apache.hadoop.hdfs.security.token.block.BlockKey; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey; @@ -220,6 +226,8 @@ public class PBHelper { AclEntryType.values(); private static final FsAction[] FSACTION_VALUES = FsAction.values(); + private static final XAttr.NameSpace[] XATTR_NAMESPACE_VALUES = + XAttr.NameSpace.values(); private PBHelper() { /** Hidden constructor */ @@ -2006,6 +2014,14 @@ public class PBHelper { private static AclEntryType convert(AclEntryTypeProto v) { return castEnum(v, ACL_ENTRY_TYPE_VALUES); } + + private static XAttrNamespaceProto convert(XAttr.NameSpace v) { + return XAttrNamespaceProto.valueOf(v.ordinal()); + } + + private static XAttr.NameSpace convert(XAttrNamespaceProto v) { + return castEnum(v, XATTR_NAMESPACE_VALUES); + } private static FsActionProto convert(FsAction v) { return FsActionProto.valueOf(v != null ? v.ordinal() : 0); @@ -2059,6 +2075,108 @@ public class PBHelper { .addAllEntries(convertAclEntryProto(e.getEntries())).build(); return GetAclStatusResponseProto.newBuilder().setResult(r).build(); } + + public static XAttrProto convertXAttrProto(XAttr a) { + XAttrProto.Builder builder = XAttrProto.newBuilder(); + builder.setNamespace(convert(a.getNameSpace())); + if (a.getName() != null) { + builder.setName(a.getName()); + } + if (a.getValue() != null) { + builder.setValue(getByteString(a.getValue())); + } + return builder.build(); + } + + public static List<XAttrProto> convertXAttrProto( + List<XAttr> xAttrSpec) { + ArrayList<XAttrProto> xAttrs = Lists.newArrayListWithCapacity( + xAttrSpec.size()); + for (XAttr a : xAttrSpec) { + XAttrProto.Builder builder = XAttrProto.newBuilder(); + builder.setNamespace(convert(a.getNameSpace())); + if (a.getName() != null) { + builder.setName(a.getName()); + } + if (a.getValue() != null) { + builder.setValue(getByteString(a.getValue())); + } + xAttrs.add(builder.build()); + } + return xAttrs; + } + + /** + * The flag field in PB is a bitmask whose values are the same a the + * emum values of XAttrSetFlag + */ + public static int convert(EnumSet<XAttrSetFlag> flag) { + int value = 0; + if (flag.contains(XAttrSetFlag.CREATE)) { + value |= XAttrSetFlagProto.XATTR_CREATE.getNumber(); + } + if (flag.contains(XAttrSetFlag.REPLACE)) { + value |= XAttrSetFlagProto.XATTR_REPLACE.getNumber(); + } + return value; + } + + public static EnumSet<XAttrSetFlag> convert(int flag) { + EnumSet<XAttrSetFlag> result = + EnumSet.noneOf(XAttrSetFlag.class); + if ((flag & XAttrSetFlagProto.XATTR_CREATE_VALUE) == + XAttrSetFlagProto.XATTR_CREATE_VALUE) { + result.add(XAttrSetFlag.CREATE); + } + if ((flag & XAttrSetFlagProto.XATTR_REPLACE_VALUE) == + XAttrSetFlagProto.XATTR_REPLACE_VALUE) { + result.add(XAttrSetFlag.REPLACE); + } + return result; + } + + public static XAttr convertXAttr(XAttrProto a) { + XAttr.Builder builder = new XAttr.Builder(); + builder.setNameSpace(convert(a.getNamespace())); + if (a.hasName()) { + builder.setName(a.getName()); + } + if (a.hasValue()) { + builder.setValue(a.getValue().toByteArray()); + } + return builder.build(); + } + + public static List<XAttr> convertXAttrs(List<XAttrProto> xAttrSpec) { + ArrayList<XAttr> xAttrs = Lists.newArrayListWithCapacity(xAttrSpec.size()); + for (XAttrProto a : xAttrSpec) { + XAttr.Builder builder = new XAttr.Builder(); + builder.setNameSpace(convert(a.getNamespace())); + if (a.hasName()) { + builder.setName(a.getName()); + } + if (a.hasValue()) { + builder.setValue(a.getValue().toByteArray()); + } + xAttrs.add(builder.build()); + } + return xAttrs; + } + + public static List<XAttr> convert(GetXAttrsResponseProto a) { + List<XAttrProto> xAttrs = a.getXAttrsList(); + return convertXAttrs(xAttrs); + } + + public static GetXAttrsResponseProto convertXAttrsResponse( + List<XAttr> xAttrs) { + GetXAttrsResponseProto.Builder builder = GetXAttrsResponseProto + .newBuilder(); + if (xAttrs != null) { + builder.addAllXAttrs(convertXAttrProto(xAttrs)); + } + return builder.build(); + } public static ShortCircuitShmSlotProto convert(SlotId slotId) { return ShortCircuitShmSlotProto.newBuilder(). Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java?rev=1590974&r1=1590973&r2=1590974&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java (original) +++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java Tue Apr 29 13:08:03 2014 @@ -49,6 +49,8 @@ import org.apache.hadoop.fs.Options; import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnresolvedLinkException; +import org.apache.hadoop.fs.XAttr; +import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsPermission; @@ -1378,5 +1380,23 @@ class NameNodeRpcServer implements Namen public AclStatus getAclStatus(String src) throws IOException { return namesystem.getAclStatus(src); } + + @Override + public void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag) + throws IOException { + //TODO: will be implemented as part of HDFS-6258 + } + + @Override + public List<XAttr> getXAttrs(String src, List<XAttr> xAttrs) + throws IOException { + //TODO: will be implemented as part of HDFS-6258 + return null; + } + + @Override + public void removeXAttr(String src, XAttr xAttr) throws IOException { + //TODO: will be implemented as part of HDFS-6258 + } } Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto?rev=1590974&r1=1590973&r2=1590974&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto (original) +++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto Tue Apr 29 13:08:03 2014 @@ -31,6 +31,7 @@ package hadoop.hdfs; import "Security.proto"; import "hdfs.proto"; import "acl.proto"; +import "xattr.proto"; /** * The ClientNamenodeProtocol Service defines the interface between a client @@ -755,4 +756,10 @@ service ClientNamenodeProtocol { returns(SetAclResponseProto); rpc getAclStatus(GetAclStatusRequestProto) returns(GetAclStatusResponseProto); + rpc setXAttr(SetXAttrRequestProto) + returns(SetXAttrResponseProto); + rpc getXAttrs(GetXAttrsRequestProto) + returns(GetXAttrsResponseProto); + rpc removeXAttr(RemoveXAttrRequestProto) + returns(RemoveXAttrResponseProto); } Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto?rev=1590974&r1=1590973&r2=1590974&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto (original) +++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto Tue Apr 29 13:08:03 2014 @@ -23,6 +23,7 @@ package hadoop.hdfs.fsimage; import "hdfs.proto"; import "acl.proto"; +import "xattr.proto"; /** * This file defines the on-disk layout of the file system image. The @@ -106,7 +107,23 @@ message INodeSection { */ repeated fixed32 entries = 2 [packed = true]; } - + + message XAttrCompactProto { + /** + * + * [0:2) -- the namespace of XAttr (XAttrNamespaceProto) + * [2:26) -- the name of the entry, which is an ID that points to a + * string in the StringTableSection. + * [26:32) -- reserved for future uses. + */ + required fixed32 name = 1; + optional bytes value = 2; + } + + message XAttrFeatureProto { + repeated XAttrCompactProto xAttrs = 1; + } + message INodeFile { optional uint32 replication = 1; optional uint64 modificationTime = 2; @@ -116,6 +133,7 @@ message INodeSection { repeated BlockProto blocks = 6; optional FileUnderConstructionFeature fileUC = 7; optional AclFeatureProto acl = 8; + optional XAttrFeatureProto xAttrs = 9; } message INodeDirectory { @@ -126,6 +144,7 @@ message INodeSection { optional uint64 dsQuota = 3; optional fixed64 permission = 4; optional AclFeatureProto acl = 5; + optional XAttrFeatureProto xAttrs = 6; } message INodeSymlink { Added: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/xattr.proto URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/xattr.proto?rev=1590974&view=auto ============================================================================== --- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/xattr.proto (added) +++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/xattr.proto Tue Apr 29 13:08:03 2014 @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +option java_package = "org.apache.hadoop.hdfs.protocol.proto"; +option java_outer_classname = "XAttrProtos"; +option java_generate_equals_and_hash = true; +package hadoop.hdfs; + +message XAttrProto { + enum XAttrNamespaceProto { + USER = 0; + TRUSTED = 1; + SECURITY = 2; + SYSTEM = 3; + } + + required XAttrNamespaceProto namespace = 1; + required string name = 2; + optional bytes value = 3; +} + +message XAttrEditLogProto { + required string src = 1; + repeated XAttrProto xAttrs = 2; +} + +enum XAttrSetFlagProto { + XATTR_CREATE = 0x01; + XATTR_REPLACE = 0x02; +} + +message SetXAttrRequestProto { + required string src = 1; + optional XAttrProto xAttr = 2; + optional uint32 flag = 3; //bits set using XAttrSetFlagProto +} + +message SetXAttrResponseProto { +} + +message GetXAttrsRequestProto { + required string src = 1; + repeated XAttrProto xAttrs = 2; +} + +message GetXAttrsResponseProto { + repeated XAttrProto xAttrs = 1; +} + +message RemoveXAttrRequestProto { + required string src = 1; + optional XAttrProto xAttr = 2; +} + +message RemoveXAttrResponseProto { +}