Author: szetszwo
Date: Fri Aug 1 01:29:49 2014
New Revision: 1615020
URL: http://svn.apache.org/r1615020
Log:
Merge r1609845 through r1615019 from trunk.
Added:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java
- copied unchanged from r1615019,
hadoop/common/trunk/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/test/resources/log4j-kmsaudit.properties
- copied unchanged from r1615019,
hadoop/common/trunk/hadoop-common-project/hadoop-kms/src/test/resources/log4j-kmsaudit.properties
Modified:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/CHANGES.txt
(contents, props changed)
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/
(props changed)
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
Modified:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/CHANGES.txt
Fri Aug 1 01:29:49 2014
@@ -192,6 +192,9 @@ Trunk (Unreleased)
HADOOP-10891. Add EncryptedKeyVersion factory method to
KeyProviderCryptoExtension. (wang)
+ HADOOP-10756. KMS audit log should consolidate successful similar
requests.
+ (asuresh via tucu)
+
BUG FIXES
HADOOP-9451. Fault single-layer config if node group topology is enabled.
@@ -460,6 +463,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10882. Move DirectBufferPool into common util. (todd)
+ HADOOP-8069. Enable TCP_NODELAY by default for IPC. (Todd Lipcon via
+ Arpit Agarwal)
+
OPTIMIZATIONS
BUG FIXES
@@ -810,6 +816,8 @@ Release 2.5.0 - UNRELEASED
HADOOP-10894. Fix dead link in ToolRunner documentation. (Akira Ajisaka
via Arpit Agarwal)
+ HADOOP-10910. Increase findbugs maxHeap size. (wang)
+
BREAKDOWN OF HADOOP-10514 SUBTASKS AND RELATED JIRAS
HADOOP-10520. Extended attributes definition and FileSystem APIs for
Propchange:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1614232-1615019
Propchange:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1614232-1615019
Modified:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
Fri Aug 1 01:29:49 2014
@@ -1844,6 +1844,38 @@ public class Configuration implements It
}
/**
+ * Get the socket address for <code>hostProperty</code> as a
+ * <code>InetSocketAddress</code>. If <code>hostProperty</code> is
+ * <code>null</code>, <code>addressProperty</code> will be used. This
+ * is useful for cases where we want to differentiate between host
+ * bind address and address clients should use to establish connection.
+ *
+ * @param hostProperty bind host property name.
+ * @param addressProperty address property name.
+ * @param defaultAddressValue the default value
+ * @param defaultPort the default port
+ * @return InetSocketAddress
+ */
+ public InetSocketAddress getSocketAddr(
+ String hostProperty,
+ String addressProperty,
+ String defaultAddressValue,
+ int defaultPort) {
+
+ InetSocketAddress bindAddr = getSocketAddr(
+ addressProperty, defaultAddressValue, defaultPort);
+
+ final String host = get(hostProperty);
+
+ if (host == null || host.isEmpty()) {
+ return bindAddr;
+ }
+
+ return NetUtils.createSocketAddr(
+ host, bindAddr.getPort(), hostProperty);
+ }
+
+ /**
* Get the socket address for <code>name</code> property as a
* <code>InetSocketAddress</code>.
* @param name property name.
@@ -1864,6 +1896,40 @@ public class Configuration implements It
public void setSocketAddr(String name, InetSocketAddress addr) {
set(name, NetUtils.getHostPortString(addr));
}
+
+ /**
+ * Set the socket address a client can use to connect for the
+ * <code>name</code> property as a <code>host:port</code>. The wildcard
+ * address is replaced with the local host's address. If the host and address
+ * properties are configured the host component of the address will be
combined
+ * with the port component of the addr to generate the address. This is to
allow
+ * optional control over which host name is used in multi-home bind-host
+ * cases where a host can have multiple names
+ * @param hostProperty the bind-host configuration name
+ * @param addressProperty the service address configuration name
+ * @param defaultAddressValue the service default address configuration value
+ * @param addr InetSocketAddress of the service listener
+ * @return InetSocketAddress for clients to connect
+ */
+ public InetSocketAddress updateConnectAddr(
+ String hostProperty,
+ String addressProperty,
+ String defaultAddressValue,
+ InetSocketAddress addr) {
+
+ final String host = get(hostProperty);
+ final String connectHostPort = getTrimmed(addressProperty,
defaultAddressValue);
+
+ if (host == null || host.isEmpty() || connectHostPort == null ||
connectHostPort.isEmpty()) {
+ //not our case, fall back to original logic
+ return updateConnectAddr(addressProperty, addr);
+ }
+
+ final String connectHost = connectHostPort.split(":")[0];
+ // Create connect address using client address hostname and server port.
+ return updateConnectAddr(addressProperty, NetUtils.createSocketAddrForHost(
+ connectHost, addr.getPort()));
+ }
/**
* Set the socket address a client can use to connect for the
Modified:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
Fri Aug 1 01:29:49 2014
@@ -43,6 +43,7 @@ import org.apache.hadoop.fs.Options.Crea
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.security.AccessControlException;
@@ -805,6 +806,18 @@ public abstract class AbstractFileSystem
/**
* The specification of this method matches that of
+ * {@link FileContext#access(Path, FsAction)}
+ * except that an UnresolvedLinkException may be thrown if a symlink is
+ * encountered in the path.
+ */
+ @InterfaceAudience.LimitedPrivate({"HDFS", "Hive"})
+ public void access(Path path, FsAction mode) throws AccessControlException,
+ FileNotFoundException, UnresolvedLinkException, IOException {
+ FileSystem.checkAccessPermissions(this.getFileStatus(path), mode);
+ }
+
+ /**
+ * The specification of this method matches that of
* {@link FileContext#getFileLinkStatus(Path)}
* except that an UnresolvedLinkException may be thrown if a symlink is
* encountered in the path leading up to the final path component.
@@ -1040,21 +1053,10 @@ public abstract class AbstractFileSystem
/**
* Set an xattr of a file or directory.
- * The name must be prefixed with user/trusted/security/system and
- * followed by ".". For example, "user.attr".
- * <p/>
- * A regular user can only set an xattr for the "user" namespace.
- * The super user can set an xattr of either the "user" or "trusted"
namespaces.
- * The xattrs of the "security" and "system" namespaces are only
used/exposed
- * internally by/to the FS impl.
- * <p/>
- * The access permissions of an xattr in the "user" namespace are
- * defined by the file and directory permission bits.
- * An xattr can only be set when the logged-in user has the correct
permissions.
- * If the xattr exists, it will be replaced.
+ * The name must be prefixed with the namespace followed by ".". For example,
+ * "user.attr".
* <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes";>
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to modify
* @param name xattr name.
@@ -1069,21 +1071,10 @@ public abstract class AbstractFileSystem
/**
* Set an xattr of a file or directory.
- * The name must be prefixed with user/trusted/security/system and
- * followed by ".". For example, "user.attr".
+ * The name must be prefixed with the namespace followed by ".". For example,
+ * "user.attr".
* <p/>
- * A regular user can only set an xattr for the "user" namespace.
- * The super user can set an xattr of either the "user" or "trusted"
namespaces.
- * The xattrs of the "security" and "system" namespaces are only
used/exposed
- * internally by/to the FS impl.
- * <p/>
- * The access permissions of an xattr in the "user" namespace are
- * defined by the file and directory permission bits.
- * An xattr can only be set when the logged-in user has the correct
permissions.
- * If the xattr exists, it will be replaced.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes";>
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to modify
* @param name xattr name.
@@ -1099,18 +1090,10 @@ public abstract class AbstractFileSystem
/**
* Get an xattr for a file or directory.
- * The name must be prefixed with user/trusted/security/system and
- * followed by ".". For example, "user.attr".
- * <p/>
- * A regular user can only get an xattr for the "user" namespace.
- * The super user can get an xattr of either the "user" or "trusted"
namespaces.
- * The xattrs of the "security" and "system" namespaces are only
used/exposed
- * internally by/to the FS impl.
+ * The name must be prefixed with the namespace followed by ".". For example,
+ * "user.attr".
* <p/>
- * An xattr will only be returned when the logged-in user has the correct
permissions.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes";>
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attribute
* @param name xattr name.
@@ -1127,13 +1110,7 @@ public abstract class AbstractFileSystem
* Only those xattrs for which the logged-in user has permissions to view
* are returned.
* <p/>
- * A regular user can only get xattrs for the "user" namespace.
- * The super user can only get xattrs for "user" and "trusted" namespaces.
- * The xattr of "security" and "system" namespaces are only used/exposed
- * internally by/to the FS impl.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes";>
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
* @return Map<String, byte[]> describing the XAttrs of the file or directory
@@ -1149,13 +1126,7 @@ public abstract class AbstractFileSystem
* Only those xattrs for which the logged-in user has permissions to view
* are returned.
* <p/>
- * A regular user can only get xattrs for the "user" namespace.
- * The super user can only get xattrs for "user" and "trusted" namespaces.
- * The xattr of "security" and "system" namespaces are only used/exposed
- * internally by/to the FS impl.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes";>
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
* @param names XAttr names.
@@ -1173,14 +1144,7 @@ public abstract class AbstractFileSystem
* Only the xattr names for which the logged-in user has permissions to view
* are returned.
* <p/>
- * A regular user can only get xattr names for the "user" namespace.
- * The super user can only get xattr names for the "user" and "trusted"
- * namespaces.
- * The xattr names in the "security" and "system" namespaces are only
- * used/exposed internally by/to the FS impl.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes";>
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
* @return Map<String, byte[]> describing the XAttrs of the file or directory
@@ -1194,21 +1158,10 @@ public abstract class AbstractFileSystem
/**
* Remove an xattr of a file or directory.
- * The name must be prefixed with user/trusted/security/system and
- * followed by ".". For example, "user.attr".
- * <p/>
- * A regular user can only remove an xattr for the "user" namespace.
- * The super user can remove an xattr of either the "user" or "trusted"
namespaces.
- * The xattrs of the "security" and "system" namespaces are only
used/exposed
- * internally by/to the FS impl.
- * <p/>
- * The access permissions of an xattr in the "user" namespace are
- * defined by the file and directory permission bits.
- * An xattr can only be set when the logged-in user has the correct
permissions.
- * If the xattr exists, it will be replaced.
+ * The name must be prefixed with the namespace followed by ".". For example,
+ * "user.attr".
* <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes";>
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to remove extended attribute
* @param name xattr name
Modified:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
Fri Aug 1 01:29:49 2014
@@ -207,7 +207,7 @@ public class CommonConfigurationKeysPubl
public static final String IPC_CLIENT_TCPNODELAY_KEY =
"ipc.client.tcpnodelay";
/** Defalt value for IPC_CLIENT_TCPNODELAY_KEY */
- public static final boolean IPC_CLIENT_TCPNODELAY_DEFAULT = false;
+ public static final boolean IPC_CLIENT_TCPNODELAY_DEFAULT = true;
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
public static final String IPC_SERVER_LISTEN_QUEUE_SIZE_KEY =
"ipc.server.listen.queue.size";
@@ -226,7 +226,7 @@ public class CommonConfigurationKeysPubl
public static final String IPC_SERVER_TCPNODELAY_KEY =
"ipc.server.tcpnodelay";
/** Default value for IPC_SERVER_TCPNODELAY_KEY */
- public static final boolean IPC_SERVER_TCPNODELAY_DEFAULT = false;
+ public static final boolean IPC_SERVER_TCPNODELAY_DEFAULT = true;
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
public static final String HADOOP_RPC_SOCKET_FACTORY_CLASS_DEFAULT_KEY =
Modified:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
Fri Aug 1 01:29:49 2014
@@ -44,6 +44,7 @@ import org.apache.hadoop.fs.FileSystem.S
import org.apache.hadoop.fs.Options.CreateOpts;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT;
@@ -1109,6 +1110,55 @@ public final class FileContext {
}
/**
+ * Checks if the user can access a path. The mode specifies which access
+ * checks to perform. If the requested permissions are granted, then the
+ * method returns normally. If access is denied, then the method throws an
+ * {@link AccessControlException}.
+ * <p/>
+ * The default implementation of this method calls {@link
#getFileStatus(Path)}
+ * and checks the returned permissions against the requested permissions.
+ * Note that the getFileStatus call will be subject to authorization checks.
+ * Typically, this requires search (execute) permissions on each directory in
+ * the path's prefix, but this is implementation-defined. Any file system
+ * that provides a richer authorization model (such as ACLs) may override the
+ * default implementation so that it checks against that model instead.
+ * <p>
+ * In general, applications should avoid using this method, due to the risk
of
+ * time-of-check/time-of-use race conditions. The permissions on a file may
+ * change immediately after the access call returns. Most applications
should
+ * prefer running specific file system actions as the desired user
represented
+ * by a {@link UserGroupInformation}.
+ *
+ * @param path Path to check
+ * @param mode type of access to check
+ * @throws AccessControlException if access is denied
+ * @throws FileNotFoundException if the path does not exist
+ * @throws UnsupportedFileSystemException if file system for
<code>path</code>
+ * is not supported
+ * @throws IOException see specific implementation
+ *
+ * Exceptions applicable to file systems accessed over RPC:
+ * @throws RpcClientException If an exception occurred in the RPC client
+ * @throws RpcServerException If an exception occurred in the RPC server
+ * @throws UnexpectedServerException If server implementation throws
+ * undeclared exception to RPC server
+ */
+ @InterfaceAudience.LimitedPrivate({"HDFS", "Hive"})
+ public void access(final Path path, final FsAction mode)
+ throws AccessControlException, FileNotFoundException,
+ UnsupportedFileSystemException, IOException {
+ final Path absPath = fixRelativePart(path);
+ new FSLinkResolver<Void>() {
+ @Override
+ public Void next(AbstractFileSystem fs, Path p) throws IOException,
+ UnresolvedLinkException {
+ fs.access(p, mode);
+ return null;
+ }
+ }.resolve(this, absPath);
+ }
+
+ /**
* Return a file status object that represents the path. If the path
* refers to a symlink then the FileStatus of the symlink is returned.
* The behavior is equivalent to #getFileStatus() if the underlying
@@ -2297,21 +2347,10 @@ public final class FileContext {
/**
* Set an xattr of a file or directory.
- * The name must be prefixed with user/trusted/security/system and
- * followed by ".". For example, "user.attr".
- * <p/>
- * A regular user can only set an xattr for the "user" namespace.
- * The super user can set an xattr of either the "user" or "trusted"
namespaces.
- * The xattrs of the "security" and "system" namespaces are only
used/exposed
- * internally by/to the FS impl.
+ * The name must be prefixed with the namespace followed by ".". For example,
+ * "user.attr".
* <p/>
- * The access permissions of an xattr in the "user" namespace are
- * defined by the file and directory permission bits.
- * An xattr can only be set when the logged-in user has the correct
permissions.
- * If the xattr exists, it will be replaced.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes";>
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to modify
* @param name xattr name.
@@ -2326,21 +2365,10 @@ public final class FileContext {
/**
* Set an xattr of a file or directory.
- * The name must be prefixed with user/trusted/security/system and
- * followed by ".". For example, "user.attr".
- * <p/>
- * A regular user can only set an xattr for the "user" namespace.
- * The super user can set an xattr of either the "user" or "trusted"
namespaces.
- * The xattrs of the "security" and "system" namespaces are only
used/exposed
- * internally by/to the FS impl.
+ * The name must be prefixed with the namespace followed by ".". For example,
+ * "user.attr".
* <p/>
- * The access permissions of an xattr in the "user" namespace are
- * defined by the file and directory permission bits.
- * An xattr can only be set when the logged-in user has the correct
permissions.
- * If the xattr exists, it will be replaced.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes";>
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to modify
* @param name xattr name.
@@ -2363,19 +2391,10 @@ public final class FileContext {
/**
* Get an xattr for a file or directory.
- * The name must be prefixed with user/trusted/security/system and
- * followed by ".". For example, "user.attr".
+ * The name must be prefixed with the namespace followed by ".". For example,
+ * "user.attr".
* <p/>
- *
- * A regular user can only get an xattr for the "user" namespace.
- * The super user can get an xattr of either the "user" or "trusted"
namespaces.
- * The xattrs of the "security" and "system" namespaces are only
used/exposed
- * internally by/to the FS impl.
- * <p/>
- * An xattr will only be returned when the logged-in user has the correct
permissions.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes";>
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attribute
* @param name xattr name.
@@ -2398,13 +2417,7 @@ public final class FileContext {
* Only those xattrs for which the logged-in user has permissions to view
* are returned.
* <p/>
- * A regular user can only get xattrs for the "user" namespace.
- * The super user can only get xattrs for "user" and "trusted" namespaces.
- * The xattr of "security" and "system" namespaces are only used/exposed
- * internally by/to the FS impl.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes";>
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
* @return Map<String, byte[]> describing the XAttrs of the file or directory
@@ -2426,13 +2439,7 @@ public final class FileContext {
* Only those xattrs for which the logged-in user has permissions to view
* are returned.
* <p/>
- * A regular user can only get xattrs for the "user" namespace.
- * The super user can only get xattrs for "user" and "trusted" namespaces.
- * The xattr of "security" and "system" namespaces are only used/exposed
- * internally by/to the FS impl.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes";>
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
* @param names XAttr names.
@@ -2453,21 +2460,10 @@ public final class FileContext {
/**
* Remove an xattr of a file or directory.
- * The name must be prefixed with user/trusted/security/system and
- * followed by ".". For example, "user.attr".
+ * The name must be prefixed with the namespace followed by ".". For example,
+ * "user.attr".
* <p/>
- * A regular user can only remove an xattr for the "user" namespace.
- * The super user can remove an xattr of either the "user" or "trusted"
namespaces.
- * The xattrs of the "security" and "system" namespaces are only
used/exposed
- * internally by/to the FS impl.
- * <p/>
- * The access permissions of an xattr in the "user" namespace are
- * defined by the file and directory permission bits.
- * An xattr can only be set when the logged-in user has the correct
permissions.
- * If the xattr exists, it will be replaced.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes";>
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to remove extended attribute
* @param name xattr name
@@ -2490,14 +2486,7 @@ public final class FileContext {
* Only those xattr names which the logged-in user has permissions to view
* are returned.
* <p/>
- * A regular user can only get xattr names for the "user" namespace.
- * The super user can only get xattr names for "user" and "trusted"
- * namespaces.
- * The xattrs of the "security" and "system" namespaces are only
- * used/exposed internally by/to the FS impl.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes";>
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
* @return List<String> of the XAttr names of the file or directory
Modified:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
Fri Aug 1 01:29:49 2014
@@ -25,6 +25,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
@@ -50,6 +51,7 @@ import org.apache.hadoop.fs.Options.Chec
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.MultipleIOException;
import org.apache.hadoop.io.Text;
@@ -2073,6 +2075,71 @@ public abstract class FileSystem extends
public abstract FileStatus getFileStatus(Path f) throws IOException;
/**
+ * Checks if the user can access a path. The mode specifies which access
+ * checks to perform. If the requested permissions are granted, then the
+ * method returns normally. If access is denied, then the method throws an
+ * {@link AccessControlException}.
+ * <p/>
+ * The default implementation of this method calls {@link
#getFileStatus(Path)}
+ * and checks the returned permissions against the requested permissions.
+ * Note that the getFileStatus call will be subject to authorization checks.
+ * Typically, this requires search (execute) permissions on each directory in
+ * the path's prefix, but this is implementation-defined. Any file system
+ * that provides a richer authorization model (such as ACLs) may override the
+ * default implementation so that it checks against that model instead.
+ * <p>
+ * In general, applications should avoid using this method, due to the risk
of
+ * time-of-check/time-of-use race conditions. The permissions on a file may
+ * change immediately after the access call returns. Most applications
should
+ * prefer running specific file system actions as the desired user
represented
+ * by a {@link UserGroupInformation}.
+ *
+ * @param path Path to check
+ * @param mode type of access to check
+ * @throws AccessControlException if access is denied
+ * @throws FileNotFoundException if the path does not exist
+ * @throws IOException see specific implementation
+ */
+ @InterfaceAudience.LimitedPrivate({"HDFS", "Hive"})
+ public void access(Path path, FsAction mode) throws AccessControlException,
+ FileNotFoundException, IOException {
+ checkAccessPermissions(this.getFileStatus(path), mode);
+ }
+
+ /**
+ * This method provides the default implementation of
+ * {@link #access(Path, FsAction)}.
+ *
+ * @param stat FileStatus to check
+ * @param mode type of access to check
+ * @throws IOException for any error
+ */
+ @InterfaceAudience.Private
+ static void checkAccessPermissions(FileStatus stat, FsAction mode)
+ throws IOException {
+ FsPermission perm = stat.getPermission();
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ String user = ugi.getShortUserName();
+ List<String> groups = Arrays.asList(ugi.getGroupNames());
+ if (user.equals(stat.getOwner())) {
+ if (perm.getUserAction().implies(mode)) {
+ return;
+ }
+ } else if (groups.contains(stat.getGroup())) {
+ if (perm.getGroupAction().implies(mode)) {
+ return;
+ }
+ } else {
+ if (perm.getOtherAction().implies(mode)) {
+ return;
+ }
+ }
+ throw new AccessControlException(String.format(
+ "Permission denied: user=%s, path=\"%s\":%s:%s:%s%s", user,
stat.getPath(),
+ stat.getOwner(), stat.getGroup(), stat.isDirectory() ? "d" : "-", perm));
+ }
+
+ /**
* See {@link FileContext#fixRelativePart}
*/
protected Path fixRelativePart(Path p) {
@@ -2364,21 +2431,10 @@ public abstract class FileSystem extends
/**
* Set an xattr of a file or directory.
- * The name must be prefixed with user/trusted/security/system and
- * followed by ".". For example, "user.attr".
- * <p/>
- * A regular user can only set an xattr for the "user" namespace.
- * The super user can set an xattr of either the "user" or "trusted"
namespaces.
- * The xattrs of the "security" and "system" namespaces are only
used/exposed
- * internally by/to the FS impl.
+ * The name must be prefixed with the namespace followed by ".". For example,
+ * "user.attr".
* <p/>
- * The access permissions of an xattr in the "user" namespace are
- * defined by the file and directory permission bits.
- * An xattr can only be set when the logged-in user has the correct
permissions.
- * If the xattr exists, it will be replaced.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes";>
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to modify
* @param name xattr name.
@@ -2393,21 +2449,10 @@ public abstract class FileSystem extends
/**
* Set an xattr of a file or directory.
- * The name must be prefixed with user/trusted/security/system and
- * followed by ".". For example, "user.attr".
- * <p/>
- * A regular user can only set an xattr for the "user" namespace.
- * The super user can set an xattr of either the "user" or "trusted"
namespaces.
- * The xattrs of the "security" and "system" namespaces are only
used/exposed
- * internally by/to the FS impl.
+ * The name must be prefixed with the namespace followed by ".". For example,
+ * "user.attr".
* <p/>
- * The access permissions of an xattr in the "user" namespace are
- * defined by the file and directory permission bits.
- * An xattr can only be set if the logged-in user has the correct
permissions.
- * If the xattr exists, it is replaced.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes";>
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to modify
* @param name xattr name.
@@ -2423,20 +2468,10 @@ public abstract class FileSystem extends
/**
* Get an xattr name and value for a file or directory.
- * The name must be prefixed with user/trusted/security/system and
- * followed by ".". For example, "user.attr".
- * <p/>
- *
- * A regular user can only get an xattr for the "user" namespace.
- * The super user can get an xattr of either the "user" or "trusted"
namespaces.
- * The xattrs of the "security" and "system" namespaces are only
used/exposed
- * internally by/to the FS impl.
+ * The name must be prefixed with the namespace followed by ".". For example,
+ * "user.attr".
* <p/>
- * An xattr will only be returned if the logged-in user has the
- * correct permissions.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes";>
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attribute
* @param name xattr name.
@@ -2453,13 +2488,7 @@ public abstract class FileSystem extends
* Only those xattrs which the logged-in user has permissions to view
* are returned.
* <p/>
- * A regular user can only get xattrs for the "user" namespace.
- * The super user can only get xattrs for "user" and "trusted" namespaces.
- * The xattrs of the "security" and "system" namespaces are only used/exposed
- * internally by/to the FS impl.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes";>
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
* @return Map<String, byte[]> describing the XAttrs of the file or directory
@@ -2475,13 +2504,7 @@ public abstract class FileSystem extends
* Only those xattrs which the logged-in user has permissions to view
* are returned.
* <p/>
- * A regular user can only get xattrs for the "user" namespace.
- * The super user can only get xattrs for "user" and "trusted" namespaces.
- * The xattrs of the "security" and "system" namespaces are only used/exposed
- * internally by/to the FS impl.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes";>
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
* @param names XAttr names.
@@ -2499,14 +2522,7 @@ public abstract class FileSystem extends
* Only those xattr names which the logged-in user has permissions to view
* are returned.
* <p/>
- * A regular user can only get xattr names for the "user" namespace.
- * The super user can only get xattr names for "user" and "trusted"
- * namespaces.
- * The xattrs of the "security" and "system" namespaces are only
- * used/exposed internally by/to the FS impl.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes";>
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
* @return List<String> of the XAttr names of the file or directory
@@ -2519,21 +2535,10 @@ public abstract class FileSystem extends
/**
* Remove an xattr of a file or directory.
- * The name must be prefixed with user/trusted/security/system and
- * followed by ".". For example, "user.attr".
- * <p/>
- * A regular user can only remove an xattr for the "user" namespace.
- * The super user can remove an xattr of either the "user" or "trusted"
namespaces.
- * The xattrs of the "security" and "system" namespaces are only
used/exposed
- * internally by/to the FS impl.
- * <p/>
- * The access permissions of an xattr in the "user" namespace are
- * defined by the file and directory permission bits.
- * An xattr can only be set when the logged-in user has the correct
permissions.
- * If the xattr exists, it will be replaced.
+ * The name must be prefixed with the namespace followed by ".". For example,
+ * "user.attr".
* <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes";>
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to remove extended attribute
* @param name xattr name
Modified:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
Fri Aug 1 01:29:49 2014
@@ -30,6 +30,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.security.AccessControlException;
@@ -397,6 +398,12 @@ public class FilterFileSystem extends Fi
return fs.getFileStatus(f);
}
+ @Override
+ public void access(Path path, FsAction mode) throws AccessControlException,
+ FileNotFoundException, IOException {
+ fs.access(path, mode);
+ }
+
public void createSymlink(final Path target, final Path link,
final boolean createParent) throws AccessControlException,
FileAlreadyExistsException, FileNotFoundException,
Modified:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
Fri Aug 1 01:29:49 2014
@@ -29,6 +29,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.fs.FileSystem.Statistics;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.security.AccessControlException;
@@ -120,6 +121,13 @@ public abstract class FilterFs extends A
}
@Override
+ public void access(Path path, FsAction mode) throws AccessControlException,
+ FileNotFoundException, UnresolvedLinkException, IOException {
+ checkPath(path);
+ myFs.access(path, mode);
+ }
+
+ @Override
public FileStatus getFileLinkStatus(final Path f)
throws IOException, UnresolvedLinkException {
checkPath(f);
Modified:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
Fri Aug 1 01:29:49 2014
@@ -41,7 +41,9 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.util.Progressable;
/**
@@ -223,6 +225,12 @@ class ChRootedFileSystem extends FilterF
}
@Override
+ public void access(Path path, FsAction mode) throws AccessControlException,
+ FileNotFoundException, IOException {
+ super.access(fullPath(path), mode);
+ }
+
+ @Override
public FsStatus getStatus(Path p) throws IOException {
return super.getStatus(fullPath(p));
}
Modified:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
Fri Aug 1 01:29:49 2014
@@ -41,7 +41,9 @@ import org.apache.hadoop.fs.UnresolvedLi
import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Progressable;
@@ -200,6 +202,11 @@ class ChRootedFs extends AbstractFileSys
return myFs.getFileStatus(fullPath(f));
}
+ public void access(Path path, FsAction mode) throws AccessControlException,
+ FileNotFoundException, UnresolvedLinkException, IOException {
+ myFs.access(fullPath(path), mode);
+ }
+
@Override
public FileStatus getFileLinkStatus(final Path f)
throws IOException, UnresolvedLinkException {
Modified:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
Fri Aug 1 01:29:49 2014
@@ -51,6 +51,7 @@ import org.apache.hadoop.fs.XAttrSetFlag
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.AclUtil;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.viewfs.InodeTree.INode;
import org.apache.hadoop.fs.viewfs.InodeTree.INodeLink;
@@ -359,7 +360,14 @@ public class ViewFileSystem extends File
return new ViewFsFileStatus(status, this.makeQualified(f));
}
-
+ @Override
+ public void access(Path path, FsAction mode) throws AccessControlException,
+ FileNotFoundException, IOException {
+ InodeTree.ResolveResult<FileSystem> res =
+ fsState.resolve(getUriPath(path), true);
+ res.targetFileSystem.access(res.remainingPath, mode);
+ }
+
@Override
public FileStatus[] listStatus(final Path f) throws AccessControlException,
FileNotFoundException, IOException {
Modified:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
Fri Aug 1 01:29:49 2014
@@ -54,6 +54,7 @@ import org.apache.hadoop.fs.local.LocalC
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclUtil;
import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.viewfs.InodeTree.INode;
import org.apache.hadoop.fs.viewfs.InodeTree.INodeLink;
@@ -353,6 +354,14 @@ public class ViewFs extends AbstractFile
}
@Override
+ public void access(Path path, FsAction mode) throws AccessControlException,
+ FileNotFoundException, UnresolvedLinkException, IOException {
+ InodeTree.ResolveResult<AbstractFileSystem> res =
+ fsState.resolve(getUriPath(path), true);
+ res.targetFileSystem.access(res.remainingPath, mode);
+ }
+
+ @Override
public FileStatus getFileLinkStatus(final Path f)
throws AccessControlException, FileNotFoundException,
UnsupportedFileSystemException, IOException {
Modified:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
Fri Aug 1 01:29:49 2014
@@ -27,6 +27,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
@@ -377,6 +378,19 @@ public class StringUtils {
return str.trim().split("\\s*,\\s*");
}
+ /**
+ * Trims all the strings in a Collection<String> and returns a Set<String>.
+ * @param strings
+ * @return
+ */
+ public static Set<String> getTrimmedStrings(Collection<String> strings) {
+ Set<String> trimmedStrings = new HashSet<String>();
+ for (String string: strings) {
+ trimmedStrings.add(string.trim());
+ }
+ return trimmedStrings;
+ }
+
final public static String[] emptyStringArray = {};
final public static char COMMA = ',';
final public static String COMMA_STR = ",";
Modified:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
Fri Aug 1 01:29:49 2014
@@ -807,25 +807,6 @@ for ldap providers in the same way as ab
</description>
</property>
-<property>
- <name>ipc.server.tcpnodelay</name>
- <value>false</value>
- <description>Turn on/off Nagle's algorithm for the TCP socket connection on
- the server. Setting to true disables the algorithm and may decrease latency
- with a cost of more/smaller packets.
- </description>
-</property>
-
-<property>
- <name>ipc.client.tcpnodelay</name>
- <value>false</value>
- <description>Turn on/off Nagle's algorithm for the TCP socket connection on
- the client. Setting to true disables the algorithm and may decrease latency
- with a cost of more/smaller packets.
- </description>
-</property>
-
-
<!-- Proxy Configuration -->
<property>
Modified:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
Fri Aug 1 01:29:49 2014
@@ -23,6 +23,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
@@ -201,6 +202,8 @@ public class TestHarFileSystem {
public void removeXAttr(Path path, String name) throws IOException;
public AclStatus getAclStatus(Path path) throws IOException;
+
+ public void access(Path path, FsAction mode) throws IOException;
}
@Test
Modified:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
Fri Aug 1 01:29:49 2014
@@ -20,6 +20,7 @@ package org.apache.hadoop.crypto.key.kms
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import
org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
import org.apache.hadoop.crypto.key.kms.KMSRESTConstants;
@@ -27,7 +28,6 @@ import org.apache.hadoop.security.Access
import
org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
-import org.apache.hadoop.util.StringUtils;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
@@ -59,22 +59,25 @@ import java.util.Map;
@Path(KMSRESTConstants.SERVICE_VERSION)
@InterfaceAudience.Private
public class KMS {
- private static final String CREATE_KEY = "CREATE_KEY";
- private static final String DELETE_KEY = "DELETE_KEY";
- private static final String ROLL_NEW_VERSION = "ROLL_NEW_VERSION";
- private static final String GET_KEYS = "GET_KEYS";
- private static final String GET_KEYS_METADATA = "GET_KEYS_METADATA";
- private static final String GET_KEY_VERSION = "GET_KEY_VERSION";
- private static final String GET_CURRENT_KEY = "GET_CURRENT_KEY";
- private static final String GET_KEY_VERSIONS = "GET_KEY_VERSIONS";
- private static final String GET_METADATA = "GET_METADATA";
- private static final String GENERATE_EEK = "GENERATE_EEK";
- private static final String DECRYPT_EEK = "DECRYPT_EEK";
-
+ public static final String CREATE_KEY = "CREATE_KEY";
+ public static final String DELETE_KEY = "DELETE_KEY";
+ public static final String ROLL_NEW_VERSION = "ROLL_NEW_VERSION";
+ public static final String GET_KEYS = "GET_KEYS";
+ public static final String GET_KEYS_METADATA = "GET_KEYS_METADATA";
+ public static final String GET_KEY_VERSIONS = "GET_KEY_VERSIONS";
+ public static final String GET_METADATA = "GET_METADATA";
+
+ public static final String GET_KEY_VERSION = "GET_KEY_VERSION";
+ public static final String GET_CURRENT_KEY = "GET_CURRENT_KEY";
+ public static final String GENERATE_EEK = "GENERATE_EEK";
+ public static final String DECRYPT_EEK = "DECRYPT_EEK";
+
private KeyProviderCryptoExtension provider;
+ private KMSAudit kmsAudit;
public KMS() throws Exception {
provider = KMSWebApp.getKeyProvider();
+ kmsAudit= KMSWebApp.getKMSAudit();
}
private static Principal getPrincipal(SecurityContext securityContext)
@@ -86,13 +89,26 @@ public class KMS {
return user;
}
- private static void assertAccess(KMSACLs.Type aclType, Principal principal,
+
+ private static final String UNAUTHORIZED_MSG_WITH_KEY =
+ "User:{0} not allowed to do ''{1}'' on ''{2}''";
+
+ private static final String UNAUTHORIZED_MSG_WITHOUT_KEY =
+ "User:{0} not allowed to do ''{1}''";
+
+ private void assertAccess(KMSACLs.Type aclType, Principal principal,
+ String operation) throws AccessControlException {
+ assertAccess(aclType, principal, operation, null);
+ }
+
+ private void assertAccess(KMSACLs.Type aclType, Principal principal,
String operation, String key) throws AccessControlException {
if (!KMSWebApp.getACLs().hasAccess(aclType, principal.getName())) {
KMSWebApp.getUnauthorizedCallsMeter().mark();
- KMSAudit.unauthorized(principal, operation, key);
+ kmsAudit.unauthorized(principal, operation, key);
throw new AuthorizationException(MessageFormat.format(
- "User:{0} not allowed to do ''{1}'' on ''{2}''",
+ (key != null) ? UNAUTHORIZED_MSG_WITH_KEY
+ : UNAUTHORIZED_MSG_WITHOUT_KEY,
principal.getName(), operation, key));
}
}
@@ -149,7 +165,7 @@ public class KMS {
provider.flush();
- KMSAudit.ok(user, CREATE_KEY, name, "UserProvidedMaterial:" +
+ kmsAudit.ok(user, CREATE_KEY, name, "UserProvidedMaterial:" +
(material != null) + " Description:" + description);
if (!KMSWebApp.getACLs().hasAccess(KMSACLs.Type.GET, user.getName())) {
@@ -175,7 +191,7 @@ public class KMS {
provider.deleteKey(name);
provider.flush();
- KMSAudit.ok(user, DELETE_KEY, name, "");
+ kmsAudit.ok(user, DELETE_KEY, name, "");
return Response.ok().build();
}
@@ -203,7 +219,7 @@ public class KMS {
provider.flush();
- KMSAudit.ok(user, ROLL_NEW_VERSION, name, "UserProvidedMaterial:" +
+ kmsAudit.ok(user, ROLL_NEW_VERSION, name, "UserProvidedMaterial:" +
(material != null) + " NewVersion:" + keyVersion.getVersionName());
if (!KMSWebApp.getACLs().hasAccess(KMSACLs.Type.GET, user.getName())) {
@@ -222,11 +238,10 @@ public class KMS {
KMSWebApp.getAdminCallsMeter().mark();
Principal user = getPrincipal(securityContext);
String[] keyNames = keyNamesList.toArray(new String[keyNamesList.size()]);
- String names = StringUtils.arrayToString(keyNames);
- assertAccess(KMSACLs.Type.GET_METADATA, user, GET_KEYS_METADATA, names);
+ assertAccess(KMSACLs.Type.GET_METADATA, user, GET_KEYS_METADATA);
KeyProvider.Metadata[] keysMeta = provider.getKeysMetadata(keyNames);
Object json = KMSServerJSONUtils.toJSON(keyNames, keysMeta);
- KMSAudit.ok(user, GET_KEYS_METADATA, names, "");
+ kmsAudit.ok(user, GET_KEYS_METADATA, "");
return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
}
@@ -237,9 +252,9 @@ public class KMS {
throws Exception {
KMSWebApp.getAdminCallsMeter().mark();
Principal user = getPrincipal(securityContext);
- assertAccess(KMSACLs.Type.GET_KEYS, user, GET_KEYS, "*");
+ assertAccess(KMSACLs.Type.GET_KEYS, user, GET_KEYS);
Object json = provider.getKeys();
- KMSAudit.ok(user, GET_KEYS, "*", "");
+ kmsAudit.ok(user, GET_KEYS, "");
return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
}
@@ -263,7 +278,7 @@ public class KMS {
KMSWebApp.getAdminCallsMeter().mark();
assertAccess(KMSACLs.Type.GET_METADATA, user, GET_METADATA, name);
Object json = KMSServerJSONUtils.toJSON(name, provider.getMetadata(name));
- KMSAudit.ok(user, GET_METADATA, name, "");
+ kmsAudit.ok(user, GET_METADATA, name, "");
return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
}
@@ -279,7 +294,7 @@ public class KMS {
KMSWebApp.getKeyCallsMeter().mark();
assertAccess(KMSACLs.Type.GET, user, GET_CURRENT_KEY, name);
Object json = KMSServerJSONUtils.toJSON(provider.getCurrentKey(name));
- KMSAudit.ok(user, GET_CURRENT_KEY, name, "");
+ kmsAudit.ok(user, GET_CURRENT_KEY, name, "");
return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
}
@@ -292,9 +307,12 @@ public class KMS {
Principal user = getPrincipal(securityContext);
KMSClientProvider.checkNotEmpty(versionName, "versionName");
KMSWebApp.getKeyCallsMeter().mark();
- assertAccess(KMSACLs.Type.GET, user, GET_KEY_VERSION, versionName);
- Object json =
KMSServerJSONUtils.toJSON(provider.getKeyVersion(versionName));
- KMSAudit.ok(user, GET_KEY_VERSION, versionName, "");
+ KeyVersion keyVersion = provider.getKeyVersion(versionName);
+ assertAccess(KMSACLs.Type.GET, user, GET_KEY_VERSION);
+ if (keyVersion != null) {
+ kmsAudit.ok(user, GET_KEY_VERSION, keyVersion.getName(), "");
+ }
+ Object json = KMSServerJSONUtils.toJSON(keyVersion);
return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
}
@@ -327,7 +345,7 @@ public class KMS {
} catch (Exception e) {
throw new IOException(e);
}
- KMSAudit.ok(user, GENERATE_EEK, name, "");
+ kmsAudit.ok(user, GENERATE_EEK, name, "");
retJSON = new ArrayList();
for (EncryptedKeyVersion edek : retEdeks) {
((ArrayList)retJSON).add(KMSServerJSONUtils.toJSON(edek));
@@ -362,7 +380,7 @@ public class KMS {
(String) jsonPayload.get(KMSRESTConstants.MATERIAL_FIELD);
Object retJSON;
if (eekOp.equals(KMSRESTConstants.EEK_DECRYPT)) {
- assertAccess(KMSACLs.Type.DECRYPT_EEK, user, DECRYPT_EEK, versionName);
+ assertAccess(KMSACLs.Type.DECRYPT_EEK, user, DECRYPT_EEK, keyName);
KMSClientProvider.checkNotNull(ivStr, KMSRESTConstants.IV_FIELD);
byte[] iv = Base64.decodeBase64(ivStr);
KMSClientProvider.checkNotNull(encMaterialStr,
@@ -373,7 +391,7 @@ public class KMS {
new KMSClientProvider.KMSEncryptedKeyVersion(keyName,
versionName,
iv, KeyProviderCryptoExtension.EEK, encMaterial));
retJSON = KMSServerJSONUtils.toJSON(retKeyVersion);
- KMSAudit.ok(user, DECRYPT_EEK, versionName, "");
+ kmsAudit.ok(user, DECRYPT_EEK, keyName, "");
} else {
throw new IllegalArgumentException("Wrong " + KMSRESTConstants.EEK_OP +
" value, it must be " + KMSRESTConstants.EEK_GENERATE + " or " +
@@ -396,7 +414,7 @@ public class KMS {
KMSWebApp.getKeyCallsMeter().mark();
assertAccess(KMSACLs.Type.GET, user, GET_KEY_VERSIONS, name);
Object json = KMSServerJSONUtils.toJSON(provider.getKeyVersions(name));
- KMSAudit.ok(user, GET_KEY_VERSIONS, name, "");
+ kmsAudit.ok(user, GET_KEY_VERSIONS, name, "");
return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
}
Modified:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
Fri Aug 1 01:29:49 2014
@@ -20,43 +20,202 @@ package org.apache.hadoop.crypto.key.kms
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Joiner;
+import com.google.common.base.Strings;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
import java.security.Principal;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
/**
* Provides convenience methods for audit logging consistently the different
* types of events.
*/
public class KMSAudit {
+
+ private static class AuditEvent {
+ private final AtomicLong accessCount = new AtomicLong(-1);
+ private final String keyName;
+ private final String user;
+ private final String op;
+ private final String extraMsg;
+ private final long startTime = System.currentTimeMillis();
+
+ private AuditEvent(String keyName, String user, String op, String msg) {
+ this.keyName = keyName;
+ this.user = user;
+ this.op = op;
+ this.extraMsg = msg;
+ }
+
+ public String getExtraMsg() {
+ return extraMsg;
+ }
+
+ public AtomicLong getAccessCount() {
+ return accessCount;
+ }
+
+ public String getKeyName() {
+ return keyName;
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+ public String getOp() {
+ return op;
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+ }
+
+ public static enum OpStatus {
+ OK, UNAUTHORIZED, UNAUTHENTICATED, ERROR;
+ }
+
+ private static Set<String> AGGREGATE_OPS_WHITELIST = Sets.newHashSet(
+ KMS.GET_KEY_VERSION, KMS.GET_CURRENT_KEY, KMS.DECRYPT_EEK, KMS.GENERATE_EEK
+ );
+
+ private Cache<String, AuditEvent> cache;
+
+ private ScheduledExecutorService executor;
+
public static final String KMS_LOGGER_NAME = "kms-audit";
private static Logger AUDIT_LOG = LoggerFactory.getLogger(KMS_LOGGER_NAME);
- private static void op(String status, String op, Principal user, String key,
- String extraMsg) {
- AUDIT_LOG.info("Status:{} User:{} Op:{} Name:{}{}", status, user.getName(),
- op, key, extraMsg);
+ KMSAudit(long delay) {
+ cache = CacheBuilder.newBuilder()
+ .expireAfterWrite(delay, TimeUnit.MILLISECONDS)
+ .removalListener(
+ new RemovalListener<String, AuditEvent>() {
+ @Override
+ public void onRemoval(
+ RemovalNotification<String, AuditEvent> entry) {
+ AuditEvent event = entry.getValue();
+ if (event.getAccessCount().get() > 0) {
+ KMSAudit.this.logEvent(event);
+ event.getAccessCount().set(0);
+ KMSAudit.this.cache.put(entry.getKey(), event);
+ }
+ }
+ }).build();
+ executor = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder()
+ .setDaemon(true).setNameFormat(KMS_LOGGER_NAME + "_thread").build());
+ executor.scheduleAtFixedRate(new Runnable() {
+ @Override
+ public void run() {
+ cache.cleanUp();
+ }
+ }, delay / 10, delay / 10, TimeUnit.MILLISECONDS);
}
- public static void ok(Principal user, String op, String key,
+ private void logEvent(AuditEvent event) {
+ AUDIT_LOG.info(
+ "OK[op={}, key={}, user={}, accessCount={}, interval={}ms] {}",
+ event.getOp(), event.getKeyName(), event.getUser(),
+ event.getAccessCount().get(),
+ (System.currentTimeMillis() - event.getStartTime()),
+ event.getExtraMsg());
+ }
+
+ private void op(OpStatus opStatus, final String op, final String user,
+ final String key, final String extraMsg) {
+ if (!Strings.isNullOrEmpty(user) && !Strings.isNullOrEmpty(key)
+ && !Strings.isNullOrEmpty(op)
+ && AGGREGATE_OPS_WHITELIST.contains(op)) {
+ String cacheKey = createCacheKey(user, key, op);
+ if (opStatus == OpStatus.UNAUTHORIZED) {
+ cache.invalidate(cacheKey);
+ AUDIT_LOG.info("UNAUTHORIZED[op={}, key={}, user={}] {}", op, key,
user,
+ extraMsg);
+ } else {
+ try {
+ AuditEvent event = cache.get(cacheKey, new Callable<AuditEvent>() {
+ @Override
+ public AuditEvent call() throws Exception {
+ return new AuditEvent(key, user, op, extraMsg);
+ }
+ });
+ // Log first access (initialized as -1 so
+ // incrementAndGet() == 0 implies first access)
+ if (event.getAccessCount().incrementAndGet() == 0) {
+ event.getAccessCount().incrementAndGet();
+ logEvent(event);
+ }
+ } catch (ExecutionException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+ } else {
+ List<String> kvs = new LinkedList<String>();
+ if (!Strings.isNullOrEmpty(op)) {
+ kvs.add("op=" + op);
+ }
+ if (!Strings.isNullOrEmpty(key)) {
+ kvs.add("key=" + key);
+ }
+ if (!Strings.isNullOrEmpty(user)) {
+ kvs.add("user=" + user);
+ }
+ if (kvs.size() == 0) {
+ AUDIT_LOG.info("{} {}", opStatus.toString(), extraMsg);
+ } else {
+ String join = Joiner.on(", ").join(kvs);
+ AUDIT_LOG.info("{}[{}] {}", opStatus.toString(), join, extraMsg);
+ }
+ }
+ }
+
+ public void ok(Principal user, String op, String key,
String extraMsg) {
- op("OK", op, user, key, extraMsg);
+ op(OpStatus.OK, op, user.getName(), key, extraMsg);
+ }
+
+ public void ok(Principal user, String op, String extraMsg) {
+ op(OpStatus.OK, op, user.getName(), null, extraMsg);
}
- public static void unauthorized(Principal user, String op, String key) {
- op("UNAUTHORIZED", op, user, key, "");
+ public void unauthorized(Principal user, String op, String key) {
+ op(OpStatus.UNAUTHORIZED, op, user.getName(), key, "");
}
- public static void error(Principal user, String method, String url,
+ public void error(Principal user, String method, String url,
String extraMsg) {
- AUDIT_LOG.info("Status:ERROR User:{} Method:{} URL:{} Exception:'{}'",
- user.getName(), method, url, extraMsg);
+ op(OpStatus.ERROR, null, user.getName(), null, "Method:'" + method
+ + "' Exception:'" + extraMsg + "'");
}
- public static void unauthenticated(String remoteHost, String method,
+ public void unauthenticated(String remoteHost, String method,
String url, String extraMsg) {
- AUDIT_LOG.info(
- "Status:UNAUTHENTICATED RemoteHost:{} Method:{} URL:{} ErrorMsg:'{}'",
- remoteHost, method, url, extraMsg);
+ op(OpStatus.UNAUTHENTICATED, null, null, null, "RemoteHost:"
+ + remoteHost + " Method:" + method
+ + " URL:" + url + " ErrorMsg:'" + extraMsg + "'");
}
+ private static String createCacheKey(String user, String key, String op) {
+ return user + "#" + key + "#" + op;
+ }
+
+ public void shutdown() {
+ executor.shutdownNow();
+ }
}
Modified:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
Fri Aug 1 01:29:49 2014
@@ -115,8 +115,10 @@ public class KMSAuthenticationFilter ext
if (queryString != null) {
requestURL.append("?").append(queryString);
}
- KMSAudit.unauthenticated(request.getRemoteHost(), method,
- requestURL.toString(), kmsResponse.msg);
+
+ KMSWebApp.getKMSAudit().unauthenticated(
+ request.getRemoteHost(), method, requestURL.toString(),
+ kmsResponse.msg);
}
}
Modified:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
Fri Aug 1 01:29:49 2014
@@ -43,12 +43,17 @@ public class KMSConfiguration {
// TImeout for the Current Key cache
public static final String CURR_KEY_CACHE_TIMEOUT_KEY = CONFIG_PREFIX +
"current.key.cache.timeout.ms";
-
+ // Delay for Audit logs that need aggregation
+ public static final String KMS_AUDIT_AGGREGATION_DELAY = CONFIG_PREFIX +
+ "aggregation.delay.ms";
+
public static final boolean KEY_CACHE_ENABLE_DEFAULT = true;
// 10 mins
public static final long KEY_CACHE_TIMEOUT_DEFAULT = 10 * 60 * 1000;
// 30 secs
public static final long CURR_KEY_CACHE_TIMEOUT_DEFAULT = 30 * 1000;
+ // 10 secs
+ public static final long KMS_AUDIT_AGGREGATION_DELAY_DEFAULT = 10000;
static Configuration getConfiguration(boolean loadHadoopDefaults,
String ... resources) {
Modified:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
Fri Aug 1 01:29:49 2014
@@ -20,9 +20,11 @@ package org.apache.hadoop.crypto.key.kms
import org.apache.hadoop.classification.InterfaceAudience;
import com.sun.jersey.api.container.ContainerException;
+
import org.apache.hadoop.crypto.key.kms.KMSRESTConstants;
import org.apache.hadoop.security.AccessControlException;
import
org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authorize.AuthorizationException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,6 +32,7 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.ext.ExceptionMapper;
import javax.ws.rs.ext.Provider;
+
import java.io.IOException;
import java.security.Principal;
import java.util.LinkedHashMap;
@@ -83,6 +86,10 @@ public class KMSExceptionsProvider imple
status = Response.Status.FORBIDDEN;
// we don't audit here because we did it already when checking access
doAudit = false;
+ } else if (throwable instanceof AuthorizationException) {
+ status = Response.Status.UNAUTHORIZED;
+ // we don't audit here because we did it already when checking access
+ doAudit = false;
} else if (throwable instanceof AccessControlException) {
status = Response.Status.FORBIDDEN;
} else if (exception instanceof IOException) {
@@ -95,7 +102,8 @@ public class KMSExceptionsProvider imple
status = Response.Status.INTERNAL_SERVER_ERROR;
}
if (doAudit) {
- KMSAudit.error(KMSMDCFilter.getPrincipal(), KMSMDCFilter.getMethod(),
+ KMSWebApp.getKMSAudit().error(KMSMDCFilter.getPrincipal(),
+ KMSMDCFilter.getMethod(),
KMSMDCFilter.getURL(), getOneLineMessage(exception));
}
return createResponse(status, throwable);
Modified:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
Fri Aug 1 01:29:49 2014
@@ -76,6 +76,7 @@ public class KMSWebApp implements Servle
private static Meter decryptEEKCallsMeter;
private static Meter generateEEKCallsMeter;
private static Meter invalidCallsMeter;
+ private static KMSAudit kmsAudit;
private static KeyProviderCryptoExtension keyProviderCryptoExtension;
static {
@@ -144,6 +145,11 @@ public class KMSWebApp implements Servle
unauthenticatedCallsMeter = metricRegistry.register(
UNAUTHENTICATED_CALLS_METER, new Meter());
+ kmsAudit =
+ new KMSAudit(kmsConf.getLong(
+ KMSConfiguration.KMS_AUDIT_AGGREGATION_DELAY,
+ KMSConfiguration.KMS_AUDIT_AGGREGATION_DELAY_DEFAULT));
+
// this is required for the the JMXJsonServlet to work properly.
// the JMXJsonServlet is behind the authentication filter,
// thus the '*' ACL.
@@ -199,6 +205,7 @@ public class KMSWebApp implements Servle
@Override
public void contextDestroyed(ServletContextEvent sce) {
+ kmsAudit.shutdown();
acls.stopReloader();
jmxReporter.stop();
jmxReporter.close();
@@ -245,4 +252,8 @@ public class KMSWebApp implements Servle
public static KeyProviderCryptoExtension getKeyProvider() {
return keyProviderCryptoExtension;
}
+
+ public static KMSAudit getKMSAudit() {
+ return kmsAudit;
+ }
}
Modified:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm?rev=1615020&r1=1615019&r2=1615020&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
Fri Aug 1 01:29:49 2014
@@ -104,6 +104,25 @@ Hadoop Key Management Server (KMS) - Doc
</property>
+---+
+** KMS Aggregated Audit logs
+
+Audit logs are aggregated for API accesses to the GET_KEY_VERSION,
+GET_CURRENT_KEY, DECRYPT_EEK, GENERATE_EEK operations.
+
+Entries are grouped by the (user,key,operation) combined key for a configurable
+aggregation interval after which the number of accesses to the specified
+end-point by the user for a given key is flushed to the audit log.
+
+The Aggregation interval is configured via the property :
+
++---+
+ <property>
+ <name>hadoop.kms.aggregation.delay.ms</name>
+ <value>10000</value>
+ </property>
++---+
+
+
** Start/Stop the KMS
To start/stop KMS use KMS's bin/kms.sh script. For example:
