Author: namit
Date: Wed Feb 9 21:01:46 2011
New Revision: 1069093
URL: http://svn.apache.org/viewvc?rev=1069093&view=rev
Log:
HIVE-1948 Add audit logging in the metastore
(Devaraj Das via namit)
Modified:
hive/trunk/CHANGES.txt
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
Modified: hive/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hive/trunk/CHANGES.txt?rev=1069093&r1=1069092&r2=1069093&view=diff
==============================================================================
--- hive/trunk/CHANGES.txt (original)
+++ hive/trunk/CHANGES.txt Wed Feb 9 21:01:46 2011
@@ -184,6 +184,8 @@ Trunk - Unreleased
HIVE-1971 Verbose/echo mode for the Hive CLI
(Jonathan Natkins via Ning Zhang)
+ HIVE-1948 Add audit logging in the metastore
+ (Devaraj Das via namit)
IMPROVEMENTS
Modified:
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL:
http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1069093&r1=1069092&r2=1069093&view=diff
==============================================================================
---
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
(original)
+++
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
Wed Feb 9 21:01:46 2011
@@ -24,7 +24,10 @@ import static org.apache.hadoop.hive.met
import static org.apache.hadoop.hive.metastore.MetaStoreUtils.validateName;
import java.io.IOException;
+import java.net.InetAddress;
+import java.net.Socket;
import java.util.ArrayList;
+import java.util.Formatter;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
@@ -77,15 +80,18 @@ import org.apache.hadoop.hive.serde2.Ser
import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.thrift.TException;
import org.apache.thrift.TProcessor;
import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.protocol.TProtocol;
import org.apache.thrift.server.TServer;
import org.apache.thrift.server.TThreadPoolServer;
import org.apache.thrift.transport.TServerSocket;
import org.apache.thrift.transport.TServerTransport;
+import org.apache.thrift.transport.TSocket;
import org.apache.thrift.transport.TTransportFactory;
import com.facebook.fb303.FacebookBase;
@@ -127,6 +133,38 @@ public class HiveMetaStore extends Thrif
}
};
+ public static final String AUDIT_FORMAT =
+ "ugi=%s\t" + // ugi
+ "ip=%s\t" + // remote IP
+ "cmd=%s\t"; // command
+ public static final Log auditLog = LogFactory.getLog(
+ HiveMetaStore.class.getName() + ".audit");
+ private static final ThreadLocal<Formatter> auditFormatter =
+ new ThreadLocal<Formatter>() {
+ @Override
+ protected Formatter initialValue() {
+ return new Formatter(new StringBuilder(AUDIT_FORMAT.length() * 4));
+ }
+ };
+
+ private final void logAuditEvent(String cmd) {
+ if (!ShimLoader.getHadoopShims().isSecureShimImpl() || cmd == null) {
+ return;
+ }
+
+ UserGroupInformation ugi;
+ try {
+ ugi = ShimLoader.getHadoopShims().getUGIForConf(getConf());
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ InetAddress addr = TLoggingProcessor.getRemoteAddress();
+ final Formatter fmt = auditFormatter.get();
+ ((StringBuilder)fmt.out()).setLength(0);
+ auditLog.info(fmt.format(AUDIT_FORMAT, ugi.getUserName(),
+ addr == null ? "unknown-ip-addr" : addr.toString(), cmd).toString());
+ }
+
// The next serial number to be assigned
private boolean checkForDefaultDb;
private static int nextSerialNum = 0;
@@ -439,6 +477,7 @@ public class HiveMetaStore extends Thrif
private void logInfo(String m) {
LOG.info(threadLocalId.get().toString() + ": " + m);
+ logAuditEvent(m);
}
public String startFunction(String function, String extraLogInfo) {
@@ -3067,7 +3106,8 @@ public class HiveMetaStore extends Thrif
TServerTransport serverTransport = tcpKeepAlive ?
new TServerSocketKeepAlive(port) : new TServerSocket(port);
- TProcessor processor = new ThriftHiveMetastore.Processor(handler);
+ TProcessor processor =
+ new TLoggingProcessor(new ThriftHiveMetastore.Processor(handler));
TTransportFactory transFactory;
if (useSasl) {
saslServer = bridge.createServer(
@@ -3102,4 +3142,29 @@ public class HiveMetaStore extends Thrif
throw x;
}
}
+ //Assists audit logger - gets the remote client's IP address.
+ private static class TLoggingProcessor implements TProcessor {
+ private final static ThreadLocal<InetAddress> remoteAddress =
+ new ThreadLocal<InetAddress>() {
+ @Override
+ protected synchronized InetAddress initialValue() {
+ return null;
+ }
+ };
+ TProcessor wrapped;
+ TLoggingProcessor(TProcessor wrapped) {
+ this.wrapped = wrapped;
+ }
+ static InetAddress getRemoteAddress() {
+ return remoteAddress.get();
+ }
+ public boolean process(final TProtocol inProt, final TProtocol outProt)
+ throws TException {
+ if (TSocket.class.isAssignableFrom(inProt.getTransport().getClass())) {
+ Socket socket = ((TSocket)inProt.getTransport()).getSocket();
+ remoteAddress.set(socket.getInetAddress());
+ }
+ return wrapped.process(inProt, outProt);
+ }
+ }
}
Modified:
hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
URL:
http://svn.apache.org/viewvc/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1069093&r1=1069092&r2=1069093&view=diff
==============================================================================
---
hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
(original)
+++
hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
Wed Feb 9 21:01:46 2011
@@ -449,6 +449,11 @@ public class Hadoop20Shims implements Ha
}
return ugi;
}
+
+ @Override
+ public boolean isSecureShimImpl() {
+ return false;
+ }
@Override
public String getTokenStrForm(String tokenSignature) throws IOException {
Modified:
hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
URL:
http://svn.apache.org/viewvc/hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1069093&r1=1069092&r2=1069093&view=diff
==============================================================================
---
hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
(original)
+++
hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
Wed Feb 9 21:01:46 2011
@@ -446,6 +446,11 @@ public class Hadoop20SShims implements H
public UserGroupInformation getUGIForConf(Configuration conf) throws
IOException {
return UserGroupInformation.getCurrentUser();
}
+
+ @Override
+ public boolean isSecureShimImpl() {
+ return true;
+ }
@Override
public String getTokenStrForm(String tokenSignature) throws IOException {
Modified:
hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL:
http://svn.apache.org/viewvc/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1069093&r1=1069092&r2=1069093&view=diff
==============================================================================
---
hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
(original)
+++
hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
Wed Feb 9 21:01:46 2011
@@ -158,6 +158,10 @@ public interface HadoopShims {
*/
public UserGroupInformation getUGIForConf(Configuration conf) throws
LoginException, IOException;
+ /**
+ * Return true if the Shim is based on Hadoop Security APIs.
+ */
+ public boolean isSecureShimImpl();
/**
* Get the string form of the token given a token signature.
