HIVE-16297: Improving hive logging configuration variables (Vihang Karajgaonkar, reviewed by Peter Vary & Aihua Xu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4e60ea3f Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4e60ea3f Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4e60ea3f Branch: refs/heads/master Commit: 4e60ea3f786ed597b0594c2452dd7d64c44645b9 Parents: c7a44eb Author: Aihua Xu <aihu...@apache.org> Authored: Tue Apr 4 19:14:28 2017 -0400 Committer: Aihua Xu <aihu...@apache.org> Committed: Tue Apr 4 19:28:46 2017 -0400 ---------------------------------------------------------------------- .../java/org/apache/hadoop/hive/conf/HiveConfUtil.java | 13 ++++++++++++- common/src/java/org/apache/hive/http/ConfServlet.java | 10 +++++++--- .../apache/hadoop/hive/ql/exec/FileSinkOperator.java | 10 ++++------ .../hive/ql/exec/spark/RemoteHiveSparkClient.java | 9 +++------ 4 files changed, 26 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/4e60ea3f/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java index 9ba08e5..dc02803 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java @@ -94,11 +94,22 @@ public class HiveConfUtil { public static void stripConfigurations(Configuration conf, Set<String> hiddenSet) { for (String name : hiddenSet) { if (conf.get(name) != null) { - conf.set(name, ""); + conf.set(name, StringUtils.EMPTY); } } } + /** + * Searches the given configuration object and replaces all the configuration values for keys + * defined hive.conf.hidden.list by empty String + * + * @param conf - Configuration object which needs to be modified to remove sensitive keys + */ + public static void stripConfigurations(Configuration conf) { + Set<String> hiddenSet = getHiddenSet(conf); + stripConfigurations(conf, hiddenSet); + } + public static void dumpConfig(Configuration originalConf, StringBuilder sb) { Set<String> hiddenSet = getHiddenSet(originalConf); sb.append("Values omitted for security reason if present: ").append(hiddenSet).append("\n"); http://git-wip-us.apache.org/repos/asf/hive/blob/4e60ea3f/common/src/java/org/apache/hive/http/ConfServlet.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hive/http/ConfServlet.java b/common/src/java/org/apache/hive/http/ConfServlet.java index 253df4f..856a5d2 100644 --- a/common/src/java/org/apache/hive/http/ConfServlet.java +++ b/common/src/java/org/apache/hive/http/ConfServlet.java @@ -26,6 +26,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConfUtil; /** * A servlet to print out the running configuration data. @@ -81,11 +82,14 @@ public class ConfServlet extends HttpServlet { * Guts of the servlet - extracted for easy testing. */ static void writeResponse(Configuration conf, Writer out, String format) - throws IOException, BadFormatException { + throws IOException, BadFormatException { + //redact the sensitive information from the configuration values + Configuration hconf = new Configuration(conf); + HiveConfUtil.stripConfigurations(hconf); if (FORMAT_JSON.equals(format)) { - Configuration.dumpConfiguration(conf, out); + Configuration.dumpConfiguration(hconf, out); } else if (FORMAT_XML.equals(format)) { - conf.writeXml(out); + hconf.writeXml(out); } else { throw new BadFormatException("Bad format: " + format); } http://git-wip-us.apache.org/repos/asf/hive/blob/4e60ea3f/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java index a9d03d0..4d727ba 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java @@ -26,6 +26,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfUtil; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.ErrorMsg; @@ -463,13 +464,10 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements } private void logOutputFormatError(Configuration hconf, HiveException ex) { - StringWriter errorWriter = new StringWriter(); + StringBuilder errorWriter = new StringBuilder(); errorWriter.append("Failed to create output format; configuration: "); - try { - Configuration.dumpConfiguration(hconf, errorWriter); - } catch (IOException ex2) { - errorWriter.append("{ failed to dump configuration: " + ex2.getMessage() + " }"); - } + // redact sensitive information before logging + HiveConfUtil.dumpConfig(hconf, errorWriter); Properties tdp = null; if (this.conf.getTableInfo() != null && (tdp = this.conf.getTableInfo().getProperties()) != null) { http://git-wip-us.apache.org/repos/asf/hive/blob/4e60ea3f/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java index 4c69899..4ca8f93 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java @@ -356,12 +356,9 @@ public class RemoteHiveSparkClient implements HiveSparkClient { private void logConfigurations(JobConf localJobConf) { if (LOG.isInfoEnabled()) { LOG.info("Logging job configuration: "); - StringWriter outWriter = new StringWriter(); - try { - Configuration.dumpConfiguration(localJobConf, outWriter); - } catch (IOException e) { - LOG.warn("Error logging job configuration", e); - } + StringBuilder outWriter = new StringBuilder(); + // redact sensitive information before logging + HiveConfUtil.dumpConfig(localJobConf, outWriter); LOG.info(outWriter.toString()); } }
