Repository: hive Updated Branches: refs/heads/branch-2.1 80156443d -> 6bf2790d5
Revert "HIVE-14426: Extensive logging on info level in WebHCat (Peter Vary via Chaoyu Tang)" This reverts commit 80156443db185ca85db0928e609f6cbe2f6ceb3f. Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6bf2790d Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6bf2790d Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6bf2790d Branch: refs/heads/branch-2.1 Commit: 6bf2790d5a64c448b554df85d9c184bfd8c32008 Parents: 8015644 Author: ctang <[email protected]> Authored: Wed Sep 28 11:15:50 2016 -0400 Committer: ctang <[email protected]> Committed: Wed Sep 28 11:15:50 2016 -0400 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/common/LogUtils.java | 22 ----------- .../org/apache/hadoop/hive/conf/HiveConf.java | 20 ++++++++-- .../apache/hadoop/hive/conf/HiveConfUtil.java | 39 +------------------- .../apache/hadoop/hive/common/TestLogUtils.java | 34 ----------------- .../hive/hcatalog/templeton/AppConfig.java | 15 +------- .../hcatalog/templeton/tool/TempletonUtils.java | 4 +- .../apache/hadoop/hive/ql/exec/Utilities.java | 7 ++++ .../hadoop/hive/ql/exec/mr/MapredLocalTask.java | 3 +- .../ql/exec/spark/HiveSparkClientFactory.java | 11 +++--- 9 files changed, 34 insertions(+), 121 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/6bf2790d/common/src/java/org/apache/hadoop/hive/common/LogUtils.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/common/LogUtils.java b/common/src/java/org/apache/hadoop/hive/common/LogUtils.java index c2a0d9a..599e798 100644 --- a/common/src/java/org/apache/hadoop/hive/common/LogUtils.java +++ b/common/src/java/org/apache/hadoop/hive/common/LogUtils.java @@ -42,12 +42,6 @@ public class LogUtils { private static final String HIVE_EXEC_L4J = "hive-exec-log4j2.properties"; private static final Logger l4j = LoggerFactory.getLogger(LogUtils.class); - /** - * Constants for log masking - */ - private static String KEY_TO_MASK_WITH = "password"; - private static String MASKED_VALUE = "###_MASKED_###"; - @SuppressWarnings("serial") public static class LogInitializationException extends Exception { public LogInitializationException(String msg) { @@ -177,20 +171,4 @@ public class LogUtils { + conf.getHiveSiteLocation().getPath()); } } - - /** - * Returns MASKED_VALUE if the key contains KEY_TO_MASK_WITH or the original property otherwise. - * Used to mask environment variables, and properties in logs which contain passwords - * @param key The property key to check - * @param value The original value of the property - * @return The masked property value - */ - public static String maskIfPassword(String key, String value) { - if (key!=null && value!=null) { - if (key.toLowerCase().indexOf(KEY_TO_MASK_WITH) != -1) { - return MASKED_VALUE; - } - } - return value; - } } http://git-wip-us.apache.org/repos/asf/hive/blob/6bf2790d/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 5a29684..178afbc 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -3777,8 +3777,8 @@ public class HiveConf extends Configuration { // setup list of conf vars that are not allowed to change runtime setupRestrictList(); - hiddenSet.clear(); - hiddenSet.addAll(HiveConfUtil.getHiddenSet(this)); + setupHiddenSet(); + } /** @@ -4130,11 +4130,25 @@ public class HiveConf extends Configuration { restrictList.add(ConfVars.HIVE_CONF_INTERNAL_VARIABLE_LIST.varname); } + private void setupHiddenSet() { + String hiddenListStr = this.getVar(ConfVars.HIVE_CONF_HIDDEN_LIST); + hiddenSet.clear(); + if (hiddenListStr != null) { + for (String entry : hiddenListStr.split(",")) { + hiddenSet.add(entry.trim()); + } + } + } + /** * Strips hidden config entries from configuration */ public void stripHiddenConfigurations(Configuration conf) { - HiveConfUtil.stripConfigurations(conf, hiddenSet); + for (String name : hiddenSet) { + if (conf.get(name) != null) { + conf.set(name, ""); + } + } } /** http://git-wip-us.apache.org/repos/asf/hive/blob/6bf2790d/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java index 16c2eaf..073a978 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java @@ -25,11 +25,9 @@ import java.io.File; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.StringTokenizer; /** @@ -61,42 +59,7 @@ public class HiveConfUtil { dumpConfig(conf, sb); return sb.append("END========\"new HiveConf()\"========\n"); } - - /** - * Getting the set of the hidden configurations - * @param configuration The original configuration - * @return The list of the configuration values to hide - */ - public static Set<String> getHiddenSet(Configuration configuration) { - Set<String> hiddenSet = new HashSet<String>(); - String hiddenListStr = HiveConf.getVar(configuration, HiveConf.ConfVars.HIVE_CONF_HIDDEN_LIST); - if (hiddenListStr != null) { - for (String entry : hiddenListStr.split(",")) { - hiddenSet.add(entry.trim()); - } - } - return hiddenSet; - } - - /** - * Strips hidden config entries from configuration - * @param conf The configuration to strip from - * @param hiddenSet The values to strip - */ - public static void stripConfigurations(Configuration conf, Set<String> hiddenSet) { - for (String name : hiddenSet) { - if (conf.get(name) != null) { - conf.set(name, ""); - } - } - } - - public static void dumpConfig(Configuration originalConf, StringBuilder sb) { - Set<String> hiddenSet = getHiddenSet(originalConf); - sb.append("Values omitted for security reason if present: ").append(hiddenSet).append("\n"); - Configuration conf = new Configuration(originalConf); - stripConfigurations(conf, hiddenSet); - + public static void dumpConfig(Configuration conf, StringBuilder sb) { Iterator<Map.Entry<String, String>> configIter = conf.iterator(); List<Map.Entry<String, String>> configVals = new ArrayList<>(); while(configIter.hasNext()) { http://git-wip-us.apache.org/repos/asf/hive/blob/6bf2790d/common/src/test/org/apache/hadoop/hive/common/TestLogUtils.java ---------------------------------------------------------------------- diff --git a/common/src/test/org/apache/hadoop/hive/common/TestLogUtils.java b/common/src/test/org/apache/hadoop/hive/common/TestLogUtils.java deleted file mode 100644 index 923ac2d..0000000 --- a/common/src/test/org/apache/hadoop/hive/common/TestLogUtils.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.apache.hadoop.hive.common; -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import org.junit.Assert; -import org.junit.Test; - -public class TestLogUtils { - @Test - public void testMaskIfPassword() { - Assert.assertNull(LogUtils.maskIfPassword("",null)); - Assert.assertNull(LogUtils.maskIfPassword(null,null)); - Assert.assertEquals("test", LogUtils.maskIfPassword(null,"test")); - Assert.assertEquals("test2", LogUtils.maskIfPassword("any","test2")); - Assert.assertEquals("###_MASKED_###", LogUtils.maskIfPassword("password","test3")); - Assert.assertEquals("###_MASKED_###", LogUtils.maskIfPassword("a_passWord","test4")); - Assert.assertEquals("###_MASKED_###", LogUtils.maskIfPassword("password_a","test5")); - Assert.assertEquals("###_MASKED_###", LogUtils.maskIfPassword("a_PassWord_a","test6")); - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/6bf2790d/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java ---------------------------------------------------------------------- diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java index 54d0907..dd1208b 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java @@ -200,19 +200,14 @@ public class AppConfig extends Configuration { * been installed. We need pass some properties to that client to make sure it connects to the * right Metastore, configures Tez, etc. Here we look for such properties in hive config, * and set a comma-separated list of key values in {@link #HIVE_PROPS_NAME}. - * The HIVE_CONF_HIDDEN_LIST should be handled separately too - this also should be copied from - * the hive config to the webhcat config if not defined there. * Note that the user may choose to set the same keys in HIVE_PROPS_NAME directly, in which case * those values should take precedence. */ private void handleHiveProperties() { HiveConf hiveConf = new HiveConf();//load hive-site.xml from classpath List<String> interestingPropNames = Arrays.asList( - HiveConf.ConfVars.METASTOREURIS.varname, - HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL.varname, - HiveConf.ConfVars.METASTORE_EXECUTE_SET_UGI.varname, - HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname, - HiveConf.ConfVars.HIVE_CONF_HIDDEN_LIST.varname); + "hive.metastore.uris","hive.metastore.sasl.enabled", + "hive.metastore.execute.setugi","hive.execution.engine"); //each items is a "key=value" format List<String> webhcatHiveProps = new ArrayList<String>(hiveProps()); @@ -237,12 +232,6 @@ public class AppConfig extends Configuration { hiveProps.append(hiveProps.length() > 0 ? "," : "").append(StringUtils.escapeString(whProp)); } set(HIVE_PROPS_NAME, hiveProps.toString()); - // Setting the hidden list - String hiddenProperties = hiveConf.get(HiveConf.ConfVars.HIVE_CONF_HIDDEN_LIST.varname); - if (this.get(HiveConf.ConfVars.HIVE_CONF_HIDDEN_LIST.varname) == null - && hiddenProperties!=null) { - set(HiveConf.ConfVars.HIVE_CONF_HIDDEN_LIST.varname, hiddenProperties); - } } private static void logConfigLoadAttempt(String path) { http://git-wip-us.apache.org/repos/asf/hive/blob/6bf2790d/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java ---------------------------------------------------------------------- diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java index f629a78..83584d3 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java @@ -42,7 +42,6 @@ import java.util.regex.Pattern; import javax.ws.rs.core.UriBuilder; -import org.apache.hadoop.hive.common.LogUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -484,8 +483,7 @@ public class TempletonUtils { } } else { - sb.append(propKey).append('=').append(LogUtils.maskIfPassword(propKey, map.get(propKey))); - sb.append('\n'); + sb.append(propKey).append('=').append(map.get(propKey)).append('\n'); } } return sb.append("END").append(header).append('\n'); http://git-wip-us.apache.org/repos/asf/hive/blob/6bf2790d/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 3b6fdeb..202adf3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -225,6 +225,13 @@ public final class Utilities { public static String REDUCENAME = "Reducer "; /** + * Constants for log masking + */ + private static String KEY_TO_MASK_WITH = "password"; + private static String MASKED_VALUE = "###_MASKED_###"; + + + /** * ReduceField: * KEY: record key * VALUE: record value http://git-wip-us.apache.org/repos/asf/hive/blob/6bf2790d/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java index 127d3e6..d4b17d7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java @@ -41,7 +41,6 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.LogUtils; import org.apache.hadoop.hive.common.io.CachingPrintStream; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; @@ -310,7 +309,7 @@ public class MapredLocalTask extends Task<MapredLocalWork> implements Serializab String name = entry.getKey(); String value = entry.getValue(); env[pos++] = name + "=" + value; - LOG.debug("Setting env: " + name + "=" + LogUtils.maskIfPassword(name, value)); + LOG.debug("Setting env: " + name + "=" + Utilities.maskIfPassword(name, value)); } LOG.info("Executing: " + cmdLine); http://git-wip-us.apache.org/repos/asf/hive/blob/6bf2790d/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java index 8867415..ed87adb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java @@ -28,7 +28,6 @@ import java.util.Set; import org.apache.commons.compress.utils.CharsetNames; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.hive.common.LogUtils; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.session.SessionState; import org.slf4j.Logger; @@ -100,7 +99,7 @@ public class HiveSparkClientFactory { sparkConf.put(propertyName, properties.getProperty(propertyName)); LOG.info(String.format( "load spark property from %s (%s -> %s).", - SPARK_DEFAULT_CONF_FILE, propertyName, LogUtils.maskIfPassword(propertyName,value))); + SPARK_DEFAULT_CONF_FILE, propertyName, Utilities.maskIfPassword(propertyName,value))); } } } @@ -137,7 +136,7 @@ public class HiveSparkClientFactory { sparkConf.put(propertyName, value); LOG.info(String.format( "load spark property from hive configuration (%s -> %s).", - propertyName, LogUtils.maskIfPassword(propertyName,value))); + propertyName, Utilities.maskIfPassword(propertyName,value))); } else if (propertyName.startsWith("yarn") && (sparkMaster.equals("yarn-client") || sparkMaster.equals("yarn-cluster"))) { String value = hiveConf.get(propertyName); @@ -147,7 +146,7 @@ public class HiveSparkClientFactory { sparkConf.put("spark.hadoop." + propertyName, value); LOG.info(String.format( "load yarn property from hive configuration in %s mode (%s -> %s).", - sparkMaster, propertyName, LogUtils.maskIfPassword(propertyName,value))); + sparkMaster, propertyName, Utilities.maskIfPassword(propertyName,value))); } else if (propertyName.equals(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY)) { String value = hiveConf.get(propertyName); if (value != null && !value.isEmpty()) { @@ -160,7 +159,7 @@ public class HiveSparkClientFactory { String value = hiveConf.get(propertyName); sparkConf.put("spark.hadoop." + propertyName, value); LOG.info(String.format( - "load HBase configuration (%s -> %s).", propertyName, LogUtils.maskIfPassword(propertyName,value))); + "load HBase configuration (%s -> %s).", propertyName, Utilities.maskIfPassword(propertyName,value))); } if (RpcConfiguration.HIVE_SPARK_RSC_CONFIGS.contains(propertyName)) { @@ -168,7 +167,7 @@ public class HiveSparkClientFactory { sparkConf.put(propertyName, value); LOG.info(String.format( "load RPC property from hive configuration (%s -> %s).", - propertyName, LogUtils.maskIfPassword(propertyName,value))); + propertyName, Utilities.maskIfPassword(propertyName,value))); } }
