Repository: hive
Updated Branches:
  refs/heads/branch-2.1 b424fd097 -> 0646cc2d0


HIVE-14098: Logging task properties, and environment variables might contain 
passwords (Peter Vary, reviewed by Sergio Pena)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0646cc2d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0646cc2d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0646cc2d

Branch: refs/heads/branch-2.1
Commit: 0646cc2d051b75bd3a042d1ea6d9f0292c505321
Parents: b424fd0
Author: Peter Vary <pv...@cloudera.com>
Authored: Wed Sep 21 15:22:41 2016 -0500
Committer: Sergio Pena <sergio.p...@cloudera.com>
Committed: Wed Sep 21 15:22:41 2016 -0500

----------------------------------------------------------------------
 .../apache/hadoop/hive/ql/exec/Utilities.java   | 23 ++++++++++++++++++++
 .../hadoop/hive/ql/exec/mr/MapredLocalTask.java |  2 +-
 .../ql/exec/spark/HiveSparkClientFactory.java   | 11 +++++-----
 .../hadoop/hive/ql/exec/TestUtilities.java      | 12 ++++++++++
 4 files changed, 42 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/0646cc2d/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 8f7bbb2..202adf3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -225,6 +225,13 @@ public final class Utilities {
   public static String REDUCENAME = "Reducer ";
 
   /**
+   * Constants for log masking
+   */
+  private static String KEY_TO_MASK_WITH = "password";
+  private static String MASKED_VALUE = "###_MASKED_###";
+
+
+  /**
    * ReduceField:
    * KEY: record key
    * VALUE: record value
@@ -3697,4 +3704,20 @@ public final class Utilities {
     }
     return result;
   }
+
+  /**
+   * Returns MASKED_VALUE if the key contains KEY_TO_MASK_WITH or the original 
property otherwise.
+   * Used to mask environment variables, and properties in logs which contain 
passwords
+   * @param key The property key to check
+   * @param value The original value of the property
+   * @return The masked property value
+   */
+  public static String maskIfPassword(String key, String value) {
+    if (key!=null && value!=null) {
+      if (key.toLowerCase().indexOf(KEY_TO_MASK_WITH) != -1) {
+        return MASKED_VALUE;
+      }
+    }
+    return value;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/0646cc2d/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
index f4d3d88..d4b17d7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
@@ -309,7 +309,7 @@ public class MapredLocalTask extends Task<MapredLocalWork> 
implements Serializab
         String name = entry.getKey();
         String value = entry.getValue();
         env[pos++] = name + "=" + value;
-        LOG.debug("Setting env: " + env[pos-1]);
+        LOG.debug("Setting env: " + name + "=" + 
Utilities.maskIfPassword(name, value));
       }
 
       LOG.info("Executing: " + cmdLine);

http://git-wip-us.apache.org/repos/asf/hive/blob/0646cc2d/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
----------------------------------------------------------------------
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
index b36c60e..ed87adb 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
@@ -28,6 +28,7 @@ import java.util.Set;
 
 import org.apache.commons.compress.utils.CharsetNames;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -98,7 +99,7 @@ public class HiveSparkClientFactory {
             sparkConf.put(propertyName, properties.getProperty(propertyName));
             LOG.info(String.format(
               "load spark property from %s (%s -> %s).",
-              SPARK_DEFAULT_CONF_FILE, propertyName, value));
+              SPARK_DEFAULT_CONF_FILE, propertyName, 
Utilities.maskIfPassword(propertyName,value)));
           }
         }
       }
@@ -135,7 +136,7 @@ public class HiveSparkClientFactory {
         sparkConf.put(propertyName, value);
         LOG.info(String.format(
           "load spark property from hive configuration (%s -> %s).",
-          propertyName, value));
+          propertyName, Utilities.maskIfPassword(propertyName,value)));
       } else if (propertyName.startsWith("yarn") &&
         (sparkMaster.equals("yarn-client") || 
sparkMaster.equals("yarn-cluster"))) {
         String value = hiveConf.get(propertyName);
@@ -145,7 +146,7 @@ public class HiveSparkClientFactory {
         sparkConf.put("spark.hadoop." + propertyName, value);
         LOG.info(String.format(
           "load yarn property from hive configuration in %s mode (%s -> %s).",
-          sparkMaster, propertyName, value));
+          sparkMaster, propertyName, 
Utilities.maskIfPassword(propertyName,value)));
       } else if 
(propertyName.equals(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY)) {
         String value = hiveConf.get(propertyName);
         if (value != null && !value.isEmpty()) {
@@ -158,7 +159,7 @@ public class HiveSparkClientFactory {
         String value = hiveConf.get(propertyName);
         sparkConf.put("spark.hadoop." + propertyName, value);
         LOG.info(String.format(
-          "load HBase configuration (%s -> %s).", propertyName, value));
+          "load HBase configuration (%s -> %s).", propertyName, 
Utilities.maskIfPassword(propertyName,value)));
       }
 
       if (RpcConfiguration.HIVE_SPARK_RSC_CONFIGS.contains(propertyName)) {
@@ -166,7 +167,7 @@ public class HiveSparkClientFactory {
         sparkConf.put(propertyName, value);
         LOG.info(String.format(
           "load RPC property from hive configuration (%s -> %s).",
-          propertyName, value));
+          propertyName, Utilities.maskIfPassword(propertyName,value)));
       }
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/0646cc2d/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java 
b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
index d2060a1..b095608 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
@@ -246,4 +246,16 @@ public class TestUtilities {
     FileSystem.getLocal(hconf).create(taskOutputPath).close();
     return tempDirPath;
   }
+
+  @Test
+  public void testMaskIfPassword() {
+    Assert.assertNull(Utilities.maskIfPassword("",null));
+    Assert.assertNull(Utilities.maskIfPassword(null,null));
+    Assert.assertEquals("test",Utilities.maskIfPassword(null,"test"));
+    Assert.assertEquals("test2",Utilities.maskIfPassword("any","test2"));
+    
Assert.assertEquals("###_MASKED_###",Utilities.maskIfPassword("password","test3"));
+    
Assert.assertEquals("###_MASKED_###",Utilities.maskIfPassword("a_passWord","test4"));
+    
Assert.assertEquals("###_MASKED_###",Utilities.maskIfPassword("password_a","test5"));
+    
Assert.assertEquals("###_MASKED_###",Utilities.maskIfPassword("a_PassWord_a","test6"));
+  }
 }

Reply via email to