Repository: oozie Updated Branches: refs/heads/master 77817e735 -> ccbf692d9
OOZIE-2803 Mask passwords when printing out configs/args in MapReduceMain and SparkMain (pbacsko via rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/ccbf692d Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/ccbf692d Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/ccbf692d Branch: refs/heads/master Commit: ccbf692d90f8e52ab30afcdc7e411209f3d2b94b Parents: 77817e7 Author: Robert Kanter <rkan...@apache.org> Authored: Fri Feb 24 13:18:11 2017 -0800 Committer: Robert Kanter <rkan...@apache.org> Committed: Fri Feb 24 13:18:11 2017 -0800 ---------------------------------------------------------------------- .../org/apache/oozie/util/Instrumentation.java | 1 + .../org/apache/oozie/util/PasswordMasker.java | 121 ---------------- .../apache/oozie/util/TestPasswordMasker.java | 92 ------------ .../test/resources/instrumentation-os-env.json | 47 ------ .../instrumentation-system-properties.json | 88 ------------ release-log.txt | 1 + .../oozie/action/hadoop/MapReduceMain.java | 13 +- .../oozie/action/hadoop/PasswordMasker.java | 144 +++++++++++++++++++ .../oozie/action/hadoop/TestPasswordMasker.java | 140 ++++++++++++++++++ .../test/resources/instrumentation-os-env.json | 47 ++++++ .../instrumentation-system-properties.json | 88 ++++++++++++ .../apache/oozie/action/hadoop/SparkMain.java | 3 +- 12 files changed, 434 insertions(+), 351 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/ccbf692d/core/src/main/java/org/apache/oozie/util/Instrumentation.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/util/Instrumentation.java b/core/src/main/java/org/apache/oozie/util/Instrumentation.java index 55e00d4..45219a9 100644 --- a/core/src/main/java/org/apache/oozie/util/Instrumentation.java +++ b/core/src/main/java/org/apache/oozie/util/Instrumentation.java @@ -20,6 +20,7 @@ package org.apache.oozie.util; import com.google.common.collect.Maps; import org.apache.hadoop.conf.Configuration; +import org.apache.oozie.action.hadoop.PasswordMasker; import org.apache.oozie.service.ConfigurationService; import org.apache.oozie.service.Services; http://git-wip-us.apache.org/repos/asf/oozie/blob/ccbf692d/core/src/main/java/org/apache/oozie/util/PasswordMasker.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/util/PasswordMasker.java b/core/src/main/java/org/apache/oozie/util/PasswordMasker.java deleted file mode 100644 index 1f8a0ab..0000000 --- a/core/src/main/java/org/apache/oozie/util/PasswordMasker.java +++ /dev/null @@ -1,121 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.oozie.util; - -import com.google.common.collect.Maps; - -import javax.annotation.Nullable; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import static com.google.common.base.Preconditions.checkNotNull; - -/** - * A generic password masker that masks {@code Map<String, String>} values given that its keys are considered password keys. - * <p/> - * Tested with {@see System#getProperties()} and {@see System#getenv()}. - */ -class PasswordMasker { - - /** - * The mask that is applied to recognized passwords. - **/ - private static final String PASSWORD_MASK = "*****"; - - /** - * A key is considered a password key, if it contains {{pass}}, case ignored. - **/ - private static final String PASSWORD_KEY = "pass"; - - /** - * Tells us whether an OS environment variable that contains a password fragment. - * <p/> - * E.g. {{-Djavax.net.ssl.trustStorePassword=password}} from {{$CATALINA_OPTS}}. - **/ - private static final String REGEX_CONTAINING_PASSWORD_FRAGMENT_OS_ENV_STYLE = - ".*[((\\s)+-[D|X][\\w[.\\w]*]*(?i)pass[\\w[.\\w]*]*=)([\\w]+)]+.*"; - - /** - * Extracts a password fragment from an OS environment variable. Can be used iteratively to get all fragments. - * <p/> - * E.g. {{-Doozie.https.keystore.pass=password}} and {{-Djavax.net.ssl.trustStorePassword=password}} from {{$CATALINA_OPTS}}. - * {@see java.util.Matcher#find()} - **/ - private static final String REGEX_EXTRACTING_PASSWORD_FRAGMENTS_OS_ENV_STYLE = - "((\\s)+-[D|X][\\w[.\\w]*]*(?i)pass[\\w[.\\w]*]*=)([\\w]+)"; - - private static final Pattern PATTERN_CONTAINING_PASSWORD_FRAGMENTS = Pattern - .compile(REGEX_CONTAINING_PASSWORD_FRAGMENT_OS_ENV_STYLE); - - private static final Pattern PATTERN_EXTRACTING_PASSWORD_FRAGMENTS = Pattern - .compile(REGEX_EXTRACTING_PASSWORD_FRAGMENTS_OS_ENV_STYLE); - - Map<String, String> mask(Map<String, String> unmasked) { - return Maps.transformEntries(unmasked, new Maps.EntryTransformer<String, String, String>() { - @Override - public String transformEntry(@Nullable String key, @Nullable String value) { - checkNotNull(key, "key has to be set"); - checkNotNull(value, "value has to be set"); - - if (isPasswordKey(key)) { - return PASSWORD_MASK; - } - - if (containsPasswordFragment(value)) { - return maskPasswordFragments(value); - } - - return value; - } - }); - } - - private boolean isPasswordKey(String key) { - return key.toLowerCase().contains(PASSWORD_KEY); - - } - - private boolean containsPasswordFragment(String maybePasswordFragments) { - return PATTERN_CONTAINING_PASSWORD_FRAGMENTS - .matcher(maybePasswordFragments) - .matches(); - } - - private String maskPasswordFragments(String maybePasswordFragments) { - StringBuilder maskedBuilder = new StringBuilder(); - Matcher passwordFragmentsMatcher = PATTERN_EXTRACTING_PASSWORD_FRAGMENTS - .matcher(maybePasswordFragments); - - int start = 0, end; - while (passwordFragmentsMatcher.find()) { - end = passwordFragmentsMatcher.start(); - - maskedBuilder.append(maybePasswordFragments.substring(start, end)); - maskedBuilder.append(passwordFragmentsMatcher.group(1)); - maskedBuilder.append(PASSWORD_MASK); - - start = passwordFragmentsMatcher.end(); - } - - maskedBuilder.append(maybePasswordFragments.substring(start)); - - return maskedBuilder.toString(); - } -} http://git-wip-us.apache.org/repos/asf/oozie/blob/ccbf692d/core/src/test/java/org/apache/oozie/util/TestPasswordMasker.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/util/TestPasswordMasker.java b/core/src/test/java/org/apache/oozie/util/TestPasswordMasker.java deleted file mode 100644 index b00cce7..0000000 --- a/core/src/test/java/org/apache/oozie/util/TestPasswordMasker.java +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.oozie.util; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.Test; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class TestPasswordMasker { - - @Test - public void testWhenJavaSystemPropertiesAreAskedPasswordsAppearMasked() throws Exception { - Map<String, String> masked = new PasswordMasker().mask(jsonToMap("/instrumentation-system-properties.json")); - - assertPasswordValueIsMasked(masked, "javax.net.ssl.trustStorePassword"); - assertPasswordValueIsMasked(masked, "oozie.https.keystore.pass"); - } - - @Test - public void testWhenOSEnvIsAskedPasswordsAppearMasked() throws Exception { - Map<String, String> masked = new PasswordMasker().mask(jsonToMap("/instrumentation-os-env.json")); - - assertPasswordValueIsMasked(masked, "HADOOP_CREDSTORE_PASSWORD"); - assertPasswordValueIsMasked(masked, "OOZIE_HTTPS_KEYSTORE_PASSWORD"); - assertPasswordValueIsMasked(masked, "OOZIE_HTTPS_TRUSTSTORE_PASSWORD"); - - assertPasswordValueFragmentIsMasked(masked, "CATALINA_OPTS", "-Doozie.https.keystore.pass="); - assertPasswordValueFragmentIsMasked(masked, "CATALINA_OPTS", "-Djavax.net.ssl.trustStorePassword="); - - assertValueFragmentIsPresent(masked, "CATALINA_OPTS", "-Xmx1024m"); - assertValueFragmentIsPresent(masked, "CATALINA_OPTS", "-Doozie.https.keystore.file=/Users/forsage/.keystore"); - assertValueFragmentIsPresent(masked, "CATALINA_OPTS", "-Djava.library.path="); - } - - @SuppressWarnings("unchecked") - private Map<String, String> jsonToMap(String jsonPath) throws IOException { - return new ObjectMapper().readValue(getClass().getResourceAsStream(jsonPath), HashMap.class); - } - - private void assertPasswordValueIsMasked(Map<String, String> mapContainingMaskedPassword, String passwordKey) { - assertEquals(String.format("Value of key '%s' should be masked.", passwordKey), - "*****", - mapContainingMaskedPassword.get(passwordKey)); - } - - private void assertPasswordValueFragmentIsMasked(Map<String, String> mapContainingMaskedPassword, String passwordKey, - String passwordFragmentKey) { - assertEquals( - String.format("Value fragment of password key '%s' and password fragment key '%s' should be masked.", - passwordKey, - passwordFragmentKey), - "*****", - getFragmentValue(mapContainingMaskedPassword.get(passwordKey), passwordFragmentKey)); - } - - private String getFragmentValue(String base, String fragmentKey) { - for (String fragment : base.split(" ")) { - if (fragment.startsWith(fragmentKey)) { - return fragment.substring(fragmentKey.length()); - } - } - - return null; - } - - private void assertValueFragmentIsPresent(Map<String, String> masked, String key, String valueFragment) { - assertTrue(String.format("For key '%s' value fragment '%s' should be present.", key, valueFragment), - masked.get(key).contains(valueFragment)); - } -} http://git-wip-us.apache.org/repos/asf/oozie/blob/ccbf692d/core/src/test/resources/instrumentation-os-env.json ---------------------------------------------------------------------- diff --git a/core/src/test/resources/instrumentation-os-env.json b/core/src/test/resources/instrumentation-os-env.json deleted file mode 100644 index e85cd8d..0000000 --- a/core/src/test/resources/instrumentation-os-env.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "HADOOP_CREDSTORE_PASSWORD": "password", - "OOZIE_HTTPS_KEYSTORE_PASSWORD": "password", - "OOZIE_HTTPS_TRUSTSTORE_PASSWORD": "password", - "PATH": "/opt/local/bin:/opt/local/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin", - "HISTCONTROL": "ignoreboth", - "OOZIE_DATA": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/data", - "CATALINA_PID": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/temp/oozie.pid", - "MC_SID": "8597", - "OOZIE_INSTANCE_ID": "Budapests-MacBook-Pro.local", - "OOZIE_HTTP_HOSTNAME": "Budapests-MacBook-Pro.local", - "JAVA_HOME": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home", - "CATALINA_OUT": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs/catalina.out", - "TERM": "xterm-256color", - "LANG": "en_US.UTF-8", - "CATALINA_BASE": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server", - "OOZIE_CONFIG_FILE": "oozie-site.xml", - "LOGNAME": "forsage", - "OOZIE_HOME": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT", - "XPC_SERVICE_NAME": "0", - "PWD": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT", - "TERM_PROGRAM_VERSION": "361.1", - "JAVA_MAIN_CLASS_33220": "org.apache.catalina.startup.Bootstrap", - "_": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/bin/java", - "SHELL": "/bin/bash", - "OOZIE_CONFIG": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/conf", - "TERM_PROGRAM": "Apple_Terminal", - "OOZIE_ADMIN_PORT": "11001", - "CATALINA_OPTS": " -Xmx1024m -Dderby.stream.error.file=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs/derby.log -Doozie.home.dir=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT -Doozie.config.dir=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/conf -Doozie.log.dir=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs -Doozie.data.dir=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/data -Doozie.instance.id=Budapests-MacBook-Pro.local -Doozie.config.file=oozie-site.xml -Doozie.log4j.file=oozie-log4j.properties -Doozie.log4j.reload=10 -Doozie.http.hostname=Budapests-MacBook-Pro.local -Doozie.admin.port=11001 -Doozie.http.port=11000 -Doozie.https.port=11443 -Doozie.base.url=http://Budapests-MacBook-Pro.local:11000/oozie -Doozie.https.keystore.file=/Users /forsage/.keystore -Doozie.https.keystore.pass=password -Djavax.net.ssl.trustStorePassword=password -Djava.library.path=", - "USER": "forsage", - "OOZIE_LOG": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs", - "OOZIE_LOG4J_RELOAD": "10", - "TMPDIR": "/var/folders/yy/gkvmmzn91vv_lb2_bmymxz600000gp/T/", - "SSH_AUTH_SOCK": "/private/tmp/com.apple.launchd.NvNvd0j95Z/Listeners", - "MC_TMPDIR": "/var/folders/yy/gkvmmzn91vv_lb2_bmymxz600000gp/T/mc-forsage", - "XPC_FLAGS": "0x0", - "OOZIE_BASE_URL": "http://Budapests-MacBook-Pro.local:11000/oozie", - "TERM_SESSION_ID": "283A05FC-7501-4B9D-B3E3-BDDD3521593C", - "OOZIE_HTTPS_KEYSTORE_FILE": "/Users/forsage/.keystore", - "__CF_USER_TEXT_ENCODING": "0x1F6:0x0:0x0", - "Apple_PubSub_Socket_Render": "/private/tmp/com.apple.launchd.6kR2bgiMHn/Render", - "OOZIE_HTTP_PORT": "11000", - "OOZIE_HTTPS_PORT": "11443", - "SHLVL": "3", - "HOME": "/Users/forsage", - "OOZIE_LOG4J_FILE": "oozie-log4j.properties" -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/ccbf692d/core/src/test/resources/instrumentation-system-properties.json ---------------------------------------------------------------------- diff --git a/core/src/test/resources/instrumentation-system-properties.json b/core/src/test/resources/instrumentation-system-properties.json deleted file mode 100644 index 61430d2..0000000 --- a/core/src/test/resources/instrumentation-system-properties.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "javax.net.ssl.trustStorePassword": "password", - "oozie.https.keystore.pass": "password", - "gopherProxySet": "false", - "awt.toolkit": "sun.lwawt.macosx.LWCToolkit", - "oozie.base.url": "http://Budapests-MacBook-Pro.local:11000/oozie", - "file.encoding.pkg": "sun.io", - "java.specification.version": "1.8", - "sun.cpu.isalist": "", - "sun.jnu.encoding": "UTF-8", - "java.class.path": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/bin/bootstrap.jar", - "java.vm.vendor": "Oracle Corporation", - "sun.arch.data.model": "64", - "sun.font.fontmanager": "sun.font.CFontManager", - "catalina.useNaming": "true", - "java.vendor.url": "http://java.oracle.com/", - "user.timezone": "Europe/Budapest", - "os.name": "Mac OS X", - "java.vm.specification.version": "1.8", - "oozie.http.hostname": "Budapests-MacBook-Pro.local", - "oozie.instance.id": "Budapests-MacBook-Pro.local", - "sun.java.launcher": "SUN_STANDARD", - "user.country": "US", - "oozie.log.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs", - "oozie.home.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT", - "sun.boot.library.path": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib", - "sun.java.command": "org.apache.catalina.startup.Bootstrap start", - "http.nonProxyHosts": "local|*.local|169.254/16|*.169.254/16", - "sun.cpu.endian": "little", - "user.home": "/Users/forsage", - "user.language": "en", - "java.specification.vendor": "Oracle Corporation", - "java.naming.factory.url.pkgs": "org.apache.naming", - "java.home": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre", - "oozie.config.file": "oozie-site.xml", - "oozie.log4j.reload": "10", - "file.separator": "/", - "oozie.https.keystore.file": "/Users/forsage/.keystore", - "line.separator": "\n", - "java.vm.specification.vendor": "Oracle Corporation", - "java.specification.name": "Java Platform API Specification", - "derby.stream.error.file": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs/derby.log", - "oozie.log4j.file": "oozie-log4j.properties", - "oozie.admin.port": "11001", - "java.awt.graphicsenv": "sun.awt.CGraphicsEnvironment", - "package.access": "sun.,org.apache.catalina.,org.apache.coyote.,org.apache.jasper.,org.apache.naming.resources.,org.apache.tomcat.,sun.beans.", - "package.definition": "sun.,java.,org.apache.catalina.,org.apache.coyote.,org.apache.jasper.,org.apache.naming.,org.apache.tomcat.", - "sun.boot.class.path": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/resources.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/rt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/sunrsasign.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/jsse.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/jce.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/charsets.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/jfr.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/classes", - "server.loader": "", - "java.util.logging.config.file": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/conf/logging.properties", - "sun.management.compiler": "HotSpot 64-Bit Tiered Compilers", - "oozie.data.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/data", - "ftp.nonProxyHosts": "local|*.local|169.254/16|*.169.254/16", - "java.runtime.version": "1.8.0_102-b14", - "java.naming.factory.initial": "org.apache.naming.java.javaURLContextFactory", - "user.name": "forsage", - "oozie.https.port": "11443", - "path.separator": ":", - "common.loader": "${catalina.base}/lib,${catalina.base}/lib/*.jar,${catalina.home}/lib,${catalina.home}/lib/*.jar", - "os.version": "10.11.6", - "java.endorsed.dirs": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/endorsed", - "java.runtime.name": "Java(TM) SE Runtime Environment", - "file.encoding": "UTF-8", - "java.vm.name": "Java HotSpot(TM) 64-Bit Server VM", - "java.vendor.url.bug": "http://bugreport.sun.com/bugreport/", - "java.io.tmpdir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/temp", - "oozie.http.port": "11000", - "catalina.home": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server", - "java.version": "1.8.0_102", - "user.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT", - "oozie.config.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/conf", - "os.arch": "x86_64", - "java.vm.specification.name": "Java Virtual Machine Specification", - "java.awt.printerjob": "sun.lwawt.macosx.CPrinterJob", - "sun.os.patch.level": "unknown", - "catalina.base": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server", - "shared.loader": "", - "java.util.logging.manager": "org.apache.juli.ClassLoaderLogManager", - "java.library.path": "", - "java.vendor": "Oracle Corporation", - "java.vm.info": "mixed mode", - "java.vm.version": "25.102-b14", - "sun.io.unicode.encoding": "UnicodeBig", - "java.ext.dirs": "/Users/forsage/Library/Java/Extensions:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/ext:/Library/Java/Extensions:/Network/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java", - "tomcat.util.buf.StringCache.byte.enabled": "true", - "java.class.version": "52.0", - "socksNonProxyHosts": "local|*.local|169.254/16|*.169.254/16" -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/ccbf692d/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index df586c9..fdf6f2b 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.4.0 release (trunk - unreleased) +OOZIE-2803 Mask passwords when printing out configs/args in MapReduceMain and SparkMain (pbacsko via rkanter) OOZIE-2799 Setting log location for spark sql on hive (satishsaley) OOZIE-2792 Hive2 action is not parsing Spark application ID from log file properly when Hive is on Spark (zhengxb2005 via rkanter) OOZIE-2788 Fix jobs API servlet mapping for EmbeddedOozieServer (abhishekbafna via rkanter) http://git-wip-us.apache.org/repos/asf/oozie/blob/ccbf692d/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java index 23447cf..d376057 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java @@ -23,8 +23,10 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RunningJob; -import java.util.HashSet; + +import java.util.ArrayList; import java.util.Map; +import java.util.Map.Entry; import java.io.FileOutputStream; import java.io.IOException; import java.io.File; @@ -55,7 +57,14 @@ public class MapReduceMain extends LauncherMain { // Run a config class if given to update the job conf runConfigClass(jobConf); - logMasking("Map-Reduce job configuration:", new HashSet<String>(), jobConf); + PasswordMasker passwordMasker = new PasswordMasker(); + // Temporary JobConf object, we mask out possible passwords before we print key-value pairs + JobConf maskedJobConf = new JobConf(false); + for (Entry<String, String> entry : jobConf) { + maskedJobConf.set(entry.getKey(), passwordMasker.maskPasswordsIfNecessary(entry.getValue())); + } + + logMasking("Map-Reduce job configuration:", new ArrayList<String>(), maskedJobConf); File idFile = new File(System.getProperty(LauncherMapper.ACTION_PREFIX + LauncherMapper.ACTION_DATA_NEW_ID)); System.out.println("Submitting Oozie action Map-Reduce job"); http://git-wip-us.apache.org/repos/asf/oozie/blob/ccbf692d/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/PasswordMasker.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/PasswordMasker.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/PasswordMasker.java new file mode 100644 index 0000000..eb60aac --- /dev/null +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/PasswordMasker.java @@ -0,0 +1,144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.action.hadoop; + +import com.google.common.collect.Maps; + +import javax.annotation.Nonnull; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.google.common.base.Preconditions.checkNotNull; + +/** + * A generic password masker that masks {@code Map<String, String>} values given that its keys are considered password keys. + * <p/> + * Tested with {@see System#getProperties()} and {@see System#getenv()}. + */ +public class PasswordMasker { + + /** + * The mask that is applied to recognized passwords. + **/ + private static final String PASSWORD_MASK = "*****"; + + /** + * A key is considered a password key, if it contains {{pass}}, case ignored. + **/ + private static final String PASSWORD_KEY = "pass"; + + /** + * Tells us whether a given string contains a password fragment. A password fragment is something that looks + * like {{-Djavax.net.ssl.trustStorePassword=password}} or {{HADOOP_CREDSTORE_PASSWORD=pwd123}} + * + **/ + private static final String PASSWORD_CONTAINING_REGEX = + "(.*)([\\w[.\\w]*]*(?i)" + PASSWORD_KEY + "[\\w]*=)([\\w]+)(.*)"; + + private static final Pattern PASSWORD_CONTAINING_PATTERN = Pattern + .compile(PASSWORD_CONTAINING_REGEX); + + /** + * Extracts a password fragment from a given string. + * <p/> + * {@see java.util.Matcher#find()} + **/ + private static final String PASSWORD_EXTRACTING_REGEX = + "([\\w[.\\w]*]*(?i)pass[\\w]*=)([\\w]+)"; + + private static final Pattern PASSWORD_EXTRACTING_PATTERN = Pattern + .compile(PASSWORD_EXTRACTING_REGEX); + + /** + * Returns a map where keys are masked if they are considered a password. + * There are two cases when passwords are masked: + * 1. The key contains the string "pass". In this case, the entire value is considered a password and replaced completely with + * a masking string. + * 2. The value matches a regular expression. Strings like "HADOOP_CREDSTORE_PASSWORD=pwd123" or + * "-Djavax.net.ssl.trustStorePassword=password" are considered password definition strings and the text after the equal sign + * is replaced with a masking string. + * + * @param unmasked key-value map + * @return A new map where values are changed based on the replace algorithm described above + */ + public Map<String, String> mask(Map<String, String> unmasked) { + return Maps.transformEntries(unmasked, new Maps.EntryTransformer<String, String, String>() { + @Override + public String transformEntry(@Nonnull String key, @Nonnull String value) { + checkNotNull(key, "key has to be set"); + checkNotNull(value, "value has to be set"); + + if (isPasswordKey(key)) { + return PASSWORD_MASK; + } + + return maskPasswordsIfNecessary(value); + } + }); + } + + /** + * Masks passwords inside a string. A substring is subject to password masking if it looks like + * "HADOOP_CREDSTORE_PASSWORD=pwd123" or "-Djavax.net.ssl.trustStorePassword=password". The text after the equal sign is + * replaced with a masking string. + * + * @param unmasked String which might contain passwords + * @return The same string where passwords are replaced with a masking string. If there is no password inside, the original + * string is returned. + */ + public String maskPasswordsIfNecessary(String unmasked) { + if (containsPasswordFragment(unmasked)) { + return maskPasswordFragments(unmasked); + } else { + return unmasked; + } + } + + private boolean isPasswordKey(String key) { + return key.toLowerCase().contains(PASSWORD_KEY); + } + + private boolean containsPasswordFragment(String maybePasswordFragments) { + return PASSWORD_CONTAINING_PATTERN + .matcher(maybePasswordFragments) + .matches(); + } + + private String maskPasswordFragments(String maybePasswordFragments) { + StringBuilder maskedBuilder = new StringBuilder(); + Matcher passwordFragmentsMatcher = PASSWORD_EXTRACTING_PATTERN + .matcher(maybePasswordFragments); + + int start = 0, end; + while (passwordFragmentsMatcher.find()) { + end = passwordFragmentsMatcher.start(); + + maskedBuilder.append(maybePasswordFragments.substring(start, end)); + maskedBuilder.append(passwordFragmentsMatcher.group(1)); + maskedBuilder.append(PASSWORD_MASK); + + start = passwordFragmentsMatcher.end(); + } + + maskedBuilder.append(maybePasswordFragments.substring(start)); + + return maskedBuilder.toString(); + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/ccbf692d/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestPasswordMasker.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestPasswordMasker.java b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestPasswordMasker.java new file mode 100644 index 0000000..08e55e1 --- /dev/null +++ b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestPasswordMasker.java @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.action.hadoop; + +import org.codehaus.jackson.map.ObjectMapper; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestPasswordMasker { + private PasswordMasker passwordMasker; + + @Before + public void setup() { + passwordMasker = new PasswordMasker(); + } + + @Test + public void testWhenJavaSystemPropertiesAreAskedPasswordsAppearMasked() throws Exception { + Map<String, String> masked = passwordMasker.mask(jsonToMap("/instrumentation-system-properties.json")); + + assertPasswordValueIsMasked(masked, "javax.net.ssl.trustStorePassword"); + assertPasswordValueIsMasked(masked, "oozie.https.keystore.pass"); + } + + @Test + public void testWhenOSEnvIsAskedPasswordsAppearMasked() throws Exception { + Map<String, String> masked = passwordMasker.mask(jsonToMap("/instrumentation-os-env.json")); + + assertPasswordValueIsMasked(masked, "HADOOP_CREDSTORE_PASSWORD"); + assertPasswordValueIsMasked(masked, "OOZIE_HTTPS_KEYSTORE_PASSWORD"); + assertPasswordValueIsMasked(masked, "OOZIE_HTTPS_TRUSTSTORE_PASSWORD"); + + assertPasswordValueFragmentIsMasked(masked, "CATALINA_OPTS", "-Doozie.https.keystore.pass="); + assertPasswordValueFragmentIsMasked(masked, "CATALINA_OPTS", "-Djavax.net.ssl.trustStorePassword="); + + assertValueFragmentIsPresent(masked, "CATALINA_OPTS", "-Xmx1024m"); + assertValueFragmentIsPresent(masked, "CATALINA_OPTS", "-Doozie.https.keystore.file=/Users/forsage/.keystore"); + assertValueFragmentIsPresent(masked, "CATALINA_OPTS", "-Djava.library.path="); + } + + @Test + public void testMaskNothing() { + assertEquals("abcd", passwordMasker.maskPasswordsIfNecessary("abcd")); + assertEquals("abcd abcd", passwordMasker.maskPasswordsIfNecessary("abcd abcd")); + assertEquals("-Djava.net.pasX=pwd1", passwordMasker.maskPasswordsIfNecessary("-Djava.net.pasX=pwd1")); + } + + @Test + public void testMaskJavaSystemProp() { + assertEquals("-Djava.sysprop.password=*****", passwordMasker.maskPasswordsIfNecessary("-Djava.sysprop.password=pwd123")); + } + + @Test + public void testMaskJavaSystemPropWithWhiteSpaces() { + assertEquals(" -Djava.sysprop.password=***** ", + passwordMasker.maskPasswordsIfNecessary(" -Djava.sysprop.password=pwd123 ")); + } + + @Test + public void testMaskTwoJavaSystemProps() { + assertEquals("-Djava.sysprop.password=***** -Djava.another.password=*****", + passwordMasker.maskPasswordsIfNecessary("-Djava.sysprop.password=pwd123 -Djava.another.password=pwd456")); + } + + @Test + public void testMaskEnvironmentVariable() { + assertEquals("DUMMY_PASSWORD=*****", passwordMasker.maskPasswordsIfNecessary("DUMMY_PASSWORD=dummy")); + } + + @Test + public void testMaskTwoEnvironmentVariables() { + assertEquals("DUMMY_PASSWORD=*****:ANOTHER_PASSWORD=*****", + passwordMasker.maskPasswordsIfNecessary("DUMMY_PASSWORD=dummy:ANOTHER_PASSWORD=pwd123")); + } + + @Test + public void testMaskRandomMatchingStuff() { + assertEquals("aa -Djava.sysprop.password=***** bb DUMMY_PASSWORD=***** cc", + passwordMasker.maskPasswordsIfNecessary("aa -Djava.sysprop.password=1234 bb DUMMY_PASSWORD=dummy cc")); + } + + @SuppressWarnings("unchecked") + private Map<String, String> jsonToMap(String jsonPath) throws IOException { + return new ObjectMapper().readValue(getClass().getResourceAsStream(jsonPath), HashMap.class); + } + + private void assertPasswordValueIsMasked(Map<String, String> mapContainingMaskedPassword, String passwordKey) { + assertEquals(String.format("Value of key '%s' should be masked.", passwordKey), + "*****", + mapContainingMaskedPassword.get(passwordKey)); + } + + private void assertPasswordValueFragmentIsMasked(Map<String, String> mapContainingMaskedPassword, String passwordKey, + String passwordFragmentKey) { + assertEquals( + String.format("Value fragment of password key '%s' and password fragment key '%s' should be masked.", + passwordKey, + passwordFragmentKey), + "*****", + getFragmentValue(mapContainingMaskedPassword.get(passwordKey), passwordFragmentKey)); + } + + private String getFragmentValue(String base, String fragmentKey) { + for (String fragment : base.split(" ")) { + if (fragment.startsWith(fragmentKey)) { + return fragment.substring(fragmentKey.length()); + } + } + + return null; + } + + private void assertValueFragmentIsPresent(Map<String, String> masked, String key, String valueFragment) { + assertTrue(String.format("For key '%s' value fragment '%s' should be present.", key, valueFragment), + masked.get(key).contains(valueFragment)); + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/ccbf692d/sharelib/oozie/src/test/resources/instrumentation-os-env.json ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/test/resources/instrumentation-os-env.json b/sharelib/oozie/src/test/resources/instrumentation-os-env.json new file mode 100644 index 0000000..e85cd8d --- /dev/null +++ b/sharelib/oozie/src/test/resources/instrumentation-os-env.json @@ -0,0 +1,47 @@ +{ + "HADOOP_CREDSTORE_PASSWORD": "password", + "OOZIE_HTTPS_KEYSTORE_PASSWORD": "password", + "OOZIE_HTTPS_TRUSTSTORE_PASSWORD": "password", + "PATH": "/opt/local/bin:/opt/local/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin", + "HISTCONTROL": "ignoreboth", + "OOZIE_DATA": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/data", + "CATALINA_PID": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/temp/oozie.pid", + "MC_SID": "8597", + "OOZIE_INSTANCE_ID": "Budapests-MacBook-Pro.local", + "OOZIE_HTTP_HOSTNAME": "Budapests-MacBook-Pro.local", + "JAVA_HOME": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home", + "CATALINA_OUT": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs/catalina.out", + "TERM": "xterm-256color", + "LANG": "en_US.UTF-8", + "CATALINA_BASE": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server", + "OOZIE_CONFIG_FILE": "oozie-site.xml", + "LOGNAME": "forsage", + "OOZIE_HOME": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT", + "XPC_SERVICE_NAME": "0", + "PWD": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT", + "TERM_PROGRAM_VERSION": "361.1", + "JAVA_MAIN_CLASS_33220": "org.apache.catalina.startup.Bootstrap", + "_": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/bin/java", + "SHELL": "/bin/bash", + "OOZIE_CONFIG": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/conf", + "TERM_PROGRAM": "Apple_Terminal", + "OOZIE_ADMIN_PORT": "11001", + "CATALINA_OPTS": " -Xmx1024m -Dderby.stream.error.file=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs/derby.log -Doozie.home.dir=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT -Doozie.config.dir=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/conf -Doozie.log.dir=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs -Doozie.data.dir=/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/data -Doozie.instance.id=Budapests-MacBook-Pro.local -Doozie.config.file=oozie-site.xml -Doozie.log4j.file=oozie-log4j.properties -Doozie.log4j.reload=10 -Doozie.http.hostname=Budapests-MacBook-Pro.local -Doozie.admin.port=11001 -Doozie.http.port=11000 -Doozie.https.port=11443 -Doozie.base.url=http://Budapests-MacBook-Pro.local:11000/oozie -Doozie.https.keystore.file=/Users /forsage/.keystore -Doozie.https.keystore.pass=password -Djavax.net.ssl.trustStorePassword=password -Djava.library.path=", + "USER": "forsage", + "OOZIE_LOG": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs", + "OOZIE_LOG4J_RELOAD": "10", + "TMPDIR": "/var/folders/yy/gkvmmzn91vv_lb2_bmymxz600000gp/T/", + "SSH_AUTH_SOCK": "/private/tmp/com.apple.launchd.NvNvd0j95Z/Listeners", + "MC_TMPDIR": "/var/folders/yy/gkvmmzn91vv_lb2_bmymxz600000gp/T/mc-forsage", + "XPC_FLAGS": "0x0", + "OOZIE_BASE_URL": "http://Budapests-MacBook-Pro.local:11000/oozie", + "TERM_SESSION_ID": "283A05FC-7501-4B9D-B3E3-BDDD3521593C", + "OOZIE_HTTPS_KEYSTORE_FILE": "/Users/forsage/.keystore", + "__CF_USER_TEXT_ENCODING": "0x1F6:0x0:0x0", + "Apple_PubSub_Socket_Render": "/private/tmp/com.apple.launchd.6kR2bgiMHn/Render", + "OOZIE_HTTP_PORT": "11000", + "OOZIE_HTTPS_PORT": "11443", + "SHLVL": "3", + "HOME": "/Users/forsage", + "OOZIE_LOG4J_FILE": "oozie-log4j.properties" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/ccbf692d/sharelib/oozie/src/test/resources/instrumentation-system-properties.json ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/test/resources/instrumentation-system-properties.json b/sharelib/oozie/src/test/resources/instrumentation-system-properties.json new file mode 100644 index 0000000..61430d2 --- /dev/null +++ b/sharelib/oozie/src/test/resources/instrumentation-system-properties.json @@ -0,0 +1,88 @@ +{ + "javax.net.ssl.trustStorePassword": "password", + "oozie.https.keystore.pass": "password", + "gopherProxySet": "false", + "awt.toolkit": "sun.lwawt.macosx.LWCToolkit", + "oozie.base.url": "http://Budapests-MacBook-Pro.local:11000/oozie", + "file.encoding.pkg": "sun.io", + "java.specification.version": "1.8", + "sun.cpu.isalist": "", + "sun.jnu.encoding": "UTF-8", + "java.class.path": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/bin/bootstrap.jar", + "java.vm.vendor": "Oracle Corporation", + "sun.arch.data.model": "64", + "sun.font.fontmanager": "sun.font.CFontManager", + "catalina.useNaming": "true", + "java.vendor.url": "http://java.oracle.com/", + "user.timezone": "Europe/Budapest", + "os.name": "Mac OS X", + "java.vm.specification.version": "1.8", + "oozie.http.hostname": "Budapests-MacBook-Pro.local", + "oozie.instance.id": "Budapests-MacBook-Pro.local", + "sun.java.launcher": "SUN_STANDARD", + "user.country": "US", + "oozie.log.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs", + "oozie.home.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT", + "sun.boot.library.path": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib", + "sun.java.command": "org.apache.catalina.startup.Bootstrap start", + "http.nonProxyHosts": "local|*.local|169.254/16|*.169.254/16", + "sun.cpu.endian": "little", + "user.home": "/Users/forsage", + "user.language": "en", + "java.specification.vendor": "Oracle Corporation", + "java.naming.factory.url.pkgs": "org.apache.naming", + "java.home": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre", + "oozie.config.file": "oozie-site.xml", + "oozie.log4j.reload": "10", + "file.separator": "/", + "oozie.https.keystore.file": "/Users/forsage/.keystore", + "line.separator": "\n", + "java.vm.specification.vendor": "Oracle Corporation", + "java.specification.name": "Java Platform API Specification", + "derby.stream.error.file": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/logs/derby.log", + "oozie.log4j.file": "oozie-log4j.properties", + "oozie.admin.port": "11001", + "java.awt.graphicsenv": "sun.awt.CGraphicsEnvironment", + "package.access": "sun.,org.apache.catalina.,org.apache.coyote.,org.apache.jasper.,org.apache.naming.resources.,org.apache.tomcat.,sun.beans.", + "package.definition": "sun.,java.,org.apache.catalina.,org.apache.coyote.,org.apache.jasper.,org.apache.naming.,org.apache.tomcat.", + "sun.boot.class.path": "/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/resources.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/rt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/sunrsasign.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/jsse.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/jce.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/charsets.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/jfr.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/classes", + "server.loader": "", + "java.util.logging.config.file": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/conf/logging.properties", + "sun.management.compiler": "HotSpot 64-Bit Tiered Compilers", + "oozie.data.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/data", + "ftp.nonProxyHosts": "local|*.local|169.254/16|*.169.254/16", + "java.runtime.version": "1.8.0_102-b14", + "java.naming.factory.initial": "org.apache.naming.java.javaURLContextFactory", + "user.name": "forsage", + "oozie.https.port": "11443", + "path.separator": ":", + "common.loader": "${catalina.base}/lib,${catalina.base}/lib/*.jar,${catalina.home}/lib,${catalina.home}/lib/*.jar", + "os.version": "10.11.6", + "java.endorsed.dirs": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/endorsed", + "java.runtime.name": "Java(TM) SE Runtime Environment", + "file.encoding": "UTF-8", + "java.vm.name": "Java HotSpot(TM) 64-Bit Server VM", + "java.vendor.url.bug": "http://bugreport.sun.com/bugreport/", + "java.io.tmpdir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server/temp", + "oozie.http.port": "11000", + "catalina.home": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server", + "java.version": "1.8.0_102", + "user.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT", + "oozie.config.dir": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/conf", + "os.arch": "x86_64", + "java.vm.specification.name": "Java Virtual Machine Specification", + "java.awt.printerjob": "sun.lwawt.macosx.CPrinterJob", + "sun.os.patch.level": "unknown", + "catalina.base": "/Users/forsage/Workspace/oozie/distro/target/oozie-4.3.0-SNAPSHOT-distro/oozie-4.3.0-SNAPSHOT/oozie-server", + "shared.loader": "", + "java.util.logging.manager": "org.apache.juli.ClassLoaderLogManager", + "java.library.path": "", + "java.vendor": "Oracle Corporation", + "java.vm.info": "mixed mode", + "java.vm.version": "25.102-b14", + "sun.io.unicode.encoding": "UnicodeBig", + "java.ext.dirs": "/Users/forsage/Library/Java/Extensions:/Library/Java/JavaVirtualMachines/jdk1.8.0_102.jdk/Contents/Home/jre/lib/ext:/Library/Java/Extensions:/Network/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java", + "tomcat.util.buf.StringCache.byte.enabled": "true", + "java.class.version": "52.0", + "socksNonProxyHosts": "local|*.local|169.254/16|*.169.254/16" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/ccbf692d/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java ---------------------------------------------------------------------- diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java index db1e197..88ac64e 100644 --- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java +++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java @@ -263,8 +263,9 @@ public class SparkMain extends LauncherMain { System.out.println("Oozie Spark action configuration"); System.out.println("================================================================="); System.out.println(); + PasswordMasker passwordMasker = new PasswordMasker(); for (String arg : sparkArgs) { - System.out.println(" " + arg); + System.out.println(" " + passwordMasker.maskPasswordsIfNecessary(arg)); } System.out.println(); try {