Repository: tez Updated Branches: refs/heads/master a33d2213b -> a2b6eac1c
TEZ-3537. ArrayIndexOutOfBoundsException with empty environment variables/Port YARN-3768 to Tez (Eric Badger via jeagles) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/a2b6eac1 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/a2b6eac1 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/a2b6eac1 Branch: refs/heads/master Commit: a2b6eac1cc4483126afb96e84c355f253dcfbd85 Parents: a33d221 Author: Jonathan Eagles <[email protected]> Authored: Wed Dec 7 16:59:04 2016 -0600 Committer: Jonathan Eagles <[email protected]> Committed: Wed Dec 7 16:59:04 2016 -0600 ---------------------------------------------------------------------- CHANGES.txt | 3 ++ .../org/apache/tez/common/TezYARNUtils.java | 25 ++++++++----- .../org/apache/tez/common/TestTezYARNUtils.java | 37 ++++++++++++++++++++ 3 files changed, 57 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/a2b6eac1/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 6bbbfb2..b15686e 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3537. ArrayIndexOutOfBoundsException with empty environment variables/Port YARN-3768 to Tez TEZ-3271. Provide mapreduce failures.maxpercent equivalent. TEZ-3222. Reduce messaging overhead for auto-reduce parallelism case. TEZ-3547. Add TaskAssignment Analyzer. @@ -154,6 +155,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3537. ArrayIndexOutOfBoundsException with empty environment variables/Port YARN-3768 to Tez TEZ-3536. NPE in WebUIService start when host resolution fails. TEZ-3534. Differentiate thread names on Fetchers, minor changes to shuffle shutdown code. TEZ-3491. Tez job can hang due to container priority inversion. @@ -662,6 +664,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3537. ArrayIndexOutOfBoundsException with empty environment variables/Port YARN-3768 to Tez TEZ-3536. NPE in WebUIService start when host resolution fails. TEZ-3493. DAG submit timeout cannot be set to a month TEZ-3505. Move license to the file header for TezBytesWritableSerialization http://git-wip-us.apache.org/repos/asf/tez/blob/a2b6eac1/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java index 145f928..d5da672 100644 --- a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java +++ b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java @@ -38,8 +38,17 @@ import org.apache.tez.dag.api.TezConstants; @Private public class TezYARNUtils { private static Logger LOG = LoggerFactory.getLogger(TezYARNUtils.class); - - private static Pattern ENV_VARIABLE_PATTERN = Pattern.compile(Shell.getEnvironmentVariableRegex()); + + public static final String ENV_NAME_REGEX = "[A-Za-z_][A-Za-z0-9_]*"; + + private static final Pattern VAR_SUBBER = + Pattern.compile(Shell.getEnvironmentVariableRegex()); + private static final Pattern VARVAL_SPLITTER = Pattern.compile( + "(?<=^|,)" // preceded by ',' or line begin + + '(' + ENV_NAME_REGEX + ')' // var group + + '=' + + "([^,]*)" // val group + ); public static String getFrameworkClasspath(Configuration conf, boolean usingArchive) { StringBuilder classpathBuilder = new StringBuilder(); @@ -126,10 +135,10 @@ public class TezYARNUtils { public static void appendToEnvFromInputString(Map<String, String> env, String envString, String classPathSeparator) { if (envString != null && envString.length() > 0) { - String childEnvs[] = envString.split(","); - for (String cEnv : childEnvs) { - String[] parts = cEnv.split("="); // split on '=' - Matcher m = ENV_VARIABLE_PATTERN.matcher(parts[1]); + Matcher varValMatcher = VARVAL_SPLITTER.matcher(envString); + while (varValMatcher.find()) { + String envVar = varValMatcher.group(1); + Matcher m = VAR_SUBBER.matcher(varValMatcher.group(2)); StringBuffer sb = new StringBuffer(); while (m.find()) { String var = m.group(1); @@ -145,7 +154,7 @@ public class TezYARNUtils { m.appendReplacement(sb, Matcher.quoteReplacement(replace)); } m.appendTail(sb); - addToEnvironment(env, parts[0], sb.toString(), classPathSeparator); + addToEnvironment(env, envVar, sb.toString(), classPathSeparator); } } } @@ -156,7 +165,7 @@ public class TezYARNUtils { String childEnvs[] = envString.split(","); for (String cEnv : childEnvs) { String[] parts = cEnv.split("="); // split on '=' - Matcher m = ENV_VARIABLE_PATTERN.matcher(parts[1]); + Matcher m = VAR_SUBBER .matcher(parts[1]); StringBuffer sb = new StringBuffer(); while (m.find()) { String var = m.group(1); http://git-wip-us.apache.org/repos/asf/tez/blob/a2b6eac1/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java index 092bd50..9f02801 100644 --- a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java +++ b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java @@ -19,16 +19,21 @@ package org.apache.tez.common; import java.io.File; +import java.util.HashMap; import java.util.Map; import java.util.TreeMap; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.TezConstants; import org.junit.Assert; import org.junit.Test; +import static org.apache.tez.common.TezYARNUtils.appendToEnvFromInputString; +import static org.junit.Assert.assertEquals; + public class TestTezYARNUtils { @Test(timeout = 5000) @@ -104,4 +109,36 @@ public class TestTezYARNUtils { Assert.assertTrue(classpath.indexOf("foobar") > classpath.indexOf(Environment.PWD.$())); } + + @Test + public void testSetEnvFromInputString() { + Map<String, String> environment = new HashMap<String, String>(); + environment.put("JAVA_HOME", "/path/jdk"); + String goodEnv = "a1=1,b_2=2,_c=3,d=4,e=,f_win=%JAVA_HOME%" + + ",g_nix=$JAVA_HOME"; + appendToEnvFromInputString(environment, goodEnv, File.pathSeparator); + assertEquals("1", environment.get("a1")); + assertEquals("2", environment.get("b_2")); + assertEquals("3", environment.get("_c")); + assertEquals("4", environment.get("d")); + assertEquals("", environment.get("e")); + if (Shell.WINDOWS) { + assertEquals("$JAVA_HOME", environment.get("g_nix")); + assertEquals("/path/jdk", environment.get("f_win")); + } else { + assertEquals("/path/jdk", environment.get("g_nix")); + assertEquals("%JAVA_HOME%", environment.get("f_win")); + } + String badEnv = "0=,1,,2=a=b,3=a=,4==,5==a,==,c-3=3,="; + environment.clear(); + appendToEnvFromInputString(environment, badEnv, File.pathSeparator); + assertEquals(environment.size(), 0); + + // Test "=" in the value part + environment.clear(); + appendToEnvFromInputString(environment, "b1,e1==,e2=a1=a2,b2", + File.pathSeparator); + assertEquals("=", environment.get("e1")); + assertEquals("a1=a2", environment.get("e2")); + } }
