Repository: incubator-apex-core Updated Branches: refs/heads/devel-3 3d0dd624a -> 48cd2e8ff
APEX-24 #resolve took out rhino dependency as it is GPL Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/commit/48cd2e8f Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/tree/48cd2e8f Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/diff/48cd2e8f Branch: refs/heads/devel-3 Commit: 48cd2e8ffd7afd9cc7aa2aa8a8a44235616ece43 Parents: 3d0dd62 Author: David Yan <[email protected]> Authored: Thu Oct 8 12:37:36 2015 -0700 Committer: David Yan <[email protected]> Committed: Thu Oct 8 12:37:36 2015 -0700 ---------------------------------------------------------------------- engine/pom.xml | 5 -- .../java/com/datatorrent/stram/StramClient.java | 1 - .../stram/client/StramClientUtils.java | 92 +++++++++++--------- .../stram/client/EvalPropertiesTest.java | 68 --------------- .../stram/client/StramClientUtilsTest.java | 37 ++++++++ 5 files changed, 87 insertions(+), 116 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/48cd2e8f/engine/pom.xml ---------------------------------------------------------------------- diff --git a/engine/pom.xml b/engine/pom.xml index 752fc0d..7974313 100644 --- a/engine/pom.xml +++ b/engine/pom.xml @@ -278,11 +278,6 @@ <type>jar</type> </dependency> <dependency> - <groupId>org.mozilla</groupId> - <artifactId>rhino</artifactId> - <version>1.7R4</version> - </dependency> - <dependency> <!-- We need this direct dependency because without this, it throws method not found exception on Amazon EMR version of hadoop 2.4. Amazon EMR hadoop has commons-codec-1.2 in its classpath. --> <groupId>commons-codec</groupId> http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/48cd2e8f/engine/src/main/java/com/datatorrent/stram/StramClient.java ---------------------------------------------------------------------- diff --git a/engine/src/main/java/com/datatorrent/stram/StramClient.java b/engine/src/main/java/com/datatorrent/stram/StramClient.java index 7abfc82..9a570e0 100644 --- a/engine/src/main/java/com/datatorrent/stram/StramClient.java +++ b/engine/src/main/java/com/datatorrent/stram/StramClient.java @@ -121,7 +121,6 @@ public class StramClient org.apache.http.message.BasicHeaderValueParser.class, com.esotericsoftware.minlog.Log.class, org.apache.xbean.asm5.tree.ClassNode.class, - org.mozilla.javascript.Scriptable.class, // The jersey client inclusion is only for Hadoop-2.2 and should be removed when we upgrade our Hadoop // dependency version since Hadoop-2.3 onwards has jersey client bundled com.sun.jersey.api.client.ClientHandler.class, http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/48cd2e8f/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java ---------------------------------------------------------------------- diff --git a/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java b/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java index 4ac6487..27f58f0 100644 --- a/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java +++ b/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java @@ -25,14 +25,17 @@ import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.mozilla.javascript.Scriptable; +import javax.script.ScriptEngine; +import javax.script.ScriptEngineManager; +import javax.script.ScriptException; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.StringEscapeUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; @@ -647,58 +650,63 @@ public class StramClientUtils public static void evalProperties(Properties target, Configuration vars) { - Pattern substitionPattern = Pattern.compile("\\$\\{(.+?)\\}"); + ScriptEngine engine = new ScriptEngineManager().getEngineByName("javascript"); + + Pattern substitutionPattern = Pattern.compile("\\$\\{(.+?)\\}"); Pattern evalPattern = Pattern.compile("\\{% (.+?) %\\}"); - org.mozilla.javascript.Context context = org.mozilla.javascript.Context.enter(); - context.setOptimizationLevel(-1); - Scriptable scope = context.initStandardObjects(); try { - context.evaluateString(scope, "var _prop = {}", "EvalLaunchProperties", 0, null); + engine.eval("var _prop = {}"); for (Map.Entry<String, String> entry : vars) { - LOG.info("Evaluating: {}", "_prop[\"" + entry.getKey() + "\"] = " + entry.getValue()); - context.evaluateString(scope, "_prop[\"" + StringEscapeUtils.escapeJava(entry.getKey()) + "\"] = \"" + StringEscapeUtils.escapeJava(entry.getValue()) + "\"", "EvalLaunchProperties", 0, null); + String evalString = String.format("_prop[\"%s\"] = \"%s\"", StringEscapeUtils.escapeJava(entry.getKey()), StringEscapeUtils.escapeJava(entry.getValue())); + LOG.debug("Evaluating: {}", evalString); + engine.eval(evalString); + } + } catch (ScriptException ex) { + LOG.warn("Javascript error: {}", ex.getMessage()); + } + + for (Map.Entry<Object, Object> entry : target.entrySet()) { + String value = entry.getValue().toString(); + + Matcher matcher = substitutionPattern.matcher(value); + if (matcher.find()) { + StringBuilder newValue = new StringBuilder(); + int cursor = 0; + do { + newValue.append(value.substring(cursor, matcher.start())); + String subst = vars.get(matcher.group(1)); + if (subst != null) { + newValue.append(subst); + } + cursor = matcher.end(); + } while (matcher.find()); + newValue.append(value.substring(cursor)); + target.put(entry.getKey(), newValue.toString()); } - for (Map.Entry<Object, Object> entry : target.entrySet()) { - String value = entry.getValue().toString(); - - Matcher matcher = substitionPattern.matcher(value); - if (matcher.find()) { - StringBuilder newValue = new StringBuilder(); - int cursor = 0; - do { - newValue.append(value.substring(cursor, matcher.start())); - String subst = vars.get(matcher.group(1)); - if (subst != null) { - newValue.append(subst); - } - cursor = matcher.end(); - } while (matcher.find()); - newValue.append(value.substring(cursor)); - target.put(entry.getKey(), newValue.toString()); - } + matcher = evalPattern.matcher(value); + if (matcher.find()) { + StringBuilder newValue = new StringBuilder(); + int cursor = 0; + do { + newValue.append(value.substring(cursor, matcher.start())); + try { + Object result = engine.eval(matcher.group(1)); + String eval = result.toString(); - matcher = evalPattern.matcher(value); - if (matcher.find()) { - StringBuilder newValue = new StringBuilder(); - int cursor = 0; - do { - newValue.append(value.substring(cursor, matcher.start())); - String eval = context.evaluateString(scope, matcher.group(1), "EvalLaunchProperties", 0, null).toString(); if (eval != null) { newValue.append(eval); } - cursor = matcher.end(); - } while (matcher.find()); - newValue.append(value.substring(cursor)); - target.put(entry.getKey(), newValue.toString()); - } + } catch (ScriptException ex) { + LOG.warn("JavaScript exception {}", ex.getMessage()); + } + cursor = matcher.end(); + } while (matcher.find()); + newValue.append(value.substring(cursor)); + target.put(entry.getKey(), newValue.toString()); } } - finally { - org.mozilla.javascript.Context.exit(); - } } public static <T> T doAs(String userName, PrivilegedExceptionAction<T> action) throws Exception http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/48cd2e8f/engine/src/test/java/com/datatorrent/stram/client/EvalPropertiesTest.java ---------------------------------------------------------------------- diff --git a/engine/src/test/java/com/datatorrent/stram/client/EvalPropertiesTest.java b/engine/src/test/java/com/datatorrent/stram/client/EvalPropertiesTest.java deleted file mode 100644 index 816e705..0000000 --- a/engine/src/test/java/com/datatorrent/stram/client/EvalPropertiesTest.java +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.datatorrent.stram.client; - -import java.util.Properties; -import org.apache.hadoop.conf.Configuration; -import org.junit.Assert; -import org.junit.Test; - -/** - * - */ - - -public class EvalPropertiesTest -{ - @Test - public void testEvalExpression() throws Exception - { - Configuration conf = new Configuration(); - conf.set("a.b.c", "123"); - conf.set("d.e.f", "456"); - conf.set("x.y.z", "foobar"); - - Properties prop = new Properties(); - prop.put("product.result", "Product result is {% (_prop[\"a.b.c\"] * _prop[\"d.e.f\"]).toFixed(0) %}..."); - prop.put("concat.result", "Concat result is {% _prop[\"x.y.z\"] %} ... {% _prop[\"a.b.c\"] %} blah"); - - StramClientUtils.evalProperties(prop, conf); - - Assert.assertEquals("Product result is " + (123 * 456) + "...", prop.get("product.result")); - Assert.assertEquals("Concat result is foobar ... 123 blah", prop.get("concat.result")); - Assert.assertEquals("123", conf.get("a.b.c")); - Assert.assertEquals("456", conf.get("d.e.f")); - Assert.assertEquals("foobar", conf.get("x.y.z")); - } - - @Test - public void testVariableSubstitution() throws Exception - { - Configuration conf = new Configuration(); - conf.set("a.b.c", "123"); - conf.set("x.y.z", "foobar"); - - Properties prop = new Properties(); - prop.put("var.result", "1111 ${a.b.c} xxx ${x.y.z} yyy"); - - StramClientUtils.evalProperties(prop, conf); - - Assert.assertEquals("1111 123 xxx foobar yyy", prop.get("var.result")); - } -} http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/48cd2e8f/engine/src/test/java/com/datatorrent/stram/client/StramClientUtilsTest.java ---------------------------------------------------------------------- diff --git a/engine/src/test/java/com/datatorrent/stram/client/StramClientUtilsTest.java b/engine/src/test/java/com/datatorrent/stram/client/StramClientUtilsTest.java index f9fca05..51193ef 100644 --- a/engine/src/test/java/com/datatorrent/stram/client/StramClientUtilsTest.java +++ b/engine/src/test/java/com/datatorrent/stram/client/StramClientUtilsTest.java @@ -22,6 +22,7 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.util.List; +import java.util.Properties; import org.junit.Assert; import org.junit.Test; @@ -39,6 +40,42 @@ import com.datatorrent.stram.util.ConfigUtils; public class StramClientUtilsTest { + @Test + public void testEvalExpression() throws Exception + { + Configuration conf = new Configuration(); + conf.set("a.b.c", "123"); + conf.set("d.e.f", "456"); + conf.set("x.y.z", "foobar"); + + Properties prop = new Properties(); + prop.put("product.result", "Product result is {% (_prop[\"a.b.c\"] * _prop[\"d.e.f\"]).toFixed(0) %}..."); + prop.put("concat.result", "Concat result is {% _prop[\"x.y.z\"] %} ... {% _prop[\"a.b.c\"] %} blah"); + + StramClientUtils.evalProperties(prop, conf); + + Assert.assertEquals("Product result is " + (123 * 456) + "...", prop.get("product.result")); + Assert.assertEquals("Concat result is foobar ... 123 blah", prop.get("concat.result")); + Assert.assertEquals("123", conf.get("a.b.c")); + Assert.assertEquals("456", conf.get("d.e.f")); + Assert.assertEquals("foobar", conf.get("x.y.z")); + } + + @Test + public void testVariableSubstitution() throws Exception + { + Configuration conf = new Configuration(); + conf.set("a.b.c", "123"); + conf.set("x.y.z", "foobar"); + + Properties prop = new Properties(); + prop.put("var.result", "1111 ${a.b.c} xxx ${x.y.z} yyy"); + + StramClientUtils.evalProperties(prop, conf); + + Assert.assertEquals("1111 123 xxx foobar yyy", prop.get("var.result")); + } + private String getHostString(String host) throws UnknownHostException { InetAddress address = InetAddress.getByName(host);
