Repository: apex-core Updated Branches: refs/heads/master 5f95ee0e9 -> b4a4e0517
APEXCORE-593 apex cli get-app-package-info could not retrieve properties defined in properties.xml Project: http://git-wip-us.apache.org/repos/asf/apex-core/repo Commit: http://git-wip-us.apache.org/repos/asf/apex-core/commit/82f2761d Tree: http://git-wip-us.apache.org/repos/asf/apex-core/tree/82f2761d Diff: http://git-wip-us.apache.org/repos/asf/apex-core/diff/82f2761d Branch: refs/heads/master Commit: 82f2761dd5c31968e95d85b51a2103b7ebe2f293 Parents: 16d1bf6 Author: vikram <[email protected]> Authored: Fri Mar 24 10:58:20 2017 +0530 Committer: vikram <[email protected]> Committed: Fri Mar 24 12:17:58 2017 +0530 ---------------------------------------------------------------------- .../datatorrent/stram/client/AppPackage.java | 4 ++ .../stram/client/AppPackageTest.java | 71 +++++++++++++++++--- .../java/com/example/mydtapp/Application.java | 6 +- .../java/com/example/mydtapp/Application2.java | 2 +- .../com/example/mydtapp/StdoutOperator.java | 12 ++++ .../java/com/example/mydtapp/TestModule.java | 57 ++++++++++++++++ .../com/example/mydtapp/TestModuleOperator.java | 48 +++++++++++++ .../src/main/resources/META-INF/properties.xml | 8 +++ 8 files changed, 195 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/apex-core/blob/82f2761d/engine/src/main/java/com/datatorrent/stram/client/AppPackage.java ---------------------------------------------------------------------- diff --git a/engine/src/main/java/com/datatorrent/stram/client/AppPackage.java b/engine/src/main/java/com/datatorrent/stram/client/AppPackage.java index fd95649..238b646 100644 --- a/engine/src/main/java/com/datatorrent/stram/client/AppPackage.java +++ b/engine/src/main/java/com/datatorrent/stram/client/AppPackage.java @@ -380,6 +380,10 @@ public class AppPackage extends JarFile Configuration config = new Configuration(); + for (Map.Entry<String, PropertyInfo> entry : defaultProperties.entrySet()) { + config.set(entry.getKey(), entry.getValue().getValue()); + } + List<String> absClassPath = new ArrayList<>(classPath); for (int i = 0; i < absClassPath.size(); i++) { String path = absClassPath.get(i); http://git-wip-us.apache.org/repos/asf/apex-core/blob/82f2761d/engine/src/test/java/com/datatorrent/stram/client/AppPackageTest.java ---------------------------------------------------------------------- diff --git a/engine/src/test/java/com/datatorrent/stram/client/AppPackageTest.java b/engine/src/test/java/com/datatorrent/stram/client/AppPackageTest.java index aae3913..20550a9 100644 --- a/engine/src/test/java/com/datatorrent/stram/client/AppPackageTest.java +++ b/engine/src/test/java/com/datatorrent/stram/client/AppPackageTest.java @@ -27,6 +27,7 @@ import java.util.Set; import org.codehaus.jackson.JsonGenerationException; import org.codehaus.jackson.map.JsonMappingException; +import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.junit.AfterClass; @@ -95,22 +96,57 @@ public class AppPackageTest Assert.assertEquals(System.getProperty("apex.version", "3.4.0"), json.getString("dtEngineVersion")); Assert.assertEquals("lib/*.jar", json.getJSONArray("classPath").getString(0)); - JSONObject application1 = json.getJSONArray("applications").getJSONObject(0); - JSONObject application2 = json.getJSONArray("applications").getJSONObject(1); + // Test if there are fixed number of applications + Assert.assertEquals("Number of applications", 2, json.getJSONArray("applications").length()); + JSONArray applications = json.getJSONArray("applications"); Map<String, JSONObject> apps = new HashMap<>(); - apps.put(application1.getString("name"), application1); - apps.put(application2.getString("name"), application2); + for (int i = 0; i < applications.length(); i++) { + JSONObject application = applications.getJSONObject(i); + apps.put(application.getString("name"), application); + } + + // Retrieve applications found + JSONObject myFirstApplication = apps.get("MyFirstApplication"); + JSONObject mySecondApplication = apps.get("MySecondApplication"); + + // Testing if expected applications are found + Assert.assertNotNull("MyFirstApplication not found", myFirstApplication); + Assert.assertNotNull("MySecondApplication not found", mySecondApplication); + + // Tests related to MyFirstApplication start + Assert.assertEquals("mydtapp-1.0-SNAPSHOT.jar", myFirstApplication.getString("file")); + String errorStackTrace = myFirstApplication.getString("errorStackTrace"); + Assert.assertEquals("ErrorStackTrace", "null", errorStackTrace); + + JSONObject dag = myFirstApplication.getJSONObject("dag"); + Assert.assertNotNull("MyFirstApplication DAG does not exist", dag); + Assert.assertEquals("Number of streams", 1, dag.getJSONArray("streams").length()); + Assert.assertEquals("Number of operators", 3, dag.getJSONArray("operators").length()); + + JSONArray operatorsJSONArray = dag.getJSONArray("operators"); + Map<String, JSONObject> operatorMap = new HashMap<>(); - Assert.assertEquals(true, apps.containsKey("MyFirstApplication")); - Assert.assertEquals(true, apps.containsKey("MySecondApplication")); + for (int i = 0; i < operatorsJSONArray.length(); i++) { + JSONObject operator = operatorsJSONArray.getJSONObject(i); + operatorMap.put(operator.getString("name"), operator); + } + + JSONObject operator = operatorMap.get("rand"); + Assert.assertNotNull("Input Operator not found", operator); + Assert.assertEquals("Input operator class", "com.example.mydtapp.RandomNumberGenerator", operator.getJSONObject("properties").get("@class")); - Assert.assertEquals("mydtapp-1.0-SNAPSHOT.jar", apps.get("MyFirstApplication").getString("file")); + operator = operatorMap.get("stdout"); + Assert.assertNotNull("Output Operator not found", operator); + Assert.assertEquals("Value for Output Operator's testInput:", "default-value-1", operator.getJSONObject("properties").get("testInput")); + Assert.assertEquals("Output Operator class", "com.example.mydtapp.StdoutOperator", operator.getJSONObject("properties").get("@class")); - JSONObject dag = apps.get("MyFirstApplication").getJSONObject("dag"); - Assert.assertTrue("There is at least one stream", dag.getJSONArray("streams").length() >= 1); - Assert.assertEquals("There are two operator", 2, dag.getJSONArray("operators").length()); + operator = operatorMap.get("testModule$testModuleOperator"); + Assert.assertNotNull("Test Module Operator not found", operator); + Assert.assertEquals("Test Module Operator class", "com.example.mydtapp.TestModuleOperator", operator.getJSONObject("properties").get("@class")); + Assert.assertEquals("Value for Test Module Operator's testInput:", "default-value-1", operator.getJSONObject("properties").get("testInput")); + // Tests related to MyFirstApplication end Assert.assertTrue("app package extraction folder should be retained", new File("target/testapp").exists()); yap.cleanContent(); @@ -158,4 +194,19 @@ public class AppPackageTest result = jomp.getContext(null).writeValueAsString(propertyInfo); Assert.assertEquals("{\"value\":\"test-value\",\"description\":\"test-description\"}", result); } + + public void testDefaultProperties() + { + + Map<String, AppPackage.PropertyInfo> defaultProperties = ap.getDefaultProperties(); + Assert.assertEquals(8, defaultProperties.size()); + Assert.assertEquals("package-default", defaultProperties.get("dt.test.1").getValue()); + Assert.assertEquals("package-default", defaultProperties.get("dt.test.2").getValue()); + Assert.assertEquals("package-default", defaultProperties.get("dt.test.3").getValue()); + Assert.assertEquals("package-default", defaultProperties.get("dt.test.4").getValue()); + Assert.assertEquals("package-default", defaultProperties.get("dt.test.5").getValue()); + Assert.assertEquals("package-default", defaultProperties.get("dt.test.6").getValue()); + Assert.assertEquals("default-value-1", defaultProperties.get("dt.operator.testModule.prop.testInput").getValue()); + Assert.assertEquals("default-value-1", defaultProperties.get("dt.operator.stdout.prop.testInput").getValue()); + } } http://git-wip-us.apache.org/repos/asf/apex-core/blob/82f2761d/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/Application.java ---------------------------------------------------------------------- diff --git a/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/Application.java b/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/Application.java index 67b2569..1431beb 100644 --- a/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/Application.java +++ b/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/Application.java @@ -20,12 +20,13 @@ package com.example.mydtapp; import org.apache.hadoop.conf.Configuration; +import com.datatorrent.api.Module; import com.datatorrent.api.annotation.ApplicationAnnotation; import com.datatorrent.api.StreamingApplication; import com.datatorrent.api.DAG; import com.datatorrent.api.DAG.Locality; -@ApplicationAnnotation(name="MyFirstApplication") +@ApplicationAnnotation(name = "MyFirstApplication") public class Application implements StreamingApplication { @@ -37,7 +38,8 @@ public class Application implements StreamingApplication RandomNumberGenerator rand = dag.addOperator("rand", RandomNumberGenerator.class); StdoutOperator stdout = dag.addOperator("stdout", new StdoutOperator()); - + // This module will be added to dag for testing purpose but will not be connected in a dag. + Module testModule = dag.addModule("testModule", com.example.mydtapp.TestModule.class); dag.addStream("data", rand.out, stdout.in).setLocality(Locality.CONTAINER_LOCAL); } } http://git-wip-us.apache.org/repos/asf/apex-core/blob/82f2761d/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/Application2.java ---------------------------------------------------------------------- diff --git a/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/Application2.java b/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/Application2.java index cc4b6d5..76d4777 100644 --- a/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/Application2.java +++ b/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/Application2.java @@ -25,7 +25,7 @@ import com.datatorrent.api.StreamingApplication; import com.datatorrent.api.DAG; import com.datatorrent.api.DAG.Locality; -@ApplicationAnnotation(name="MySecondApplication") +@ApplicationAnnotation(name = "MySecondApplication") public class Application2 implements StreamingApplication { http://git-wip-us.apache.org/repos/asf/apex-core/blob/82f2761d/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/StdoutOperator.java ---------------------------------------------------------------------- diff --git a/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/StdoutOperator.java b/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/StdoutOperator.java index b4729c9..6e6c4f6 100644 --- a/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/StdoutOperator.java +++ b/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/StdoutOperator.java @@ -27,6 +27,18 @@ import com.datatorrent.common.util.BaseOperator; public class StdoutOperator extends BaseOperator { + private String testInput; + + public String getTestInput() + { + return testInput; + } + + public void setTestInput(String testInput) + { + this.testInput = testInput; + } + public final transient DefaultInputPort<Object> in = new DefaultInputPort<Object>() { @Override http://git-wip-us.apache.org/repos/asf/apex-core/blob/82f2761d/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/TestModule.java ---------------------------------------------------------------------- diff --git a/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/TestModule.java b/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/TestModule.java new file mode 100644 index 0000000..6aab91a --- /dev/null +++ b/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/TestModule.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.example.mydtapp; + +import org.apache.hadoop.conf.Configuration; +import com.datatorrent.api.DAG; +import com.datatorrent.api.Module; + +/** + * This is a test module which can be added to DAG . It will add one dummy operator as well. + */ +public class TestModule implements Module +{ + private String testInput; + + public final transient ProxyOutputPort<String> outputs = new ProxyOutputPort<String>(); + + public final transient ProxyInputPort<String> inputs = new ProxyInputPort<String>(); + + @Override + public void populateDAG(DAG dag, Configuration configuration) + { + if (testInput == null) { + throw new NullPointerException(); + } + + TestModuleOperator testModuleOperator = dag.addOperator("testModuleOperator", TestModuleOperator.class); + testModuleOperator.setTestInput(this.getTestInput()); + } + + public String getTestInput() + { + return testInput; + } + + public void setTestInput(String testInput) + { + this.testInput = testInput; + } +} http://git-wip-us.apache.org/repos/asf/apex-core/blob/82f2761d/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/TestModuleOperator.java ---------------------------------------------------------------------- diff --git a/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/TestModuleOperator.java b/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/TestModuleOperator.java new file mode 100644 index 0000000..bc09e53 --- /dev/null +++ b/engine/src/test/resources/testAppPackage/mydtapp/src/main/java/com/example/mydtapp/TestModuleOperator.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.example.mydtapp; + +import com.datatorrent.api.DefaultOutputPort; +import com.datatorrent.api.InputOperator; +import com.datatorrent.common.util.BaseOperator; + + +public class TestModuleOperator extends BaseOperator implements InputOperator +{ + + private String testInput; + + public String getTestInput() + { + return testInput; + } + + public void setTestInput(String testInput) + { + this.testInput = testInput; + } + + public final transient DefaultOutputPort<Object> out = new DefaultOutputPort<Object>(); + + @Override + public void emitTuples() + { + + } +} http://git-wip-us.apache.org/repos/asf/apex-core/blob/82f2761d/engine/src/test/resources/testAppPackage/mydtapp/src/main/resources/META-INF/properties.xml ---------------------------------------------------------------------- diff --git a/engine/src/test/resources/testAppPackage/mydtapp/src/main/resources/META-INF/properties.xml b/engine/src/test/resources/testAppPackage/mydtapp/src/main/resources/META-INF/properties.xml index e074fbe..937a803 100644 --- a/engine/src/test/resources/testAppPackage/mydtapp/src/main/resources/META-INF/properties.xml +++ b/engine/src/test/resources/testAppPackage/mydtapp/src/main/resources/META-INF/properties.xml @@ -50,4 +50,12 @@ <property> <name>dt.test.required.2</name> </property> + <property> + <name>dt.operator.testModule.prop.testInput</name> + <value>default-value-1</value> + </property> + <property> + <name>dt.operator.stdout.prop.testInput</name> + <value>default-value-1</value> + </property> </configuration>
