Repository: oozie Updated Branches: refs/heads/master 69c5091ad -> 8b247f28f
http://git-wip-us.apache.org/repos/asf/oozie/blob/8b247f28/core/src/test/java/org/apache/oozie/service/TestSchemaService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestSchemaService.java b/core/src/test/java/org/apache/oozie/service/TestSchemaService.java index 88a10db..940868a 100644 --- a/core/src/test/java/org/apache/oozie/service/TestSchemaService.java +++ b/core/src/test/java/org/apache/oozie/service/TestSchemaService.java @@ -18,11 +18,8 @@ package org.apache.oozie.service; -import org.apache.oozie.service.Services; import org.apache.oozie.service.SchemaService.SchemaName; import org.apache.oozie.test.XTestCase; -import org.apache.oozie.util.XmlUtils; -import org.jdom.Element; import javax.xml.validation.Validator; import javax.xml.transform.stream.StreamSource; @@ -32,26 +29,32 @@ import java.io.StringReader; public class TestSchemaService extends XTestCase { private static final String APP1 = "<workflow-app xmlns='uri:oozie:workflow:0.1' name='app'>" + - "<start to='end'/>" + - "<end name='end'/>" + + " <start to='end'/>" + + " <end name='end'/>" + "</workflow-app>"; private static final String APP_V2 = "<workflow-app xmlns='uri:oozie:workflow:0.2' name='app'>" + "<start to='end'/>" + "<end name='end'/>" + "</workflow-app>"; - private static final String APP_V25 = "<workflow-app xmlns='uri:oozie:workflow:0.2.5' name='app'>" + "<credentials></credentials>"+"<start to='end'/>" - + "<end name='end'/>" + "</workflow-app>"; + private static final String APP_V25 = "<?xml version=\"1.0\"?>\n" + + "<workflow-app xmlns=\"uri:oozie:workflow:0.2.5\" name=\"app\">\n" + + " <credentials/>\n" + + " <start to=\"end\"/>\n" + + " <end name=\"end\"/>\n" + + "</workflow-app>"; - private static final String WF_SLA_APP = "<workflow-app xmlns='uri:oozie:workflow:0.2' name='app' xmlns:sla='uri:oozie:sla:0.1'>" - + "<start to='end'/>" - + "<end name='end'/>" - + "<sla:info> <sla:app-name>5</sla:app-name> <sla:nominal-time>2009-03-06T010:00Z</sla:nominal-time> " - + "<sla:should-start>5</sla:should-start> <sla:should-end>50</sla:should-end> " + private static final String WF_SLA_APP = "<workflow-app xmlns='uri:oozie:workflow:0.2' name='app'" + + " xmlns:sla='uri:oozie:sla:0.1'>" + + " <start to='end'/>" + + " <end name='end'/>" + + " <sla:info><sla:app-name>5</sla:app-name> <sla:nominal-time>2009-03-06T010:00Z</sla:nominal-time> " + + " <sla:should-start>5</sla:should-start> <sla:should-end>50</sla:should-end> " + "<sla:alert-contact>a...@example.com</sla:alert-contact> <sla:dev-contact>a...@example.com</sla:dev-contact>" + " <sla:qa-contact>a...@example.com</sla:qa-contact> <sla:se-contact>a...@example.com</sla:se-contact>" + "</sla:info>" + "</workflow-app>"; - private static final String WF_SLA_APP_NW = "<workflow-app xmlns='uri:oozie:workflow:0.1' name='app' xmlns:sla='uri:oozie:sla:0.1'>" + private static final String WF_SLA_APP_NW = "<workflow-app xmlns='uri:oozie:workflow:0.1' name='app'" + + " xmlns:sla='uri:oozie:sla:0.1'>" + "<start to='end'/>" + "<end name='end'/>" + "<sla:info> <sla:app-name>5</sla:app-name> <sla:nominal-time>2009-03-06T010:00Z</sla:nominal-time> " @@ -60,14 +63,6 @@ public class TestSchemaService extends XTestCase { + " <sla:qa-contact>a...@example.com</sla:qa-contact> <sla:se-contact>a...@example.com</sla:se-contact>" + "</sla:info>" + "</workflow-app>"; - private static final String COORD_APP1 = "<coordinator-app name=\"NAME\" frequency=\"${coord:days(1)}\" start=\"${start}\" end=\"${end}\" timezone=\"${timezone}\" xmlns=\"uri:oozie:coordinator:0.2\">" - + "<controls> <timeout>${timeout}</timeout> <concurrency>${concurrency_level}</concurrency> <execution>${execution_order}</execution> </controls>" - + "<datasets> <include>${include_ds_files}</include> <!-- Synchronous datasets --> <dataset name=\"local_a\" frequency=\"${coord:days(7)}\" initial-instance=\"${start}\" timezone=\"${timezone}\"> " - + "<uri-template>${baseFsURI}/${YEAR}/${DAY}</uri-template> </dataset> </datasets> " - + "<input-events> <data-in name=\"A\" dataset=\"a\"> <instance>${coord:latest(0)}</instance> </data-in> <data-in name=\"B\" dataset=\"b\"> <start-instance>${coord:current(-2)}</start-instance> <end-instance>${coord:current(0)}</end-instance> </data-in> </input-events> <output-events> " - + "<data-out name=\"LOCAL_A\" dataset=\"local_a\"> <instance>${coord:current(0)}</instance> </data-out> </output-events> <action> <workflow> <app-path>${app_path}</app-path> <configuration> <property> <name>inputA</name> <value>${coord:dataIn('A')}</value> </property> <property> <name>inputB</name> <value>${coord:dataIn('B')}</value> </property> <property> <name>inputB</name> <value>${coord:dataOut('LOCAL_A')}</value> </property> <property> <name>TESTING</name> <value>${start}</value> </property> </configuration> </workflow> </action> </coordinator-app>"; - - private static final String APP2 = "<workflow-app xmlns='uri:oozie:workflow:0.1' name='app'>" + "<start to='a'/>" + "<action name='a'>" + @@ -101,10 +96,230 @@ public class TestSchemaService extends XTestCase { "<end name='end'/>" + "</workflow-app>"; + private static final String WF_GLOBAL_LAUNCHER_CONF = "<workflow-app xmlns=\"uri:oozie:workflow:1.0\" name=\"test-wf\">\n" + + " <global>\n" + + " <launcher>\n" + + " <memory.mb>1024</memory.mb>\n" + + " <vcores>2</vcores>\n" + + " <java-opts>dummyJavaOpts</java-opts>\n" + + " <env>dummyEnv</env>\n" + + " <queue>dummyQueue</queue>\n" + + " <sharelib>a,b,c</sharelib>\n" + + " </launcher>\n" + + " </global>\n" + + " <start to=\"a\"/>\n" + + " <action name=\"a\">\n" + + " <fs>\n" + + " <mkdir path='/tmp'/>\n" + + " </fs>\n" + + " <ok to=\"e\"/>\n" + + " <error to=\"k\"/>\n" + + " </action>\n" + + " <kill name=\"k\">\n" + + " <message>kill</message>\n" + + " </kill>\n" + + " <end name=\"e\"/>\n" + + "</workflow-app>\n"; + + private static final String HIVE_ACTION_LAUNCHER_CONF = + "<workflow-app xmlns=\"uri:oozie:workflow:1.0\" name=\"hive-wf\">\n" + + " <start to=\"hive-node\"/>\n" + + " <action name=\"hive-node\">\n" + + " <hive xmlns=\"uri:oozie:hive-action:1.0\">\n" + + " <job-tracker>${jobTracker}</job-tracker>\n" + + " <name-node>${nameNode}</name-node>\n" + + " <prepare>\n" + + " <delete path=\"${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/hive\"/>\n" + + " <mkdir path=\"${nameNode}/user/${wf:user()}/${examplesRoot}/output-data\"/>\n" + + " </prepare>\n" + + " <launcher>\n" + + " <memory.mb>1024</memory.mb>\n" + + " <vcores>2</vcores>\n" + + " <java-opts>dummyJavaOpts</java-opts>\n" + + " <env>dummyEnv</env>\n" + + " <queue>dummyQueue</queue>\n" + + " <sharelib>a,b,c</sharelib>\n" + + " </launcher>\n" + + " <configuration>\n" + + " <property>\n" + + " <name>mapred.job.queue.name</name>\n" + + " <value>${queueName}</value>\n" + + " </property>\n" + + " </configuration>\n" + + " <script>script.q</script>\n" + + " <param>INPUT=/user/${wf:user()}/${examplesRoot}/input-data/table</param>\n" + + " <param>OUTPUT=/user/${wf:user()}/${examplesRoot}/output-data/hive</param>\n" + + " </hive>\n" + + " <ok to=\"end\"/>\n" + + " <error to=\"fail\"/>\n" + + " </action>\n" + + " <kill name=\"fail\">\n" + + " <message>Hive failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>\n" + + " </kill>\n" + + " <end name=\"end\"/>\n" + + "</workflow-app>\n"; + + private static final String HIVE2_ACTION_LAUNCHER_CONF = + "<workflow-app xmlns=\"uri:oozie:workflow:1.0\" name=\"hive2-wf\">\n" + + " <start to=\"hive2-node\"/>\n" + + " <action name=\"hive2-node\">\n" + + " <hive2 xmlns=\"uri:oozie:hive2-action:1.0\">\n" + + " <job-tracker>${jobTracker}</job-tracker>\n" + + " <name-node>${nameNode}</name-node>\n" + + " <prepare>\n" + + " <delete path=\"${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/hive2\"/>\n" + + " <mkdir path=\"${nameNode}/user/${wf:user()}/${examplesRoot}/output-data\"/>\n" + + " </prepare>\n" + + " <launcher>\n" + + " <memory.mb>1024</memory.mb>\n" + + " <vcores>2</vcores>\n" + + " <java-opts>dummyJavaOpts</java-opts>\n" + + " <env>dummyEnv</env>\n" + + " <queue>dummyQueue</queue>\n" + + " <sharelib>a,b,c</sharelib>\n" + + " </launcher>\n" + + " <configuration>\n" + + " <property>\n" + + " <name>mapred.job.queue.name</name>\n" + + " <value>${queueName}</value>\n" + + " </property>\n" + + " </configuration>\n" + + " <jdbc-url>${jdbcURL}</jdbc-url>\n" + + " <script>script.q</script>\n" + + " <param>INPUT=/user/${wf:user()}/${examplesRoot}/input-data/table</param>\n" + + " <param>OUTPUT=/user/${wf:user()}/${examplesRoot}/output-data/hive2</param>\n" + + " </hive2>\n" + + " <ok to=\"end\"/>\n" + + " <error to=\"fail\"/>\n" + + " </action>\n" + + " <kill name=\"fail\">\n" + + " <message>Hive2 (Beeline) action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>\n" + + " </kill>\n" + + " <end name=\"end\"/>\n" + + "</workflow-app>\n"; + + private static final String SHELL_ACTION_LAUNCHER_CONF = "<workflow-app xmlns=\"uri:oozie:workflow:1.0\" " + + " name=\"shell-wf\">\n" + + " <start to=\"shell-node\"/>\n" + + " <action name=\"shell-node\">\n" + + " <shell xmlns=\"uri:oozie:shell-action:1.0\">\n" + + " <job-tracker>${jobTracker}</job-tracker>\n" + + " <name-node>${nameNode}</name-node>\n" + + " <launcher>\n" + + " <memory.mb>1024</memory.mb>\n" + + " </launcher>\n" + + " <configuration>\n" + + " <property>\n" + + " <name>mapred.job.queue.name</name>\n" + + " <value>${queueName}</value>\n" + + " </property>\n" + + " </configuration>\n" + + " <exec>echo</exec>\n" + + " <argument>my_output=Hello Oozie</argument>\n" + + " <capture-output/>\n" + + " </shell>\n" + + " <ok to=\"check-output\"/>\n" + + " <error to=\"fail\"/>\n" + + " </action>\n" + + " <decision name=\"check-output\">\n" + + " <switch>\n" + + " <case to=\"end\">\n" + + " ${wf:actionData('shell-node')['my_output'] eq 'Hello Oozie'}\n" + + " </case>\n" + + " <default to=\"fail-output\"/>\n" + + " </switch>\n" + + " </decision>\n" + + " <kill name=\"fail\">\n" + + " <message>Shell action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>\n" + + " </kill>\n" + + " <kill name=\"fail-output\">\n" + + " <message>Incorrect output</message>\n" + + " </kill>\n" + + " <end name=\"end\"/>\n" + + "</workflow-app>"; + + private static final String SQQP_ACTION_LAUNCHER_CONF = "<workflow-app xmlns=\"uri:oozie:workflow:1.0\" name=\"sqoop-wf\">\n" + + " <start to=\"sqoop-node\"/>\n" + + " <action name=\"sqoop-node\">\n" + + " <sqoop xmlns=\"uri:oozie:sqoop-action:1.0\">\n" + + " <job-tracker>${jobTracker}</job-tracker>\n" + + " <name-node>${nameNode}</name-node>\n" + + " <prepare>\n" + + " <delete path=\"${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/sqoop\"/>\n" + + " <mkdir path=\"${nameNode}/user/${wf:user()}/${examplesRoot}/output-data\"/>\n" + + " </prepare>\n" + + " <launcher>\n" + + " <memory.mb>1024</memory.mb>\n" + + " </launcher>\n" + + " <configuration>\n" + + " <property>\n" + + " <name>mapred.job.queue.name</name>\n" + + " <value>${queueName}</value>\n" + + " </property>\n" + + " </configuration>\n" + + " <command>import --connect jdbc:hsqldb:file:db.hsqldb --table TT --target-dir " + + "/user/${wf:user()}/${examplesRoot}/output-data/sqoop -m 1</command>\n" + + " <file>db.hsqldb.properties#db.hsqldb.properties</file>\n" + + " <file>db.hsqldb.script#db.hsqldb.script</file>\n" + + " </sqoop>\n" + + " <ok to=\"end\"/>\n" + + " <error to=\"fail\"/>\n" + + " </action>\n" + + " <kill name=\"fail\">\n" + + " <message>Sqoop failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>\n" + + " </kill>\n" + + " <end name=\"end\"/>\n" + + "</workflow-app>"; + + private static final String SPARK_ACTION_LAUNCHER_CONF = "<workflow-app xmlns='uri:oozie:workflow:1.0' " + + "name='SparkFileCopy'>\n" + + " <start to='spark-node' />\n" + + " <action name='spark-node'>\n" + + " <spark xmlns=\"uri:oozie:spark-action:1.0\">\n" + + " <job-tracker>${jobTracker}</job-tracker>\n" + + " <name-node>${nameNode}</name-node>\n" + + " <prepare>\n" + + " <delete path=\"${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/spark\"/>\n" + + " </prepare>\n" + + " <launcher>\n" + + " <memory.mb>1024</memory.mb>\n" + + " <vcores>2</vcores>\n" + + " <java-opts>dummyJavaOpts</java-opts>\n" + + " <env>dummyEnv</env>\n" + + " <queue>dummyQueue</queue>\n" + + " <sharelib>a,b,c</sharelib>\n" + + " </launcher>\n" + + " <master>${master}</master>\n" + + " <name>Spark-FileCopy</name>\n" + + " <class>org.apache.oozie.example.SparkFileCopy</class>\n" + + " <jar>${nameNode}/user/${wf:user()}/${examplesRoot}/apps/spark/lib/oozie-examples.jar</jar>\n" + + " <arg>${nameNode}/user/${wf:user()}/${examplesRoot}/input-data/text/data.txt</arg>\n" + + " <arg>${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/spark</arg>\n" + + " </spark>\n" + + " <ok to=\"end\" />\n" + + " <error to=\"fail\" />\n" + + " </action>\n" + + " <kill name=\"fail\">\n" + + " <message>Workflow failed, error\n" + + " message[${wf:errorMessage(wf:lastErrorNode())}]\n" + + " </message>\n" + + " </kill>\n" + + " <end name='end' />\n" + + "</workflow-app>\n"; + + private SchemaService wss; + private Validator workflowValidator; + private Validator coordinatorValidator; + private Validator bundleValidator; + @Override protected void setUp() throws Exception { super.setUp(); new Services().init(); + wss = Services.get().get(SchemaService.class); + workflowValidator = wss.getSchema(SchemaName.WORKFLOW).newValidator(); + coordinatorValidator = wss.getSchema(SchemaName.COORDINATOR).newValidator(); + bundleValidator = wss.getSchema(SchemaName.BUNDLE).newValidator(); } @Override @@ -118,27 +333,19 @@ public class TestSchemaService extends XTestCase { } public void testWfSchema() throws Exception { - SchemaService wss = Services.get().get(SchemaService.class); - Validator validator = wss.getSchema(SchemaName.WORKFLOW).newValidator(); - validator.validate(new StreamSource(new StringReader(APP1))); + workflowValidator.validate(new StreamSource(new StringReader(APP1))); } public void testWfMultipleJavaOpts() throws Exception { - SchemaService wss = Services.get().get(SchemaService.class); - Validator validator = wss.getSchema(SchemaName.WORKFLOW).newValidator(); - validator.validate(new StreamSource(new StringReader(WF_4_MULTIPLE_JAVA_OPTS))); + workflowValidator.validate(new StreamSource(new StringReader(WF_4_MULTIPLE_JAVA_OPTS))); } public void testWfSchemaV2() throws Exception { - SchemaService wss = Services.get().get(SchemaService.class); - Validator validator = wss.getSchema(SchemaName.WORKFLOW).newValidator(); - validator.validate(new StreamSource(new StringReader(APP_V2))); + workflowValidator.validate(new StreamSource(new StringReader(APP_V2))); } public void testWfSchemaV25() throws Exception { - SchemaService wss = Services.get().get(SchemaService.class); - Validator validator = wss.getSchema(SchemaName.WORKFLOW).newValidator(); - validator.validate(new StreamSource(new StringReader(APP_V25))); + workflowValidator.validate(new StreamSource(new StringReader(APP_V25))); } public void testExtSchema() throws Exception { @@ -151,16 +358,12 @@ public class TestSchemaService extends XTestCase { } public void testWfSLASchema() throws Exception { - SchemaService wss = Services.get().get(SchemaService.class); - Validator validator = wss.getSchema(SchemaName.WORKFLOW).newValidator(); - validator.validate(new StreamSource(new StringReader(WF_SLA_APP))); + workflowValidator.validate(new StreamSource(new StringReader(WF_SLA_APP))); } public void testWfSLASchemaNW() throws Exception { - SchemaService wss = Services.get().get(SchemaService.class); - Validator validator = wss.getSchema(SchemaName.WORKFLOW).newValidator(); try { - validator.validate(new StreamSource(new StringReader(WF_SLA_APP_NW))); + workflowValidator.validate(new StreamSource(new StringReader(WF_SLA_APP_NW))); fail("Schema service check does not work"); } catch (Exception ex) { @@ -169,59 +372,199 @@ public class TestSchemaService extends XTestCase { } public void testCoordSchema() throws Exception { - SchemaService wss = Services.get().get(SchemaService.class); - Validator validator = wss.getSchema(SchemaName.COORDINATOR).newValidator(); - String COORD_APP1 = "<coordinator-app name='NAME' frequency='${coord:days(1)}' start='2009-02-01T01:00Z' end='2009-02-03T23:59Z' timezone='UTC' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns='uri:oozie:coordinator:0.1' xmlns:sla='uri:oozie:sla:0.1'> " - + "<controls> <timeout>10</timeout> <concurrency>2</concurrency> <execution>LIFO</execution> </controls> <datasets> <dataset name='a' frequency='${coord:days(7)}' initial-instance='2009-02-01T01:00Z' timezone='UTC'> <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template> </dataset> <dataset name='local_a' frequency='${coord:days(7)}' initial-instance='2009-02-01T01:00Z' timezone='UTC'> <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template> </dataset> </datasets> <input-events> <data-in name='A' dataset='a'> <instance>${coord:latest(0)}</instance> </data-in> </input-events> <output-events> <data-out name='LOCAL_A' dataset='local_a'> <instance>${coord:current(-1)}</instance> </data-out> </output-events> <action> <workflow> <app-path>hdfs:///tmp/workflows/</app-path> <configuration> <property> <name>inputA</name> <value>${coord:dataIn('A')}</value> </property> <property> <name>inputB</name> <value>${coord:dataOut('LOCAL_A')}</value> </p roperty></configuration> </workflow> " - + "</action> </coordinator-app>"; - - Element e = XmlUtils.parseXml(COORD_APP1); - //System.out.println("XML :"+ XmlUtils.prettyPrint(e)); - validator.validate(new StreamSource(new StringReader(COORD_APP1))); + String COORD_APP1 = "<?xml version=\"1.0\"?>\n" + + "<coordinator-app xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"; xmlns=\"uri:oozie:coordinator:0.1\"" + + " xmlns:sla=\"uri:oozie:sla:0.1\" name=\"NAME\" frequency=\"${coord:days(1)}\" start=\"2009-02-01T01:00Z\"" + + " end=\"2009-02-03T23:59Z\" timezone=\"UTC\">\n" + + " <controls>\n" + + " <timeout>10</timeout>\n" + + " <concurrency>2</concurrency>\n" + + " <execution>LIFO</execution>\n" + + " </controls>\n" + + " <datasets>\n" + + " <dataset name=\"a\" frequency=\"${coord:days(7)}\" initial-instance=\"2009-02-01T01:00Z\"" + + " timezone=\"UTC\">\n" + + " <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template>\n" + + " </dataset>\n" + + " <dataset name=\"local_a\" frequency=\"${coord:days(7)}\" initial-instance=\"2009-02-01T01:00Z\"" + + " timezone=\"UTC\">\n" + + " <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template>\n" + + " </dataset>\n" + + " </datasets>\n" + + " <input-events>\n" + + " <data-in name=\"A\" dataset=\"a\">\n" + + " <instance>${coord:latest(0)}</instance>\n" + + " </data-in>\n" + + " </input-events>\n" + + " <output-events>\n" + + " <data-out name=\"LOCAL_A\" dataset=\"local_a\">\n" + + " <instance>${coord:current(-1)}</instance>\n" + + " </data-out>\n" + + " </output-events>\n" + + " <action>\n" + + " <workflow>\n" + + " <app-path>hdfs:///tmp/workflows/</app-path>\n" + + " <configuration>\n" + + " <property>\n" + + " <name>inputA</name>\n" + + " <value>${coord:dataIn('A')}</value>\n" + + " </property>\n" + + " <property>\n" + + " <name>inputB</name>\n" + + " <value>${coord:dataOut('LOCAL_A')}</value>\n" + + " </property>\n" + + " </configuration>\n" + + " </workflow>\n" + + " </action>\n" + + "</coordinator-app>"; + + coordinatorValidator.validate(new StreamSource(new StringReader(COORD_APP1))); } public void testCoordSchema2() throws Exception { - SchemaService wss = Services.get().get(SchemaService.class); - Validator validator = wss.getSchema(SchemaName.COORDINATOR).newValidator(); - String COORD_APP1 = "<coordinator-app name='NAME' frequency='${coord:days(1)}' start='2009-02-01T01:00Z' end='2009-02-03T23:59Z' timezone='UTC' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns='uri:oozie:coordinator:0.2' xmlns:sla='uri:oozie:sla:0.1'> " - + "<controls> <timeout>10</timeout> <concurrency>2</concurrency> <execution>LIFO</execution> <throttle>3</throttle></controls> <datasets> <dataset name='a' frequency='${coord:days(7)}' initial-instance='2009-02-01T01:00Z' timezone='UTC'> <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template> </dataset> <dataset name='local_a' frequency='${coord:days(7)}' initial-instance='2009-02-01T01:00Z' timezone='UTC'> <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template> </dataset> </datasets> <input-events> <data-in name='A' dataset='a'> <instance>${coord:latest(0)}</instance> </data-in> </input-events> <output-events> <data-out name='LOCAL_A' dataset='local_a'> <instance>${coord:current(-1)}</instance> </data-out> </output-events> <action> <workflow> <app-path>hdfs:///tmp/workflows/</app-path> <configuration> <property> <name>inputA</name> <value>${coord:dataIn('A')}</value> </property> <property> <name>inputB</name> <value>${coord:dataOut( 'LOCAL_A')}</value> </property></configuration> </workflow> " - + "</action> </coordinator-app>"; - - Element e = XmlUtils.parseXml(COORD_APP1); - //System.out.println("XML :"+ XmlUtils.prettyPrint(e)); - validator.validate(new StreamSource(new StringReader(COORD_APP1))); + String COORD_APP1 = "<coordinator-app xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\""; + + " xmlns=\"uri:oozie:coordinator:0.2\" xmlns:sla=\"uri:oozie:sla:0.1\" name=\"NAME\"" + + " frequency=\"${coord:days(1)}\" start=\"2009-02-01T01:00Z\" end=\"2009-02-03T23:59Z\" timezone=\"UTC\">\n" + + " <controls>\n" + + " <timeout>10</timeout>\n" + + " <concurrency>2</concurrency>\n" + + " <execution>LIFO</execution>\n" + + " <throttle>3</throttle>\n" + + " </controls>\n" + + " <datasets>\n" + + " <dataset name=\"a\" frequency=\"${coord:days(7)}\" initial-instance=\"2009-02-01T01:00Z\"" + + " timezone=\"UTC\">\n" + + " <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template>\n" + + " </dataset>\n" + + " <dataset name=\"local_a\" frequency=\"${coord:days(7)}\" initial-instance=\"2009-02-01T01:00Z\"" + + " timezone=\"UTC\">\n" + + " <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template>\n" + + " </dataset>\n" + + " </datasets>\n" + + " <input-events>\n" + + " <data-in name=\"A\" dataset=\"a\">\n" + + " <instance>${coord:latest(0)}</instance>\n" + + " </data-in>\n" + + " </input-events>\n" + + " <output-events>\n" + + " <data-out name=\"LOCAL_A\" dataset=\"local_a\">\n" + + " <instance>${coord:current(-1)}</instance>\n" + + " </data-out>\n" + + " </output-events>\n" + + " <action>\n" + + " <workflow>\n" + + " <app-path>hdfs:///tmp/workflows/</app-path>\n" + + " <configuration>\n" + + " <property>\n" + + " <name>inputA</name>\n" + + " <value>${coord:dataIn('A')}</value>\n" + + " </property>\n" + + " <property>\n" + + " <name>inputB</name>\n" + + " <value>${coord:dataOut('LOCAL_A')}</value>\n" + + " </property>\n" + + " </configuration>\n" + + " </workflow>\n" + + " </action>\n" + + "</coordinator-app>"; + + coordinatorValidator.validate(new StreamSource(new StringReader(COORD_APP1))); } public void testCoordSLASchema() throws Exception { - SchemaService wss = Services.get().get(SchemaService.class); - Validator validator = wss.getSchema(SchemaName.COORDINATOR) - .newValidator(); - String COORD_APP1 = "<coordinator-app name='NAME' frequency='${coord:days(1)}' start='2009-02-01T01:00Z' end='2009-02-03T23:59Z' timezone='UTC' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns='uri:oozie:coordinator:0.2' xmlns:sla='uri:oozie:sla:0.1'> " - + "<controls> <timeout>10</timeout> <concurrency>2</concurrency> <execution>LIFO</execution> </controls> <datasets> <dataset name='a' frequency='${coord:days(7)}' initial-instance='2009-02-01T01:00Z' timezone='UTC'> <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template> </dataset> <dataset name='local_a' frequency='${coord:days(7)}' initial-instance='2009-02-01T01:00Z' timezone='UTC'> <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template> </dataset> </datasets> <input-events> <data-in name='A' dataset='a'> <instance>${coord:latest(0)}</instance> </data-in> </input-events> <output-events> <data-out name='LOCAL_A' dataset='local_a'> <instance>${coord:current(-1)}</instance> </data-out> </output-events> <action> <workflow> <app-path>hdfs:///tmp/workflows/</app-path> <configuration> <property> <name>inputA</name> <value>${coord:dataIn('A')}</value> </property> <property> <name>inputB</name> <value>${coord:dataOut('LOCAL_A')}</value> </p roperty></configuration> </workflow> " - + "<sla:info> <sla:app-name>5</sla:app-name> <sla:nominal-time>2009-03-06T010:00Z</sla:nominal-time> " - + "<sla:should-start>5</sla:should-start> <sla:should-end>50</sla:should-end> " - + "<sla:alert-contact>a...@example.com</sla:alert-contact> <sla:dev-contact>a...@example.com</sla:dev-contact>" - + " <sla:qa-contact>a...@example.com</sla:qa-contact> <sla:se-contact>a...@example.com</sla:se-contact>" - + "</sla:info>" + "</action> </coordinator-app>"; - - Element e = XmlUtils.parseXml(COORD_APP1); - // System.out.println("XML :"+ XmlUtils.prettyPrint(e)); - validator.validate(new StreamSource(new StringReader(COORD_APP1))); + String COORD_APP1 = "<?xml version=\"1.0\"?>\n" + + "<coordinator-app xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"; xmlns=\"uri:oozie:coordinator:0.2\"" + + " xmlns:sla=\"uri:oozie:sla:0.1\" name=\"NAME\" frequency=\"${coord:days(1)}\" start=\"2009-02-01T01:00Z\"" + + " end=\"2009-02-03T23:59Z\" timezone=\"UTC\">\n" + + " <controls>\n" + + " <timeout>10</timeout>\n" + + " <concurrency>2</concurrency>\n" + + " <execution>LIFO</execution>\n" + + " </controls>\n" + + " <datasets>\n" + + " <dataset name=\"a\" frequency=\"${coord:days(7)}\" initial-instance=\"2009-02-01T01:00Z\"" + + " timezone=\"UTC\">\n" + + " <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template>\n" + + " </dataset>\n" + + " <dataset name=\"local_a\" frequency=\"${coord:days(7)}\" initial-instance=\"2009-02-01T01:00Z\"" + + " timezone=\"UTC\">\n" + + " <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template>\n" + + " </dataset>\n" + + " </datasets>\n" + + " <input-events>\n" + + " <data-in name=\"A\" dataset=\"a\">\n" + + " <instance>${coord:latest(0)}</instance>\n" + + " </data-in>\n" + + " </input-events>\n" + + " <output-events>\n" + + " <data-out name=\"LOCAL_A\" dataset=\"local_a\">\n" + + " <instance>${coord:current(-1)}</instance>\n" + + " </data-out>\n" + + " </output-events>\n" + + " <action>\n" + + " <workflow>\n" + + " <app-path>hdfs:///tmp/workflows/</app-path>\n" + + " <configuration>\n" + + " <property>\n" + + " <name>inputA</name>\n" + + " <value>${coord:dataIn('A')}</value>\n" + + " </property>\n" + + " <property>\n" + + " <name>inputB</name>\n" + + " <value>${coord:dataOut('LOCAL_A')}</value>\n" + + " </property>\n" + + " </configuration>\n" + + " </workflow>\n" + + " <sla:info>\n" + + " <sla:app-name>5</sla:app-name>\n" + + " <sla:nominal-time>2009-03-06T010:00Z</sla:nominal-time>\n" + + " <sla:should-start>5</sla:should-start>\n" + + " <sla:should-end>50</sla:should-end>\n" + + " <sla:alert-contact>a...@example.com</sla:alert-contact>\n" + + " <sla:dev-contact>a...@example.com</sla:dev-contact>\n" + + " <sla:qa-contact>a...@example.com</sla:qa-contact>\n" + + " <sla:se-contact>a...@example.com</sla:se-contact>\n" + + " </sla:info>\n" + + " </action>\n" + + "</coordinator-app>"; + + coordinatorValidator.validate(new StreamSource(new StringReader(COORD_APP1))); } public void testBundleSchema() throws Exception { - SchemaService wss = Services.get().get(SchemaService.class); - Validator validator = wss.getSchema(SchemaName.BUNDLE).newValidator(); - String BUNDLE_APP = "<bundle-app name='NAME' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns='uri:oozie:bundle:0.1'> " + String BUNDLE_APP = "<bundle-app name='NAME' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' " + + " xmlns='uri:oozie:bundle:0.1'> " + "<controls> <kick-off-time>2009-02-02T00:00Z</kick-off-time> </controls> " + "<coordinator name='c12'> " + "<app-path>hdfs://localhost:9001/tmp/bundle-apps/coordinator1.xml</app-path>" + "<configuration> " + "<property> <name>START_TIME</name> <value>2009-02-01T00:00Z</value> </property> </configuration> " + "</coordinator></bundle-app>"; - Element e = XmlUtils.parseXml(BUNDLE_APP); - // System.out.println("XML :"+ XmlUtils.prettyPrint(e)); - validator.validate(new StreamSource(new StringReader(BUNDLE_APP))); + + bundleValidator.validate(new StreamSource(new StringReader(BUNDLE_APP))); } + public void testWfLauncherConfig() throws Exception { + workflowValidator.validate(new StreamSource(new StringReader(WF_GLOBAL_LAUNCHER_CONF))); + } + + public void testHiveActionLauncherConfig() throws Exception { + workflowValidator.validate(new StreamSource(new StringReader(HIVE_ACTION_LAUNCHER_CONF))); + } + + public void testHive2ActionLauncherConfig() throws Exception { + workflowValidator.validate(new StreamSource(new StringReader(HIVE2_ACTION_LAUNCHER_CONF))); + } + + public void testShellActionLauncherConfig() throws Exception { + workflowValidator.validate(new StreamSource(new StringReader(SHELL_ACTION_LAUNCHER_CONF))); + } + + public void testSqoopActionLauncherConfig() throws Exception { + workflowValidator.validate(new StreamSource(new StringReader(SQQP_ACTION_LAUNCHER_CONF))); + } + + public void testSparkActionLauncherConfig() throws Exception { + workflowValidator.validate(new StreamSource(new StringReader(SPARK_ACTION_LAUNCHER_CONF))); + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/8b247f28/core/src/test/java/org/apache/oozie/util/TestMetricsInstrumentation.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/util/TestMetricsInstrumentation.java b/core/src/test/java/org/apache/oozie/util/TestMetricsInstrumentation.java index a882c82..ab97443 100644 --- a/core/src/test/java/org/apache/oozie/util/TestMetricsInstrumentation.java +++ b/core/src/test/java/org/apache/oozie/util/TestMetricsInstrumentation.java @@ -228,11 +228,11 @@ public class TestMetricsInstrumentation extends XTestCase { //Setting the id of the VM unique, so we can find it. String uniqueId = UUID.randomUUID().toString(); - System.setProperty("process.unique.id", uniqueId); + System.setProperty("processSettings.unique.id", uniqueId); //Finding our own VM by the id. for(VirtualMachineDescriptor d : VirtualMachine.list()) { - String remoteUniqueId = VirtualMachine.attach(d).getSystemProperties().getProperty("process.unique.id"); + String remoteUniqueId = VirtualMachine.attach(d).getSystemProperties().getProperty("processSettings.unique.id"); if(remoteUniqueId != null && remoteUniqueId.equals(uniqueId)) { descriptor = d; http://git-wip-us.apache.org/repos/asf/oozie/blob/8b247f28/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java b/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java index 2133240..a361078 100644 --- a/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java +++ b/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java @@ -25,15 +25,20 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hadoop.conf.Configuration; import org.apache.oozie.ErrorCode; +import org.apache.oozie.action.hadoop.LauncherAM; import org.apache.oozie.service.ConfigurationService; import org.apache.oozie.service.LiteWorkflowStoreService; import org.apache.oozie.service.SchemaService; import org.apache.oozie.service.Services; import org.apache.oozie.test.XTestCase; import org.apache.oozie.util.IOUtils; +import org.apache.oozie.util.XConfiguration; +import org.apache.oozie.util.XmlUtils; import org.apache.oozie.workflow.WorkflowException; import org.apache.oozie.workflow.lite.TestLiteWorkflowLib.TestActionNodeHandler; import org.apache.oozie.workflow.lite.TestLiteWorkflowLib.TestDecisionNodeHandler; +import org.jdom.Element; +import org.jdom.Namespace; public class TestLiteWorkflowAppParser extends XTestCase { public static String dummyConf = "<java></java>"; @@ -652,6 +657,42 @@ public class TestLiteWorkflowAppParser extends XTestCase { } } + public void testParserGlobalLauncherAM() throws Exception { + LiteWorkflowAppParser parser = new LiteWorkflowAppParser(null, + LiteWorkflowStoreService.LiteControlNodeHandler.class, + LiteWorkflowStoreService.LiteDecisionHandler.class, + LiteWorkflowStoreService.LiteActionHandler.class); + + LiteWorkflowApp workflowApp = parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-global-launcherconf.xml", -1), new Configuration()); + + XConfiguration xconf = extractConfig(workflowApp, "action1"); + + assertEquals("Vcores", 2, xconf.getInt(LauncherAM.OOZIE_LAUNCHER_VCORES_PROPERTY, Integer.MIN_VALUE)); + assertEquals("Memory", 1024, xconf.getInt(LauncherAM.OOZIE_LAUNCHER_MEMORY_MB_PROPERTY, Integer.MIN_VALUE)); + assertEquals("Env", "dummyEnv", xconf.get(LauncherAM.OOZIE_LAUNCHER_ENV_PROPERTY)); + assertEquals("Queue", "dummyQueue", xconf.get(LauncherAM.OOZIE_LAUNCHER_QUEUE_PROPERTY)); + assertEquals("Java opts", "dummyJavaOpts", xconf.get(LauncherAM.OOZIE_LAUNCHER_JAVAOPTS_PROPERTY)); + assertEquals("Sharelib", "a,b,c", xconf.get(LauncherAM.OOZIE_LAUNCHER_SHARELIB_PROPERTY)); + } + + public void testParserGlobalLauncherAMOverridden() throws Exception { + LiteWorkflowAppParser parser = new LiteWorkflowAppParser(null, + LiteWorkflowStoreService.LiteControlNodeHandler.class, + LiteWorkflowStoreService.LiteDecisionHandler.class, + LiteWorkflowStoreService.LiteActionHandler.class); + + LiteWorkflowApp workflowApp = parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-global-launcherconf-override.xml", -1), new Configuration()); + + XConfiguration xconf = extractConfig(workflowApp, "a"); + + assertEquals("Vcores", 1, xconf.getInt(LauncherAM.OOZIE_LAUNCHER_VCORES_PROPERTY, Integer.MIN_VALUE)); + assertEquals("Memory", 2048, xconf.getInt(LauncherAM.OOZIE_LAUNCHER_MEMORY_MB_PROPERTY, Integer.MIN_VALUE)); + assertEquals("Java opts", "dummyJavaOpts", xconf.get(LauncherAM.OOZIE_LAUNCHER_JAVAOPTS_PROPERTY)); + assertNull("Queue", xconf.get(LauncherAM.OOZIE_LAUNCHER_QUEUE_PROPERTY)); + assertNull("Env", xconf.get(LauncherAM.OOZIE_LAUNCHER_ENV_PROPERTY)); + assertNull("Sharelib", xconf.get(LauncherAM.OOZIE_LAUNCHER_SHARELIB_PROPERTY)); + } + /* * 1->ok->2 * 2->ok->end @@ -1604,4 +1645,14 @@ public class TestLiteWorkflowAppParser extends XTestCase { assertEquals(app.getNode("retry").getUserRetryInterval(), "10"); } + private XConfiguration extractConfig(LiteWorkflowApp app, String actionNode) throws Exception { + String confXML = app.getNode(actionNode).getConf(); + Element confElement = XmlUtils.parseXml(confXML); + Namespace ns = confElement.getNamespace(); + String configSection = XmlUtils.prettyPrint(confElement.getChild("configuration", ns)).toString(); + XConfiguration xconf = new XConfiguration(new StringReader(configSection)); + + return xconf; + } + } http://git-wip-us.apache.org/repos/asf/oozie/blob/8b247f28/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index db18f30..efccc34 100644 --- a/pom.xml +++ b/pom.xml @@ -107,7 +107,7 @@ <jline.version>0.9.94</jline.version> <openjpa.version>2.4.2</openjpa.version> - <xerces.version>2.10.0</xerces.version> + <xerces.version>2.11.0</xerces.version> <curator.version>2.5.0</curator.version> <jackson.version>1.9.13</jackson.version> <log4j.version>1.2.17</log4j.version> http://git-wip-us.apache.org/repos/asf/oozie/blob/8b247f28/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 1fb1a1d..0a4b600 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.0.0 release (trunk - unreleased) +OOZIE-2687 Create XML schema for launcher configurations (asasvari) OOZIE-3041 TestWorkflowActionRetryInfoXCommand fails in oozie core module (andras.piros via gezapeti) OOZIE-2916 Set a job name for the MR Action's child job (asasvari) OOZIE-2858 HiveMain, ShellMain and SparkMain should not overwrite properties and config files locally (gezapeti) http://git-wip-us.apache.org/repos/asf/oozie/blob/8b247f28/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java index 6a98d6e..3e6ffc9 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java @@ -49,6 +49,14 @@ public class LauncherAM { public static final String OOZIE_ACTION_CONF_XML = "oozie.action.conf.xml"; public static final String OOZIE_LAUNCHER_JOB_ID = "oozie.launcher.job.id"; + public static final String OOZIE_LAUNCHER_VCORES_PROPERTY = "oozie.launcher.vcores"; + public static final String OOZIE_LAUNCHER_MEMORY_MB_PROPERTY = "oozie.launcher.memory.mb"; + public static final String OOZIE_LAUNCHER_PRIORITY_PROPERTY = "oozie.launcher.priority"; + public static final String OOZIE_LAUNCHER_QUEUE_PROPERTY = "oozie.launcher.queue"; + public static final String OOZIE_LAUNCHER_JAVAOPTS_PROPERTY = "oozie.launcher.javaopts"; + public static final String OOZIE_LAUNCHER_ENV_PROPERTY = "oozie.launcher.env"; + public static final String OOZIE_LAUNCHER_SHARELIB_PROPERTY = "oozie.launcher.sharelib"; + public static final String JAVA_CLASS_PATH = "java.class.path"; public static final String OOZIE_ACTION_ID = "oozie.action.id"; public static final String OOZIE_JOB_ID = "oozie.job.id"; http://git-wip-us.apache.org/repos/asf/oozie/blob/8b247f28/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/UDFTester.java ---------------------------------------------------------------------- diff --git a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/UDFTester.java b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/UDFTester.java index ff999ff..c0e2ec3 100644 --- a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/UDFTester.java +++ b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/UDFTester.java @@ -39,7 +39,7 @@ public class UDFTester extends EvalFunc<String> { return query.toLowerCase().trim(); } catch (Exception e) { - System.err.println("ToLower: failed to process input; error - " + e.getMessage()); + System.err.println("ToLower: failed to processSettings input; error - " + e.getMessage()); return null; } }
