http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/mapping/TestSubWorkflowActionMapping.java ---------------------------------------------------------------------- diff --git a/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/mapping/TestSubWorkflowActionMapping.java b/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/mapping/TestSubWorkflowActionMapping.java new file mode 100644 index 0000000..10f673f --- /dev/null +++ b/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/mapping/TestSubWorkflowActionMapping.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.fluentjob.api.mapping; + +import org.apache.oozie.fluentjob.api.generated.workflow.SUBWORKFLOW; +import org.apache.oozie.fluentjob.api.action.SubWorkflowAction; +import org.apache.oozie.fluentjob.api.action.SubWorkflowActionBuilder; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class TestSubWorkflowActionMapping { + + @Test + public void testMappingSubWorkflowAction() { + final String appPath = "/path/to/app"; + + final SubWorkflowAction action = SubWorkflowActionBuilder.create() + .withAppPath(appPath) + .withPropagatingConfiguration() + .withConfigProperty("propertyName", "propertyValue") + .build(); + + final SUBWORKFLOW subWorkflowAction = DozerBeanMapperSingleton.instance().map(action, SUBWORKFLOW.class); + + assertEquals(appPath, subWorkflowAction.getAppPath()); + assertNotNull(subWorkflowAction.getPropagateConfiguration()); + assertNotNull(subWorkflowAction.getConfiguration()); + } +}
http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/mapping/TestWorkflowAttributesMapping.java ---------------------------------------------------------------------- diff --git a/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/mapping/TestWorkflowAttributesMapping.java b/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/mapping/TestWorkflowAttributesMapping.java new file mode 100644 index 0000000..24cbb64 --- /dev/null +++ b/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/mapping/TestWorkflowAttributesMapping.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.fluentjob.api.mapping; + +import org.apache.oozie.fluentjob.api.dag.Graph; +import org.apache.oozie.fluentjob.api.generated.workflow.WORKFLOWAPP; +import org.apache.oozie.fluentjob.api.workflow.Workflow; +import org.apache.oozie.fluentjob.api.workflow.WorkflowBuilder; +import org.junit.Test; + +import static org.junit.Assert.assertNull; + +public class TestWorkflowAttributesMapping { + + private final SourceDataFactory factory = new SourceDataFactory(); + + @Test + public void testMappingNoCredentialsToWorkflow() { + final Workflow source = new WorkflowBuilder() + .build(); + + final WORKFLOWAPP destination = DozerBeanMapperSingleton.instance().map(new Graph(source), WORKFLOWAPP.class); + + assertNull(destination.getCredentials()); + } + + @Test + public void testMappingCredentialsAsWorkflowParameters() { + final Workflow source = new WorkflowBuilder() + .withCredentials(factory.createCredentials()) + .build(); + + final WORKFLOWAPP destination = DozerBeanMapperSingleton.instance().map(new Graph(source), WORKFLOWAPP.class); + + factory.assertCredentials(destination.getCredentials()); + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/workflow/TestCredentialBuilder.java ---------------------------------------------------------------------- diff --git a/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/workflow/TestCredentialBuilder.java b/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/workflow/TestCredentialBuilder.java new file mode 100644 index 0000000..06c5b3d --- /dev/null +++ b/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/workflow/TestCredentialBuilder.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.fluentjob.api.workflow; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class TestCredentialBuilder { + + @Test + public void testCreate() { + final Credential credential = CredentialBuilder.create() + .withName("hive2") + .withType("hive") + .withConfigurationEntry("jdbcUrl", "jdbc://localhost/hive") + .build(); + + assertEquals("hive2", credential.getName()); + assertEquals("hive", credential.getType()); + assertEquals("jdbcUrl", credential.getConfigurationEntries().get(0).getName()); + assertEquals("jdbc://localhost/hive", credential.getConfigurationEntries().get(0).getValue()); + } + + @Test + public void testCreateFromExisting() { + final Credential credential = CredentialBuilder.create() + .withName("hive2") + .withType("hive") + .withConfigurationEntry("jdbcUrl", "jdbc://localhost/hive") + .build(); + + final Credential fromExisting = CredentialBuilder.createFromExisting(credential).build(); + + assertEquals("hive2", fromExisting.getName()); + assertEquals("hive", fromExisting.getType()); + assertEquals("jdbcUrl", fromExisting.getConfigurationEntries().get(0).getName()); + assertEquals("jdbc://localhost/hive", fromExisting.getConfigurationEntries().get(0).getValue()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/workflow/TestCredentialsBuilder.java ---------------------------------------------------------------------- diff --git a/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/workflow/TestCredentialsBuilder.java b/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/workflow/TestCredentialsBuilder.java new file mode 100644 index 0000000..4d3dce3 --- /dev/null +++ b/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/workflow/TestCredentialsBuilder.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.fluentjob.api.workflow; + +import com.google.common.collect.Lists; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class TestCredentialsBuilder { + + @Test + public void testCreate() { + final Credentials credentials = CredentialsBuilder.create() + .withCredential("hive2", + "hive", + Lists.newArrayList( + new ConfigurationEntry("jdbcUrl", "jdbc://localhost/hive"))) + .build(); + + assertEquals("hive2", credentials.getCredentials().get(0).getName()); + assertEquals("hive", credentials.getCredentials().get(0).getType()); + assertEquals("jdbcUrl", credentials.getCredentials().get(0).getConfigurationEntries().get(0).getName()); + assertEquals("jdbc://localhost/hive", credentials.getCredentials().get(0).getConfigurationEntries().get(0).getValue()); + } + + @Test + public void testCreateFromExisting() { + final Credentials credentials = CredentialsBuilder.create() + .withCredential("hive2", + "hive", + Lists.newArrayList( + new ConfigurationEntry("jdbcUrl", "jdbc://localhost/hive"))) + .build(); + + final Credentials fromExisting = CredentialsBuilder.createFromExisting(credentials) + .withCredential("hbase", + "hbase") + .build(); + + assertEquals("hive2", fromExisting.getCredentials().get(0).getName()); + assertEquals("hive", fromExisting.getCredentials().get(0).getType()); + assertEquals("jdbcUrl", fromExisting.getCredentials().get(0).getConfigurationEntries().get(0).getName()); + assertEquals("jdbc://localhost/hive", fromExisting.getCredentials().get(0).getConfigurationEntries().get(0).getValue()); + + assertEquals("hbase", fromExisting.getCredentials().get(1).getName()); + assertEquals("hbase", fromExisting.getCredentials().get(1).getType()); + assertEquals(0, fromExisting.getCredentials().get(1).getConfigurationEntries().size()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/workflow/TestGlobalBuilder.java ---------------------------------------------------------------------- diff --git a/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/workflow/TestGlobalBuilder.java b/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/workflow/TestGlobalBuilder.java new file mode 100644 index 0000000..c417170 --- /dev/null +++ b/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/workflow/TestGlobalBuilder.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.fluentjob.api.workflow; + +import org.apache.oozie.fluentjob.api.action.LauncherBuilder; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class TestGlobalBuilder { + + public static final String DEFAULT = "default"; + + @Test + public void testAfterCopyFieldsAreSetCorrectly() { + final Global original = GlobalBuilder.create() + .withResourceManager(DEFAULT) + .withNameNode(DEFAULT) + .withJobXml(DEFAULT) + .withConfigProperty("key1", "value1") + .withLauncher(new LauncherBuilder() + .withMemoryMb(1024L) + .withVCores(1L) + .build()) + .build(); + + assertEquals(DEFAULT, original.getResourceManager()); + assertEquals(DEFAULT, original.getNameNode()); + assertEquals(DEFAULT, original.getJobXmls().get(0)); + assertEquals("value1", original.getConfigProperty("key1")); + assertEquals(1024L, original.getLauncher().getMemoryMb()); + assertEquals(1L, original.getLauncher().getVCores()); + + final Global copied = GlobalBuilder.createFromExisting(original) + .withoutJobXml(DEFAULT) + .withConfigProperty("key1", null) + .build(); + + assertEquals(DEFAULT, copied.getResourceManager()); + assertEquals(DEFAULT, copied.getNameNode()); + assertEquals(0, copied.getJobXmls().size()); + assertNull(copied.getConfigProperty("key1")); + assertEquals(1024L, copied.getLauncher().getMemoryMb()); + assertEquals(1L, copied.getLauncher().getVCores()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/workflow/TestParametersBuilder.java ---------------------------------------------------------------------- diff --git a/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/workflow/TestParametersBuilder.java b/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/workflow/TestParametersBuilder.java new file mode 100644 index 0000000..f9405e6 --- /dev/null +++ b/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/workflow/TestParametersBuilder.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.fluentjob.api.workflow; + +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class TestParametersBuilder { + private ParametersBuilder builder; + + @Before + public void setUp() { + this.builder = ParametersBuilder.create(); + } + + @Test + public void testWithoutDescription() { + final Parameters parameters = builder + .withParameter("name1", "value1") + .withParameter("name2", "value2") + .build(); + + assertEquals("name1", parameters.getParameters().get(0).getName()); + assertEquals("value1", parameters.getParameters().get(0).getValue()); + assertNull(parameters.getParameters().get(0).getDescription()); + assertEquals("name2", parameters.getParameters().get(1).getName()); + assertEquals("value2", parameters.getParameters().get(1).getValue()); + assertNull(parameters.getParameters().get(1).getDescription()); + } + + @Test + public void testWithDescription() { + final Parameters parameters = builder + .withParameter("name1", "value1", "description1") + .withParameter("name2", "value2", "description2") + .build(); + + assertEquals("name1", parameters.getParameters().get(0).getName()); + assertEquals("value1", parameters.getParameters().get(0).getValue()); + assertEquals("description1", parameters.getParameters().get(0).getDescription()); + assertEquals("name2", parameters.getParameters().get(1).getName()); + assertEquals("value2", parameters.getParameters().get(1).getValue()); + assertEquals("description2", parameters.getParameters().get(1).getDescription()); + } + + @Test + public void testCreateFromExisting() { + final Parameters existing = builder + .withParameter("name1", "value1") + .withParameter("name2", "value2") + .build(); + + final Parameters fromExisting = ParametersBuilder.createFromExisting(existing) + .withParameter("name3", "value3") + .build(); + + assertEquals("value1", fromExisting.getParameters().get(0).getValue()); + assertEquals("value2", fromExisting.getParameters().get(1).getValue()); + assertEquals("value3", fromExisting.getParameters().get(2).getValue()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/workflow/TestWorkflowBuilder.java ---------------------------------------------------------------------- diff --git a/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/workflow/TestWorkflowBuilder.java b/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/workflow/TestWorkflowBuilder.java new file mode 100644 index 0000000..29caf27 --- /dev/null +++ b/fluent-job/fluent-job-api/src/test/java/org/apache/oozie/fluentjob/api/workflow/TestWorkflowBuilder.java @@ -0,0 +1,258 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.fluentjob.api.workflow; + +import com.google.common.collect.Lists; +import org.apache.oozie.fluentjob.api.action.MapReduceActionBuilder; +import org.apache.oozie.fluentjob.api.action.Node; +import org.apache.oozie.fluentjob.api.action.ShellAction; +import org.apache.oozie.fluentjob.api.action.ShellActionBuilder; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class TestWorkflowBuilder { + private static final String NAME = "workflow-name"; + + @Rule + public final ExpectedException expectedException = ExpectedException.none(); + + @Test + public void testAddName() { + final WorkflowBuilder builder = new WorkflowBuilder(); + builder.withName(NAME); + + final Workflow workflow = builder.build(); + + assertEquals(NAME, workflow.getName()); + } + + @Test + public void testAddDagTrivial() { + final Node mrAction1 = MapReduceActionBuilder.create() + .withName("mr1") + .withNameNode("${nameNode}") + .withResourceManager("${resourceManager}") + .withConfigProperty("mapred.output.dir", "${outputDir}") + .build(); + + final Node mrAction2 = MapReduceActionBuilder.create() + .withName("mr2") + .withNameNode("${nameNode}") + .withResourceManager("${resourceManager}") + .withConfigProperty("mapred.output.dir", "${outputDir}") + .build(); + + final WorkflowBuilder builder = new WorkflowBuilder(); + + builder.withDagContainingNode(mrAction1) + .withDagContainingNode(mrAction2); + + final Workflow workflow = builder.build(); + + final Set<Node> expectedRoots = new HashSet<>(Arrays.asList(mrAction1, mrAction2)); + assertEquals(expectedRoots, workflow.getRoots()); + + final Set<Node> expectedNodes = new HashSet<>(Arrays.asList(mrAction1, mrAction2)); + assertEquals(expectedNodes, workflow.getNodes()); + } + + @Test + public void testAddDagFindRoots() { + final Node mrAction1 = MapReduceActionBuilder.create() + .withName("mr1") + .build(); + + final Node mrAction2 = MapReduceActionBuilder.create() + .withName("mr2") + .build(); + + final Node mrAction3 = MapReduceActionBuilder.create() + .withName("mr3") + .withParent(mrAction1) + .withParent(mrAction2) + .build(); + + final WorkflowBuilder builder = new WorkflowBuilder(); + + builder.withDagContainingNode(mrAction3); + + final Workflow workflow = builder.build(); + + final Set<Node> expectedRoots = new HashSet<>(Arrays.asList(mrAction1, mrAction2)); + assertEquals(expectedRoots, workflow.getRoots()); + + final Set<Node> expectedNodes = new HashSet<>(Arrays.asList(mrAction1, mrAction2, mrAction3)); + assertEquals(expectedNodes, workflow.getNodes()); + } + + @Test + public void testAddDagThrowOnDuplicateNodeNames() { + final Node mrAction = MapReduceActionBuilder.create() + .withName("mr-action") + .build(); + + final Node mrActionWithTheSameName = MapReduceActionBuilder.create() + .withName("mr-action") + .build(); + + final WorkflowBuilder builder = new WorkflowBuilder(); + builder.withName(NAME) + .withDagContainingNode(mrAction) + .withDagContainingNode(mrActionWithTheSameName); + + expectedException.expect(IllegalArgumentException.class); + builder.build(); + } + + @Test + public void testAddDagWithConditionalChildrenAndConditionalParents() { + final String condition = "condition"; + + final Node mrAction1 = MapReduceActionBuilder.create() + .withName("mr1") + .build(); + + final Node mrAction2 = MapReduceActionBuilder.create() + .withName("mr2") + .build(); + + final Node mrAction3 = MapReduceActionBuilder.create() + .withName("mr3") + .withParentWithCondition(mrAction1, condition) + .withParent(mrAction2) + .build(); + final Node mrAction4 = MapReduceActionBuilder.create() + .withName("mr4") + .withParentWithCondition(mrAction3, condition) + .build(); + final Node mrAction5 = MapReduceActionBuilder.create() + .withName("mr5") + .withParentWithCondition(mrAction3, condition) + .build(); + + final WorkflowBuilder builder = new WorkflowBuilder(); + + builder.withDagContainingNode(mrAction3); + + final Workflow workflow = builder.build(); + + final Set<Node> expectedRoots = new HashSet<>(Arrays.asList(mrAction1, mrAction2)); + assertEquals(expectedRoots, workflow.getRoots()); + + final Set<Node> expectedNodes = new HashSet<>(Arrays.asList(mrAction1, mrAction2, mrAction3, mrAction4, mrAction5)); + assertEquals(expectedNodes, workflow.getNodes()); + } + + @Test + public void testAddMixedParameters() { + final Workflow workflow = new WorkflowBuilder() + .withParameter("name1", "value1") + .withParameter("name2", "value2", "description2") + .build(); + + assertEquals("name1", workflow.getParameters().getParameters().get(0).getName()); + assertEquals("value1", workflow.getParameters().getParameters().get(0).getValue()); + assertNull(workflow.getParameters().getParameters().get(0).getDescription()); + assertEquals("name2", workflow.getParameters().getParameters().get(1).getName()); + assertEquals("value2", workflow.getParameters().getParameters().get(1).getValue()); + assertEquals("description2", workflow.getParameters().getParameters().get(1).getDescription()); + } + + @Test + public void testAddGlobal() { + final Workflow workflow = new WorkflowBuilder() + .withGlobal(GlobalBuilder.create() + .withConfigProperty("key1", "value1") + .build()) + .build(); + + assertEquals("value1", workflow.getGlobal().getConfigProperty("key1")); + } + + @Test + public void testAddCredentials() { + final Workflow workflow = new WorkflowBuilder() + .withCredentials(CredentialsBuilder.create() + .withCredential("hbase", "hbase") + .build()) + .build(); + + assertEquals("hbase", workflow.getCredentials().getCredentials().get(0).getName()); + assertEquals("hbase", workflow.getCredentials().getCredentials().get(0).getType()); + } + + @Test + public void testOmittedCredentialsAreAutogenerated() { + final Credential hbaseCredential = CredentialBuilder.create() + .withName("hbase") + .withType("hbase") + .build(); + + final ShellAction hbaseAction = ShellActionBuilder.create() + .withName("hbase") + .withCredential(hbaseCredential) + .build(); + + final Workflow workflow = new WorkflowBuilder() + .withDagContainingNode(hbaseAction) + .build(); + + assertEquals(1, workflow.getCredentials().getCredentials().size()); + assertEquals("hbase", workflow.getCredentials().getCredentials().get(0).getName()); + assertEquals("hbase", workflow.getCredentials().getCredentials().get(0).getType()); + } + + @Test + public void testOverrideCredentialsPreviouslyAddedInActionsOnWorkflowLevel() { + final Credential hbaseCredential = CredentialBuilder.create() + .withName("hbase") + .withType("hbase") + .build(); + + final ShellAction hbaseAction = ShellActionBuilder.create() + .withName("hbase") + .withCredential(hbaseCredential) + .build(); + + final Workflow workflow = new WorkflowBuilder() + .withDagContainingNode(hbaseAction) + .withCredentials(CredentialsBuilder.create() + .withCredential("hive2", "hive2", Lists.newArrayList( + new ConfigurationEntry("jdbcUrl", "jdbc://localhost/hive"))) + .build()) + .build(); + + assertEquals(1, workflow.getCredentials().getCredentials().size()); + assertEquals("hive2", workflow.getCredentials().getCredentials().get(0).getName()); + assertEquals("hive2", workflow.getCredentials().getCredentials().get(0).getType()); + assertEquals(1, workflow.getCredentials().getCredentials().get(0).getConfigurationEntries().size()); + assertEquals("jdbcUrl", + workflow.getCredentials().getCredentials().get(0).getConfigurationEntries().get(0).getName()); + assertEquals("jdbc://localhost/hive", + workflow.getCredentials().getCredentials().get(0).getConfigurationEntries().get(0).getValue()); + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/fluent-job/fluent-job-client/pom.xml ---------------------------------------------------------------------- diff --git a/fluent-job/fluent-job-client/pom.xml b/fluent-job/fluent-job-client/pom.xml new file mode 100644 index 0000000..612fabb --- /dev/null +++ b/fluent-job/fluent-job-client/pom.xml @@ -0,0 +1,99 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>oozie-fluent-job</artifactId> + <groupId>org.apache.oozie</groupId> + <version>5.1.0-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + + <artifactId>oozie-fluent-job-client</artifactId> + <version>5.1.0-SNAPSHOT</version> + <description>Apache Oozie Fluent Job Client</description> + <name>Apache Oozie Fluent Job Client</name> + <packaging>jar</packaging> + + <dependencies> + <dependency> + <groupId>org.apache.oozie</groupId> + <artifactId>oozie-core</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.oozie</groupId> + <artifactId>oozie-core</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.derby</groupId> + <artifactId>derby</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.oozie</groupId> + <artifactId>oozie-sharelib-hive</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-minicluster</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.oozie.test</groupId> + <artifactId>oozie-mini</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.oozie.test</groupId> + <artifactId>oozie-mini</artifactId> + <classifier>tests</classifier> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.oozie</groupId> + <artifactId>oozie-client</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.xmlunit</groupId> + <artifactId>xmlunit-core</artifactId> + <scope>test</scope> + </dependency> + </dependencies> +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/jaxb/TestJAXBWorkflow.java ---------------------------------------------------------------------- diff --git a/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/jaxb/TestJAXBWorkflow.java b/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/jaxb/TestJAXBWorkflow.java new file mode 100644 index 0000000..47adb11 --- /dev/null +++ b/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/jaxb/TestJAXBWorkflow.java @@ -0,0 +1,435 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.jobs.client.jaxb; + +import org.apache.oozie.fluentjob.api.generated.workflow.ACTION; +import org.apache.oozie.fluentjob.api.generated.workflow.ACTIONTRANSITION; +import org.apache.oozie.fluentjob.api.generated.workflow.CONFIGURATION; +import org.apache.oozie.fluentjob.api.generated.workflow.DELETE; +import org.apache.oozie.fluentjob.api.generated.workflow.END; +import org.apache.oozie.fluentjob.api.generated.workflow.KILL; +import org.apache.oozie.fluentjob.api.generated.workflow.MAPREDUCE; +import org.apache.oozie.fluentjob.api.generated.workflow.ObjectFactory; +import org.apache.oozie.fluentjob.api.generated.workflow.PREPARE; +import org.apache.oozie.fluentjob.api.generated.workflow.START; +import org.apache.oozie.fluentjob.api.generated.workflow.WORKFLOWAPP; + +import org.junit.Test; +import org.xml.sax.SAXException; +import org.xmlunit.builder.DiffBuilder; +import org.xmlunit.builder.Input; +import org.xmlunit.diff.Comparison; +import org.xmlunit.diff.ComparisonResult; +import org.xmlunit.diff.ComparisonType; +import org.xmlunit.diff.Diff; +import org.xmlunit.diff.DifferenceEvaluator; +import org.xmlunit.diff.DifferenceEvaluators; + +import javax.xml.XMLConstants; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBElement; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import javax.xml.bind.Unmarshaller; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.Source; +import javax.xml.transform.stream.StreamSource; +import javax.xml.validation.Schema; +import javax.xml.validation.SchemaFactory; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.charset.Charset; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * This class tests whether the workflow.xml files are parsed correctly into JAXB objects and whether the JAXB objects + * are serialized correctly to xml. + */ +public class TestJAXBWorkflow { + private static final String GENERATED_PACKAGES_ALL = "org.apache.oozie.fluentjob.api.generated.action.distcp:" + + "org.apache.oozie.fluentjob.api.generated.action.email:" + + "org.apache.oozie.fluentjob.api.generated.action.hive2:" + + "org.apache.oozie.fluentjob.api.generated.action.hive:" + + "org.apache.oozie.fluentjob.api.generated.sla:" + + "org.apache.oozie.fluentjob.api.generated.workflow:" + + "org.apache.oozie.fluentjob.api.generated.action.shell:" + + "org.apache.oozie.fluentjob.api.generated.action.spark:" + + "org.apache.oozie.fluentjob.api.generated.action.sqoop:" + + "org.apache.oozie.fluentjob.api.generated.action.ssh"; + private static final String GENERATED_PACKAGES_WORKFLOW = "org.apache.oozie.fluentjob.api.generated.workflow"; + private static final String WORKFLOW_MAPREDUCE_ACTION = "/workflow-mapreduce-action.xml"; + private static final String WORKFLOW_ALL_ACTIONS = "/workflow-all-actions.xml"; + + /** + * Tests whether a workflow.xml object is parsed correctly into a JAXB element tree by checking some of the main + * properties. + * @throws SAXException If a SAX error occurs during parsing the schema file. + * @throws JAXBException If an error was encountered while creating the <tt>JAXBContext</tt> or the + * <tt>Unmarshaller</tt> objects. + */ + @Test + public void whenWorkflowXmlWithAllActionTypesIsUnmarshalledAttributesArePreserved() + throws SAXException, JAXBException, URISyntaxException { + final WORKFLOWAPP wf = unmarshalWorkflowWithAllActionTypes(); + + assertEquals("jaxb-example-wf", wf.getName()); + assertEquals("mr-node", wf.getStart().getTo()); + assertEquals("end", wf.getEnd().getName()); + + final List<Object> actions = wf.getDecisionOrForkOrJoin(); + + final KILL kill = (KILL) actions.get(9); + assertKill(kill); + + final MAPREDUCE mr = ((ACTION) actions.get(0)).getMapReduce(); + assertMapReduce(mr); + + final org.apache.oozie.fluentjob.api.generated.action.distcp.ACTION distcp = + (org.apache.oozie.fluentjob.api.generated.action.distcp.ACTION) + ((JAXBElement<?>) ((ACTION) actions.get(1)).getOther()).getValue(); + assertDistcp(distcp); + + final org.apache.oozie.fluentjob.api.generated.action.email.ACTION email = + (org.apache.oozie.fluentjob.api.generated.action.email.ACTION) + ((JAXBElement<?>) ((ACTION) actions.get(2)).getOther()).getValue(); + assertEmail(email); + + final org.apache.oozie.fluentjob.api.generated.action.hive2.ACTION hive2 = + (org.apache.oozie.fluentjob.api.generated.action.hive2.ACTION) + ((JAXBElement<?>) ((ACTION) actions.get(3)).getOther()).getValue(); + assertHive2(hive2); + + final org.apache.oozie.fluentjob.api.generated.action.hive.ACTION hive = + (org.apache.oozie.fluentjob.api.generated.action.hive.ACTION) + ((JAXBElement<?>) ((ACTION) actions.get(4)).getOther()).getValue(); + assertHive(hive); + + final org.apache.oozie.fluentjob.api.generated.action.shell.ACTION shell = + (org.apache.oozie.fluentjob.api.generated.action.shell.ACTION) + ((JAXBElement<?>) ((ACTION) actions.get(5)).getOther()).getValue(); + assertShell(shell); + + final org.apache.oozie.fluentjob.api.generated.action.spark.ACTION spark = + (org.apache.oozie.fluentjob.api.generated.action.spark.ACTION) + ((JAXBElement<?>) ((ACTION) actions.get(6)).getOther()).getValue(); + assertSpark(spark); + + final org.apache.oozie.fluentjob.api.generated.action.sqoop.ACTION sqoop = + (org.apache.oozie.fluentjob.api.generated.action.sqoop.ACTION) + ((JAXBElement<?>) ((ACTION) actions.get(7)).getOther()).getValue(); + assertSqoop(sqoop); + + final org.apache.oozie.fluentjob.api.generated.action.ssh.ACTION ssh = + (org.apache.oozie.fluentjob.api.generated.action.ssh.ACTION) + ((JAXBElement<?>) ((ACTION) actions.get(8)).getOther()).getValue(); + assertSsh(ssh); + } + + private void assertKill(final KILL kill) { + assertEquals("fail", kill.getName()); + assertEquals("Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]", kill.getMessage()); + } + + private void assertMapReduce(final MAPREDUCE mr) { + final PREPARE prepare = mr.getPrepare(); + assertEquals(0, prepare.getMkdir().size()); + + final List<DELETE> deleteList = prepare.getDelete(); + assertEquals(1, deleteList.size()); + + final DELETE delete = deleteList.get(0); + assertEquals("${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/${outputDir}", delete.getPath()); + + final CONFIGURATION conf = mr.getConfiguration(); + final List<CONFIGURATION.Property> properties = conf.getProperty(); + + final CONFIGURATION.Property mapper = properties.get(1); + assertEquals("mapred.mapper.class", mapper.getName()); + assertEquals("org.apache.oozie.example.SampleMapper", mapper.getValue()); + } + + private void assertDistcp(final org.apache.oozie.fluentjob.api.generated.action.distcp.ACTION distcp) { + assertEquals(1, distcp.getPrepare().getDelete().size()); + assertEquals(1, distcp.getPrepare().getMkdir().size()); + assertEquals(2, distcp.getConfiguration().getProperty().size()); + assertEquals(2, distcp.getArg().size()); + } + + private void assertEmail(final org.apache.oozie.fluentjob.api.generated.action.email.ACTION email) { + assertEquals("[email protected]", email.getTo()); + assertEquals("foo", email.getSubject()); + assertEquals("bar", email.getBody()); + } + + private void assertHive2(final org.apache.oozie.fluentjob.api.generated.action.hive2.ACTION hive2) { + assertEquals(1, hive2.getPrepare().getDelete().size()); + assertEquals(1, hive2.getConfiguration().getProperty().size()); + assertEquals(2, hive2.getParam().size()); + } + + private void assertHive(final org.apache.oozie.fluentjob.api.generated.action.hive.ACTION hive) { + assertEquals(1, hive.getPrepare().getDelete().size()); + assertEquals(1, hive.getConfiguration().getProperty().size()); + assertEquals(2, hive.getParam().size()); + } + + private void assertShell(final org.apache.oozie.fluentjob.api.generated.action.shell.ACTION shell) { + assertEquals("echo", shell.getExec()); + assertEquals(1, shell.getArgument().size()); + } + + private void assertSpark(final org.apache.oozie.fluentjob.api.generated.action.spark.ACTION spark) { + assertEquals(1, spark.getPrepare().getDelete().size()); + assertEquals(1, spark.getConfiguration().getProperty().size()); + assertEquals(2, spark.getArg().size()); + } + + private void assertSqoop(final org.apache.oozie.fluentjob.api.generated.action.sqoop.ACTION sqoop) { + assertEquals(1, sqoop.getPrepare().getDelete().size()); + assertEquals(1, sqoop.getConfiguration().getProperty().size()); + } + + private void assertSsh(final org.apache.oozie.fluentjob.api.generated.action.ssh.ACTION ssh) { + assertEquals("[email protected]", ssh.getHost()); + assertEquals("uploaddata", ssh.getCommand()); + assertEquals(2, ssh.getArgs().size()); + } + + /** + * Tests whether a programmatically built JAXB element tree is serialized correctly to xml. + * + * @throws JAXBException If an error was encountered while creating the <tt>JAXBContext</tt> + * or during the marshalling. + */ + @Test + public void marshallingWorkflowProducesCorrectXml() throws JAXBException, URISyntaxException, IOException, + ParserConfigurationException, SAXException { + final WORKFLOWAPP programmaticallyCreatedWfApp = getWfApp(); + final String outputXml = marshalWorkflowApp(programmaticallyCreatedWfApp, GENERATED_PACKAGES_WORKFLOW); + + final Diff diff = DiffBuilder.compare(Input.fromURL(getClass().getResource(WORKFLOW_MAPREDUCE_ACTION))) + .withTest(Input.fromString(outputXml)) + .ignoreComments() + .withDifferenceEvaluator(DifferenceEvaluators.chain( + DifferenceEvaluators.Default, + new IgnoreWhitespaceInTextValueDifferenceEvaluator(), + new IgnoreNamespacePrefixDifferenceEvaluator())) + .build(); + + assertFalse(diff.hasDifferences()); + } + + @Test + public void testMarshallingWorkflowWithAllActionTypesWorks() throws JAXBException, SAXException, + URISyntaxException, UnsupportedEncodingException { + final WORKFLOWAPP wf = unmarshalWorkflowWithAllActionTypes(); + final String outputXml = marshalWorkflowApp(wf, GENERATED_PACKAGES_ALL); + + final Diff diff = DiffBuilder.compare(Input.fromURL(getClass().getResource(WORKFLOW_ALL_ACTIONS))) + .withTest(Input.fromString(outputXml)) + .ignoreComments() + .withDifferenceEvaluator(DifferenceEvaluators.chain( + DifferenceEvaluators.Default, + new IgnoreWhitespaceInTextValueDifferenceEvaluator(), + new IgnoreNamespacePrefixDifferenceEvaluator())) + .build(); + + assertFalse("unmarshalled and marshalled workflow XMLs differ", diff.hasDifferences()); + } + + private static class IgnoreWhitespaceInTextValueDifferenceEvaluator implements DifferenceEvaluator { + @Override + public ComparisonResult evaluate(final Comparison comparison, final ComparisonResult comparisonResult) { + // We want to ignore whitespace differences in TEXT_VALUE comparisons but not anywhere else, + // for example not in attribute names. + if (isTextValueComparison(comparison) && expectedAndActualValueTrimmedAreEqual(comparison)) { + return ComparisonResult.EQUAL; + } else { + return comparisonResult; + } + } + + private boolean isTextValueComparison(final Comparison comparison) { + return comparison.getType().equals(ComparisonType.TEXT_VALUE); + } + + private boolean expectedAndActualValueTrimmedAreEqual(final Comparison comparison) { + final String expectedNodeValue = comparison.getControlDetails().getTarget().getNodeValue(); + final String actualNodeValue = comparison.getTestDetails().getTarget().getNodeValue(); + + if (expectedNodeValue == null || actualNodeValue == null) { + return false; + } + + return expectedNodeValue.trim().equals(actualNodeValue.trim()); + } + } + + private static class IgnoreNamespacePrefixDifferenceEvaluator implements DifferenceEvaluator { + + @Override + public ComparisonResult evaluate(final Comparison comparison, final ComparisonResult comparisonResult) { + if (isElementNodeComparison(comparison)) { + return ComparisonResult.EQUAL; + } + + return comparisonResult; + } + + private boolean isElementNodeComparison(final Comparison comparison) { + return comparison.getType().equals(ComparisonType.NAMESPACE_PREFIX); + } + } + + private WORKFLOWAPP unmarshalWorkflowWithAllActionTypes() throws SAXException, JAXBException, URISyntaxException { + final JAXBContext jc = JAXBContext.newInstance(GENERATED_PACKAGES_ALL); + final Unmarshaller u = jc.createUnmarshaller(); + final Schema wfSchema = getSchema(); + u.setSchema(wfSchema); + + final URL wfUrl = getClass().getResource(WORKFLOW_ALL_ACTIONS); + final JAXBElement element = (JAXBElement) u.unmarshal(wfUrl); + + return (WORKFLOWAPP) element.getValue(); + } + + private Schema getSchema() throws SAXException, URISyntaxException { + final SchemaFactory sf = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); + return sf.newSchema(new Source [] { + getStreamSource("/oozie-common-1.0.xsd"), + getStreamSource("/distcp-action-1.0.xsd"), + getStreamSource("/email-action-0.2.xsd"), + getStreamSource("/hive2-action-1.0.xsd"), + getStreamSource("/hive-action-1.0.xsd"), + getStreamSource("/oozie-sla-0.2.xsd"), + getStreamSource("/oozie-workflow-1.0.xsd"), + getStreamSource("/shell-action-1.0.xsd"), + getStreamSource("/spark-action-1.0.xsd"), + getStreamSource("/sqoop-action-1.0.xsd"), + getStreamSource("/ssh-action-0.2.xsd") + }); + } + + private Source getStreamSource(final String resourceURI) throws URISyntaxException { + return new StreamSource(getClass().getResource(resourceURI).toExternalForm()); + } + + private String marshalWorkflowApp(final WORKFLOWAPP wfApp, final String packages) + throws JAXBException, UnsupportedEncodingException { + final JAXBElement wfElement = new ObjectFactory().createWorkflowApp(wfApp); + + final JAXBContext jc = JAXBContext.newInstance(packages); + final Marshaller m = jc.createMarshaller(); + m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + m.marshal(wfElement, out); + + return out.toString(Charset.defaultCharset().toString()); + } + + private WORKFLOWAPP getWfApp() { + final START start = new START(); + start.setTo("mr-node"); + + final KILL kill = new KILL(); + kill.setName("fail"); + kill.setMessage("Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]"); + + final END end = new END(); + end.setName("end"); + + final WORKFLOWAPP wfApp = new WORKFLOWAPP(); + wfApp.setName("jaxb-example-wf"); + wfApp.setStart(start); + wfApp.getDecisionOrForkOrJoin().add(getMapReduceAction()); + wfApp.getDecisionOrForkOrJoin().add(kill); + wfApp.setEnd(end); + + return wfApp; + } + + private ACTION getMapReduceAction() { + final ACTION action = new ACTION(); + + action.setName("mr-node"); + action.setMapReduce(getMapreduce()); + + final ACTIONTRANSITION okTransition = new ACTIONTRANSITION(); + okTransition.setTo("end"); + action.setOk(okTransition); + + final ACTIONTRANSITION errorTransition = new ACTIONTRANSITION(); + errorTransition.setTo("fail"); + action.setError(errorTransition); + + return action; + } + + private MAPREDUCE getMapreduce() { + final MAPREDUCE mr = new MAPREDUCE(); + + mr.setResourceManager("${resourceManager}"); + mr.setNameNode("${nameNode}"); + mr.setPrepare(getPrepare()); + mr.setConfiguration(getConfiguration()); + + return mr; + } + + private CONFIGURATION getConfiguration() { + final String[][] nameValuePairs = { + {"mapred.job.queue.name", "${queueName}"}, + {"mapred.mapper.class", "org.apache.oozie.example.SampleMapper"}, + {"mapred.reducer.class", "org.apache.oozie.example.SampleReducer"}, + {"mapred.map.tasks", "1"}, + {"mapred.input.dir", "/user/${wf:user()}/${examplesRoot}/input-data/text"}, + {"mapred.output.dir", "/user/${wf:user()}/${examplesRoot}/output-data/${outputDir}"} + }; + + final CONFIGURATION config = new CONFIGURATION(); + final List<CONFIGURATION.Property> properties = config.getProperty(); + + for (final String[] pair : nameValuePairs) { + final CONFIGURATION.Property property = new CONFIGURATION.Property(); + property.setName(pair[0]); + property.setValue(pair[1]); + + properties.add(property); + } + + return config; + } + + private PREPARE getPrepare() { + final PREPARE prepare = new PREPARE(); + + final DELETE delete = new DELETE(); + delete.setPath("${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/${outputDir}"); + + prepare.getDelete().add(delete); + + return prepare; + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestDistcpAction.java ---------------------------------------------------------------------- diff --git a/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestDistcpAction.java b/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestDistcpAction.java new file mode 100644 index 0000000..e377d5c --- /dev/null +++ b/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestDistcpAction.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.jobs.client.minitest; + +import org.apache.oozie.client.OozieClientException; +import org.apache.oozie.fluentjob.api.GraphVisualization; +import org.apache.oozie.fluentjob.api.action.DistcpAction; +import org.apache.oozie.fluentjob.api.action.DistcpActionBuilder; +import org.apache.oozie.fluentjob.api.action.Prepare; +import org.apache.oozie.fluentjob.api.action.PrepareBuilder; +import org.apache.oozie.fluentjob.api.action.SshActionBuilder; +import org.apache.oozie.fluentjob.api.dag.Graph; +import org.apache.oozie.fluentjob.api.serialization.WorkflowMarshaller; +import org.apache.oozie.fluentjob.api.workflow.Workflow; +import org.apache.oozie.fluentjob.api.workflow.WorkflowBuilder; +import org.apache.oozie.test.WorkflowTestCase; + +import javax.xml.bind.JAXBException; +import java.io.IOException; + +public class TestDistcpAction extends WorkflowTestCase { + public void testForkedDistcpActions() throws IOException, JAXBException, OozieClientException { + final Prepare prepare = new PrepareBuilder() + .withDelete("hdfs://localhost:8020/user/${wf:user()}/examples/output") + .build(); + + final DistcpAction parent = DistcpActionBuilder.create() + .withResourceManager(getJobTrackerUri()) + .withNameNode(getNameNodeUri()) + .withPrepare(prepare) + .withConfigProperty("mapred.job.queue.name", "default") + .withJavaOpts("-Dopt1 -Dopt2") + .withArg("arg1") + .build(); + + // We are reusing the definition of parent and only modifying and adding what is different. + final DistcpAction leftChild = DistcpActionBuilder.createFromExistingAction(parent) + .withParent(parent) + .withoutArg("arg1") + .withArg("arg2") + .build(); + + final DistcpAction rightChild = DistcpActionBuilder.createFromExistingAction(leftChild) + .withoutArg("arg2") + .withArg("arg3") + .build(); + + final Workflow workflow = new WorkflowBuilder() + .withName("simple-distcp-example") + .withDagContainingNode(parent).build(); + + SshActionBuilder.create() + .withParent(leftChild) + .withParent(rightChild) + .withHost("localhost") + .withCommand("pwd") + .build(); + + final String xml = WorkflowMarshaller.marshal(workflow); + + System.out.println(xml); + + GraphVisualization.workflowToPng(workflow, "simple-distcp-example-workflow.png"); + + final Graph intermediateGraph = new Graph(workflow); + + GraphVisualization.graphToPng(intermediateGraph, "simple-distcp-example-graph.png"); + + log.debug("Workflow XML is:\n{0}", xml); + + validate(xml); + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestEmailAction.java ---------------------------------------------------------------------- diff --git a/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestEmailAction.java b/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestEmailAction.java new file mode 100644 index 0000000..d46cee2 --- /dev/null +++ b/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestEmailAction.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.jobs.client.minitest; + +import org.apache.oozie.client.OozieClientException; +import org.apache.oozie.fluentjob.api.GraphVisualization; +import org.apache.oozie.fluentjob.api.action.EmailActionBuilder; +import org.apache.oozie.fluentjob.api.action.Node; +import org.apache.oozie.fluentjob.api.dag.Graph; +import org.apache.oozie.fluentjob.api.serialization.WorkflowMarshaller; +import org.apache.oozie.fluentjob.api.workflow.Workflow; +import org.apache.oozie.fluentjob.api.workflow.WorkflowBuilder; +import org.apache.oozie.test.WorkflowTestCase; + +import javax.xml.bind.JAXBException; +import java.io.IOException; + +public class TestEmailAction extends WorkflowTestCase { + + public void testForkedEmailActions() throws JAXBException, IOException, OozieClientException { + final Node parent = EmailActionBuilder.create() + .withRecipient("[email protected]") + .withSubject("Subject") + .withBody("This is a wonderful e-mail.") + .build(); + + EmailActionBuilder.create() + .withParent(parent) + .withRecipient("[email protected]") + .withSubject("Re: Subject") + .withBody("This is an even more wonderful e-mail.") + .build(); + + EmailActionBuilder.create() + .withParent(parent) + .withRecipient("[email protected]") + .withSubject("Re: Subject") + .withBody("No, this is the most wonderful e-mail.") + .build(); + + final Workflow workflow = new WorkflowBuilder() + .withName("simple-email-example") + .withDagContainingNode(parent).build(); + + GraphVisualization.workflowToPng(workflow, "simple-email-example-workflow.png"); + + final Graph intermediateGraph = new Graph(workflow); + + GraphVisualization.graphToPng(intermediateGraph, "simple-email-example-graph.png"); + + final String xml = WorkflowMarshaller.marshal(workflow); + + log.debug("Workflow XML is:\n{0}", xml); + + validate(xml); + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestFSAction.java ---------------------------------------------------------------------- diff --git a/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestFSAction.java b/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestFSAction.java new file mode 100644 index 0000000..947360f --- /dev/null +++ b/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestFSAction.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.jobs.client.minitest; + +import org.apache.oozie.client.OozieClientException; +import org.apache.oozie.fluentjob.api.GraphVisualization; +import org.apache.oozie.fluentjob.api.action.Delete; +import org.apache.oozie.fluentjob.api.action.FSAction; +import org.apache.oozie.fluentjob.api.action.FSActionBuilder; +import org.apache.oozie.fluentjob.api.action.Mkdir; +import org.apache.oozie.fluentjob.api.dag.Graph; +import org.apache.oozie.fluentjob.api.serialization.WorkflowMarshaller; +import org.apache.oozie.fluentjob.api.workflow.Workflow; +import org.apache.oozie.fluentjob.api.workflow.WorkflowBuilder; +import org.apache.oozie.test.WorkflowTestCase; + +import javax.xml.bind.JAXBException; +import java.io.IOException; +import java.util.Date; + +public class TestFSAction extends WorkflowTestCase { + + public void testTwoFSActions() throws JAXBException, IOException, OozieClientException { + final String hdfsPath = getFsTestCaseDir() + "/user/${wf:user()}/examples/output_" + new Date().getTime(); + + final Delete delete = new Delete(hdfsPath, true); + + final Mkdir mkdir = new Mkdir(hdfsPath); + + final FSAction parent = FSActionBuilder.create() + .withNameNode(getNameNodeUri()) + .withDelete(delete) + .withMkdir(mkdir) + .build(); + + FSActionBuilder.createFromExistingAction(parent) + .withParent(parent) + .build(); + + final Workflow workflow = new WorkflowBuilder() + .withName("simple-fs-example") + .withDagContainingNode(parent).build(); + + final String xml = WorkflowMarshaller.marshal(workflow); + + log.debug("Workflow XML is:\n{0}", xml); + + GraphVisualization.workflowToPng(workflow, "simple-fs-example-workflow.png"); + + final Graph intermediateGraph = new Graph(workflow); + + GraphVisualization.graphToPng(intermediateGraph, "simple-fs-example-graph.png"); + + validate(xml); + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestHive2Action.java ---------------------------------------------------------------------- diff --git a/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestHive2Action.java b/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestHive2Action.java new file mode 100644 index 0000000..bb4fa98 --- /dev/null +++ b/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestHive2Action.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.jobs.client.minitest; + +import org.apache.oozie.client.OozieClientException; +import org.apache.oozie.fluentjob.api.GraphVisualization; +import org.apache.oozie.fluentjob.api.action.Hive2Action; +import org.apache.oozie.fluentjob.api.action.Hive2ActionBuilder; +import org.apache.oozie.fluentjob.api.action.Prepare; +import org.apache.oozie.fluentjob.api.action.PrepareBuilder; +import org.apache.oozie.fluentjob.api.dag.Graph; +import org.apache.oozie.fluentjob.api.serialization.WorkflowMarshaller; +import org.apache.oozie.fluentjob.api.workflow.Workflow; +import org.apache.oozie.fluentjob.api.workflow.WorkflowBuilder; +import org.apache.oozie.test.WorkflowTestCase; + +import javax.xml.bind.JAXBException; +import java.io.IOException; + +public class TestHive2Action extends WorkflowTestCase { + public void testForkedHive2Actions() throws IOException, JAXBException, OozieClientException { + final Prepare prepare = new PrepareBuilder() + .withDelete("hdfs://localhost:8020/user/${wf:user()}/examples/output") + .build(); + + final Hive2Action parent = Hive2ActionBuilder.create() + .withResourceManager(getJobTrackerUri()) + .withNameNode(getNameNodeUri()) + .withPrepare(prepare) + .withConfigProperty("mapred.job.queue.name", "default") + .withArg("arg1") + .withJdbcUrl("jdbc://hive2") + .withPassword("secret") + .withScript("hive2.sql") + .build(); + + // We are reusing the definition of parent and only modifying and adding what is different. + final Hive2Action leftChild = Hive2ActionBuilder.createFromExistingAction(parent) + .withParent(parent) + .withoutArg("arg1") + .withArg("arg2") + .build(); + + Hive2ActionBuilder.createFromExistingAction(leftChild) + .withoutArg("arg2") + .withArg("arg3") + .build(); + + final Workflow workflow = new WorkflowBuilder() + .withName("simple-hive2-example") + .withDagContainingNode(parent).build(); + + final String xml = WorkflowMarshaller.marshal(workflow); + + System.out.println(xml); + + GraphVisualization.workflowToPng(workflow, "simple-hive2-example-workflow.png"); + + final Graph intermediateGraph = new Graph(workflow); + + GraphVisualization.graphToPng(intermediateGraph, "simple-hive2-example-graph.png"); + + log.debug("Workflow XML is:\n{0}", xml); + + validate(xml); + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestHiveAction.java ---------------------------------------------------------------------- diff --git a/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestHiveAction.java b/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestHiveAction.java new file mode 100644 index 0000000..017487d --- /dev/null +++ b/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestHiveAction.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.jobs.client.minitest; + +import org.apache.oozie.client.OozieClientException; +import org.apache.oozie.fluentjob.api.GraphVisualization; +import org.apache.oozie.fluentjob.api.action.HiveAction; +import org.apache.oozie.fluentjob.api.action.HiveActionBuilder; +import org.apache.oozie.fluentjob.api.action.Prepare; +import org.apache.oozie.fluentjob.api.action.PrepareBuilder; +import org.apache.oozie.fluentjob.api.dag.Graph; +import org.apache.oozie.fluentjob.api.serialization.WorkflowMarshaller; +import org.apache.oozie.fluentjob.api.workflow.Workflow; +import org.apache.oozie.fluentjob.api.workflow.WorkflowBuilder; +import org.apache.oozie.test.WorkflowTestCase; + +import javax.xml.bind.JAXBException; +import java.io.IOException; + +public class TestHiveAction extends WorkflowTestCase { + public void testForkedHiveActions() throws IOException, JAXBException, OozieClientException { + final Prepare prepare = new PrepareBuilder() + .withDelete("hdfs://localhost:8020/user/${wf:user()}/examples/output") + .build(); + + final HiveAction parent = HiveActionBuilder.create() + .withResourceManager(getJobTrackerUri()) + .withNameNode(getNameNodeUri()) + .withPrepare(prepare) + .withConfigProperty("mapred.job.queue.name", "default") + .withArg("arg1") + .withScript("hive2.sql") + .build(); + + // We are reusing the definition of parent and only modifying and adding what is different. + final HiveAction leftChild = HiveActionBuilder.createFromExistingAction(parent) + .withParent(parent) + .withoutArg("arg1") + .withArg("arg2") + .build(); + + HiveActionBuilder.createFromExistingAction(leftChild) + .withoutArg("arg2") + .withArg("arg3") + .build(); + + final Workflow workflow = new WorkflowBuilder() + .withName("simple-hive-example") + .withDagContainingNode(parent).build(); + + final String xml = WorkflowMarshaller.marshal(workflow); + + System.out.println(xml); + + GraphVisualization.workflowToPng(workflow, "simple-hive-example-workflow.png"); + + final Graph intermediateGraph = new Graph(workflow); + + GraphVisualization.graphToPng(intermediateGraph, "simple-hive-example-graph.png"); + + log.debug("Workflow XML is:\n{0}", xml); + + validate(xml); + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestJavaAction.java ---------------------------------------------------------------------- diff --git a/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestJavaAction.java b/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestJavaAction.java new file mode 100644 index 0000000..397144b --- /dev/null +++ b/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestJavaAction.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.jobs.client.minitest; + +import org.apache.oozie.client.OozieClientException; +import org.apache.oozie.fluentjob.api.GraphVisualization; +import org.apache.oozie.fluentjob.api.action.JavaAction; +import org.apache.oozie.fluentjob.api.action.JavaActionBuilder; +import org.apache.oozie.fluentjob.api.action.Prepare; +import org.apache.oozie.fluentjob.api.action.PrepareBuilder; +import org.apache.oozie.fluentjob.api.dag.Graph; +import org.apache.oozie.fluentjob.api.serialization.WorkflowMarshaller; +import org.apache.oozie.fluentjob.api.workflow.Workflow; +import org.apache.oozie.fluentjob.api.workflow.WorkflowBuilder; +import org.apache.oozie.test.WorkflowTestCase; + +import javax.xml.bind.JAXBException; +import java.io.IOException; + +public class TestJavaAction extends WorkflowTestCase { + public void testForkedJavaActions() throws IOException, JAXBException, OozieClientException { + final Prepare prepare = new PrepareBuilder() + .withDelete("hdfs://localhost:8020/user/${wf:user()}/examples/output") + .build(); + + final JavaAction parent = JavaActionBuilder.create() + .withResourceManager(getJobTrackerUri()) + .withNameNode(getNameNodeUri()) + .withPrepare(prepare) + .withConfigProperty("mapred.job.queue.name", "default") + .withArg("arg1") + .withMainClass("org.apache.oozie.MyFirstMainClass") + .withJavaOptsString("-Dopt1a -Dopt1b") + .withCaptureOutput(true) + .build(); + + // We are reusing the definition of parent and only modifying and adding what is different. + final JavaAction leftChild = JavaActionBuilder.createFromExistingAction(parent) + .withParent(parent) + .withoutArg("arg1") + .withArg("arg2") + .withJavaOptsString(null) + .withJavaOpt("-Dopt2a") + .withJavaOpt("-Dopt2b") + .withCaptureOutput(false) + .build(); + + JavaActionBuilder.createFromExistingAction(leftChild) + .withoutArg("arg2") + .withArg("arg3") + .withJavaOptsString(null) + .withoutJavaOpt("-Dopt2a") + .withoutJavaOpt("-Dopt2b") + .withJavaOpt("-Dopt3a") + .withJavaOpt("-Dopt3b") + .build(); + + final Workflow workflow = new WorkflowBuilder() + .withName("simple-java-example") + .withDagContainingNode(parent).build(); + + final String xml = WorkflowMarshaller.marshal(workflow); + + System.out.println(xml); + + GraphVisualization.workflowToPng(workflow, "simple-java-example-workflow.png"); + + final Graph intermediateGraph = new Graph(workflow); + + GraphVisualization.graphToPng(intermediateGraph, "simple-java-example-graph.png"); + + log.debug("Workflow XML is:\n{0}", xml); + + validate(xml); + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestMapReduceAction.java ---------------------------------------------------------------------- diff --git a/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestMapReduceAction.java b/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestMapReduceAction.java new file mode 100644 index 0000000..573208b --- /dev/null +++ b/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestMapReduceAction.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.jobs.client.minitest; + +import org.apache.oozie.client.OozieClientException; +import org.apache.oozie.fluentjob.api.GraphVisualization; +import org.apache.oozie.fluentjob.api.action.MapReduceAction; +import org.apache.oozie.fluentjob.api.action.MapReduceActionBuilder; +import org.apache.oozie.fluentjob.api.action.Prepare; +import org.apache.oozie.fluentjob.api.action.PrepareBuilder; +import org.apache.oozie.fluentjob.api.dag.Graph; +import org.apache.oozie.fluentjob.api.serialization.WorkflowMarshaller; +import org.apache.oozie.fluentjob.api.workflow.Workflow; +import org.apache.oozie.fluentjob.api.workflow.WorkflowBuilder; +import org.apache.oozie.test.WorkflowTestCase; + +import javax.xml.bind.JAXBException; +import java.io.IOException; + +public class TestMapReduceAction extends WorkflowTestCase { + public void testForkedMapReduceActions() throws IOException, JAXBException, OozieClientException { + final Prepare prepare = new PrepareBuilder() + .withDelete("hdfs://localhost:8020/user/${wf:user()}/examples/output") + .build(); + + final MapReduceAction parent = MapReduceActionBuilder.create() + .withResourceManager(getJobTrackerUri()) + .withNameNode(getNameNodeUri()) + .withPrepare(prepare) + .withConfigProperty("mapred.job.queue.name", "default") + .withConfigProperty("mapred.mapper.class", "org.apache.hadoop.mapred.lib.IdentityMapper") + .withConfigProperty("mapred.input.dir", "/user/${wf:user()}/examples/input") + .withConfigProperty("mapred.output.dir", "/user/${wf:user()}/examples/output") + .build(); + + // We are reusing the definition of mrAction1 and only modifying and adding what is different. + final MapReduceAction leftChild = MapReduceActionBuilder.createFromExistingAction(parent) + .withParent(parent) + .build(); + + MapReduceActionBuilder.createFromExistingAction(leftChild) + .build(); + + final Workflow workflow = new WorkflowBuilder() + .withName("simple-map-reduce-example") + .withDagContainingNode(parent).build(); + + final String xml = WorkflowMarshaller.marshal(workflow); + + System.out.println(xml); + + GraphVisualization.workflowToPng(workflow, "simple-map-reduce-example-workflow.png"); + + final Graph intermediateGraph = new Graph(workflow); + + GraphVisualization.graphToPng(intermediateGraph, "simple-map-reduce-example-graph.png"); + + validate(xml); + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/8a0a6487/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestPigAction.java ---------------------------------------------------------------------- diff --git a/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestPigAction.java b/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestPigAction.java new file mode 100644 index 0000000..b4daf33 --- /dev/null +++ b/fluent-job/fluent-job-client/src/test/java/org/apache/oozie/jobs/client/minitest/TestPigAction.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.jobs.client.minitest; + +import org.apache.oozie.client.OozieClientException; +import org.apache.oozie.fluentjob.api.GraphVisualization; +import org.apache.oozie.fluentjob.api.action.PigAction; +import org.apache.oozie.fluentjob.api.action.PigActionBuilder; +import org.apache.oozie.fluentjob.api.action.Prepare; +import org.apache.oozie.fluentjob.api.action.PrepareBuilder; +import org.apache.oozie.fluentjob.api.dag.Graph; +import org.apache.oozie.fluentjob.api.serialization.WorkflowMarshaller; +import org.apache.oozie.fluentjob.api.workflow.Workflow; +import org.apache.oozie.fluentjob.api.workflow.WorkflowBuilder; +import org.apache.oozie.test.WorkflowTestCase; + +import javax.xml.bind.JAXBException; +import java.io.IOException; + +public class TestPigAction extends WorkflowTestCase { + public void testForkedPigActions() throws IOException, JAXBException, OozieClientException { + final Prepare prepare = new PrepareBuilder() + .withDelete("hdfs://localhost:8020/user/${wf:user()}/examples/output") + .build(); + + final PigAction parent = PigActionBuilder.create() + .withResourceManager(getJobTrackerUri()) + .withNameNode(getNameNodeUri()) + .withPrepare(prepare) + .withConfigProperty("mapred.job.queue.name", "default") + .withArg("arg1") + .withScript("pig.sql") + .build(); + + // We are reusing the definition of parent and only modifying and adding what is different. + final PigAction leftChild = PigActionBuilder.createFromExistingAction(parent) + .withParent(parent) + .withoutArg("arg1") + .withArg("arg2") + .build(); + + PigActionBuilder.createFromExistingAction(leftChild) + .withoutArg("arg2") + .withArg("arg3") + .build(); + + final Workflow workflow = new WorkflowBuilder() + .withName("simple-pig-example") + .withDagContainingNode(parent).build(); + + final String xml = WorkflowMarshaller.marshal(workflow); + + System.out.println(xml); + + GraphVisualization.workflowToPng(workflow, "simple-pig-example-workflow.png"); + + final Graph intermediateGraph = new Graph(workflow); + + GraphVisualization.graphToPng(intermediateGraph, "simple-pig-example-graph.png"); + + log.debug("Workflow XML is:\n{0}", xml); + + validate(xml); + } +}
