Repository: falcon Updated Branches: refs/heads/master 2923c04c1 -> 400ef329d
FALCON-1980 Change input and output argument order for Spark process workflow Details to perform this change has been mentioned in the FALCON-1980. Author: peeyush b <[email protected]> Reviewers: Pavan <[email protected]> Closes #160 from peeyushb/FALCON-1980 and squashes the following commits: 23d4e24 [peeyush b] Added comments for input and output argument 3d86f58 [peeyush b] FALCON-1980 : Change input and output argument order for Spark process workflow Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/400ef329 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/400ef329 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/400ef329 Branch: refs/heads/master Commit: 400ef329d1816b408e759990b880c9dd1970dae2 Parents: 2923c04 Author: Peeyush <[email protected]> Authored: Mon May 30 10:46:02 2016 +0530 Committer: peeyush b <[email protected]> Committed: Mon May 30 10:46:02 2016 +0530 ---------------------------------------------------------------------- .../oozie/process/SparkProcessWorkflowBuilder.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/400ef329/oozie/src/main/java/org/apache/falcon/oozie/process/SparkProcessWorkflowBuilder.java ---------------------------------------------------------------------- diff --git a/oozie/src/main/java/org/apache/falcon/oozie/process/SparkProcessWorkflowBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/process/SparkProcessWorkflowBuilder.java index dc5a491..8c06711 100644 --- a/oozie/src/main/java/org/apache/falcon/oozie/process/SparkProcessWorkflowBuilder.java +++ b/oozie/src/main/java/org/apache/falcon/oozie/process/SparkProcessWorkflowBuilder.java @@ -89,8 +89,9 @@ public class SparkProcessWorkflowBuilder extends ProcessExecutionWorkflowBuilder argList.addAll(sparkArgs); } - addInputFeedsAsArgument(argList, cluster); + //Adding output first so that final order must have input and then output followed by user's arguments. addOutputFeedsAsArgument(argList, cluster); + addInputFeedsAsArgument(argList, cluster); sparkAction.setJar(addUri(sparkFilePath, cluster)); @@ -145,6 +146,7 @@ public class SparkProcessWorkflowBuilder extends ProcessExecutionWorkflowBuilder return; } + //Adding to the 0th index and getting the args shifted as arguments are added to get the desired effect. int numInputFeed = entity.getInputs().getInputs().size(); while (numInputFeed > 0) { Input input = entity.getInputs().getInputs().get(numInputFeed-1); @@ -163,13 +165,17 @@ public class SparkProcessWorkflowBuilder extends ProcessExecutionWorkflowBuilder return; } - for(Output output : entity.getOutputs().getOutputs()) { + //Adding to the 0th index and getting the args shifted as arguments are added to get the desired effect. + int numOutputFeed = entity.getOutputs().getOutputs().size(); + while (numOutputFeed > 0) { + Output output = entity.getOutputs().getOutputs().get(numOutputFeed-1); Feed feed = EntityUtil.getEntity(EntityType.FEED, output.getFeed()); Storage storage = FeedHelper.createStorage(cluster, feed); final String outputName = output.getName(); if (storage.getType() == Storage.TYPE.FILESYSTEM) { - argList.add(argList.size(), "${" + outputName + "}"); + argList.add(0, "${" + outputName + "}"); } + numOutputFeed--; } }
