Repository: falcon Updated Branches: refs/heads/master ef4e8a4b1 -> 6676032c3
FALCON-1588 Add ability to provide the path for recipe files in command line. Contributed by Peeyush Bishnoi. Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/6676032c Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/6676032c Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/6676032c Branch: refs/heads/master Commit: 6676032c3e2746f965a25d02a17bd107b3074df4 Parents: ef4e8a4 Author: Ajay Yadava <[email protected]> Authored: Tue Nov 24 16:08:40 2015 +0530 Committer: Ajay Yadava <[email protected]> Committed: Tue Nov 24 16:08:40 2015 +0530 ---------------------------------------------------------------------- CHANGES.txt | 2 ++ .../org/apache/falcon/cli/FalconRecipeCLI.java | 24 ++++++++++++++++- .../org/apache/falcon/client/FalconClient.java | 28 +++++++++++++++----- src/conf/client.properties | 1 + 4 files changed, 48 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/6676032c/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 62e5725..13d3439 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -9,6 +9,8 @@ Trunk (Unreleased) INCOMPATIBLE CHANGES NEW FEATURES + FALCON-1588 Add ability to provide the path for recipe files in command line(Peeyush Bishnoi via Ajay Yadava) + FALCON-1573 Supply user-defined properties to Oozie workflows during schedule(Daniel Del Castillo via Ajay Yadava) FALCON-1559 Config changes required for native scheduler (Pallavi Rao) http://git-wip-us.apache.org/repos/asf/falcon/blob/6676032c/client/src/main/java/org/apache/falcon/cli/FalconRecipeCLI.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/falcon/cli/FalconRecipeCLI.java b/client/src/main/java/org/apache/falcon/cli/FalconRecipeCLI.java index b93fed2..82053f9 100644 --- a/client/src/main/java/org/apache/falcon/cli/FalconRecipeCLI.java +++ b/client/src/main/java/org/apache/falcon/cli/FalconRecipeCLI.java @@ -21,6 +21,7 @@ package org.apache.falcon.cli; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; +import org.apache.commons.lang3.StringUtils; import org.apache.falcon.client.FalconCLIException; import org.apache.falcon.client.FalconClient; @@ -39,6 +40,7 @@ public class FalconRecipeCLI extends FalconCLI { private static final String RECIPE_NAME = "name"; private static final String RECIPE_OPERATION= "operation"; private static final String RECIPE_TOOL_CLASS_NAME = "tool"; + private static final String RECIPE_PROPERTIES_FILE = "properties"; public Options createRecipeOptions() { Options recipeOptions = new Options(); @@ -54,6 +56,9 @@ public class FalconRecipeCLI extends FalconCLI { Option recipeOperation = new Option(RECIPE_OPERATION, true, "recipe operation"); recipeOptions.addOption(recipeOperation); + Option recipeProperties = new Option(RECIPE_PROPERTIES_FILE, true, "recipe properties file path"); + recipeOptions.addOption(recipeProperties); + Option skipDryRunOperation = new Option(SKIPDRYRUN_OPT, false, "skip dryrun operation"); recipeOptions.addOption(skipDryRunOperation); @@ -72,18 +77,20 @@ public class FalconRecipeCLI extends FalconCLI { String recipeName = commandLine.getOptionValue(RECIPE_NAME); String recipeToolClass = commandLine.getOptionValue(RECIPE_TOOL_CLASS_NAME); String recipeOperation = commandLine.getOptionValue(RECIPE_OPERATION); + String recipePropertiesFile = commandLine.getOptionValue(RECIPE_PROPERTIES_FILE); String doAsUser = commandLine.getOptionValue(DO_AS_OPT); validateNotEmpty(recipeName, RECIPE_NAME); validateNotEmpty(recipeOperation, RECIPE_OPERATION); validateRecipeOperations(recipeOperation); + validateRecipePropertiesFile(recipePropertiesFile, recipeName); Boolean skipDryRun = null; if (optionsList.contains(SKIPDRYRUN_OPT)) { skipDryRun = true; } String result = client.submitRecipe(recipeName, recipeToolClass, - recipeOperation, skipDryRun, doAsUser).toString(); + recipeOperation, recipePropertiesFile, skipDryRun, doAsUser).toString(); OUT.get().println(result); } @@ -96,4 +103,19 @@ public class FalconRecipeCLI extends FalconCLI { throw new FalconCLIException("Allowed Recipe operations: " + java.util.Arrays.asList((RecipeOperation.values()))); } + + private static void validateRecipePropertiesFile(String recipePropertiesFile, String recipeName) + throws FalconCLIException { + if (StringUtils.isBlank(recipePropertiesFile)) { + return; + } + + String []fileSplits = recipePropertiesFile.split("/"); + String recipePropertiesName = (fileSplits[fileSplits.length-1]).split("\\.")[0]; + if (recipePropertiesName.equals(recipeName)) { + return; + } + + throw new FalconCLIException("Provided properties file name do match with recipe name: " +recipeName); + } } http://git-wip-us.apache.org/repos/asf/falcon/blob/6676032c/client/src/main/java/org/apache/falcon/client/FalconClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/falcon/client/FalconClient.java b/client/src/main/java/org/apache/falcon/client/FalconClient.java index 471cab4..b4b1762 100644 --- a/client/src/main/java/org/apache/falcon/client/FalconClient.java +++ b/client/src/main/java/org/apache/falcon/client/FalconClient.java @@ -1102,21 +1102,37 @@ public class FalconClient extends AbstractFalconClient { return sendMetadataLineageRequest(MetadataOperations.EDGES, id, doAsUser); } + private String getRecipePath(String recipePropertiesFile) throws FalconCLIException { + String recipePath = null; + if (StringUtils.isNotBlank(recipePropertiesFile)) { + File file = new File(recipePropertiesFile); + if (file.exists()) { + recipePath = file.getAbsoluteFile().getParentFile().getAbsolutePath(); + } + } else { + recipePath = clientProperties.getProperty("falcon.recipe.path"); + } + + return recipePath; + } + public APIResult submitRecipe(String recipeName, String recipeToolClassName, final String recipeOperation, + String recipePropertiesFile, Boolean skipDryRun, final String doAsUser) throws FalconCLIException { - String recipePath = clientProperties.getProperty("falcon.recipe.path"); + String recipePath = getRecipePath(recipePropertiesFile); if (StringUtils.isEmpty(recipePath)) { - throw new FalconCLIException("falcon.recipe.path is not set in client.properties"); + throw new FalconCLIException("falcon.recipe.path is not set in client.properties or properties " + + " file is not provided"); } - String recipeFilePath = recipePath + File.separator + recipeName + TEMPLATE_SUFFIX; - File file = new File(recipeFilePath); + String templateFilePath = recipePath + File.separator + recipeName + TEMPLATE_SUFFIX; + File file = new File(templateFilePath); if (!file.exists()) { - throw new FalconCLIException("Recipe template file does not exist : " + recipeFilePath); + throw new FalconCLIException("Recipe template file does not exist : " + templateFilePath); } String propertiesFilePath = recipePath + File.separator + recipeName + PROPERTIES_SUFFIX; @@ -1139,7 +1155,7 @@ public class FalconClient extends AbstractFalconClient { processFile = f.getAbsolutePath(); String[] args = { - "-" + RecipeToolArgs.RECIPE_FILE_ARG.getName(), recipeFilePath, + "-" + RecipeToolArgs.RECIPE_FILE_ARG.getName(), templateFilePath, "-" + RecipeToolArgs.RECIPE_PROPERTIES_FILE_ARG.getName(), propertiesFilePath, "-" + RecipeToolArgs.RECIPE_PROCESS_XML_FILE_PATH_ARG.getName(), processFile, "-" + RecipeToolArgs.RECIPE_OPERATION_ARG.getName(), recipeOperation, http://git-wip-us.apache.org/repos/asf/falcon/blob/6676032c/src/conf/client.properties ---------------------------------------------------------------------- diff --git a/src/conf/client.properties b/src/conf/client.properties index 7925009..900e043 100644 --- a/src/conf/client.properties +++ b/src/conf/client.properties @@ -22,3 +22,4 @@ ######################################################################### falcon.url=https://localhost:15443/ +#falcon.recipe.path=
