Repository: falcon Updated Branches: refs/heads/master 2f2b816fe -> 20bc33187
FALCON-1902 Command line support for server side extension artifacts⦠⦠repository management Author: Sowmya Ramesh <[email protected]> Reviewers: Balu<[email protected]>, Ying Zheng<[email protected]>, Peeyush Bishnoi<[email protected]> Closes #99 from sowmyaramesh/FALCON-1902 and squashes the following commits: 60dc1ac [Sowmya Ramesh] FALCON-1902: Incorporated review comments 642f0ae [Sowmya Ramesh] FALCON-1902: Incorpoarted review comments d27e2db [Sowmya Ramesh] FALCON-1902: Command line support for server side extension artifacts repository management Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/20bc3318 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/20bc3318 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/20bc3318 Branch: refs/heads/master Commit: 20bc33187f41df9c4e9efb46983ba79ef20657dc Parents: 2f2b816 Author: Sowmya Ramesh <[email protected]> Authored: Thu Apr 14 16:44:57 2016 -0700 Committer: Sowmya Ramesh <[email protected]> Committed: Thu Apr 14 16:44:57 2016 -0700 ---------------------------------------------------------------------- .../java/org/apache/falcon/cli/FalconCLI.java | 6 + .../apache/falcon/cli/FalconExtensionCLI.java | 114 +++++++++++++++++++ .../org/apache/falcon/FalconCLIConstants.java | 10 +- .../org/apache/falcon/client/FalconClient.java | 58 ++++++++++ common/src/main/resources/startup.properties | 2 +- src/conf/client.properties | 1 - 6 files changed, 180 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/20bc3318/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java ---------------------------------------------------------------------- diff --git a/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java b/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java index a414c94..c1b9d8c 100644 --- a/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java +++ b/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java @@ -90,6 +90,7 @@ public class FalconCLI { FalconEntityCLI entityCLI = new FalconEntityCLI(); FalconInstanceCLI instanceCLI = new FalconInstanceCLI(); FalconMetadataCLI metadataCLI = new FalconMetadataCLI(); + FalconExtensionCLI extensionCLI = new FalconExtensionCLI(); parser.addCommand(FalconCLIConstants.ADMIN_CMD, "", "admin operations", adminCLI.createAdminOptions(), true); parser.addCommand(FalconCLIConstants.HELP_CMD, "", "display usage", new Options(), false); @@ -101,6 +102,9 @@ public class FalconCLI { instanceCLI.createInstanceOptions(), false); parser.addCommand(FalconCLIConstants.METADATA_CMD, "", "Metadata operations like list, relations", metadataCLI.createMetadataOptions(), true); + parser.addCommand(FalconCLIConstants.EXTENSION_CMD, "", + "Extension operations like enumerate, definition, describe", + extensionCLI.createExtensionOptions(), true); parser.addCommand(FalconCLIConstants.VERSION_OPT, "", "show client version", new Options(), false); try { @@ -122,6 +126,8 @@ public class FalconCLI { instanceCLI.instanceCommand(commandLine, client); } else if (command.getName().equals(FalconCLIConstants.METADATA_CMD)) { metadataCLI.metadataCommand(commandLine, client); + } else if (command.getName().equals(FalconCLIConstants.EXTENSION_CMD)) { + extensionCLI.extensionCommand(commandLine, client); } } return exitValue; http://git-wip-us.apache.org/repos/asf/falcon/blob/20bc3318/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java ---------------------------------------------------------------------- diff --git a/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java b/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java new file mode 100644 index 0000000..c85a196 --- /dev/null +++ b/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java @@ -0,0 +1,114 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.falcon.cli; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionGroup; +import org.apache.commons.cli.Options; +import org.apache.commons.lang3.StringUtils; +import org.apache.falcon.client.FalconCLIException; +import org.apache.falcon.client.FalconClient; + +import java.io.PrintStream; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Falcon extensions Command Line Interface - wraps the RESTful API for extensions. + */ +public class FalconExtensionCLI { + public static final AtomicReference<PrintStream> OUT = new AtomicReference<>(System.out); + + // Extension artifact repository Commands + public static final String ENUMERATE_OPT = "enumerate"; + public static final String DEFINITION_OPT = "definition"; + public static final String DESCRIBE_OPT = "describe"; + + public static final String ENTENSION_NAME_OPT = "extensionName"; + + public FalconExtensionCLI() { + } + + public void extensionCommand(CommandLine commandLine, FalconClient client) throws FalconCLIException { + Set<String> optionsList = new HashSet<>(); + for (Option option : commandLine.getOptions()) { + optionsList.add(option.getOpt()); + } + + String result; + String extensionName = commandLine.getOptionValue(ENTENSION_NAME_OPT); + + if (optionsList.contains(ENUMERATE_OPT)) { + result = client.enumerateExtensions(); + prettyPrintJson(result); + } else if (optionsList.contains(DEFINITION_OPT)) { + validateExtensionName(extensionName); + result = client.getExtensionDefinition(extensionName); + prettyPrintJson(result); + } else if (optionsList.contains(DESCRIBE_OPT)) { + validateExtensionName(extensionName); + result = client.getExtensionDescription(extensionName); + OUT.get().println(result); + } else { + throw new FalconCLIException("Invalid extension command"); + } + } + + public Options createExtensionOptions() { + Options extensionOptions = new Options(); + + OptionGroup group = new OptionGroup(); + Option enumerate = new Option(ENUMERATE_OPT, false, "Enumerate all extensions"); + Option definition = new Option(DEFINITION_OPT, false, "Get extension definition"); + Option describe = new Option(DESCRIBE_OPT, false, "Get extension description"); + group.addOption(enumerate); + group.addOption(definition); + group.addOption(describe); + + extensionOptions.addOptionGroup(group); + + Option name = new Option(ENTENSION_NAME_OPT, true, "Extension name"); + extensionOptions.addOption(name); + + return extensionOptions; + } + + private void validateExtensionName(final String extensionName) throws FalconCLIException { + if (StringUtils.isBlank(extensionName)) { + throw new FalconCLIException("Extension name cannot be null or empty"); + } + } + + private static void prettyPrintJson(final String jsonString) { + if (StringUtils.isBlank(jsonString)) { + OUT.get().println("No result returned"); + return; + } + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + JsonParser jp = new JsonParser(); + JsonElement je = jp.parse(jsonString.trim()); + OUT.get().println(gson.toJson(je)); + } +} http://git-wip-us.apache.org/repos/asf/falcon/blob/20bc3318/client/src/main/java/org/apache/falcon/FalconCLIConstants.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/falcon/FalconCLIConstants.java b/client/src/main/java/org/apache/falcon/FalconCLIConstants.java index eb4cf3f..5f1fb3c 100644 --- a/client/src/main/java/org/apache/falcon/FalconCLIConstants.java +++ b/client/src/main/java/org/apache/falcon/FalconCLIConstants.java @@ -34,7 +34,7 @@ public final class FalconCLIConstants { public static final String METADATA_CMD = "metadata"; public static final String ENTITY_CMD = "entity"; public static final String INSTANCE_CMD = "instance"; - public static final String RECIPE_CMD = "recipe"; + public static final String EXTENSION_CMD = "extension"; public static final String TYPE_OPT = "type"; public static final String COLO_OPT = "colo"; @@ -65,12 +65,4 @@ public final class FalconCLIConstants { public static final String RELATIONS_OPT = "relations"; public static final String PIPELINE_OPT = "pipeline"; public static final String NAME_OPT = "name"; - - /** - * Recipe operation enum. - */ - public enum RecipeOperation { - HDFS_REPLICATION, - HIVE_DISASTER_RECOVERY - } } http://git-wip-us.apache.org/repos/asf/falcon/blob/20bc3318/client/src/main/java/org/apache/falcon/client/FalconClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/falcon/client/FalconClient.java b/client/src/main/java/org/apache/falcon/client/FalconClient.java index 0aaf0ec..02908d9 100644 --- a/client/src/main/java/org/apache/falcon/client/FalconClient.java +++ b/client/src/main/java/org/apache/falcon/client/FalconClient.java @@ -326,6 +326,26 @@ public class FalconClient extends AbstractFalconClient { } } + /** + * Methods allowed on Extension Resources. + */ + protected static enum ExtensionOperations { + + ENUMERATE("api/extensions/enumerate/", HttpMethod.GET, MediaType.APPLICATION_JSON), + DESCRIBE("api/extensions/describe/", HttpMethod.GET, MediaType.TEXT_PLAIN), + DEFINITION("api/extensions/definition", HttpMethod.GET, MediaType.APPLICATION_JSON); + + private String path; + private String method; + private String mimeType; + + ExtensionOperations(String path, String method, String mimeType) { + this.path = path; + this.method = method; + this.mimeType = mimeType; + } + } + public String notEmpty(String str, String name) { if (str == null) { @@ -946,6 +966,44 @@ public class FalconClient extends AbstractFalconClient { return sendMetadataLineageRequest(MetadataOperations.EDGES, id, doAsUser); } + public String enumerateExtensions() throws FalconCLIException { + return sendExtensionRequest(ExtensionOperations.ENUMERATE, null); + } + + public String getExtensionDefinition(final String extensionName) throws FalconCLIException { + return sendExtensionRequest(ExtensionOperations.DEFINITION, extensionName); + } + + public String getExtensionDescription(final String extensionName) throws FalconCLIException { + return sendExtensionRequest(ExtensionOperations.DESCRIBE, extensionName); + } + + private String sendExtensionRequest(final ExtensionOperations operation, + final String extensionName) throws FalconCLIException { + WebResource resource; + switch (operation) { + case ENUMERATE: + resource = service.path(operation.path); + break; + + case DESCRIBE: + case DEFINITION: + resource = service.path(operation.path).path(extensionName); + break; + + default: + throw new FalconCLIException("Invalid extension client Operation " + operation.toString()); + } + + ClientResponse clientResponse = resource + .header("Cookie", AUTH_COOKIE_EQ + authenticationToken) + .accept(operation.mimeType).type(operation.mimeType) + .method(operation.method, ClientResponse.class); + + checkIfSuccessful(clientResponse); + return clientResponse.getEntity(String.class); + } + private String sendMetadataLineageRequest(MetadataOperations job, String id, String doAsUser) throws FalconCLIException { ClientResponse clientResponse = new ResourceBuilder().path(job.path, id).addQueryParam(DO_AS_OPT, doAsUser) http://git-wip-us.apache.org/repos/asf/falcon/blob/20bc3318/common/src/main/resources/startup.properties ---------------------------------------------------------------------- diff --git a/common/src/main/resources/startup.properties b/common/src/main/resources/startup.properties index dd51963..b7bac73 100644 --- a/common/src/main/resources/startup.properties +++ b/common/src/main/resources/startup.properties @@ -100,7 +100,7 @@ debug.libext.feed.retention.paths=${falcon.libext} debug.libext.feed.replication.paths=${falcon.libext} debug.libext.process.paths=${falcon.libext} -debug.extension.store.uri=file://${user.dir}/target/recipe/store +debug.extension.store.uri=file://${user.dir}/target/extension/store #Configurations used in ITs it.config.store.uri=file://${user.dir}/target/store http://git-wip-us.apache.org/repos/asf/falcon/blob/20bc3318/src/conf/client.properties ---------------------------------------------------------------------- diff --git a/src/conf/client.properties b/src/conf/client.properties index 900e043..7925009 100644 --- a/src/conf/client.properties +++ b/src/conf/client.properties @@ -22,4 +22,3 @@ ######################################################################### falcon.url=https://localhost:15443/ -#falcon.recipe.path=
