Repository: falcon Updated Branches: refs/heads/master 2c9295a9d -> b11e0570f
FALCON-2184 Support to delete a non trusted recipe metadata from server Author: Praveen Adlakha <[email protected]> Reviewers: @pallavi-rao Closes #300 from PraveenAdlakha/755 and squashes the following commits: 74e7383 [Praveen Adlakha] checkstyle issue fixed acf2b5b [Praveen Adlakha] exception handling changed fb3028d [Praveen Adlakha] comment's addressed 6269ba8 [Praveen Adlakha] checkstyle issues fixed cd79f5d [Praveen Adlakha] FALCON-2181 Support for storing metadata of non trusted recipe c483e4f [Praveen Adlakha] comments addressed a158993 [Praveen Adlakha] test cases added and comments addressed 8fbb608 [Praveen Adlakha] WIP 63a0cce [Praveen Adlakha] sandeep's comment addressed 9ff91e0 [Praveen Adlakha] FALCON-2181 Support for storing metadata of non trusted recipe Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/b11e0570 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/b11e0570 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/b11e0570 Branch: refs/heads/master Commit: b11e0570f74c7355b0e4ef648f80c1e6ca9a3523 Parents: 2c9295a Author: Praveen Adlakha <[email protected]> Authored: Fri Nov 18 14:11:35 2016 +0530 Committer: Pallavi Rao <[email protected]> Committed: Fri Nov 18 14:11:35 2016 +0530 ---------------------------------------------------------------------- .../java/org/apache/falcon/cli/FalconCLI.java | 2 +- .../apache/falcon/cli/FalconExtensionCLI.java | 6 +++++ .../falcon/client/FalconCLIConstants.java | 1 + .../org/apache/falcon/client/FalconClient.java | 10 ++++++++- .../persistence/ExtensionMetadataBean.java | 4 +++- .../persistence/PersistenceConstants.java | 2 ++ .../apache/falcon/extensions/ExtensionType.java | 2 +- .../extensions/jdbc/ExtensionMetaStore.java | 23 ++++++++++++++++++++ .../falcon/extensions/store/ExtensionStore.java | 15 +++++++++++++ .../extensions/jdbc/ExtensionMetaStoreTest.java | 2 +- .../store/AbstractTestExtensionStore.java | 1 - .../resource/extensions/ExtensionManager.java | 16 +++++++++++++- 12 files changed, 77 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/b11e0570/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java ---------------------------------------------------------------------- diff --git a/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java b/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java index 0dd11f6..5b5f9ff 100644 --- a/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java +++ b/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java @@ -100,7 +100,7 @@ public class FalconCLI { metadataCLI.createMetadataOptions(), true); parser.addCommand(FalconCLIConstants.EXTENSION_CMD, "", "Extension operations like enumerate, definition, describe, list, instances, " - + "submit, submitAndSchedule, schedule, suspend, resume, delete, update, validate", + + "submit, submitAndSchedule, schedule, suspend, resume, delete, update, validate,unregister", extensionCLI.createExtensionOptions(), true); parser.addCommand(FalconCLIConstants.VERSION_OPT, "", "show client version", new Options(), false); http://git-wip-us.apache.org/repos/asf/falcon/blob/b11e0570/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java ---------------------------------------------------------------------- diff --git a/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java b/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java index dafd7a8..1bb27a0 100644 --- a/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java +++ b/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java @@ -49,6 +49,7 @@ public class FalconExtensionCLI { public static final String DEFINITION_OPT = "definition"; public static final String DESCRIBE_OPT = "describe"; public static final String INSTANCES_OPT = "instances"; + public static final String UNREGISTER_OPT = "unregister"; // Input parameters public static final String ENTENSION_NAME_OPT = "extensionName"; @@ -79,6 +80,9 @@ public class FalconExtensionCLI { } else if (optionsList.contains(DESCRIBE_OPT)) { validateRequiredParameter(extensionName, ENTENSION_NAME_OPT); result = client.getExtensionDescription(extensionName); + } else if (optionsList.contains(UNREGISTER_OPT)) { + validateRequiredParameter(extensionName, ENTENSION_NAME_OPT); + result = client.unregisterExtension(extensionName); } else if (optionsList.contains(FalconCLIConstants.SUBMIT_OPT)) { validateRequiredParameter(extensionName, ENTENSION_NAME_OPT); validateRequiredParameter(filePath, FalconCLIConstants.FILE_PATH_OPT); @@ -153,6 +157,7 @@ public class FalconExtensionCLI { Option suspend = new Option(FalconCLIConstants.SUSPEND_OPT, false, "Suspend an extension job"); Option resume = new Option(FalconCLIConstants.RESUME_OPT, false, "Resume an extension job"); Option delete = new Option(FalconCLIConstants.DELETE_OPT, false, "Delete an extension job"); + Option unregister = new Option(FalconCLIConstants.UREGISTER, false, "Delete metadata of extension job"); OptionGroup group = new OptionGroup(); group.addOption(enumerate); @@ -168,6 +173,7 @@ public class FalconExtensionCLI { group.addOption(suspend); group.addOption(resume); group.addOption(delete); + group.addOption(unregister); extensionOptions.addOptionGroup(group); Option url = new Option(FalconCLIConstants.URL_OPTION, true, "Falcon URL"); http://git-wip-us.apache.org/repos/asf/falcon/blob/b11e0570/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java b/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java index 04f1599..8b15f86 100644 --- a/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java +++ b/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java @@ -217,4 +217,5 @@ public final class FalconCLIConstants { public static final String DIRECTION_OPT_DESCRIPTION = "edge direction property"; public static final String DEBUG_OPTION_DESCRIPTION = "Use debug mode to see debugging statements on stdout"; public static final String DO_AS_DESCRIPTION = "doAs user"; + public static final String UREGISTER = "unregister"; } http://git-wip-us.apache.org/repos/asf/falcon/blob/b11e0570/client/src/main/java/org/apache/falcon/client/FalconClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/falcon/client/FalconClient.java b/client/src/main/java/org/apache/falcon/client/FalconClient.java index 8f77fad..ef9df7f 100644 --- a/client/src/main/java/org/apache/falcon/client/FalconClient.java +++ b/client/src/main/java/org/apache/falcon/client/FalconClient.java @@ -346,7 +346,8 @@ public class FalconClient extends AbstractFalconClient { SCHEDULE("api/extension/schedule", HttpMethod.POST, MediaType.TEXT_XML), SUSPEND("api/extension/suspend", HttpMethod.POST, MediaType.TEXT_XML), RESUME("api/extension/resume", HttpMethod.POST, MediaType.TEXT_XML), - DELETE("api/extension/delete", HttpMethod.POST, MediaType.TEXT_XML); + DELETE("api/extension/delete", HttpMethod.POST, MediaType.TEXT_XML), + UNREGISTER("api/extension/unregister/", HttpMethod.POST, MediaType.TEXT_PLAIN); private String path; private String method; @@ -1021,6 +1022,13 @@ public class FalconClient extends AbstractFalconClient { return getResponse(String.class, clientResponse); } + public String unregisterExtension(final String extensionName) { + ClientResponse clientResponse = new ResourceBuilder() + .path(ExtensionOperations.UNREGISTER.path, extensionName) + .call(ExtensionOperations.UNREGISTER); + return getResponse(String.class, clientResponse); + } + public String getExtensionDefinition(final String extensionName) { ClientResponse clientResponse = new ResourceBuilder() .path(ExtensionOperations.DEFINITION.path, extensionName) http://git-wip-us.apache.org/repos/asf/falcon/blob/b11e0570/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java b/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java index ac1ded6..f4208ad 100644 --- a/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java +++ b/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java @@ -38,7 +38,9 @@ import java.util.Date; @NamedQueries({ @NamedQuery(name = PersistenceConstants.GET_ALL_EXTENSIONS, query = "select OBJECT(a) from ExtensionMetadataBean a "), @NamedQuery(name = PersistenceConstants.GET_EXTENSION_LOCATION, query = "select a.location from ExtensionMetadataBean a where a.extensionName = :extensionName"), - @NamedQuery(name = PersistenceConstants.DELETE_EXTENSIONS_OF_TYPE, query = "delete from ExtensionMetadataBean a where a.extensionType = :extensionType ") + @NamedQuery(name = PersistenceConstants.DELETE_EXTENSIONS_OF_TYPE, query = "delete from ExtensionMetadataBean a where a.extensionType = :extensionType "), + @NamedQuery(name = PersistenceConstants.DELETE_EXTENSION, query = "delete from ExtensionMetadataBean a where a.extensionName = :extensionName "), + @NamedQuery(name = PersistenceConstants.GET_EXTENSION, query = "select OBJECT(a) from ExtensionMetadataBean a where a.extensionName = :extensionName") }) //RESUME CHECKSTYLE CHECK LineLengthCheck public class ExtensionMetadataBean { http://git-wip-us.apache.org/repos/asf/falcon/blob/b11e0570/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java b/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java index dbbf06d..d0e8e9c 100644 --- a/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java +++ b/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java @@ -70,4 +70,6 @@ public final class PersistenceConstants { public static final String GET_ALL_EXTENSIONS = "GET_ALL_EXTENSIONS"; public static final String GET_EXTENSION_LOCATION = "GET_EXTENSION_LOCATION"; public static final String DELETE_EXTENSIONS_OF_TYPE = "DELETE_EXTENSIONS_OF_TYPE"; + public static final String DELETE_EXTENSION = "DELETE_EXTENSION"; + public static final String GET_EXTENSION = "GET_EXTENSION"; } http://git-wip-us.apache.org/repos/asf/falcon/blob/b11e0570/extensions/src/main/java/org/apache/falcon/extensions/ExtensionType.java ---------------------------------------------------------------------- diff --git a/extensions/src/main/java/org/apache/falcon/extensions/ExtensionType.java b/extensions/src/main/java/org/apache/falcon/extensions/ExtensionType.java index c7e45cc..c4621cc 100644 --- a/extensions/src/main/java/org/apache/falcon/extensions/ExtensionType.java +++ b/extensions/src/main/java/org/apache/falcon/extensions/ExtensionType.java @@ -34,4 +34,4 @@ public enum ExtensionType { public String toString(){ return text; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/falcon/blob/b11e0570/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java ---------------------------------------------------------------------- diff --git a/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java b/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java index b2f7e7d..2c6003d 100644 --- a/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java +++ b/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java @@ -53,6 +53,17 @@ public class ExtensionMetaStore { } } + public Boolean checkIfExtensionExists(String extensionName){ + EntityManager entityManager = getEntityManager(); + beginTransaction(entityManager); + Query q = entityManager.createNamedQuery(PersistenceConstants.GET_EXTENSION); + q.setParameter("extensionName", extensionName); + if (q.getResultList().size() > 0){ + return true; + } + return false; + } + public List<ExtensionMetadataBean> getAllExtensions(){ EntityManager entityManager = getEntityManager(); beginTransaction(entityManager); @@ -88,6 +99,18 @@ public class ExtensionMetaStore { } } + public void deleteExtensionMetadata(String extensionName){ + EntityManager entityManager = getEntityManager(); + beginTransaction(entityManager); + Query q = entityManager.createNamedQuery(PersistenceConstants.DELETE_EXTENSION); + q.setParameter("extensionName", extensionName); + try{ + q.executeUpdate(); + } finally { + commitAndCloseTransaction(entityManager); + } + } + private void beginTransaction(EntityManager entityManager) { entityManager.getTransaction().begin(); } http://git-wip-us.apache.org/repos/asf/falcon/blob/b11e0570/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java ---------------------------------------------------------------------- diff --git a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java index 61804f9..ffb4341 100644 --- a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java +++ b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java @@ -24,6 +24,7 @@ import org.apache.falcon.extensions.AbstractExtension; import org.apache.falcon.extensions.ExtensionType; import org.apache.falcon.extensions.jdbc.ExtensionMetaStore; import org.apache.falcon.hadoop.HadoopClientFactory; +import org.apache.falcon.entity.parser.ValidationException; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; @@ -250,6 +251,20 @@ public final class ExtensionStore { return extesnionList; } + public String deleteExtensionMetadata(final String extensionName) throws ValidationException{ + ExtensionType extensionType = AbstractExtension.isExtensionTrusted(extensionName) ? ExtensionType.TRUSTED + : ExtensionType.CUSTOM; + if (extensionType.equals(ExtensionType.TRUSTED)){ + throw new ValidationException(extensionName + " is trusted cannot be deleted."); + } + if (metaStore.checkIfExtensionExists(extensionName)) { + metaStore.deleteExtensionMetadata(extensionName); + return "Deleted extension:" + extensionName; + }else { + return "Extension:" + extensionName + " is not registered with Falcon."; + } + } + public String getResource(final String extensionName, final String resourceName) throws StoreAccessException { Map<String, String> resources = getExtensionArtifacts(extensionName); if (resources.isEmpty()) { http://git-wip-us.apache.org/repos/asf/falcon/blob/b11e0570/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java ---------------------------------------------------------------------- diff --git a/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java b/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java index 61b26d6..7e0db0d 100644 --- a/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java +++ b/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java @@ -1,4 +1,3 @@ - /** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -60,6 +59,7 @@ public class ExtensionMetaStoreTest extends AbstractTestExtensionStore { public void dbOpertaions(){ //insert stateStore.storeExtensionMetadataBean("test1", "test_location", ExtensionType.TRUSTED, "test_description"); + Assert.assertEquals(stateStore.getAllExtensions().size(), 1); //check data Assert.assertEquals(stateStore.getLocation("test1"), "test_location"); http://git-wip-us.apache.org/repos/asf/falcon/blob/b11e0570/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java ---------------------------------------------------------------------- diff --git a/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java b/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java index cd94eca..97fe287 100644 --- a/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java +++ b/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java @@ -83,7 +83,6 @@ public class AbstractTestExtensionStore { falconJPAService.init(); new ExtensionService().init(); store = ExtensionService.getExtensionStore(); - FalconJPAService falconJPAService = FalconJPAService.get(); fileSystem = HadoopClientFactory.get().createFalconFileSystem(new Configuration(true)); extensionStorePath = new URI(StartupProperties.get().getProperty(ExtensionStore.EXTENSION_STORE_URI)).getPath(); extensionStoreSetup(resourceClass); http://git-wip-us.apache.org/repos/asf/falcon/blob/b11e0570/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java ---------------------------------------------------------------------- diff --git a/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java b/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java index 138109c..5669b8f 100644 --- a/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java +++ b/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java @@ -420,6 +420,21 @@ public class ExtensionManager extends AbstractSchedulableEntityManager { } } + @POST + @Path("unregister/{extension-name}") + @Consumes({MediaType.TEXT_XML, MediaType.TEXT_PLAIN}) + @Produces(MediaType.TEXT_PLAIN) + public String deleteExtensionMetadata( + @PathParam("extension-name") String extensionName){ + checkIfExtensionServiceIsEnabled(); + validateExtensionName(extensionName); + try { + return ExtensionStore.get().deleteExtensionMetadata(extensionName); + } catch (Throwable e) { + throw FalconWebException.newAPIException(e, Response.Status.INTERNAL_SERVER_ERROR); + } + } + @GET @Path("definition/{extension-name}") @Produces({MediaType.APPLICATION_JSON}) @@ -463,7 +478,6 @@ public class ExtensionManager extends AbstractSchedulableEntityManager { return results; } - private List<Entity> generateEntities(String extensionName, HttpServletRequest request) throws FalconException, IOException { // get entities for extension job
