FALCON-2276 Falcon Trusted extensions submission and validation was failing
Author: sandeep <[email protected]> Reviewers: @pallavi-rao Closes #360 from sandeepSamudrala/FALCON-2276 and squashes the following commits: 84169d6 [sandeep] FALCON-2276 Falcon Trusted extensions submission and validation was failing a3bd0e9 [sandeep] Merge branch 'master' of https://github.com/apache/falcon db425c5 [sandeep] Merge branch 'master' of https://github.com/apache/falcon 3f67fed [sandeep] Merge branch 'master' of https://github.com/apache/falcon cb2b00d [sandeep] Merge branch 'master' of https://github.com/apache/falcon 79e8d64 [sandeep] Merge branch 'master' of https://github.com/apache/falcon 7de7798 [sandeep] go -b FALCON-2263Merge branch 'master' of https://github.com/apache/falcon c5da0a2 [sandeep] Merge branch 'master' of https://github.com/apache/falcon 7e16263 [sandeep] Merge branch 'master' of https://github.com/apache/falcon a234d94 [sandeep] FALCON-2231 Incoporated review comments and small fixes for duplicate submission and colo addition to schedule command 26e3350 [sandeep] Merge branch 'master' of https://github.com/apache/falcon 73fbf75 [sandeep] Merge branch 'master' of https://github.com/apache/falcon cc28658 [sandeep] Merge branch 'master' of https://github.com/apache/falcon 089b10d [sandeep] Merge branch 'master' of https://github.com/apache/falcon 456d4ee [sandeep] Merge branch 'master' of https://github.com/apache/falcon 0cf9af6 [sandeep] Merge branch 'master' of https://github.com/apache/falcon 4a2e23e [sandeep] Merge branch 'master' of https://github.com/apache/falcon b1546ed [sandeep] Merge branch 'master' of https://github.com/apache/falcon 0a433fb [sandeep] Merge branch 'master' of https://github.com/apache/falcon 194f36a [sandeep] Merge branch 'master' of https://github.com/apache/falcon e0ad358 [sandeep] Merge branch 'master' of https://github.com/apache/falcon f96a084 [sandeep] Merge branch 'master' of https://github.com/apache/falcon 9cf36e9 [sandeep] Merge branch 'master' of https://github.com/apache/falcon bbca081 [sandeep] Merge branch 'master' of https://github.com/apache/falcon 48f6afa [sandeep] Merge branch 'master' of https://github.com/apache/falcon 250cc46 [sandeep] Merge branch 'master' of https://github.com/apache/falcon d0393e9 [sandeep] Merge branch 'master' of https://github.com/apache/falcon a178805 [sandeep] Merge branch 'master' of https://github.com/apache/falcon d6dc8bf [sandeep] Merge branch 'master' of https://github.com/apache/falcon 1bb8d3c [sandeep] Merge branch 'master' of https://github.com/apache/falcon c065566 [sandeep] reverting last line changes made 1a4dcd2 [sandeep] rebased and resolved the conflicts from master 271318b [sandeep] FALCON-2097. Adding UT to the new method for getting next instance time with Delay. a94d4fe [sandeep] rebasing from master 9e68a57 [sandeep] FALCON-298. Feed update with replication delay creates holes (cherry picked from commit bf027e16995a0487296517fc0024831733e4002a) Signed-off-by: Pallavi Rao <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/6c105297 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/6c105297 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/6c105297 Branch: refs/heads/master Commit: 6c105297e05a49e09828c4caa23433b2f38d7846 Parents: 2368bcc Author: sandeep <[email protected]> Authored: Fri Feb 10 08:41:40 2017 +0530 Committer: Pallavi Rao <[email protected]> Committed: Fri Feb 10 08:42:01 2017 +0530 ---------------------------------------------------------------------- .../org/apache/falcon/client/FalconClient.java | 4 ++-- .../org/apache/falcon/extensions/Extension.java | 4 ++++ .../resource/proxy/ExtensionManagerProxy.java | 19 ++----------------- 3 files changed, 8 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/6c105297/client/src/main/java/org/apache/falcon/client/FalconClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/falcon/client/FalconClient.java b/client/src/main/java/org/apache/falcon/client/FalconClient.java index 25eaeb5..cf02619 100644 --- a/client/src/main/java/org/apache/falcon/client/FalconClient.java +++ b/client/src/main/java/org/apache/falcon/client/FalconClient.java @@ -1169,7 +1169,7 @@ public class FalconClient extends AbstractFalconClient { private List<Entity> getEntities(String extensionName, String jobName, InputStream configStream, String extensionType, String extensionBuildLocation) { List<Entity> entities = null; - if (!extensionType.equals(ExtensionType.CUSTOM.name())) { + if (!extensionType.equals(ExtensionType.TRUSTED.toString())) { try { entities = ExtensionHandler.loadAndPrepare(extensionName, jobName, configStream, extensionBuildLocation); @@ -1209,7 +1209,7 @@ public class FalconClient extends AbstractFalconClient { final String configPath, final String doAsUser) { String extensionType = ExtensionHandler.getExtensionType(extensionName, getExtensionDetailJson(extensionName)); InputStream configStream = getServletInputStream(configPath); - if (ExtensionType.TRUSTED.name().equalsIgnoreCase(extensionType)) { + if (extensionType.equals(ExtensionType.TRUSTED.toString())) { ClientResponse clientResponse = new ResourceBuilder() .path(ExtensionOperations.VALIDATE.path, extensionName) .addQueryParam(DO_AS_OPT, doAsUser) http://git-wip-us.apache.org/repos/asf/falcon/blob/6c105297/extensions/src/main/java/org/apache/falcon/extensions/Extension.java ---------------------------------------------------------------------- diff --git a/extensions/src/main/java/org/apache/falcon/extensions/Extension.java b/extensions/src/main/java/org/apache/falcon/extensions/Extension.java index 3869718..6b94323 100644 --- a/extensions/src/main/java/org/apache/falcon/extensions/Extension.java +++ b/extensions/src/main/java/org/apache/falcon/extensions/Extension.java @@ -21,6 +21,7 @@ package org.apache.falcon.extensions; import org.apache.commons.lang3.StringUtils; import org.apache.falcon.FalconException; import org.apache.falcon.Pair; +import org.apache.falcon.entity.EntityUtil; import org.apache.falcon.entity.v0.Entity; import org.apache.falcon.entity.v0.feed.Schema; import org.apache.falcon.extensions.store.ExtensionStore; @@ -109,6 +110,9 @@ public class Extension implements ExtensionBuilder { throw new FalconException("Entity created from the extension template cannot be null"); } LOG.info("Extension processing complete"); + // add tags on extension name and job + String jobName = configProperties.getProperty(ExtensionProperties.JOB_NAME.getName()); + EntityUtil.applyTags(extensionName, jobName, Collections.singletonList(entity)); return Collections.singletonList(entity); } http://git-wip-us.apache.org/repos/asf/falcon/blob/6c105297/prism/src/main/java/org/apache/falcon/resource/proxy/ExtensionManagerProxy.java ---------------------------------------------------------------------- diff --git a/prism/src/main/java/org/apache/falcon/resource/proxy/ExtensionManagerProxy.java b/prism/src/main/java/org/apache/falcon/resource/proxy/ExtensionManagerProxy.java index 5652ca4..65618fe 100644 --- a/prism/src/main/java/org/apache/falcon/resource/proxy/ExtensionManagerProxy.java +++ b/prism/src/main/java/org/apache/falcon/resource/proxy/ExtensionManagerProxy.java @@ -28,7 +28,6 @@ import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Properties; import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; @@ -58,7 +57,6 @@ import org.apache.falcon.entity.v0.EntityType; import org.apache.falcon.entity.v0.feed.Feed; import org.apache.falcon.entity.v0.process.Process; import org.apache.falcon.extensions.Extension; -import org.apache.falcon.extensions.ExtensionProperties; import org.apache.falcon.extensions.ExtensionService; import org.apache.falcon.extensions.ExtensionType; import org.apache.falcon.extensions.jdbc.ExtensionMetaStore; @@ -300,7 +298,7 @@ public class ExtensionManagerProxy extends AbstractExtensionManager { List<Entity> entities; TreeMap<EntityType, List<Entity>> entityMap = new TreeMap<>(); if (ExtensionType.TRUSTED.equals(extensionType)) { - entities = generateEntities(extensionName, config); + entities = extension.getEntities(extensionName, config); List<Entity> trustedFeeds = new ArrayList<>(); List<Entity> trustedProcesses = new ArrayList<>(); for (Entity entity : entities) { @@ -568,7 +566,7 @@ public class ExtensionManagerProxy extends AbstractExtensionManager { throw FalconWebException.newAPIException("Extension validation is supported only for trusted extensions"); } try { - List<Entity> entities = generateEntities(extensionName, request.getInputStream()); + List<Entity> entities = extension.getEntities(extensionName, request.getInputStream()); for (Entity entity : entities) { super.validate(entity); } @@ -719,19 +717,6 @@ public class ExtensionManagerProxy extends AbstractExtensionManager { } } - private List<Entity> generateEntities(String extensionName, InputStream configStream) - throws FalconException, IOException { - // get entities for extension job - Properties properties = new Properties(); - properties.load(configStream); - List<Entity> entities = extension.getEntities(extensionName, configStream); - - // add tags on extension name and job - String jobName = properties.getProperty(ExtensionProperties.JOB_NAME.getName()); - EntityUtil.applyTags(extensionName, jobName, entities); - return entities; - } - private static void checkIfExtensionServiceIsEnabled() { if (!Services.get().isRegistered(ExtensionService.SERVICE_NAME)) { LOG.error(ExtensionService.SERVICE_NAME + " is not enabled.");
