Repository: falcon Updated Branches: refs/heads/master ae510daa2 -> 6e162747b
FALCON-2258 Improve error message for definition/describe api in case extension does not exist Author: sandeep <[email protected]> Reviewers: @pallavi-rao Closes #341 from sandeepSamudrala/FALCON-2258 and squashes the following commits: 6b10c9d [sandeep] Corrected error messages b7fd5ab [sandeep] Incorporated review comments 9c99133 [sandeep] Incorporated review comments 80e533b [sandeep] FALCON-2258 Improve error message for definition/describe api in case extension does not exist 7e16263 [sandeep] Merge branch 'master' of https://github.com/apache/falcon a234d94 [sandeep] FALCON-2231 Incoporated review comments and small fixes for duplicate submission and colo addition to schedule command 26e3350 [sandeep] Merge branch 'master' of https://github.com/apache/falcon 73fbf75 [sandeep] Merge branch 'master' of https://github.com/apache/falcon cc28658 [sandeep] Merge branch 'master' of https://github.com/apache/falcon 089b10d [sandeep] Merge branch 'master' of https://github.com/apache/falcon 456d4ee [sandeep] Merge branch 'master' of https://github.com/apache/falcon 0cf9af6 [sandeep] Merge branch 'master' of https://github.com/apache/falcon 4a2e23e [sandeep] Merge branch 'master' of https://github.com/apache/falcon b1546ed [sandeep] Merge branch 'master' of https://github.com/apache/falcon 0a433fb [sandeep] Merge branch 'master' of https://github.com/apache/falcon 194f36a [sandeep] Merge branch 'master' of https://github.com/apache/falcon e0ad358 [sandeep] Merge branch 'master' of https://github.com/apache/falcon f96a084 [sandeep] Merge branch 'master' of https://github.com/apache/falcon 9cf36e9 [sandeep] Merge branch 'master' of https://github.com/apache/falcon bbca081 [sandeep] Merge branch 'master' of https://github.com/apache/falcon 48f6afa [sandeep] Merge branch 'master' of https://github.com/apache/falcon 250cc46 [sandeep] Merge branch 'master' of https://github.com/apache/falcon d0393e9 [sandeep] Merge branch 'master' of https://github.com/apache/falcon a178805 [sandeep] Merge branch 'master' of https://github.com/apache/falcon d6dc8bf [sandeep] Merge branch 'master' of https://github.com/apache/falcon 1bb8d3c [sandeep] Merge branch 'master' of https://github.com/apache/falcon c065566 [sandeep] reverting last line changes made 1a4dcd2 [sandeep] rebased and resolved the conflicts from master 271318b [sandeep] FALCON-2097. Adding UT to the new method for getting next instance time with Delay. a94d4fe [sandeep] rebasing from master 9e68a57 [sandeep] FALCON-298. Feed update with replication delay creates holes Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/6e162747 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/6e162747 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/6e162747 Branch: refs/heads/master Commit: 6e162747b090b6433d2e1813938d4f28f6926af3 Parents: ae510da Author: sandeep <[email protected]> Authored: Thu Jan 19 10:50:13 2017 +0530 Committer: Pallavi Rao <[email protected]> Committed: Thu Jan 19 10:50:13 2017 +0530 ---------------------------------------------------------------------- docs/src/site/twiki/FalconUserExtension.twiki | 0 .../extensions/jdbc/ExtensionMetaStore.java | 9 +++-- .../falcon/extensions/store/ExtensionStore.java | 38 ++++++++++++-------- .../resource/AbstractExtensionManager.java | 22 ++++++++---- .../resource/proxy/ExtensionManagerProxy.java | 4 +++ .../service/BacklogMetricEmitterService.java | 2 +- 6 files changed, 51 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/6e162747/docs/src/site/twiki/FalconUserExtension.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/FalconUserExtension.twiki b/docs/src/site/twiki/FalconUserExtension.twiki new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/falcon/blob/6e162747/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java ---------------------------------------------------------------------- diff --git a/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java b/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java index 9126b67..df5d6c9 100644 --- a/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java +++ b/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java @@ -102,7 +102,7 @@ public class ExtensionMetaStore { beginTransaction(entityManager); Query q = entityManager.createNamedQuery(PersistenceConstants.GET_ALL_EXTENSIONS); try { - return q.getResultList(); + return (List<ExtensionBean>)q.getResultList(); } finally { commitAndCloseTransaction(entityManager); } @@ -126,7 +126,12 @@ public class ExtensionMetaStore { Query q = entityManager.createNamedQuery(PersistenceConstants.GET_EXTENSION); q.setParameter(EXTENSION_NAME, extensionName); try { - return (ExtensionBean)q.getSingleResult(); + List resultList = q.getResultList(); + if (!resultList.isEmpty()) { + return (ExtensionBean)resultList.get(0); + } else { + return null; + } } finally { commitAndCloseTransaction(entityManager); } http://git-wip-us.apache.org/repos/asf/falcon/blob/6e162747/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java ---------------------------------------------------------------------- diff --git a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java index 32b0cfd..c50d6de 100644 --- a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java +++ b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java @@ -75,7 +75,7 @@ public final class ExtensionStore { private static final String RESOURCES_DIR = "resources"; private static final String LIBS_DIR = "libs"; - public static final String EXTENSION_STORE_URI = "extension.store.uri"; + static final String EXTENSION_STORE_URI = "extension.store.uri"; private static final ExtensionStore STORE = new ExtensionStore(); @@ -103,10 +103,9 @@ public final class ExtensionStore { ExtensionType extensionType = AbstractExtension.isExtensionTrusted(extension) ? ExtensionType.TRUSTED : ExtensionType.CUSTOM; String description = getShortDescription(extension); - String recipeName = extension; String location = storePath.toString() + '/' + extension; String extensionOwner = System.getProperty("user.name"); - metaStore.storeExtensionBean(recipeName, location, extensionType, description, extensionOwner); + metaStore.storeExtensionBean(extension, location, extensionType, description, extensionOwner); } } catch (FalconException e) { LOG.error("Exception in ExtensionMetaStore:", e); @@ -143,17 +142,20 @@ public final class ExtensionStore { } } - public Map<String, String> getExtensionArtifacts(final String extensionName) throws - FalconException { + private Map<String, String> getExtensionArtifacts(final String extensionName) throws + FalconException { Map<String, String> extensionFileMap = new HashMap<>(); Path extensionPath; try { RemoteIterator<LocatedFileStatus> fileStatusListIterator; - if (AbstractExtension.isExtensionTrusted(extensionName)){ + if (AbstractExtension.isExtensionTrusted(extensionName)) { extensionPath = new Path(storePath, extensionName.toLowerCase()); fileStatusListIterator = fs.listFiles(extensionPath, true); - }else{ + } else { ExtensionBean extensionBean = metaStore.getDetail(extensionName); + if (null == extensionBean) { + throw new StoreAccessException("Extension not found:" + extensionName); + } extensionPath = new Path(extensionBean.getLocation()); FileSystem fileSystem = getHdfsFileSystem(extensionBean.getLocation()); fileStatusListIterator = fileSystem.listFiles(extensionPath, true); @@ -175,7 +177,6 @@ public final class ExtensionStore { } - public Map<String, String> getExtensionResources(final String extensionName) throws StoreAccessException { Map<String, String> extensionFileMap = new HashMap<>(); try { @@ -243,10 +244,10 @@ public final class ExtensionStore { InputStream data; ByteArrayOutputStream writer = new ByteArrayOutputStream(); - if (resourcePath.startsWith("file")){ + if (resourcePath.startsWith("file")) { data = fs.open(resourceFile); IOUtils.copyBytes(data, writer, fs.getConf(), true); - }else{ + } else { FileSystem fileSystem = getHdfsFileSystem(resourcePath); data = fileSystem.open(resourceFile); IOUtils.copyBytes(data, writer, fileSystem.getConf(), true); @@ -257,7 +258,7 @@ public final class ExtensionStore { } } - public List<String> getTrustedExtensions() throws StoreAccessException { + private List<String> getTrustedExtensions() throws StoreAccessException { List<String> extensionList = new ArrayList<>(); try { FileStatus[] fileStatuses = fs.listStatus(storePath); @@ -297,17 +298,18 @@ public final class ExtensionStore { throw new ValidationException(msg); } } + private FileSystem getHdfsFileSystem(String path) throws FalconException { Configuration conf = new Configuration(); URI uri; try { uri = new URI(path); - } catch (URISyntaxException e){ + } catch (URISyntaxException e) { LOG.error("Exception : ", e); throw new FalconException(e); } conf.set("fs.default.name", uri.getScheme() + "://" + uri.getAuthority()); - return HadoopClientFactory.get().createFalconFileSystem(uri); + return HadoopClientFactory.get().createFalconFileSystem(uri); } @@ -362,7 +364,8 @@ public final class ExtensionStore { LOG.info("Extension :" + extensionName + " registered successfully."); return "Extension :" + extensionName + " registered successfully."; } - public String getResource(final String extensionName, final String resourceName) throws FalconException { + + public String getResource(final String extensionName, final String resourceName) throws FalconException { Map<String, String> resources = getExtensionArtifacts(extensionName); if (resources.isEmpty()) { throw new StoreAccessException("No extension resources found for " + extensionName); @@ -382,7 +385,12 @@ public final class ExtensionStore { public String updateExtensionStatus(final String extensionName, String currentUser, ExtensionStatus status) throws FalconException { validateStatusChange(extensionName, currentUser); - if (metaStore.getDetail(extensionName).getStatus().equals(status)) { + ExtensionBean extensionBean = metaStore.getDetail(extensionName); + if (extensionBean == null) { + LOG.error("Extension not found: " + extensionName); + throw new FalconException("Extension not found:" + extensionName); + } + if (extensionBean.getStatus().equals(status)) { throw new ValidationException(extensionName + " is already in " + status.toString() + " state."); } else { metaStore.updateExtensionStatus(extensionName, status); http://git-wip-us.apache.org/repos/asf/falcon/blob/6e162747/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java ---------------------------------------------------------------------- diff --git a/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java b/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java index f7bf8e8..63974f2 100644 --- a/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java +++ b/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java @@ -180,13 +180,17 @@ public class AbstractExtensionManager extends AbstractSchedulableEntityManager { throw new ValidationException("No extension resources found for " + extensionName); } - ExtensionBean bean = metaStore.getDetail(extensionName); + ExtensionBean extensionBean = metaStore.getDetail(extensionName); + if (extensionBean == null) { + LOG.error("Extension not found: " + extensionName); + throw new FalconException("Extension not found:" + extensionName); + } JSONObject resultObject = new JSONObject(); try { - resultObject.put(NAME, bean.getExtensionName()); - resultObject.put(EXTENSION_TYPE, bean.getExtensionType()); - resultObject.put(EXTENSION_DESC, bean.getDescription()); - resultObject.put(EXTENSION_LOCATION, bean.getLocation()); + resultObject.put(NAME, extensionBean.getExtensionName()); + resultObject.put(EXTENSION_TYPE, extensionBean.getExtensionType()); + resultObject.put(EXTENSION_DESC, extensionBean.getDescription()); + resultObject.put(EXTENSION_LOCATION, extensionBean.getLocation()); } catch (JSONException e) { LOG.error("Exception in buildDetailResults:", e); throw new FalconException(e); @@ -216,7 +220,13 @@ public class AbstractExtensionManager extends AbstractSchedulableEntityManager { protected static void checkIfExtensionIsEnabled(String extensionName) { ExtensionMetaStore metaStore = ExtensionStore.getMetaStore(); - if (!metaStore.getDetail(extensionName).getStatus().equals(ExtensionStatus.ENABLED)) { + ExtensionBean extensionBean = metaStore.getDetail(extensionName); + if (extensionBean == null) { + LOG.error("Extension not found: " + extensionName); + throw FalconWebException.newAPIException("Extension not found:" + extensionName, + Response.Status.NOT_FOUND); + } + if (!extensionBean.getStatus().equals(ExtensionStatus.ENABLED)) { LOG.error("Extension: " + extensionName + " is in disabled state."); throw FalconWebException.newAPIException("Extension: " + extensionName + " is in disabled state.", Response.Status.INTERNAL_SERVER_ERROR); http://git-wip-us.apache.org/repos/asf/falcon/blob/6e162747/prism/src/main/java/org/apache/falcon/resource/proxy/ExtensionManagerProxy.java ---------------------------------------------------------------------- diff --git a/prism/src/main/java/org/apache/falcon/resource/proxy/ExtensionManagerProxy.java b/prism/src/main/java/org/apache/falcon/resource/proxy/ExtensionManagerProxy.java index 6c9087e..1008884 100644 --- a/prism/src/main/java/org/apache/falcon/resource/proxy/ExtensionManagerProxy.java +++ b/prism/src/main/java/org/apache/falcon/resource/proxy/ExtensionManagerProxy.java @@ -662,6 +662,8 @@ public class ExtensionManagerProxy extends AbstractExtensionManager { validateExtensionName(extensionName); try { return new APIResult(APIResult.Status.SUCCEEDED, ExtensionStore.get().getResource(extensionName, README)); + } catch (FalconException e) { + throw FalconWebException.newAPIException(e, Response.Status.BAD_REQUEST); } catch (Throwable e) { throw FalconWebException.newAPIException(e, Response.Status.INTERNAL_SERVER_ERROR); } @@ -731,6 +733,8 @@ public class ExtensionManagerProxy extends AbstractExtensionManager { try { return new APIResult(APIResult.Status.SUCCEEDED, ExtensionStore.get().getResource(extensionName, extensionName.toLowerCase() + EXTENSION_PROPERTY_JSON_SUFFIX)); + } catch (FalconException e) { + throw FalconWebException.newAPIException(e, Response.Status.BAD_REQUEST); } catch (Throwable e) { throw FalconWebException.newAPIException(e, Response.Status.INTERNAL_SERVER_ERROR); } http://git-wip-us.apache.org/repos/asf/falcon/blob/6e162747/prism/src/main/java/org/apache/falcon/service/BacklogMetricEmitterService.java ---------------------------------------------------------------------- diff --git a/prism/src/main/java/org/apache/falcon/service/BacklogMetricEmitterService.java b/prism/src/main/java/org/apache/falcon/service/BacklogMetricEmitterService.java index 5a323fd..2480c96 100644 --- a/prism/src/main/java/org/apache/falcon/service/BacklogMetricEmitterService.java +++ b/prism/src/main/java/org/apache/falcon/service/BacklogMetricEmitterService.java @@ -421,7 +421,7 @@ public final class BacklogMetricEmitterService implements FalconService, if (status.getInstances().length > 0 && status.getInstances()[0].status == InstancesResult. WorkflowStatus.SUCCEEDED) { - LOG.debug("Instance of nominaltime {} of entity {} has succeeded, removing " + LOG.debug("Instance of nominal time {} of entity {} has succeeded, removing " + "from backlog entries", nominalTimeStr, entity.getName()); backlogMetricStore.deleteMetricInstance(entity.getName(), metricInfo.getCluster(), nominalTime, entity.getEntityType());
