Repository: falcon Updated Branches: refs/heads/master 299b82701 -> b48f2df48
FALCON-2227 Fixing error throws to throw exceptions and also added a check for META-INF/services file to be present in the extensions Jar in the extension build path Author: sandeep <[email protected]> Reviewers: @pallavi-rao Closes #326 from sandeepSamudrala/FALCON-2227 and squashes the following commits: 6c8fc5f [sandeep] FALCON-2227 Incorporated review comments 174c63a [sandeep] FALCON-2227 Fixing error throws to throw exceptions and also added a check for META-INF/services file to be present in the extensions Jar b1546ed [sandeep] Merge branch 'master' of https://github.com/apache/falcon 0a433fb [sandeep] Merge branch 'master' of https://github.com/apache/falcon 194f36a [sandeep] Merge branch 'master' of https://github.com/apache/falcon e0ad358 [sandeep] Merge branch 'master' of https://github.com/apache/falcon f96a084 [sandeep] Merge branch 'master' of https://github.com/apache/falcon 9cf36e9 [sandeep] Merge branch 'master' of https://github.com/apache/falcon bbca081 [sandeep] Merge branch 'master' of https://github.com/apache/falcon 48f6afa [sandeep] Merge branch 'master' of https://github.com/apache/falcon 250cc46 [sandeep] Merge branch 'master' of https://github.com/apache/falcon d0393e9 [sandeep] Merge branch 'master' of https://github.com/apache/falcon a178805 [sandeep] Merge branch 'master' of https://github.com/apache/falcon d6dc8bf [sandeep] Merge branch 'master' of https://github.com/apache/falcon 1bb8d3c [sandeep] Merge branch 'master' of https://github.com/apache/falcon c065566 [sandeep] reverting last line changes made 1a4dcd2 [sandeep] rebased and resolved the conflicts from master 271318b [sandeep] FALCON-2097. Adding UT to the new method for getting next instance time with Delay. a94d4fe [sandeep] rebasing from master 9e68a57 [sandeep] FALCON-298. Feed update with replication delay creates holes Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/b48f2df4 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/b48f2df4 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/b48f2df4 Branch: refs/heads/master Commit: b48f2df4868350c4a69d26f52ac1f69971722762 Parents: 299b827 Author: sandeep <[email protected]> Authored: Mon Dec 26 15:02:49 2016 +0530 Committer: Pallavi Rao <[email protected]> Committed: Mon Dec 26 15:02:49 2016 +0530 ---------------------------------------------------------------------- .../org/apache/falcon/ExtensionHandler.java | 29 +++--- .../org/apache/falcon/client/FalconClient.java | 92 +++++++++----------- .../org/apache/falcon/ExtensionHandlerTest.java | 6 +- 3 files changed, 62 insertions(+), 65 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/b48f2df4/client/src/main/java/org/apache/falcon/ExtensionHandler.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/falcon/ExtensionHandler.java b/client/src/main/java/org/apache/falcon/ExtensionHandler.java index 8168b23..11122e2 100644 --- a/client/src/main/java/org/apache/falcon/ExtensionHandler.java +++ b/client/src/main/java/org/apache/falcon/ExtensionHandler.java @@ -44,8 +44,6 @@ import java.util.ArrayList; import java.util.List; import java.util.ServiceLoader; -import static org.apache.falcon.client.FalconClient.OUT; - /** * Handler class that is responsible for preparing Extension entities. */ @@ -56,9 +54,13 @@ public final class ExtensionHandler { private static final String TMP_BASE_DIR = String.format("file://%s", System.getProperty("java.io.tmpdir")); private static final String LOCATION = "location"; private static final String TYPE = "type"; + private static final String EXTENSION_BUILDER_INTERFACE_SERVICE_FILE = + "META-INF/services/org.apache.falcon.extensions.ExtensionBuilder"; + + private ExtensionHandler(){} - public List<Entity> getEntities(ClassLoader extensionClassloader, String extensionName, String jobName, - InputStream configStream) throws IOException, FalconException { + private List<Entity> getEntities(ClassLoader extensionClassloader, String extensionName, String jobName, + InputStream configStream) throws IOException, FalconException { Thread.currentThread().setContextClassLoader(extensionClassloader); ServiceLoader<ExtensionBuilder> extensionBuilders = ServiceLoader.load(ExtensionBuilder.class); @@ -78,6 +80,7 @@ public final class ExtensionHandler { ExtensionBuilder extensionBuilder = null; try { + @SuppressWarnings("unchecked") Class<ExtensionBuilder> clazz = (Class<ExtensionBuilder>) extensionClassloader .loadClass(result.get(0).getCanonicalName()); extensionBuilder = clazz.newInstance(); @@ -86,9 +89,8 @@ public final class ExtensionHandler { } extensionBuilder.validateExtensionConfig(extensionName, configStream); - List<Entity> entities = extensionBuilder.getEntities(jobName, configStream); - return entities; + return extensionBuilder.getEntities(jobName, configStream); } public static List<Entity> loadAndPrepare(String extensionName, String jobName, InputStream configStream, @@ -106,6 +108,10 @@ public final class ExtensionHandler { public static List<Entity> prepare(String extensionName, String jobName, InputStream configStream, List<URL> urls) throws IOException, FalconException { ClassLoader extensionClassLoader = ExtensionClassLoader.load(urls); + if (extensionClassLoader.getResourceAsStream(EXTENSION_BUILDER_INTERFACE_SERVICE_FILE) == null) { + throw new FalconCLIException("The extension build time jars do not contain " + + EXTENSION_BUILDER_INTERFACE_SERVICE_FILE); + } ExtensionHandler extensionHandler = new ExtensionHandler(); return extensionHandler.getEntities(extensionClassLoader, extensionName, jobName, configStream); @@ -145,7 +151,7 @@ public final class ExtensionHandler { return stagePath; } - public static List<URL> copyExtensionPackage(String extensionBuildUrl, FileSystem fs, String stagePath) + private static List<URL> copyExtensionPackage(String extensionBuildUrl, FileSystem fs, String stagePath) throws IOException { Path libsPath = new Path(extensionBuildUrl, FalconExtensionConstants.LIBS); @@ -167,7 +173,7 @@ public final class ExtensionHandler { return urls; } - public static List<URL> getFilesInPath(URL fileURL) throws MalformedURLException { + static List<URL> getFilesInPath(URL fileURL) throws MalformedURLException { List<URL> urls = new ArrayList<>(); File file = new File(fileURL.getPath()); @@ -199,8 +205,8 @@ public final class ExtensionHandler { try { extensionBuildPath = extensionDetailJson.get(LOCATION).toString(); } catch (JSONException e) { - OUT.get().print("Error. " + extensionName + " not found "); - throw new FalconCLIException("Failed to get extension type for the given extension"); + throw new FalconCLIException("Failed to get extension location for the given extension:" + extensionName, + e); } return extensionBuildPath; } @@ -210,8 +216,7 @@ public final class ExtensionHandler { try { extensionType = extensionDetailJson.get(TYPE).toString(); } catch (JSONException e) { - OUT.get().print("Error. " + extensionName + " not found "); - throw new FalconCLIException("Failed to get extension type for the given extension"); + throw new FalconCLIException("Failed to get extension type for the given extension:" + extensionName, e); } return extensionType; } http://git-wip-us.apache.org/repos/asf/falcon/blob/b48f2df4/client/src/main/java/org/apache/falcon/client/FalconClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/falcon/client/FalconClient.java b/client/src/main/java/org/apache/falcon/client/FalconClient.java index 0ccbe48..1614b24 100644 --- a/client/src/main/java/org/apache/falcon/client/FalconClient.java +++ b/client/src/main/java/org/apache/falcon/client/FalconClient.java @@ -84,49 +84,47 @@ public class FalconClient extends AbstractFalconClient { public static final String WS_HEADER_PREFIX = "header:"; public static final String USER = System.getProperty("user.name"); - public static final String AUTH_URL = "api/options?" + PseudoAuthenticator.USER_NAME + "=" + USER; + private static final String AUTH_URL = "api/options?" + PseudoAuthenticator.USER_NAME + "=" + USER; - - - public static final String PATH = "path"; - public static final String COLO = "colo"; + private static final String PATH = "path"; + private static final String COLO = "colo"; private static final String KEY = "key"; private static final String VALUE = "value"; public static final String CLUSTER = "cluster"; - public static final String RUN_ID = "runid"; - public static final String FORCE = "force"; - public static final String SHOW_SCHEDULER = "showScheduler"; - public static final String ENTITY_NAME = "name"; - public static final String ENTITY_TYPE = "type"; - public static final String SKIP_DRYRUN = "skipDryRun"; - public static final String FILTER_BY = "filterBy"; - public static final String ORDER_BY = "orderBy"; - public static final String SORT_ORDER = "sortOrder"; - public static final String OFFSET = "offset"; - public static final String NUM_RESULTS = "numResults"; - public static final String START = "start"; - public static final String END = "end"; - public static final String INSTANCE_TIME = "instanceTime"; - public static final String INSTANCE_STATUS = "instanceStatus"; - public static final String PROPERTIES = "properties"; + private static final String RUN_ID = "runid"; + private static final String FORCE = "force"; + private static final String SHOW_SCHEDULER = "showScheduler"; + private static final String ENTITY_NAME = "name"; + private static final String ENTITY_TYPE = "type"; + private static final String SKIP_DRYRUN = "skipDryRun"; + private static final String FILTER_BY = "filterBy"; + private static final String ORDER_BY = "orderBy"; + private static final String SORT_ORDER = "sortOrder"; + private static final String OFFSET = "offset"; + private static final String NUM_RESULTS = "numResults"; + private static final String START = "start"; + private static final String END = "end"; + private static final String INSTANCE_TIME = "instanceTime"; + private static final String INSTANCE_STATUS = "instanceStatus"; + private static final String PROPERTIES = "properties"; private static final String FIELDS = "fields"; private static final String NAME_SUBSEQUENCE = "nameseq"; private static final String FILTER_TAGS = "tags"; private static final String TAG_KEYWORDS = "tagkeys"; private static final String LIFECYCLE = "lifecycle"; private static final String NUM_INSTANCES = "numInstances"; - public static final String ALL_ATTEMPTS = "allAttempts"; + private static final String ALL_ATTEMPTS = "allAttempts"; - public static final String DO_AS_OPT = "doAs"; - public static final String JOB_NAME_OPT = "jobName"; + private static final String DO_AS_OPT = "doAs"; + private static final String JOB_NAME_OPT = "jobName"; public static final String ENTITIES_OPT = "entities"; /** * Name of the HTTP cookie used for the authentication token between the client and the server. */ - public static final String AUTH_COOKIE = "hadoop.auth"; + private static final String AUTH_COOKIE = "hadoop.auth"; private static final String AUTH_COOKIE_EQ = AUTH_COOKIE + "="; private static final KerberosAuthenticator AUTHENTICATOR = new KerberosAuthenticator(); @@ -134,7 +132,7 @@ public class FalconClient extends AbstractFalconClient { private static final String PROPERTIES_SUFFIX = ".properties"; - public static final HostnameVerifier ALL_TRUSTING_HOSTNAME_VERIFIER = new HostnameVerifier() { + private static final HostnameVerifier ALL_TRUSTING_HOSTNAME_VERIFIER = new HostnameVerifier() { public boolean verify(String hostname, SSLSession sslSession) { return true; } @@ -371,9 +369,8 @@ public class FalconClient extends AbstractFalconClient { } } - public String notEmpty(String str, String name) { + private String notEmpty(String str, String name) { if (str == null) { - throw new IllegalArgumentException(name + " cannot be null"); } if (str.length() == 0) { @@ -774,28 +771,28 @@ public class FalconClient extends AbstractFalconClient { return this; } - public ResourceBuilder addQueryParam(String paramName, Integer value) { + private ResourceBuilder addQueryParam(String paramName, Integer value) { if (value != null) { resource = resource.queryParam(paramName, value.toString()); } return this; } - public ResourceBuilder addQueryParam(String paramName, Boolean paramValue) { + private ResourceBuilder addQueryParam(String paramName, Boolean paramValue) { if (paramValue != null) { resource = resource.queryParam(paramName, String.valueOf(paramValue)); } return this; } - public ResourceBuilder addQueryParam(String paramName, String paramValue) { + private ResourceBuilder addQueryParam(String paramName, String paramValue) { if (StringUtils.isNotBlank(paramValue)) { resource = resource.queryParam(paramName, paramValue); } return this; } - public ResourceBuilder addQueryParam(String paramName, List<LifeCycle> lifeCycles, + private ResourceBuilder addQueryParam(String paramName, List<LifeCycle> lifeCycles, String type) { if (lifeCycles != null) { checkLifeCycleOption(lifeCycles, type); @@ -812,7 +809,7 @@ public class FalconClient extends AbstractFalconClient { .method(entities.method, ClientResponse.class); } - public ClientResponse call(AdminOperations operation) { + private ClientResponse call(AdminOperations operation) { return resource.header("Cookie", AUTH_COOKIE_EQ + authenticationToken) .accept(operation.mimeType).type(MediaType.TEXT_XML) .method(operation.method, ClientResponse.class); @@ -824,37 +821,37 @@ public class FalconClient extends AbstractFalconClient { .method(metadataOperations.method, ClientResponse.class); } - public ClientResponse call(Instances operation) { + private ClientResponse call(Instances operation) { return resource.header("Cookie", AUTH_COOKIE_EQ + authenticationToken) .accept(operation.mimeType).type(MediaType.TEXT_XML) .method(operation.method, ClientResponse.class); } - public ClientResponse call(ExtensionOperations operation) { + private ClientResponse call(ExtensionOperations operation) { return resource.header("Cookie", AUTH_COOKIE_EQ + authenticationToken) .accept(operation.mimeType).type(MediaType.TEXT_XML) .method(operation.method, ClientResponse.class); } - public ClientResponse call(Entities operation, InputStream entityStream) { + private ClientResponse call(Entities operation, InputStream entityStream) { return resource.header("Cookie", AUTH_COOKIE_EQ + authenticationToken) .accept(operation.mimeType).type(MediaType.TEXT_XML) .method(operation.method, ClientResponse.class, entityStream); } - public ClientResponse call(Instances operation, InputStream entityStream) { + private ClientResponse call(Instances operation, InputStream entityStream) { return resource.header("Cookie", AUTH_COOKIE_EQ + authenticationToken) .accept(operation.mimeType).type(MediaType.TEXT_XML) .method(operation.method, ClientResponse.class, entityStream); } - public ClientResponse call(ExtensionOperations operation, InputStream entityStream) { + private ClientResponse call(ExtensionOperations operation, InputStream entityStream) { return resource.header("Cookie", AUTH_COOKIE_EQ + authenticationToken) .accept(operation.mimeType).type(MediaType.TEXT_XML) .method(operation.method, ClientResponse.class, entityStream); } - public ClientResponse call(ExtensionOperations submit, FormDataMultiPart formDataMultiPart) { + private ClientResponse call(ExtensionOperations submit, FormDataMultiPart formDataMultiPart) { return resource.header("Cookie", AUTH_COOKIE_EQ + authenticationToken) .accept(submit.mimeType).type(MediaType.MULTIPART_FORM_DATA) .method(submit.method, ClientResponse.class, formDataMultiPart); @@ -902,7 +899,7 @@ public class FalconClient extends AbstractFalconClient { } } - protected void checkType(String type) { + private void checkType(String type) { if (type == null || type.isEmpty()) { throw new FalconCLIException("entity type is empty"); } else { @@ -1036,7 +1033,7 @@ public class FalconClient extends AbstractFalconClient { return getResponse(String.class, clientResponse); } - public ClientResponse getExtensionDetailResponse(final String extensionName) { + private ClientResponse getExtensionDetailResponse(final String extensionName) { return new ResourceBuilder().path(ExtensionOperations.DETAIL.path, extensionName) .call(ExtensionOperations.DETAIL); } @@ -1093,7 +1090,6 @@ public class FalconClient extends AbstractFalconClient { try { formDataMultiPart.close(); } catch (IOException e) { - OUT.get().print("Submit failed. Failed to submit entities"); throw new FalconCLIException("Submit failed. Failed to submit entities", e); } return formDataMultiPart; @@ -1104,9 +1100,8 @@ public class FalconClient extends AbstractFalconClient { JSONObject extensionDetailJson = getExtensionDetailJson(extensionName); String extensionType = ExtensionHandler.getExtensionType(extensionName, extensionDetailJson); String extensionBuildLocation = ExtensionHandler.getExtensionLocation(extensionName, extensionDetailJson); - List<Entity> entities = getEntities(extensionName, jobName, configStream, extensionType, + return getEntities(extensionName, jobName, configStream, extensionType, extensionBuildLocation); - return entities; } private JSONObject getExtensionDetailJson(String extensionName) { @@ -1115,8 +1110,7 @@ public class FalconClient extends AbstractFalconClient { try { extensionDetailJson = new JSONObject(clientResponse.getEntity(String.class)); } catch (JSONException e) { - OUT.get().print("Failed to get details for the given extension"); - throw new FalconCLIException("Failed to get details for the given extension"); + throw new FalconCLIException("Failed to get details for the given extension", e); } return extensionDetailJson; } @@ -1129,12 +1123,10 @@ public class FalconClient extends AbstractFalconClient { entities = ExtensionHandler.loadAndPrepare(extensionName, jobName, configStream, extensionBuildLocation); } catch (Exception e) { - OUT.get().println("Error in building the extension"); - throw new FalconCLIException("Failed to prepare entities for the given extension"); + throw new FalconCLIException("Error in building the extension", e); } if (entities == null || entities.isEmpty()) { - OUT.get().println("No entities got built"); - throw new FalconCLIException("Failed to prepare entities for the given extension"); + throw new FalconCLIException("No entities got built for the given extension"); } } return entities; http://git-wip-us.apache.org/repos/asf/falcon/blob/b48f2df4/unit/src/test/java/org/apache/falcon/ExtensionHandlerTest.java ---------------------------------------------------------------------- diff --git a/unit/src/test/java/org/apache/falcon/ExtensionHandlerTest.java b/unit/src/test/java/org/apache/falcon/ExtensionHandlerTest.java index 03294bc..bd3b1bb 100644 --- a/unit/src/test/java/org/apache/falcon/ExtensionHandlerTest.java +++ b/unit/src/test/java/org/apache/falcon/ExtensionHandlerTest.java @@ -36,9 +36,9 @@ import java.util.List; * Test Class for validating Extension util helper methods. */ public class ExtensionHandlerTest { - public static final String PROCESS_XML = "/extension-example-duplicate.xml"; - public static final String JARS_DIR = "file:///" + System.getProperty("user.dir") + "/src/test/resources"; - public static final String CONFIG_PATH = "file:///" + System.getProperty("user.dir") + private static final String PROCESS_XML = "/extension-example-duplicate.xml"; + private static final String JARS_DIR = "file:///" + System.getProperty("user.dir") + "/src/test/resources"; + private static final String CONFIG_PATH = "file:///" + System.getProperty("user.dir") + "/src/test/resources/extension.properties"; @Test
