Repository: falcon Updated Branches: refs/heads/master 9da6d3804 -> e132065f1
FALCON-2261 Job submission fails with IllegalArgumentException. Creating falconHdfsFilesystem based on the uri Author: sandeep <[email protected]> Reviewers: @pallavi-rao Closes #346 from sandeepSamudrala/FALCON-2261 and squashes the following commits: 3fefbc7 [sandeep] Fixed checkstyle issues 24b3da9 [sandeep] FALCON-2261 Job submission fails with IllegalArgumentException. Creating falcon hdfs filesystem based on the uri c5da0a2 [sandeep] Merge branch 'master' of https://github.com/apache/falcon 7e16263 [sandeep] Merge branch 'master' of https://github.com/apache/falcon a234d94 [sandeep] FALCON-2231 Incoporated review comments and small fixes for duplicate submission and colo addition to schedule command 26e3350 [sandeep] Merge branch 'master' of https://github.com/apache/falcon 73fbf75 [sandeep] Merge branch 'master' of https://github.com/apache/falcon cc28658 [sandeep] Merge branch 'master' of https://github.com/apache/falcon 089b10d [sandeep] Merge branch 'master' of https://github.com/apache/falcon 456d4ee [sandeep] Merge branch 'master' of https://github.com/apache/falcon 0cf9af6 [sandeep] Merge branch 'master' of https://github.com/apache/falcon 4a2e23e [sandeep] Merge branch 'master' of https://github.com/apache/falcon b1546ed [sandeep] Merge branch 'master' of https://github.com/apache/falcon 0a433fb [sandeep] Merge branch 'master' of https://github.com/apache/falcon 194f36a [sandeep] Merge branch 'master' of https://github.com/apache/falcon e0ad358 [sandeep] Merge branch 'master' of https://github.com/apache/falcon f96a084 [sandeep] Merge branch 'master' of https://github.com/apache/falcon 9cf36e9 [sandeep] Merge branch 'master' of https://github.com/apache/falcon bbca081 [sandeep] Merge branch 'master' of https://github.com/apache/falcon 48f6afa [sandeep] Merge branch 'master' of https://github.com/apache/falcon 250cc46 [sandeep] Merge branch 'master' of https://github.com/apache/falcon d0393e9 [sandeep] Merge branch 'master' of https://github.com/apache/falcon a178805 [sandeep] Merge branch 'master' of https://github.com/apache/falcon d6dc8bf [sandeep] Merge branch 'master' of https://github.com/apache/falcon 1bb8d3c [sandeep] Merge branch 'master' of https://github.com/apache/falcon c065566 [sandeep] reverting last line changes made 1a4dcd2 [sandeep] rebased and resolved the conflicts from master 271318b [sandeep] FALCON-2097. Adding UT to the new method for getting next instance time with Delay. a94d4fe [sandeep] rebasing from master 9e68a57 [sandeep] FALCON-298. Feed update with replication delay creates holes Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/e132065f Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/e132065f Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/e132065f Branch: refs/heads/master Commit: e132065f1e814f86e0dbc21f473e19c695a01a21 Parents: 9da6d38 Author: sandeep <[email protected]> Authored: Tue Jan 24 16:35:09 2017 +0530 Committer: Pallavi Rao <[email protected]> Committed: Tue Jan 24 16:35:09 2017 +0530 ---------------------------------------------------------------------- .../java/org/apache/falcon/ExtensionHandler.java | 16 +++++++++------- .../falcon/extensions/store/ExtensionStore.java | 5 ----- .../org/apache/falcon/unit/FalconUnitClient.java | 3 ++- 3 files changed, 11 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/e132065f/client/src/main/java/org/apache/falcon/ExtensionHandler.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/falcon/ExtensionHandler.java b/client/src/main/java/org/apache/falcon/ExtensionHandler.java index f6f3346..7d63d11 100644 --- a/client/src/main/java/org/apache/falcon/ExtensionHandler.java +++ b/client/src/main/java/org/apache/falcon/ExtensionHandler.java @@ -24,7 +24,7 @@ import org.apache.falcon.client.FalconExtensionConstants; import org.apache.falcon.entity.v0.Entity; import org.apache.falcon.entity.v0.EntityType; import org.apache.falcon.extensions.ExtensionBuilder; -import org.apache.hadoop.conf.Configuration; +import org.apache.falcon.hadoop.HadoopClientFactory; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.codehaus.jettison.json.JSONException; @@ -38,6 +38,8 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.net.URLEncoder; import java.util.ArrayList; @@ -95,11 +97,10 @@ public final class ExtensionHandler { } public static List<Entity> loadAndPrepare(String extensionName, String jobName, InputStream configStream, - String extensionBuildLocation) throws IOException, FalconException { - Configuration conf = new Configuration(); - FileSystem fs = FileSystem.get(conf); + String extensionBuildLocation) + throws IOException, FalconException, URISyntaxException { String stagePath = createStagePath(extensionName, jobName); - List<URL> urls = ExtensionHandler.copyExtensionPackage(extensionBuildLocation, fs, stagePath); + List<URL> urls = ExtensionHandler.copyExtensionPackage(extensionBuildLocation, stagePath); List<Entity> entities = prepare(extensionName, jobName, configStream, urls); ExtensionHandler.stageEntities(entities, stagePath); @@ -152,9 +153,10 @@ public final class ExtensionHandler { return stagePath; } - private static List<URL> copyExtensionPackage(String extensionBuildUrl, FileSystem fs, String stagePath) - throws IOException { + private static List<URL> copyExtensionPackage(String extensionBuildUrl, String stagePath) + throws IOException, FalconException, URISyntaxException { + FileSystem fs = HadoopClientFactory.get().createProxiedFileSystem(new URI(extensionBuildUrl)); Path libsPath = new Path(extensionBuildUrl, FalconExtensionConstants.LIBS); Path buildLibsPath = new Path(libsPath, FalconExtensionConstants.BUILD); Path localStagePath = new Path(stagePath); http://git-wip-us.apache.org/repos/asf/falcon/blob/e132065f/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java ---------------------------------------------------------------------- diff --git a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java index c50d6de..1d71651 100644 --- a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java +++ b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java @@ -29,7 +29,6 @@ import org.apache.falcon.extensions.jdbc.ExtensionMetaStore; import org.apache.falcon.hadoop.HadoopClientFactory; import org.apache.falcon.persistence.ExtensionBean; import org.apache.falcon.util.StartupProperties; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; @@ -300,7 +299,6 @@ public final class ExtensionStore { } private FileSystem getHdfsFileSystem(String path) throws FalconException { - Configuration conf = new Configuration(); URI uri; try { uri = new URI(path); @@ -308,19 +306,16 @@ public final class ExtensionStore { LOG.error("Exception : ", e); throw new FalconException(e); } - conf.set("fs.default.name", uri.getScheme() + "://" + uri.getAuthority()); return HadoopClientFactory.get().createFalconFileSystem(uri); } public String registerExtension(final String extensionName, final String path, final String description, String extensionOwner) throws URISyntaxException, FalconException { - Configuration conf = new Configuration(); URI uri = new URI(path); assertURI("Scheme", uri.getScheme()); assertURI("Authority", uri.getAuthority()); assertURI("Path", uri.getPath()); - conf.set("fs.defaultFS", uri.getScheme() + "://" + uri.getAuthority()); FileSystem fileSystem = getHdfsFileSystem(path); try { fileSystem.listStatus(new Path(uri.getPath() + "/README")); http://git-wip-us.apache.org/repos/asf/falcon/blob/e132065f/unit/src/main/java/org/apache/falcon/unit/FalconUnitClient.java ---------------------------------------------------------------------- diff --git a/unit/src/main/java/org/apache/falcon/unit/FalconUnitClient.java b/unit/src/main/java/org/apache/falcon/unit/FalconUnitClient.java index 9358958..75aeba0 100644 --- a/unit/src/main/java/org/apache/falcon/unit/FalconUnitClient.java +++ b/unit/src/main/java/org/apache/falcon/unit/FalconUnitClient.java @@ -56,6 +56,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; +import java.net.URISyntaxException; import java.util.TreeMap; import java.util.ArrayList; import java.util.Properties; @@ -330,7 +331,7 @@ public class FalconUnitClient extends AbstractFalconClient { try { entities = ExtensionHandler.loadAndPrepare(extensionName, jobName, configStream, packagePath); - } catch (FalconException | IOException e) { + } catch (FalconException | IOException | URISyntaxException e) { throw new FalconCLIException("Failed in generating entities" + jobName); } return entities;
