Repository: metamodel Updated Branches: refs/heads/master 2a4b85410 -> 16f3b0a00
METAMODEL-220: Fixed Fixes #79 Project: http://git-wip-us.apache.org/repos/asf/metamodel/repo Commit: http://git-wip-us.apache.org/repos/asf/metamodel/commit/16f3b0a0 Tree: http://git-wip-us.apache.org/repos/asf/metamodel/tree/16f3b0a0 Diff: http://git-wip-us.apache.org/repos/asf/metamodel/diff/16f3b0a0 Branch: refs/heads/master Commit: 16f3b0a005c115b5f70d89425cd615e79865b908 Parents: 2a4b854 Author: Kasper Sørensen <[email protected]> Authored: Mon Dec 14 11:00:02 2015 +0100 Committer: Kasper Sørensen <[email protected]> Committed: Mon Dec 14 11:00:02 2015 +0100 ---------------------------------------------------------------------- CHANGES.md | 1 + .../org/apache/metamodel/util/HdfsResource.java | 73 ++++++++++++++------ .../apache/metamodel/util/HdfsResourceTest.java | 15 ++++ 3 files changed, 68 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metamodel/blob/16f3b0a0/CHANGES.md ---------------------------------------------------------------------- diff --git a/CHANGES.md b/CHANGES.md index 2474450..b1999ff 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,7 @@ * [METAMODEL-218] - Fixed conversion of STRING and NUMBER types to database-specific types in JDBC module. * [METAMODEL-205] - Added validation of Excel sheet name before attempting to create table (sheet). * [METAMODEL-219] - Made HdfsResource capable of incorporating Hadoop configuration files core-site.xml and hdfs-site.xml + * [METAMODEL-220] - Made HdfsResource capable of working with other URI schemes than 'hdfs'. ### Apache MetaModel 4.4.1 http://git-wip-us.apache.org/repos/asf/metamodel/blob/16f3b0a0/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java ---------------------------------------------------------------------- diff --git a/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java b/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java index 9523b23..b66280c 100644 --- a/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java +++ b/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java @@ -23,9 +23,8 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.Serializable; +import java.net.URI; import java.util.Objects; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; @@ -45,8 +44,14 @@ public class HdfsResource extends AbstractResource implements Serializable { public static final String SYSTEM_PROPERTY_HADOOP_CONF_DIR_ENABLED = "metamodel.hadoop.use_hadoop_conf_dir"; - private static final Pattern URL_PATTERN = Pattern.compile("hdfs://(.+):([0-9]+)/(.*)"); + public static final String SCHEME_HDFS = "hdfs"; + public static final String SCHEME_SWIFT = "swift"; + public static final String SCHEME_EMRFS = "emrfs"; + public static final String SCHEME_MAPRFS = "maprfs"; + public static final String SCHEME_S3 = "s3"; + public static final String SCHEME_FTP = "ftp"; + private final String _scheme; private final String _hadoopConfDir; private final String _hostname; private final int _port; @@ -57,7 +62,7 @@ public class HdfsResource extends AbstractResource implements Serializable { * Creates a {@link HdfsResource} * * @param url - * a URL of the form: hdfs://hostname:port/path/to/file + * a URL of the form: scheme://hostname:port/path/to/file */ public HdfsResource(String url) { this(url, null); @@ -67,7 +72,7 @@ public class HdfsResource extends AbstractResource implements Serializable { * Creates a {@link HdfsResource} * * @param url - * a URL of the form: hdfs://hostname:port/path/to/file + * a URL of the form: scheme://hostname:port/path/to/file * @param hadoopConfDir * the path of a directory containing the Hadoop and HDFS * configuration file(s). @@ -76,19 +81,18 @@ public class HdfsResource extends AbstractResource implements Serializable { if (url == null) { throw new IllegalArgumentException("Url cannot be null"); } - final Matcher matcher = URL_PATTERN.matcher(url); - if (!matcher.find()) { - throw new IllegalArgumentException( - "Cannot parse url '" + url + "'. Must follow pattern: hdfs://hostname:port/path/to/file"); - } - _hostname = matcher.group(1); - _port = Integer.parseInt(matcher.group(2)); - _filepath = '/' + matcher.group(3); + + final URI uri = URI.create(url); + + _scheme = uri.getScheme(); + _hostname = uri.getHost(); + _port = uri.getPort(); + _filepath = uri.getPath(); _hadoopConfDir = hadoopConfDir; } /** - * Creates a {@link HdfsResource} + * Creates a {@link HdfsResource} using the "hdfs" scheme * * @param hostname * the HDFS (namenode) hostname @@ -98,12 +102,15 @@ public class HdfsResource extends AbstractResource implements Serializable { * the path on HDFS to the file, starting with slash ('/') */ public HdfsResource(String hostname, int port, String filepath) { - this(hostname, port, filepath, null); + this(SCHEME_HDFS, hostname, port, filepath, null); } /** * Creates a {@link HdfsResource} * + * @param scheme + * the scheme to use (consider using {@link #SCHEME_HDFS} or any + * of the other "SCHEME_" constants). * @param hostname * the HDFS (namenode) hostname * @param port @@ -114,13 +121,23 @@ public class HdfsResource extends AbstractResource implements Serializable { * the path of a directory containing the Hadoop and HDFS * configuration file(s). */ - public HdfsResource(String hostname, int port, String filepath, String hadoopConfDir) { + public HdfsResource(String scheme, String hostname, int port, String filepath, String hadoopConfDir) { + _scheme = scheme; _hostname = hostname; _port = port; _filepath = filepath; _hadoopConfDir = hadoopConfDir; } + public String getScheme() { + if (_scheme == null) { + // should only happen for deserialized and old objects before + // METAMODEL-220 introduced dynamic schemes + return SCHEME_HDFS; + } + return _scheme; + } + public String getFilepath() { return _filepath; } @@ -148,7 +165,18 @@ public class HdfsResource extends AbstractResource implements Serializable { @Override public String getQualifiedPath() { - return "hdfs://" + _hostname + ":" + _port + _filepath; + final StringBuilder sb = new StringBuilder(); + sb.append(getScheme()); + sb.append("://"); + if (_hostname != null) { + sb.append(_hostname); + } + if (_port > 0) { + sb.append(':'); + sb.append(_port); + } + sb.append(_filepath); + return sb.toString(); } @Override @@ -252,7 +280,9 @@ public class HdfsResource extends AbstractResource implements Serializable { public Configuration getHadoopConfiguration() { final Configuration conf = new Configuration(); - conf.set("fs.defaultFS", "hdfs://" + _hostname + ":" + _port); + if (_hostname != null && _port > 0) { + conf.set("fs.defaultFS", getScheme() + "://" + _hostname + ":" + _port); + } final File hadoopConfigurationDirectory = getHadoopConfigurationDirectoryToUse(); if (hadoopConfigurationDirectory != null) { @@ -323,7 +353,7 @@ public class HdfsResource extends AbstractResource implements Serializable { @Override public int hashCode() { - return Objects.hash(_filepath, _hostname, _port, _hadoopConfDir); + return Objects.hash(getScheme(), _filepath, _hostname, _port, _hadoopConfDir); } @Override @@ -333,8 +363,9 @@ public class HdfsResource extends AbstractResource implements Serializable { } if (obj instanceof HdfsResource) { final HdfsResource other = (HdfsResource) obj; - return Objects.equals(_filepath, other._filepath) && Objects.equals(_hostname, other._hostname) - && Objects.equals(_port, other._port) && Objects.equals(_hadoopConfDir, other._hadoopConfDir); + return Objects.equals(getScheme(), other.getScheme()) && Objects.equals(_filepath, other._filepath) + && Objects.equals(_hostname, other._hostname) && Objects.equals(_port, other._port) + && Objects.equals(_hadoopConfDir, other._hadoopConfDir); } return false; } http://git-wip-us.apache.org/repos/asf/metamodel/blob/16f3b0a0/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceTest.java ---------------------------------------------------------------------- diff --git a/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceTest.java b/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceTest.java index c6004d3..ad3393f 100644 --- a/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceTest.java +++ b/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceTest.java @@ -28,18 +28,33 @@ public class HdfsResourceTest extends TestCase { final HdfsResource res1 = new HdfsResource("hdfs://localhost:9000/home/metamodel.txt"); assertEquals("hdfs://localhost:9000/home/metamodel.txt", res1.getQualifiedPath()); assertEquals("metamodel.txt", res1.getName()); + assertEquals("hdfs", res1.getScheme()); final HdfsResource res2 = new HdfsResource("localhost", 9000, "/home/metamodel.txt"); assertEquals("hdfs://localhost:9000/home/metamodel.txt", res2.getQualifiedPath()); assertEquals("metamodel.txt", res2.getName()); + assertEquals("hdfs", res2.getScheme()); assertEquals(res1, res2); final HdfsResource res3 = new HdfsResource("localhost", 9000, "/home/apache.txt"); assertEquals("hdfs://localhost:9000/home/apache.txt", res3.getQualifiedPath()); assertEquals("apache.txt", res3.getName()); + assertEquals("hdfs", res3.getScheme()); assertFalse(res3.equals(res1)); + + final HdfsResource res4 = new HdfsResource("emrfs://localhost:9000/home/metamodel.txt"); + assertEquals("emrfs://localhost:9000/home/metamodel.txt", res4.getQualifiedPath()); + assertEquals("metamodel.txt", res4.getName()); + assertEquals("emrfs", res4.getScheme()); + + assertFalse(res4.equals(res1)); + } + + public void testNoHostOrPortInUrl() throws Exception { + final HdfsResource resource = new HdfsResource("emrfs:///home/metamodel.txt"); + assertEquals("emrfs:///home/metamodel.txt", resource.getQualifiedPath()); } public void testSerialization() throws Exception {
