Made HdfsResource non-closeable to avoid statefulness. Integration test shows very similar performance metrics.
Project: http://git-wip-us.apache.org/repos/asf/metamodel/repo Commit: http://git-wip-us.apache.org/repos/asf/metamodel/commit/4621a28e Tree: http://git-wip-us.apache.org/repos/asf/metamodel/tree/4621a28e Diff: http://git-wip-us.apache.org/repos/asf/metamodel/diff/4621a28e Branch: refs/heads/master Commit: 4621a28e39383ac0509fe061efd849690e667f8f Parents: 839926b Author: Kasper Sørensen <[email protected]> Authored: Sat Jun 13 23:11:16 2015 +0200 Committer: Kasper Sørensen <[email protected]> Committed: Sat Jun 13 23:11:16 2015 +0200 ---------------------------------------------------------------------- .../org/apache/metamodel/util/HdfsResource.java | 34 ++-------- .../util/HdfsResourceIntegrationTest.java | 66 +++++++++++++------- .../apache/metamodel/util/HdfsResourceTest.java | 4 -- 3 files changed, 50 insertions(+), 54 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metamodel/blob/4621a28e/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java ---------------------------------------------------------------------- diff --git a/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java b/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java index fcc5cc4..4590bde 100644 --- a/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java +++ b/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java @@ -18,7 +18,6 @@ */ package org.apache.metamodel.util; -import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -35,16 +34,13 @@ import org.apache.metamodel.MetaModelException; * A {@link Resource} implementation that connects to Apache Hadoop's HDFS * distributed file system. */ -public class HdfsResource implements Resource, Closeable { +public class HdfsResource implements Resource { private static final Pattern URL_PATTERN = Pattern.compile("hdfs://(.+):([0-9]+)/(.*)"); private final String _hostname; private final int _port; private final String _filepath; - - private FileSystem _fileSystem; - private Path _path; /** @@ -214,14 +210,11 @@ public class HdfsResource implements Resource, Closeable { } public FileSystem getHadoopFileSystem() { - if (_fileSystem == null) { - try { - _fileSystem = FileSystem.get(getHadoopConfiguration()); - } catch (IOException e) { - throw new MetaModelException("Could not connect to HDFS: " + e.getMessage(), e); - } + try { + return FileSystem.get(getHadoopConfiguration()); + } catch (IOException e) { + throw new MetaModelException("Could not connect to HDFS: " + e.getMessage(), e); } - return _fileSystem; } public Path getHadoopPath() { @@ -232,23 +225,6 @@ public class HdfsResource implements Resource, Closeable { } @Override - public void close() throws IOException { - if (_fileSystem != null) { - try { - _fileSystem.close(); - } finally { - _fileSystem = null; - } - } - } - - @Override - protected void finalize() throws Throwable { - super.finalize(); - close(); - } - - @Override public int hashCode() { return Arrays.hashCode(new Object[] { _filepath, _hostname, _port }); } http://git-wip-us.apache.org/repos/asf/metamodel/blob/4621a28e/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java ---------------------------------------------------------------------- diff --git a/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java b/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java index 4b80eeb..5ef5e53 100644 --- a/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java +++ b/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java @@ -23,10 +23,18 @@ import java.io.FileReader; import java.io.InputStream; import java.io.OutputStream; import java.util.Properties; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Stopwatch; import junit.framework.TestCase; public class HdfsResourceIntegrationTest extends TestCase { + + private static final Logger logger = LoggerFactory.getLogger(HdfsResourceIntegrationTest.class); private boolean _configured; private Properties _properties; @@ -67,33 +75,49 @@ public class HdfsResourceIntegrationTest extends TestCase { return; } final String contentString = "fun and games with Apache MetaModel and Hadoop is what we do"; + + final Stopwatch stopwatch = Stopwatch.createStarted(); final HdfsResource res1 = new HdfsResource(_hostname, _port, _filePath); - try { - assertFalse(res1.isExists()); + logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - start"); + + assertFalse(res1.isExists()); + logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - exists"); + + res1.write(new Action<OutputStream>() { + @Override + public void run(OutputStream out) throws Exception { + out.write(contentString.getBytes()); + } + }); - res1.write(new Action<OutputStream>() { - @Override - public void run(OutputStream out) throws Exception { - out.write(contentString.getBytes()); - } - }); + logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - written"); - assertTrue(res1.isExists()); + assertTrue(res1.isExists()); - final String str = res1.read(new Func<InputStream, String>() { - @Override - public String eval(InputStream in) { - return FileHelper.readInputStreamAsString(in, "UTF8"); - } - }); + final String str1 = res1.read(new Func<InputStream, String>() { + @Override + public String eval(InputStream in) { + return FileHelper.readInputStreamAsString(in, "UTF8"); + } + }); + assertEquals(contentString, str1); + logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - read1"); - assertEquals(contentString, str); - res1.getHadoopFileSystem().delete(res1.getHadoopPath(), false); + final String str2 = res1.read(new Func<InputStream, String>() { + @Override + public String eval(InputStream in) { + return FileHelper.readInputStreamAsString(in, "UTF8"); + } + }); + assertEquals(str1, str2); + logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - read2"); - assertFalse(res1.isExists()); + res1.getHadoopFileSystem().delete(res1.getHadoopPath(), false); + logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - deleted"); - } finally { - res1.close(); - } + assertFalse(res1.isExists()); + + logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - done"); + stopwatch.stop(); } } http://git-wip-us.apache.org/repos/asf/metamodel/blob/4621a28e/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceTest.java ---------------------------------------------------------------------- diff --git a/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceTest.java b/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceTest.java index 7185e51..eace552 100644 --- a/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceTest.java +++ b/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceTest.java @@ -38,9 +38,5 @@ public class HdfsResourceTest extends TestCase { assertEquals("apache.txt", res3.getName()); assertFalse(res3.equals(res1)); - - res1.close(); - res2.close(); - res3.close(); } }
