Repository: metamodel Updated Branches: refs/heads/master cda516b16 -> 2bb1e843d
METAMODEL-182: Fixed Fixes #53 Project: http://git-wip-us.apache.org/repos/asf/metamodel/repo Commit: http://git-wip-us.apache.org/repos/asf/metamodel/commit/2bb1e843 Tree: http://git-wip-us.apache.org/repos/asf/metamodel/tree/2bb1e843 Diff: http://git-wip-us.apache.org/repos/asf/metamodel/diff/2bb1e843 Branch: refs/heads/master Commit: 2bb1e843d0584516c989a2c95b41433350747883 Parents: cda516b Author: Kasper Sørensen <[email protected]> Authored: Sun Oct 4 20:24:44 2015 +0200 Committer: Kasper Sørensen <[email protected]> Committed: Sun Oct 4 20:24:44 2015 +0200 ---------------------------------------------------------------------- CHANGES.md | 1 + .../org/apache/metamodel/util/FileResource.java | 46 ++++++++++++++--- .../apache/metamodel/util/FileResourceTest.java | 30 +++++++++-- .../org/apache/metamodel/util/HdfsResource.java | 15 +++--- .../util/HdfsResourceIntegrationTest.java | 53 +++++++++++++------- 5 files changed, 111 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metamodel/blob/2bb1e843/CHANGES.md ---------------------------------------------------------------------- diff --git a/CHANGES.md b/CHANGES.md index 5370afb..1acc26a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,6 +13,7 @@ * [METAMODEL-190] - Improved decimal number support in Excel module. * [METAMODEL-191] - Resolved a number of dependency conflicts/overlaps when combining multiple MetaModel modules. * [METAMODEL-157] - Fixed an issue in DELETE FROM statements with WHERE clauses requiring client-side data type conversion on JDBC databases. + * [METAMODEL-182] - Improved HdfsResource and FileResource directory-based implementations by adding also getSize() and getLastModified() directory-based implementations. ### Apache MetaModel 4.3.6 http://git-wip-us.apache.org/repos/asf/metamodel/blob/2bb1e843/core/src/main/java/org/apache/metamodel/util/FileResource.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/metamodel/util/FileResource.java b/core/src/main/java/org/apache/metamodel/util/FileResource.java index a5c80e9..c350fc5 100644 --- a/core/src/main/java/org/apache/metamodel/util/FileResource.java +++ b/core/src/main/java/org/apache/metamodel/util/FileResource.java @@ -34,12 +34,7 @@ public class FileResource extends AbstractResource implements Serializable { private class DirectoryInputStream extends AbstractDirectoryInputStream<File> { public DirectoryInputStream() { - final File[] unsortedFiles = _file.listFiles(new FileFilter() { - @Override - public boolean accept(final File pathname) { - return pathname.isFile(); - } - }); + final File[] unsortedFiles = getChildren(); if (unsortedFiles == null) { _files = new File[0]; @@ -90,12 +85,18 @@ public class FileResource extends AbstractResource implements Serializable { if (!isExists()) { return false; } + if (_file.isDirectory()) { + return true; + } boolean canWrite = _file.canWrite(); return !canWrite; } @Override public OutputStream write() throws ResourceException { + if (_file.isDirectory()) { + throw new ResourceException(this, "Cannot write to directory: " + _file); + } return FileHelper.getOutputStream(_file); } @@ -115,11 +116,35 @@ public class FileResource extends AbstractResource implements Serializable { @Override public long getSize() { + if (_file.isDirectory()) { + long size = 0; + final File[] children = getChildren(); + for (File file : children) { + final long length = file.length(); + if (length == -1) { + return -1; + } + size += length; + } + return size; + } return _file.length(); } @Override public long getLastModified() { + if (_file.isDirectory()) { + long lastModified = -1; + final File[] children = getChildren(); + for (File file : children) { + final long l = file.lastModified(); + if (l != 0) { + lastModified = Math.max(lastModified, l); + } + } + return lastModified; + } + final long lastModified = _file.lastModified(); if (lastModified == 0) { return -1; @@ -135,4 +160,13 @@ public class FileResource extends AbstractResource implements Serializable { final InputStream in = FileHelper.getInputStream(_file); return in; } + + private File[] getChildren() { + return _file.listFiles(new FileFilter() { + @Override + public boolean accept(final File pathname) { + return pathname.isFile(); + } + }); + } } http://git-wip-us.apache.org/repos/asf/metamodel/blob/2bb1e843/core/src/test/java/org/apache/metamodel/util/FileResourceTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/metamodel/util/FileResourceTest.java b/core/src/test/java/org/apache/metamodel/util/FileResourceTest.java index c085225..d0b8405 100644 --- a/core/src/test/java/org/apache/metamodel/util/FileResourceTest.java +++ b/core/src/test/java/org/apache/metamodel/util/FileResourceTest.java @@ -18,6 +18,8 @@ */ package org.apache.metamodel.util; +import static org.junit.Assert.*; + import java.io.InputStream; import java.io.OutputStream; import java.util.Arrays; @@ -33,14 +35,36 @@ public class FileResourceTest { public TemporaryFolder folder = new TemporaryFolder(); @Test + public void testCannotWriteToDirectory() throws Exception { + FileResource dir = new FileResource("."); + assertTrue(dir.isReadOnly()); + + try { + dir.write(); + fail("Exception expected"); + } catch (ResourceException e) { + assertEquals("Cannot write to directory: .", e.getMessage()); + } + } + + @Test + public void testSizeAndLastModifiedOfDirectory() throws Exception { + final FileResource dir = new FileResource("."); + assertTrue(dir.getLastModified() > 0); + assertTrue(dir.getSize() > 10); + } + + @Test public void testReadDirectory() throws Exception { final String contentString = "fun and games with Apache MetaModel and Hadoop is what we do"; - final String[] contents = new String[] { "fun ", "and ", "games ", "with ", "Apache ", "MetaModel ", "and ", "Hadoop ", "is ", "what ", "we ", "do" }; + final String[] contents = new String[] { "fun ", "and ", "games ", "with ", "Apache ", "MetaModel ", "and ", + "Hadoop ", "is ", "what ", "we ", "do" }; - // Reverse both filename and contents to make sure it is the name and not the creation order that is sorted on. + // Reverse both filename and contents to make sure it is the name and + // not the creation order that is sorted on. int i = contents.length; Collections.reverse(Arrays.asList(contents)); - for(final String contentPart : contents){ + for (final String contentPart : contents) { final FileResource partResource = new FileResource(folder.newFile("/part-" + String.format("%02d", i--))); partResource.write(new Action<OutputStream>() { @Override http://git-wip-us.apache.org/repos/asf/metamodel/blob/2bb1e843/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java ---------------------------------------------------------------------- diff --git a/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java b/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java index 5d273df..b481f95 100644 --- a/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java +++ b/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java @@ -136,7 +136,7 @@ public class HdfsResource extends AbstractResource implements Serializable { } } - private class HdfsDirectoryInputStream extends AbstractDirectoryInputStream<FileStatus> { + private static class HdfsDirectoryInputStream extends AbstractDirectoryInputStream<FileStatus> { private final Path _hadoopPath; private final FileSystem _fs; @@ -157,7 +157,6 @@ public class HdfsResource extends AbstractResource implements Serializable { }); // Natural ordering is the URL Arrays.sort(fileStatuses); - } catch (IOException e) { fileStatuses = new FileStatus[0]; } @@ -165,7 +164,7 @@ public class HdfsResource extends AbstractResource implements Serializable { } @Override - InputStream openStream(final int index) throws IOException { + public InputStream openStream(final int index) throws IOException { final Path nextPath = _files[index].getPath(); return _fs.open(nextPath); } @@ -198,8 +197,8 @@ public class HdfsResource extends AbstractResource implements Serializable { } final Matcher matcher = URL_PATTERN.matcher(url); if (!matcher.find()) { - throw new IllegalArgumentException("Cannot parse url '" + url - + "'. Must follow pattern: hdfs://hostname:port/path/to/file"); + throw new IllegalArgumentException( + "Cannot parse url '" + url + "'. Must follow pattern: hdfs://hostname:port/path/to/file"); } _hostname = matcher.group(1); _port = Integer.parseInt(matcher.group(2)); @@ -270,7 +269,11 @@ public class HdfsResource extends AbstractResource implements Serializable { public long getSize() { final FileSystem fs = getHadoopFileSystem(); try { - return fs.getFileStatus(getHadoopPath()).getLen(); + if (fs.isFile(getHadoopPath())) { + return fs.getFileStatus(getHadoopPath()).getLen(); + } else { + return fs.getContentSummary(getHadoopPath()).getLength(); + } } catch (Exception e) { throw wrapException(e); } finally { http://git-wip-us.apache.org/repos/asf/metamodel/blob/2bb1e843/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java ---------------------------------------------------------------------- diff --git a/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java b/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java index 18e5891..e6119b8 100644 --- a/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java +++ b/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java @@ -60,8 +60,11 @@ public class HdfsResourceIntegrationTest { configured = _filePath != null && _hostname != null && portString != null; if (configured) { _port = Integer.parseInt(portString); + } else { + System.out.println("Skipping test because HDFS file path, hostname and port is not set"); } } else { + System.out.println("Skipping test because properties file does not exist"); configured = false; } Assume.assumeTrue(configured); @@ -106,32 +109,44 @@ public class HdfsResourceIntegrationTest { } final Stopwatch stopwatch = Stopwatch.createStarted(); - final HdfsResource res1 = new HdfsResource(_hostname, _port, _filePath); + try { + logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - start"); - logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - start"); + final String str1 = res1.read(new Func<InputStream, String>() { + @Override + public String eval(InputStream in) { + return FileHelper.readInputStreamAsString(in, "UTF8"); + } + }); - final String str1 = res1.read(new Func<InputStream, String>() { - @Override - public String eval(InputStream in) { - return FileHelper.readInputStreamAsString(in, "UTF8"); - } - }); + Assert.assertEquals(contentString, str1); + logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - read1"); - Assert.assertEquals(contentString, str1); - logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - read1"); + final String str2 = res1.read(new Func<InputStream, String>() { + @Override + public String eval(InputStream in) { + return FileHelper.readInputStreamAsString(in, "UTF8"); + } + }); + Assert.assertEquals(str1, str2); + logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - read2"); - final String str2 = res1.read(new Func<InputStream, String>() { - @Override - public String eval(InputStream in) { - return FileHelper.readInputStreamAsString(in, "UTF8"); + final StringBuilder sb = new StringBuilder(); + for (String token : contents) { + sb.append(token); } - }); - Assert.assertEquals(str1, str2); - logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - read2"); + final long expectedSize = sb.length(); + Assert.assertEquals(expectedSize, res1.getSize()); + logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - getSize"); - res1.getHadoopFileSystem().delete(res1.getHadoopPath(), true); - logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - deleted"); + Assert.assertTrue(res1.getLastModified() > System.currentTimeMillis() - 10000); + logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - getLastModified"); + + } finally { + res1.getHadoopFileSystem().delete(res1.getHadoopPath(), true); + logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - deleted"); + } Assert.assertFalse(res1.isExists());
