Repository: incubator-blur Updated Branches: refs/heads/master 6d3823712 -> f02bd95a0
Dealing with differences between a local file system and a real hdfs file system. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/f02bd95a Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/f02bd95a Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/f02bd95a Branch: refs/heads/master Commit: f02bd95a025d9dfb43728e02dd55639a745abef9 Parents: 6d38237 Author: Aaron McCurry <[email protected]> Authored: Wed Mar 18 09:40:41 2015 -0400 Committer: Aaron McCurry <[email protected]> Committed: Wed Mar 18 09:40:41 2015 -0400 ---------------------------------------------------------------------- .../org/apache/blur/store/hdfs/HdfsDirectory.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f02bd95a/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java b/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java index a727768..9396638 100644 --- a/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java +++ b/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java @@ -132,11 +132,16 @@ public class HdfsDirectory extends Directory implements LastModified, HdfsSymlin protected final Map<String, Path> _symlinkPathMap = new ConcurrentHashMap<String, Path>(); protected final Map<Path, FSDataInputStream> _inputMap = new ConcurrentHashMap<Path, FSDataInputStream>(); protected final boolean _useCache = true; + protected final boolean _asyncClosing; public HdfsDirectory(Configuration configuration, Path path) throws IOException { _fileSystem = path.getFileSystem(configuration); - _path = _fileSystem.makeQualified(path); + if (_path.toUri().getScheme().equals("hdfs")) { + _asyncClosing = true; + } else { + _asyncClosing = false; + } _fileSystem.mkdirs(path); setLockFactory(NoLockFactory.getNoLockFactory()); synchronized (_metricsGroupMap) { @@ -260,9 +265,13 @@ public class HdfsDirectory extends Directory implements LastModified, HdfsSymlin @Override public void close() throws IOException { super.close(); - outputStream.sync(); _fileStatusMap.put(name, new FStat(System.currentTimeMillis(), outputStream.getPos())); - CLOSING_QUEUE.add(outputStream); + if (_asyncClosing) { + outputStream.sync(); + CLOSING_QUEUE.add(outputStream); + } else { + outputStream.close(); + } openForInput(name); }
