Updated Branches: refs/heads/0.2-dev 747caacfd -> 203b8bd55
Added some counters to aid in creating some in directory caches for performance speed up. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/203b8bd5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/203b8bd5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/203b8bd5 Branch: refs/heads/0.2-dev Commit: 203b8bd55d88b1c2b4eaca29944668b6a6940f9e Parents: a5a38ff Author: Aaron McCurry <[email protected]> Authored: Mon Feb 18 20:57:22 2013 -0500 Committer: Aaron McCurry <[email protected]> Committed: Mon Feb 18 20:57:22 2013 -0500 ---------------------------------------------------------------------- .../org/apache/blur/store/hdfs/HdfsDirectory.java | 46 +++++++++++++++ 1 files changed, 46 insertions(+), 0 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/203b8bd5/src/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java ---------------------------------------------------------------------- diff --git a/src/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java b/src/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java index f7c9d53..366e1c7 100644 --- a/src/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java +++ b/src/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java @@ -20,7 +20,10 @@ package org.apache.blur.store.hdfs; import java.io.FileNotFoundException; import java.io.IOException; import java.util.Collection; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.blur.log.Log; +import org.apache.blur.log.LogFactory; import org.apache.blur.store.blockcache.BlockDirectory; import org.apache.blur.store.buffer.ReusedBufferedIndexInput; import org.apache.hadoop.conf.Configuration; @@ -39,6 +42,41 @@ import org.apache.lucene.store.NoLockFactory; public class HdfsDirectory extends Directory { + private static final Log LOG = LogFactory.getLog(HdfsDirectory.class); + + private static AtomicLong deleteCounter = new AtomicLong(); + private static AtomicLong existsCounter = new AtomicLong(); + private static AtomicLong fileStatusCounter = new AtomicLong(); + private static AtomicLong renameCounter = new AtomicLong(); + private static AtomicLong listCounter = new AtomicLong(); + private static AtomicLong createCounter = new AtomicLong(); + private static AtomicLong isFileCounter = new AtomicLong(); + + static { + Thread thread = new Thread(new Runnable() { + @Override + public void run() { + while (true) { + LOG.info("Delete Counter [" + deleteCounter + "]"); + LOG.info("Exists Counter [" + existsCounter + "]"); + LOG.info("File Status Counter [" + fileStatusCounter + "]"); + LOG.info("Rename Counter [" + renameCounter + "]"); + LOG.info("List Counter [" + listCounter + "]"); + LOG.info("Create Counter [" + createCounter + "]"); + LOG.info("IsFile Counter [" + isFileCounter + "]"); + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + return; + } + } + } + }); + thread.setName("HDFS dir counter logger"); + thread.setDaemon(true); + thread.start(); + } + private final Path path; private final FileSystem fileSystem; @@ -97,6 +135,7 @@ public class HdfsDirectory extends Directory { throw new IOException("File [" + name + "] already exists found."); } final FSDataOutputStream outputStream = fileSystem.create(getPath(name)); + createCounter.incrementAndGet(); return new BufferedIndexOutput() { @Override @@ -133,10 +172,12 @@ public class HdfsDirectory extends Directory { @Override public String[] listAll() throws IOException { + listCounter.incrementAndGet(); FileStatus[] files = fileSystem.listStatus(path, new PathFilter() { @Override public boolean accept(Path path) { try { + isFileCounter.incrementAndGet(); return fileSystem.isFile(path); } catch (IOException e) { throw new RuntimeException(e); @@ -152,12 +193,14 @@ public class HdfsDirectory extends Directory { @Override public boolean fileExists(String name) throws IOException { + existsCounter.incrementAndGet(); return fileSystem.exists(getPath(name)); } @Override public void deleteFile(String name) throws IOException { if (fileExists(name)) { + deleteCounter.incrementAndGet(); fileSystem.delete(getPath(name), true); } else { throw new FileNotFoundException("File [" + name + "] not found"); @@ -166,6 +209,7 @@ public class HdfsDirectory extends Directory { @Override public long fileLength(String name) throws IOException { + fileStatusCounter.incrementAndGet(); FileStatus fileStatus = fileSystem.getFileStatus(getPath(name)); return fileStatus.getLen(); } @@ -189,6 +233,7 @@ public class HdfsDirectory extends Directory { throw new FileNotFoundException("File [" + name + "] not found"); } Path file = getPath(name); + fileStatusCounter.incrementAndGet(); return fileSystem.getFileStatus(file).getModificationTime(); } @@ -213,6 +258,7 @@ public class HdfsDirectory extends Directory { if (ifSameCluster(to, this)) { Path newDest = simpleTo.getPath(dest); Path oldSrc = getPath(src); + renameCounter.incrementAndGet(); return fileSystem.rename(oldSrc, newDest); } return false;
