Repository: incubator-blur Updated Branches: refs/heads/apache-blur-0.2 f935941c4 -> 70ce3b5d5
Adding a log message to know when progress is setup correctly and changed the progress index output to a buffered version so not to miss and progress events. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/00007c78 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/00007c78 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/00007c78 Branch: refs/heads/apache-blur-0.2 Commit: 00007c78d1ed5ecf58269867cf28baeea86b0d02 Parents: f935941 Author: Aaron McCurry <[email protected]> Authored: Fri Jun 27 09:22:32 2014 -0400 Committer: Aaron McCurry <[email protected]> Committed: Fri Jun 27 09:22:32 2014 -0400 ---------------------------------------------------------------------- .../mapreduce/lib/GenericBlurRecordWriter.java | 5 ++ .../mapreduce/lib/ProgressableDirectory.java | 80 ++------------------ .../mapreduce/lib/GenericBlurRecordWriter.java | 5 ++ .../mapreduce/lib/ProgressableDirectory.java | 80 ++------------------ 4 files changed, 24 insertions(+), 146 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/00007c78/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java ---------------------------------------------------------------------- diff --git a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java index ebe9a7b..0a4544c 100644 --- a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java +++ b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java @@ -142,11 +142,16 @@ public class GenericBlurRecordWriter { private Progressable _progressable = prg; private long _lastWarn = 0; + private boolean _progressSetupLogged = false; @Override public void progress() { if (_progressable != null) { _progressable.progress(); + if (!_progressSetupLogged) { + LOG.info("Progress has been setup correctly."); + _progressSetupLogged = true; + } } else { Progressable progressable = BlurOutputFormat.getProgressable(); if (progressable != null) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/00007c78/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java ---------------------------------------------------------------------- diff --git a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java index 004e1fa..15700bc 100644 --- a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java +++ b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java @@ -18,13 +18,12 @@ package org.apache.blur.mapreduce.lib; */ import java.io.IOException; import java.util.Collection; -import java.util.Map; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; import org.apache.hadoop.util.Progressable; import org.apache.lucene.store.BufferedIndexInput; -import org.apache.lucene.store.DataInput; +import org.apache.lucene.store.BufferedIndexOutput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; @@ -146,8 +145,7 @@ public class ProgressableDirectory extends Directory { return _directory.toString(); } - @SuppressWarnings("deprecation") - static class ProgressableIndexOutput extends IndexOutput { + static class ProgressableIndexOutput extends BufferedIndexOutput { private Progressable _progressable; private IndexOutput _indexOutput; @@ -158,87 +156,23 @@ public class ProgressableDirectory extends Directory { } @Override - public void close() throws IOException { - _indexOutput.close(); - _progressable.progress(); - } - - @Override - public void copyBytes(DataInput input, long numBytes) throws IOException { - _indexOutput.copyBytes(input, numBytes); + protected void flushBuffer(byte[] b, int offset, int len) throws IOException { + _indexOutput.writeBytes(b, offset, len); _progressable.progress(); } @Override - public void flush() throws IOException { - _indexOutput.flush(); - _progressable.progress(); - } - - @Override - public long getFilePointer() { - return _indexOutput.getFilePointer(); - } - - @Override public long length() throws IOException { return _indexOutput.length(); } @Override - public void seek(long pos) throws IOException { - _indexOutput.seek(pos); - _progressable.progress(); - } - - @Override - public void setLength(long length) throws IOException { - _indexOutput.setLength(length); - _progressable.progress(); - } - - @Override - public String toString() { - return _indexOutput.toString(); - } - - @Override - public void writeByte(byte b) throws IOException { - _indexOutput.writeByte(b); - } - - @Override - public void writeBytes(byte[] b, int offset, int length) throws IOException { - _indexOutput.writeBytes(b, offset, length); - _progressable.progress(); - } - - @Override - public void writeBytes(byte[] b, int length) throws IOException { - _indexOutput.writeBytes(b, length); + public void close() throws IOException { + super.close(); + _indexOutput.close(); _progressable.progress(); } - @Override - public void writeInt(int i) throws IOException { - _indexOutput.writeInt(i); - } - - @Override - public void writeLong(long i) throws IOException { - _indexOutput.writeLong(i); - } - - @Override - public void writeString(String s) throws IOException { - _indexOutput.writeString(s); - } - - @Override - public void writeStringStringMap(Map<String, String> map) throws IOException { - _indexOutput.writeStringStringMap(map); - } - } static class ProgressableIndexInput extends BufferedIndexInput { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/00007c78/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java ---------------------------------------------------------------------- diff --git a/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java b/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java index ebe9a7b..0a4544c 100644 --- a/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java +++ b/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java @@ -142,11 +142,16 @@ public class GenericBlurRecordWriter { private Progressable _progressable = prg; private long _lastWarn = 0; + private boolean _progressSetupLogged = false; @Override public void progress() { if (_progressable != null) { _progressable.progress(); + if (!_progressSetupLogged) { + LOG.info("Progress has been setup correctly."); + _progressSetupLogged = true; + } } else { Progressable progressable = BlurOutputFormat.getProgressable(); if (progressable != null) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/00007c78/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java ---------------------------------------------------------------------- diff --git a/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java b/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java index 004e1fa..15700bc 100644 --- a/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java +++ b/blur-mapred-hadoop2/src/main/java/org/apache/blur/mapreduce/lib/ProgressableDirectory.java @@ -18,13 +18,12 @@ package org.apache.blur.mapreduce.lib; */ import java.io.IOException; import java.util.Collection; -import java.util.Map; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; import org.apache.hadoop.util.Progressable; import org.apache.lucene.store.BufferedIndexInput; -import org.apache.lucene.store.DataInput; +import org.apache.lucene.store.BufferedIndexOutput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; @@ -146,8 +145,7 @@ public class ProgressableDirectory extends Directory { return _directory.toString(); } - @SuppressWarnings("deprecation") - static class ProgressableIndexOutput extends IndexOutput { + static class ProgressableIndexOutput extends BufferedIndexOutput { private Progressable _progressable; private IndexOutput _indexOutput; @@ -158,87 +156,23 @@ public class ProgressableDirectory extends Directory { } @Override - public void close() throws IOException { - _indexOutput.close(); - _progressable.progress(); - } - - @Override - public void copyBytes(DataInput input, long numBytes) throws IOException { - _indexOutput.copyBytes(input, numBytes); + protected void flushBuffer(byte[] b, int offset, int len) throws IOException { + _indexOutput.writeBytes(b, offset, len); _progressable.progress(); } @Override - public void flush() throws IOException { - _indexOutput.flush(); - _progressable.progress(); - } - - @Override - public long getFilePointer() { - return _indexOutput.getFilePointer(); - } - - @Override public long length() throws IOException { return _indexOutput.length(); } @Override - public void seek(long pos) throws IOException { - _indexOutput.seek(pos); - _progressable.progress(); - } - - @Override - public void setLength(long length) throws IOException { - _indexOutput.setLength(length); - _progressable.progress(); - } - - @Override - public String toString() { - return _indexOutput.toString(); - } - - @Override - public void writeByte(byte b) throws IOException { - _indexOutput.writeByte(b); - } - - @Override - public void writeBytes(byte[] b, int offset, int length) throws IOException { - _indexOutput.writeBytes(b, offset, length); - _progressable.progress(); - } - - @Override - public void writeBytes(byte[] b, int length) throws IOException { - _indexOutput.writeBytes(b, length); + public void close() throws IOException { + super.close(); + _indexOutput.close(); _progressable.progress(); } - @Override - public void writeInt(int i) throws IOException { - _indexOutput.writeInt(i); - } - - @Override - public void writeLong(long i) throws IOException { - _indexOutput.writeLong(i); - } - - @Override - public void writeString(String s) throws IOException { - _indexOutput.writeString(s); - } - - @Override - public void writeStringStringMap(Map<String, String> map) throws IOException { - _indexOutput.writeStringStringMap(map); - } - } static class ProgressableIndexInput extends BufferedIndexInput {
