Making the counter more clear. Adding a column counter and lucene field counter. Removing the field counter.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/08dfc4b0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/08dfc4b0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/08dfc4b0 Branch: refs/heads/master Commit: 08dfc4b0e8ec201cbc059487f57fd4a890f1abc1 Parents: 795a13a Author: Aaron McCurry <[email protected]> Authored: Mon Aug 5 07:15:20 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Mon Aug 5 07:15:20 2013 -0400 ---------------------------------------------------------------------- .../java/org/apache/blur/mapreduce/lib/BaseBlurMapper.java | 4 ++-- .../main/java/org/apache/blur/mapreduce/lib/BlurCounters.java | 2 +- .../java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java | 6 +++++- .../main/java/org/apache/blur/mapreduce/lib/CsvBlurMapper.java | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/08dfc4b0/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BaseBlurMapper.java ---------------------------------------------------------------------- diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BaseBlurMapper.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BaseBlurMapper.java index a796810..037edec 100644 --- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BaseBlurMapper.java +++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BaseBlurMapper.java @@ -32,7 +32,7 @@ public abstract class BaseBlurMapper<KEY, VALUE> extends Mapper<KEY, VALUE, Text protected BlurMutate _mutate; protected Text _key; protected Counter _recordCounter; - protected Counter _fieldCounter; + protected Counter _columnCounter; @Override protected void setup(Context context) throws IOException, InterruptedException { @@ -40,7 +40,7 @@ public abstract class BaseBlurMapper<KEY, VALUE> extends Mapper<KEY, VALUE, Text _mutate.setRecord(new BlurRecord()); _key = new Text(); _recordCounter = context.getCounter(BlurCounters.RECORD_COUNT); - _fieldCounter = context.getCounter(BlurCounters.FIELD_COUNT); + _columnCounter = context.getCounter(BlurCounters.COLUMN_COUNT); } @Override http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/08dfc4b0/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurCounters.java ---------------------------------------------------------------------- diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurCounters.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurCounters.java index 4d008e7..0691dce 100644 --- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurCounters.java +++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurCounters.java @@ -21,6 +21,6 @@ package org.apache.blur.mapreduce.lib; * The enum class used for all the internal counters during map reduce jobs. */ public enum BlurCounters { - RECORD_COUNT, FIELD_COUNT, ROW_COUNT, RECORD_RATE, COPY_RATE, ROW_RATE, RECORD_DUPLICATE_COUNT, ROW_OVERFLOW_COUNT, ROW_DELETE_COUNT + RECORD_COUNT, LUCENE_FIELD_COUNT, ROW_COUNT, RECORD_RATE, COPY_RATE, ROW_RATE, RECORD_DUPLICATE_COUNT, ROW_OVERFLOW_COUNT, ROW_DELETE_COUNT, COLUMN_COUNT } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/08dfc4b0/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java ---------------------------------------------------------------------- diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java index eb8112b..39d670b 100644 --- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java +++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java @@ -352,6 +352,7 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> { private final Path _newIndex; private final boolean _indexLocally; private final boolean _optimizeInFlight; + private Counter _columnCount = emptyCounter(); private Counter _fieldCount = emptyCounter(); private Counter _recordCount = emptyCounter(); private Counter _rowCount = emptyCounter(); @@ -367,6 +368,7 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> { private File _localTmpPath; private ProgressableDirectory _localTmpDir; private String _deletedRowId; + public BlurRecordWriter(Configuration configuration, BlurAnalyzer blurAnalyzer, int attemptId, String tmpDirName) throws IOException { @@ -424,7 +426,8 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> { private void setupCounter() { GetCounter getCounter = BlurOutputFormat.getGetCounter(); - _fieldCount = getCounter.getCounter(BlurCounters.FIELD_COUNT); + _fieldCount = getCounter.getCounter(BlurCounters.LUCENE_FIELD_COUNT); + _columnCount = getCounter.getCounter(BlurCounters.COLUMN_COUNT); _recordCount = getCounter.getCounter(BlurCounters.RECORD_COUNT); _recordDuplicateCount = getCounter.getCounter(BlurCounters.RECORD_DUPLICATE_COUNT); _rowCount = getCounter.getCounter(BlurCounters.ROW_COUNT); @@ -443,6 +446,7 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> { _deletedRowId = blurRecord.getRowId(); return; } + _columnCount.increment(record.getColumns().size()); Document document = TransactionRecorder.convert(blurRecord.getRowId(), record, _analyzer); if (_documents.size() == 0) { document.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO)); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/08dfc4b0/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurMapper.java ---------------------------------------------------------------------- diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurMapper.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurMapper.java index a8158af..8de2e98 100644 --- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurMapper.java +++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurMapper.java @@ -414,7 +414,7 @@ public class CsvBlurMapper extends BaseBlurMapper<Writable, Text> { for (int i = 0; i < columnNames.size(); i++) { record.addColumn(columnNames.get(i), list.get(i + offset)); - _fieldCounter.increment(1); + _columnCounter.increment(1); } _key.set(record.getRowId()); _mutate.setMutateType(MUTATE_TYPE.REPLACE);
