Updated Branches: refs/heads/apache-blur-0.2 30cc2277f -> e365fb1ca
Changes needed to compile for cdh4-mr1 Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/e365fb1c Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/e365fb1c Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/e365fb1c Branch: refs/heads/apache-blur-0.2 Commit: e365fb1ca408c8bb6f6e107534dbbca506cc064e Parents: 30cc227 Author: Aaron McCurry <[email protected]> Authored: Tue Oct 15 15:57:56 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Tue Oct 15 15:58:37 2013 -0400 ---------------------------------------------------------------------- .../MasterBasedDistributedLayoutFactory.java | 8 +-- blur-gui/pom.xml | 19 +++++ .../blur/mapreduce/lib/BlurOutputFormat.java | 73 ++++++++++++-------- .../blur/mapreduce/lib/CopyRateDirectory.java | 8 ++- blur-thrift/pom.xml | 24 +++++++ pom.xml | 1 + 6 files changed, 97 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e365fb1c/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java index e865da7..477619c 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java +++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/MasterBasedDistributedLayoutFactory.java @@ -153,9 +153,9 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac return newLayoutMap; } else { LOG.info("Gather counts for table [{0}]", table); - Collection<String> shardsThatAreOffline = new TreeSet<String>(); - Map<String, Integer> onlineServerShardCount = new TreeMap<String, Integer>(); - Map<String, String> existingLayoutMap = existingLayout.getLayout(); + final Collection<String> shardsThatAreOffline = new TreeSet<String>(); + final Map<String, Integer> onlineServerShardCount = new TreeMap<String, Integer>(); + final Map<String, String> existingLayoutMap = existingLayout.getLayout(); for (Entry<String, String> e : existingLayoutMap.entrySet()) { String shard = e.getKey(); String server = e.getValue(); @@ -177,7 +177,7 @@ public class MasterBasedDistributedLayoutFactory implements DistributedLayoutFac LOG.info("Assigning any missing shards [{1}] for table [{0}]", table, shardsThatAreOffline); // Assign missing shards - Map<String, String> newLayoutMap = new TreeMap<String, String>(existingLayoutMap); + final Map<String, String> newLayoutMap = new TreeMap<String, String>(existingLayoutMap); for (String offlineShard : shardsThatAreOffline) { // Find lowest shard count. String server = getServerWithTheLowest(onlineServerShardCount); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e365fb1c/blur-gui/pom.xml ---------------------------------------------------------------------- diff --git a/blur-gui/pom.xml b/blur-gui/pom.xml index 355bd69..6dc713f 100644 --- a/blur-gui/pom.xml +++ b/blur-gui/pom.xml @@ -131,4 +131,23 @@ under the License. </plugins> </pluginManagement> </build> + + <profiles> + <profile> + <id>hadoop-1x</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + </profile> + <profile> + <id>cdh4-mr1</id> + <dependencies> + <dependency> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jetty</artifactId> + <version>6.1.26</version> + </dependency> + </dependencies> + </profile> + </profiles> </project> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e365fb1c/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java ---------------------------------------------------------------------- diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java index 2f59a20..f2e689d 100644 --- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java +++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java @@ -359,16 +359,16 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> { private final Path _newIndex; private final boolean _indexLocally; private final boolean _optimizeInFlight; - private Counter _columnCount = emptyCounter(); - private Counter _fieldCount = emptyCounter(); - private Counter _recordCount = emptyCounter(); - private Counter _rowCount = emptyCounter(); - private Counter _recordDuplicateCount = emptyCounter(); - private Counter _rowOverFlowCount = emptyCounter(); - private Counter _rowDeleteCount = emptyCounter(); - private RateCounter _recordRateCounter = new RateCounter(emptyCounter()); - private RateCounter _rowRateCounter = new RateCounter(emptyCounter()); - private RateCounter _copyRateCounter = new RateCounter(emptyCounter()); + private Counter _columnCount; + private Counter _fieldCount; + private Counter _recordCount; + private Counter _rowCount; + private Counter _recordDuplicateCount; + private Counter _rowOverFlowCount; + private Counter _rowDeleteCount; + private RateCounter _recordRateCounter; + private RateCounter _rowRateCounter; + private RateCounter _copyRateCounter; private boolean _countersSetup = false; private IndexWriter _localTmpWriter; private boolean _usingLocalTmpindex; @@ -417,11 +417,6 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> { } } - private Counter emptyCounter() { - return new Counter() { - }; - } - @Override public void write(Text key, BlurMutate value) throws IOException, InterruptedException { if (!_countersSetup) { @@ -457,14 +452,18 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> { _deletedRowId = blurRecord.getRowId(); return; } - _columnCount.increment(record.getColumns().size()); + if (_countersSetup) { + _columnCount.increment(record.getColumns().size()); + } List<Field> document = TransactionRecorder.getDoc(_fieldManager, blurRecord.getRowId(), record); List<Field> dup = _documents.put(recordId, document); - if (dup != null) { - _recordDuplicateCount.increment(1); - } else { - _fieldCount.increment(document.size()); - _recordCount.increment(1); + if (_countersSetup) { + if (dup != null) { + _recordDuplicateCount.increment(1); + } else { + _fieldCount.increment(document.size()); + _recordCount.increment(1); + } } flushToTmpIndexIfNeeded(); } @@ -522,28 +521,38 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> { flushToTmpIndex(); _localTmpWriter.close(false); DirectoryReader reader = DirectoryReader.open(_localTmpDir); - _recordRateCounter.mark(reader.numDocs()); + if (_countersSetup) { + _recordRateCounter.mark(reader.numDocs()); + } _writer.addIndexes(reader); reader.close(); resetLocalTmp(); - _rowOverFlowCount.increment(1); + if (_countersSetup) { + _rowOverFlowCount.increment(1); + } } else { if (_documents.isEmpty()) { if (_deletedRowId != null) { _writer.addDocument(getDeleteDoc()); - _rowDeleteCount.increment(1); + if (_countersSetup) { + _rowDeleteCount.increment(1); + } } } else { List<List<Field>> docs = new ArrayList<List<Field>>(_documents.values()); docs.get(0).add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO)); _writer.addDocuments(docs); - _recordRateCounter.mark(_documents.size()); + if (_countersSetup) { + _recordRateCounter.mark(_documents.size()); + } _documents.clear(); } } _deletedRowId = null; - _rowRateCounter.mark(); - _rowCount.increment(1); + if (_countersSetup) { + _rowRateCounter.mark(); + _rowCount.increment(1); + } } private Document getDeleteDoc() { @@ -557,8 +566,10 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> { public void close(TaskAttemptContext context) throws IOException, InterruptedException { flush(); _writer.close(); - _recordRateCounter.close(); - _rowRateCounter.close(); + if (_countersSetup) { + _recordRateCounter.close(); + _rowRateCounter.close(); + } if (_indexLocally) { if (_optimizeInFlight) { copyAndOptimizeInFlightDir(); @@ -566,7 +577,9 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> { copyDir(); } } - _copyRateCounter.close(); + if (_countersSetup) { + _copyRateCounter.close(); + } } private void copyAndOptimizeInFlightDir() throws IOException { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e365fb1c/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CopyRateDirectory.java ---------------------------------------------------------------------- diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CopyRateDirectory.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CopyRateDirectory.java index 886f2d6..a79541c 100644 --- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CopyRateDirectory.java +++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CopyRateDirectory.java @@ -58,12 +58,16 @@ public class CopyRateDirectory extends Directory { public void copyBytes(DataInput input, long numBytes) throws IOException { _indexOutput.copyBytes(input, numBytes); - _copyRateCounter.mark(numBytes); + if (_copyRateCounter != null) { + _copyRateCounter.mark(numBytes); + } } public void writeByte(byte b) throws IOException { _indexOutput.writeByte(b); - _copyRateCounter.mark(); + if (_copyRateCounter != null) { + _copyRateCounter.mark(); + } } public void flush() throws IOException { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e365fb1c/blur-thrift/pom.xml ---------------------------------------------------------------------- diff --git a/blur-thrift/pom.xml b/blur-thrift/pom.xml index 1cdfa10..440fa8a 100644 --- a/blur-thrift/pom.xml +++ b/blur-thrift/pom.xml @@ -100,4 +100,28 @@ under the License. </plugin> </plugins> </build> + + + + + <profiles> + <profile> + <id>hadoop-1x</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + </profile> + <profile> + <id>cdh4-mr1</id> + <dependencies> + <dependency> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + <version>${servlet-api.version}</version> + <scope>provided</scope> + </dependency> + </dependencies> + </profile> + </profiles> + </project> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e365fb1c/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index d51a42b..27e7e24 100644 --- a/pom.xml +++ b/pom.xml @@ -189,6 +189,7 @@ under the License. <json.version>20090211</json.version> <mrunit.version>0.9.0-incubating</mrunit.version> <httpclient.version>4.1.3</httpclient.version> + <servlet-api.version>2.5</servlet-api.version> </properties> <modules>
