Repository: incubator-blur Updated Branches: refs/heads/apache-blur-0.2 044b569ee -> e53e35cf4
Making some minor code changes and pom updates to support hadoop2 + mr1. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/e53e35cf Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/e53e35cf Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/e53e35cf Branch: refs/heads/apache-blur-0.2 Commit: e53e35cf49f61cc1425849f84b2e4f6ad44af8bf Parents: 044b569 Author: Aaron McCurry <[email protected]> Authored: Tue Jul 1 17:11:58 2014 -0400 Committer: Aaron McCurry <[email protected]> Committed: Tue Jul 1 17:11:58 2014 -0400 ---------------------------------------------------------------------- blur-console/pom.xml | 28 ++++++++- blur-core/pom.xml | 28 ++++++++- blur-gui/pom.xml | 22 ++++++- blur-mapred-hadoop1/pom.xml | 28 ++++++++- .../mapreduce/lib/GenericBlurRecordWriter.java | 2 +- .../apache/blur/mapreduce/lib/NullCounter.java | 62 ++++++++++++++++---- .../mapreduce/lib/BlurOutputFormatTest.java | 11 ++-- blur-mapred-hadoop2/pom.xml | 2 +- blur-query/pom.xml | 15 ++++- blur-shell/pom.xml | 28 ++++++++- blur-store/pom.xml | 28 ++++++++- blur-thrift/pom.xml | 23 +++++++- blur-util/pom.xml | 15 ++++- distribution/pom.xml | 23 +++++++- pom.xml | 27 ++++++++- 15 files changed, 300 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e53e35cf/blur-console/pom.xml ---------------------------------------------------------------------- diff --git a/blur-console/pom.xml b/blur-console/pom.xml index 5084149..88cea5d 100644 --- a/blur-console/pom.xml +++ b/blur-console/pom.xml @@ -226,7 +226,7 @@ <profiles> <profile> - <id>hadoop-1x</id> + <id>hadoop1</id> <activation> <property> <name>hadoop1</name> @@ -245,7 +245,31 @@ </dependencies> </profile> <profile> - <id>hadoop-2.2</id> + <id>hadoop2-mr1</id> + <activation> + <property> + <name>hadoop2-mr1</name> + </property> + </activation> + <properties> + <projectVersion>${project.parent.version}-hadoop2-mr1</projectVersion> + </properties> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <version>${hadoop.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-minicluster</artifactId> + <version>${hadoop.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + </profile> + <profile> + <id>hadoop2</id> <activation> <property> <name>hadoop2</name> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e53e35cf/blur-core/pom.xml ---------------------------------------------------------------------- diff --git a/blur-core/pom.xml b/blur-core/pom.xml index c51fa2f..c1eb3d2 100644 --- a/blur-core/pom.xml +++ b/blur-core/pom.xml @@ -161,7 +161,7 @@ under the License. <profiles> <profile> - <id>hadoop-1x</id> + <id>hadoop1</id> <activation> <property> <name>hadoop1</name> @@ -180,7 +180,31 @@ under the License. </dependencies> </profile> <profile> - <id>hadoop-2.2</id> + <id>hadoop2-mr1</id> + <activation> + <property> + <name>hadoop2-mr1</name> + </property> + </activation> + <properties> + <projectVersion>${project.parent.version}-hadoop2-mr1</projectVersion> + </properties> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <version>${hadoop.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-minicluster</artifactId> + <version>${hadoop.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + </profile> + <profile> + <id>hadoop2</id> <activation> <property> <name>hadoop2</name> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e53e35cf/blur-gui/pom.xml ---------------------------------------------------------------------- diff --git a/blur-gui/pom.xml b/blur-gui/pom.xml index 4f41b49..e45c9af 100644 --- a/blur-gui/pom.xml +++ b/blur-gui/pom.xml @@ -148,7 +148,7 @@ under the License. <profiles> <profile> - <id>hadoop-1x</id> + <id>hadoop1</id> <activation> <property> <name>hadoop1</name> @@ -159,7 +159,25 @@ under the License. </properties> </profile> <profile> - <id>hadoop-2.2</id> + <id>hadoop2-mr1</id> + <activation> + <property> + <name>hadoop2-mr1</name> + </property> + </activation> + <properties> + <projectVersion>${project.parent.version}-hadoop2-mr1</projectVersion> + </properties> + <dependencies> + <dependency> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jetty</artifactId> + <version>6.1.26</version> + </dependency> + </dependencies> + </profile> + <profile> + <id>hadoop2</id> <activation> <property> <name>hadoop2</name> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e53e35cf/blur-mapred-hadoop1/pom.xml ---------------------------------------------------------------------- diff --git a/blur-mapred-hadoop1/pom.xml b/blur-mapred-hadoop1/pom.xml index 6aa3320..0ea1605 100644 --- a/blur-mapred-hadoop1/pom.xml +++ b/blur-mapred-hadoop1/pom.xml @@ -142,7 +142,7 @@ under the License. <profiles> <profile> - <id>hadoop-1x</id> + <id>hadoop1</id> <activation> <property> <name>hadoop1</name> @@ -167,5 +167,31 @@ under the License. </dependency> </dependencies> </profile> + <profile> + <id>hadoop2-mr1</id> + <activation> + <property> + <name>hadoop2-mr1</name> + </property> + </activation> + <properties> + <projectVersion>${project.parent.version}-hadoop2-mr1</projectVersion> + </properties> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-test</artifactId> + <version>${hadoop.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.mrunit</groupId> + <artifactId>mrunit</artifactId> + <version>${mrunit.version}</version> + <classifier>hadoop1</classifier> + <scope>test</scope> + </dependency> + </dependencies> + </profile> </profiles> </project> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e53e35cf/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java ---------------------------------------------------------------------- diff --git a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java index 0a4544c..83e011b 100644 --- a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java +++ b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java @@ -60,7 +60,7 @@ public class GenericBlurRecordWriter { private static final Log LOG = LogFactory.getLog(GenericBlurRecordWriter.class); private static final String JAVA_IO_TMPDIR = "java.io.tmpdir"; - private static final Counter NULL_COUNTER = new NullCounter(); + private static final Counter NULL_COUNTER = NullCounter.getNullCounter(); private final Text _prevKey = new Text(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e53e35cf/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/NullCounter.java ---------------------------------------------------------------------- diff --git a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/NullCounter.java b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/NullCounter.java index b13f55f..bddd73c 100644 --- a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/NullCounter.java +++ b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/NullCounter.java @@ -1,24 +1,60 @@ package org.apache.blur.mapreduce.lib; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; + import org.apache.hadoop.mapreduce.Counter; /** * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. */ -public class NullCounter extends Counter { +public class NullCounter { + + public static Counter getNullCounter() { + try { + // Try hadoop 2 version + Class<?> clazz1 = Class.forName("org.apache.hadoop.mapreduce.counters.GenericCounter"); + return (Counter) clazz1.newInstance(); + } catch (ClassNotFoundException e1) { + // Try hadoop 1 version + try { + Class<?> clazz2 = Class.forName("org.apache.hadoop.mapreduce.Counter"); + Constructor<?> constructor = clazz2.getConstructor(new Class[] {}); + return (Counter) constructor.newInstance(new Object[] {}); + } catch (ClassNotFoundException e2) { + throw new RuntimeException(e2); + } catch (NoSuchMethodException e2) { + throw new RuntimeException(e2); + } catch (SecurityException e2) { + throw new RuntimeException(e2); + } catch (InstantiationException e2) { + throw new RuntimeException(e2); + } catch (IllegalAccessException e2) { + throw new RuntimeException(e2); + } catch (IllegalArgumentException e2) { + throw new RuntimeException(e2); + } catch (InvocationTargetException e2) { + throw new RuntimeException(e2); + } + } catch (InstantiationException e) { + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e53e35cf/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java ---------------------------------------------------------------------- diff --git a/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java b/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java index 416b868..11ad0ce 100644 --- a/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java +++ b/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java @@ -43,7 +43,6 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.TestMapperReducerCleanup.TrackingTextInputFormat; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.lucene.index.DirectoryReader; @@ -163,7 +162,7 @@ public class BlurOutputFormatTest { Job job = new Job(jobConf, "blur index"); job.setJarByClass(BlurOutputFormatTest.class); job.setMapperClass(CsvBlurMapper.class); - job.setInputFormatClass(TrackingTextInputFormat.class); + job.setInputFormatClass(TextInputFormat.class); FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in")); String tableUri = new Path(TEST_ROOT_DIR + "/out").toString(); @@ -205,7 +204,7 @@ public class BlurOutputFormatTest { Job job = new Job(jobConf, "blur index"); job.setJarByClass(BlurOutputFormatTest.class); job.setMapperClass(CsvBlurMapper.class); - job.setInputFormatClass(TrackingTextInputFormat.class); + job.setInputFormatClass(TextInputFormat.class); FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in")); String tableUri = new Path(TEST_ROOT_DIR + "/out").toString(); @@ -253,7 +252,7 @@ public class BlurOutputFormatTest { Job job = new Job(jobConf, "blur index"); job.setJarByClass(BlurOutputFormatTest.class); job.setMapperClass(CsvBlurMapper.class); - job.setInputFormatClass(TrackingTextInputFormat.class); + job.setInputFormatClass(TextInputFormat.class); FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in")); String tableUri = new Path(TEST_ROOT_DIR + "/out").toString(); @@ -300,7 +299,7 @@ public class BlurOutputFormatTest { Job job = new Job(jobConf, "blur index"); job.setJarByClass(BlurOutputFormatTest.class); job.setMapperClass(CsvBlurMapper.class); - job.setInputFormatClass(TrackingTextInputFormat.class); + job.setInputFormatClass(TextInputFormat.class); FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in")); String tableUri = new Path(TEST_ROOT_DIR + "/out").toString(); @@ -334,7 +333,7 @@ public class BlurOutputFormatTest { Job job = new Job(jobConf, "blur index"); job.setJarByClass(BlurOutputFormatTest.class); job.setMapperClass(CsvBlurMapper.class); - job.setInputFormatClass(TrackingTextInputFormat.class); + job.setInputFormatClass(TextInputFormat.class); FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in")); String tableUri = new Path(TEST_ROOT_DIR + "/out").toString(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e53e35cf/blur-mapred-hadoop2/pom.xml ---------------------------------------------------------------------- diff --git a/blur-mapred-hadoop2/pom.xml b/blur-mapred-hadoop2/pom.xml index c1bb196..1bcf361 100644 --- a/blur-mapred-hadoop2/pom.xml +++ b/blur-mapred-hadoop2/pom.xml @@ -142,7 +142,7 @@ under the License. <profiles> <profile> - <id>hadoop-2.2</id> + <id>hadoop2</id> <activation> <property> <name>hadoop2</name> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e53e35cf/blur-query/pom.xml ---------------------------------------------------------------------- diff --git a/blur-query/pom.xml b/blur-query/pom.xml index 506db55..2837d7b 100644 --- a/blur-query/pom.xml +++ b/blur-query/pom.xml @@ -153,7 +153,7 @@ under the License. <profiles> <profile> - <id>hadoop-1x</id> + <id>hadoop1</id> <activation> <property> <name>hadoop1</name> @@ -164,7 +164,18 @@ under the License. </properties> </profile> <profile> - <id>hadoop-2.2</id> + <id>hadoop2-mr1</id> + <activation> + <property> + <name>hadoop2-mr1</name> + </property> + </activation> + <properties> + <projectVersion>${project.parent.version}-hadoop2-mr1</projectVersion> + </properties> + </profile> + <profile> + <id>hadoop2</id> <activation> <property> <name>hadoop2</name> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e53e35cf/blur-shell/pom.xml ---------------------------------------------------------------------- diff --git a/blur-shell/pom.xml b/blur-shell/pom.xml index f84efb3..41f2e4d 100644 --- a/blur-shell/pom.xml +++ b/blur-shell/pom.xml @@ -108,7 +108,7 @@ under the License. <profiles> <profile> - <id>hadoop-1x</id> + <id>hadoop1</id> <activation> <property> <name>hadoop1</name> @@ -132,7 +132,31 @@ under the License. </dependencies> </profile> <profile> - <id>hadoop-2.2</id> + <id>hadoop2-mr1</id> + <activation> + <property> + <name>hadoop2-mr1</name> + </property> + </activation> + <properties> + <projectVersion>${project.parent.version}-hadoop2-mr1</projectVersion> + </properties> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <version>${hadoop.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-minicluster</artifactId> + <version>${hadoop.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + </profile> + <profile> + <id>hadoop2</id> <activation> <property> <name>hadoop2</name> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e53e35cf/blur-store/pom.xml ---------------------------------------------------------------------- diff --git a/blur-store/pom.xml b/blur-store/pom.xml index 7b50345..d297035 100644 --- a/blur-store/pom.xml +++ b/blur-store/pom.xml @@ -152,7 +152,7 @@ under the License. <profiles> <profile> - <id>hadoop-1x</id> + <id>hadoop1</id> <activation> <property> <name>hadoop1</name> @@ -176,7 +176,31 @@ under the License. </dependencies> </profile> <profile> - <id>hadoop-2.2</id> + <id>hadoop2-mr1</id> + <activation> + <property> + <name>hadoop2-mr1</name> + </property> + </activation> + <properties> + <projectVersion>${project.parent.version}-hadoop2-mr1</projectVersion> + </properties> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <version>${hadoop.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-minicluster</artifactId> + <version>${hadoop.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + </profile> + <profile> + <id>hadoop2</id> <activation> <property> <name>hadoop2</name> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e53e35cf/blur-thrift/pom.xml ---------------------------------------------------------------------- diff --git a/blur-thrift/pom.xml b/blur-thrift/pom.xml index ae3482d..2934103 100644 --- a/blur-thrift/pom.xml +++ b/blur-thrift/pom.xml @@ -131,7 +131,7 @@ under the License. <profiles> <profile> - <id>hadoop-1x</id> + <id>hadoop1</id> <activation> <property> <name>hadoop1</name> @@ -142,7 +142,26 @@ under the License. </properties> </profile> <profile> - <id>hadoop-2.2</id> + <id>hadoop2-mr1</id> + <activation> + <property> + <name>hadoop2-mr1</name> + </property> + </activation> + <properties> + <projectVersion>${project.parent.version}-hadoop2-mr1</projectVersion> + </properties> + <dependencies> + <dependency> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + <version>${servlet-api.version}</version> + <scope>provided</scope> + </dependency> + </dependencies> + </profile> + <profile> + <id>hadoop2</id> <activation> <property> <name>hadoop2</name> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e53e35cf/blur-util/pom.xml ---------------------------------------------------------------------- diff --git a/blur-util/pom.xml b/blur-util/pom.xml index 5660157..6934dff 100644 --- a/blur-util/pom.xml +++ b/blur-util/pom.xml @@ -208,7 +208,7 @@ under the License. </build> <profiles> <profile> - <id>hadoop-1x</id> + <id>hadoop1</id> <activation> <property> <name>hadoop1</name> @@ -219,7 +219,18 @@ under the License. </properties> </profile> <profile> - <id>hadoop-2.2</id> + <id>hadoop2-mr1</id> + <activation> + <property> + <name>hadoop2-mr1</name> + </property> + </activation> + <properties> + <projectVersion>${project.parent.version}-hadoop2-mr1</projectVersion> + </properties> + </profile> + <profile> + <id>hadoop2</id> <activation> <property> <name>hadoop2</name> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e53e35cf/distribution/pom.xml ---------------------------------------------------------------------- diff --git a/distribution/pom.xml b/distribution/pom.xml index 789bf82..ea2b05f 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -84,7 +84,7 @@ under the License. <profiles> <profile> - <id>hadoop-1x</id> + <id>hadoop1</id> <activation> <property> <name>hadoop1</name> @@ -103,7 +103,26 @@ under the License. </properties> </profile> <profile> - <id>hadoop-2.2</id> + <id>hadoop2-mr1</id> + <activation> + <property> + <name>hadoop2-mr1</name> + </property> + </activation> + <dependencies> + <dependency> + <groupId>org.apache.blur</groupId> + <artifactId>blur-mapred-hadoop1</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + <properties> + <projectVersion>${project.parent.version}-hadoop2-mr1</projectVersion> + <bin.assembly.file>src/assemble/bin-hadoop2.xml</bin.assembly.file> + </properties> + </profile> + <profile> + <id>hadoop2</id> <activation> <property> <name>hadoop2</name> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e53e35cf/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 75e1956..eae077e 100644 --- a/pom.xml +++ b/pom.xml @@ -382,7 +382,7 @@ under the License. <profiles> <profile> - <id>hadoop-1x</id> + <id>hadoop1</id> <activation> <property> <name>hadoop1</name> @@ -405,7 +405,30 @@ under the License. </modules> </profile> <profile> - <id>hadoop-2.2</id> + <id>hadoop2-mr1</id> + <activation> + <property> + <name>hadoop2-mr1</name> + </property> + </activation> + <properties> + <hadoop.version>2.3.0-mr1-cdh5.0.2</hadoop.version> + </properties> + <modules> + <module>blur-console</module> + <module>blur-core</module> + <module>blur-thrift</module> + <module>blur-query</module> + <module>blur-store</module> + <module>blur-mapred-hadoop1</module> + <module>blur-util</module> + <module>blur-gui</module> + <module>blur-shell</module> + <module>distribution</module> + </modules> + </profile> + <profile> + <id>hadoop2</id> <activation> <property> <name>hadoop2</name>
