Updated Branches: refs/heads/master a48048fc5 -> 84eb05a1c
Updates to compile Blur against cdh4 mr-1. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/84eb05a1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/84eb05a1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/84eb05a1 Branch: refs/heads/master Commit: 84eb05a1caf3d8aac03f29b92db4dd4fe7cbce38 Parents: a48048f Author: Aaron McCurry <[email protected]> Authored: Sun Jul 14 16:03:47 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Sun Jul 14 16:03:47 2013 -0400 ---------------------------------------------------------------------- blur-core/pom.xml | 50 +++++++++++-- .../blur/thrift/ThriftBlurShardServer.java | 2 +- .../utils/TableShardCountCollapserTest.java | 2 +- .../apache/blur/gui/JSONReporterServlet.java | 40 +++++++++++ blur-mapred/pom.xml | 35 +++++++-- .../apache/blur/store/hdfs/HdfsDirectory.java | 1 + blur-util/pom.xml | 32 +++++++-- .../blur/metrics/JSONReporterServlet.java | 38 ---------- distribution/pom.xml | 10 +-- pom.xml | 76 +++++++------------- 10 files changed, 167 insertions(+), 119 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/84eb05a1/blur-core/pom.xml ---------------------------------------------------------------------- diff --git a/blur-core/pom.xml b/blur-core/pom.xml index efb2799..40137b6 100644 --- a/blur-core/pom.xml +++ b/blur-core/pom.xml @@ -75,12 +75,6 @@ under the License. <scope>provided</scope> </dependency> <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-test</artifactId> - <version>${hadoop.version}</version> - <scope>test</scope> - </dependency> - <dependency> <groupId>com.sun.jersey</groupId> <artifactId>jersey-core</artifactId> <version>${jersey.version}</version> @@ -142,4 +136,48 @@ under the License. </plugin> </plugins> </build> + + <profiles> + <profile> + <id>hadoop-1x</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-test</artifactId> + <version>${hadoop.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + </profile> + <profile> + <id>cdh4-mr1</id> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-test</artifactId> + <version>${hadoop.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-minicluster</artifactId> + <version>${hadoop.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-core</artifactId> + <version>${hadoop.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <version>${hadoop.version}</version> + </dependency> + </dependencies> + </profile> + </profiles> </project> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/84eb05a1/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java index 964e1e8..e37fc90 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java @@ -48,6 +48,7 @@ import org.apache.blur.BlurConfiguration; import org.apache.blur.concurrent.SimpleUncaughtExceptionHandler; import org.apache.blur.concurrent.ThreadWatcher; import org.apache.blur.gui.HttpJettyServer; +import org.apache.blur.gui.JSONReporterServlet; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; import org.apache.blur.manager.BlurFilterCache; @@ -62,7 +63,6 @@ import org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup; import org.apache.blur.manager.indexserver.DistributedIndexServer; import org.apache.blur.manager.writer.BlurIndexRefresher; import org.apache.blur.metrics.JSONReporter; -import org.apache.blur.metrics.JSONReporterServlet; import org.apache.blur.metrics.ReporterSetup; import org.apache.blur.server.ShardServerEventHandler; import org.apache.blur.store.blockcache.BlockCache; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/84eb05a1/blur-core/src/test/java/org/apache/blur/utils/TableShardCountCollapserTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/utils/TableShardCountCollapserTest.java b/blur-core/src/test/java/org/apache/blur/utils/TableShardCountCollapserTest.java index 79361d8..28bf718 100644 --- a/blur-core/src/test/java/org/apache/blur/utils/TableShardCountCollapserTest.java +++ b/blur-core/src/test/java/org/apache/blur/utils/TableShardCountCollapserTest.java @@ -94,7 +94,7 @@ public class TableShardCountCollapserTest { private static void createShard(Configuration configuration, int i, Path path, int totalShardCount) throws IOException { HdfsDirectory hdfsDirectory = new HdfsDirectory(configuration, path); - IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_42, new KeywordAnalyzer()); + IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer()); TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy(); mergePolicy.setUseCompoundFile(false); IndexWriter indexWriter = new IndexWriter(hdfsDirectory, conf); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/84eb05a1/blur-gui/src/main/java/org/apache/blur/gui/JSONReporterServlet.java ---------------------------------------------------------------------- diff --git a/blur-gui/src/main/java/org/apache/blur/gui/JSONReporterServlet.java b/blur-gui/src/main/java/org/apache/blur/gui/JSONReporterServlet.java new file mode 100644 index 0000000..60b8e9b --- /dev/null +++ b/blur-gui/src/main/java/org/apache/blur/gui/JSONReporterServlet.java @@ -0,0 +1,40 @@ +package org.apache.blur.gui; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.io.PrintWriter; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.blur.metrics.JSONReporter; + +public class JSONReporterServlet extends HttpServlet { + private static final long serialVersionUID = -3086441832701983642L; + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse response) throws ServletException, IOException { + response.setContentType("application/json"); + PrintWriter writer = response.getWriter(); + JSONReporter.writeJSONData(writer); + writer.flush(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/84eb05a1/blur-mapred/pom.xml ---------------------------------------------------------------------- diff --git a/blur-mapred/pom.xml b/blur-mapred/pom.xml index 0d1e579..d8187cd 100644 --- a/blur-mapred/pom.xml +++ b/blur-mapred/pom.xml @@ -83,12 +83,6 @@ under the License. <version>${mrunit.version}</version> <scope>test</scope> </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-test</artifactId> - <version>${hadoop.version}</version> - <scope>test</scope> - </dependency> </dependencies> <repositories> @@ -121,4 +115,33 @@ under the License. </plugin> </plugins> </build> + + + <profiles> + <profile> + <id>hadoop-1x</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-test</artifactId> + <version>${hadoop.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + </profile> + <profile> + <id>cdh4-mr1</id> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-test</artifactId> + <version>${hadoop.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + </profile> + </profiles> </project> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/84eb05a1/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java b/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java index 050ba6d..6804a44 100644 --- a/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java +++ b/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java @@ -118,6 +118,7 @@ public class HdfsDirectory extends Directory { public HdfsDirectory(Configuration configuration, Path path) throws IOException { this.path = path; fileSystem = path.getFileSystem(configuration); + fileSystem.mkdirs(path); setLockFactory(NoLockFactory.getNoLockFactory()); synchronized (metricsGroupMap) { URI uri = fileSystem.getUri(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/84eb05a1/blur-util/pom.xml ---------------------------------------------------------------------- diff --git a/blur-util/pom.xml b/blur-util/pom.xml index 5d3fc58..57e2abb 100644 --- a/blur-util/pom.xml +++ b/blur-util/pom.xml @@ -62,12 +62,6 @@ under the License. <scope>test</scope> </dependency> <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> - <version>${hadoop.version}</version> - <scope>compile</scope> - </dependency> - <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> <version>${slf4j.version}</version> @@ -182,4 +176,30 @@ under the License. </plugins> </pluginManagement> </build> + + <profiles> + <profile> + <id>hadoop-1x</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-core</artifactId> + <version>${hadoop.version}</version> + </dependency> + </dependencies> + </profile> + <profile> + <id>cdh4-mr1</id> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <version>${hadoop.version}</version> + </dependency> + </dependencies> + </profile> + </profiles> </project> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/84eb05a1/blur-util/src/main/java/org/apache/blur/metrics/JSONReporterServlet.java ---------------------------------------------------------------------- diff --git a/blur-util/src/main/java/org/apache/blur/metrics/JSONReporterServlet.java b/blur-util/src/main/java/org/apache/blur/metrics/JSONReporterServlet.java deleted file mode 100644 index 283817a..0000000 --- a/blur-util/src/main/java/org/apache/blur/metrics/JSONReporterServlet.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.apache.blur.metrics; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import java.io.IOException; -import java.io.PrintWriter; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -public class JSONReporterServlet extends HttpServlet { - private static final long serialVersionUID = -3086441832701983642L; - - @Override - protected void doGet(HttpServletRequest req, HttpServletResponse response) throws ServletException, IOException { - response.setContentType("application/json"); - PrintWriter writer = response.getWriter(); - JSONReporter.writeJSONData(writer); - writer.flush(); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/84eb05a1/distribution/pom.xml ---------------------------------------------------------------------- diff --git a/distribution/pom.xml b/distribution/pom.xml index 8a20a22..2e19952 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -45,13 +45,7 @@ under the License. <version>${project.version}</version> </dependency> </dependencies> - - <profiles> - <profile> - <id>distribution</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> + <build> <plugins> <plugin> @@ -74,6 +68,4 @@ under the License. </plugin> </plugins> </build> - </profile> - </profiles> </project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/84eb05a1/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 28c57af..efb24b7 100644 --- a/pom.xml +++ b/pom.xml @@ -38,7 +38,6 @@ under the License. <inceptionYear>2012</inceptionYear> <properties> - <hadoop.version>1.0.4</hadoop.version> <zookeeper.version>3.4.5</zookeeper.version> <log4j.version>1.2.15</log4j.version> <jersey.version>1.14</jersey.version> @@ -79,6 +78,17 @@ under the License. <scope>test</scope> </dependency> </dependencies> + + <repositories> + <repository> + <id>libdir</id> + <url>file://${basedir}/../lib</url> + </repository> + <repository> + <id>cloudera</id> + <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url> + </repository> + </repositories> <build> <plugins> @@ -108,6 +118,9 @@ under the License. <artifactId>maven-surefire-plugin</artifactId> <configuration> <reuseForks>false</reuseForks> + <systemPropertyVariables> + <blur.tmp.dir>${project.build.directory}/target/tmp</blur.tmp.dir> + </systemPropertyVariables> </configuration> </plugin> <!-- Ignore/Execute plugin execution --> @@ -157,64 +170,23 @@ under the License. </plugins> </pluginManagement> </build> + <profiles> <profile> - <id>default</id> + <id>hadoop-1x</id> <activation> <activeByDefault>true</activeByDefault> </activation> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <excludes> - <exclude>**/IT*.java</exclude> - </excludes> - <systemPropertyVariables> - <blur.tmp.dir>${project.build.directory}/tmp</blur.tmp.dir> - </systemPropertyVariables> - </configuration> - </plugin> - </plugins> - </build> + <properties> + <hadoop.version>1.1.2</hadoop.version> + </properties> </profile> - + <profile> - <id>integration-test</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <includes> - <include>**/IT*.java</include> - </includes> - <systemPropertyVariables> - <blur.tmp.dir>${project.build.directory}/tmp</blur.tmp.dir> - </systemPropertyVariables> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-failsafe-plugin</artifactId> - <version>2.12</version> - <executions> - <execution> - <goals> - <goal>integration-test</goal> - <goal>verify</goal> - </goals> - </execution> - </executions> - </plugin> - </plugins> - </build> + <id>cdh4-mr1</id> + <properties> + <hadoop.version>2.0.0-mr1-cdh4.3.0</hadoop.version> + </properties> </profile> </profiles> </project>
