Repository: incubator-blur Updated Branches: refs/heads/apache-blur-0.2 052c131e9 -> 2f0f94f2e
Making some pom changes and some code refactors so that the Blur can be compiled to run against hadoop 1.2.x and hadoop 2.2.x. Now to build on either plaform a mvn property switch is needed. -Dhadoop1 for hadoop 1.2.1 and -Dhadoop2 for hadoop 2.2.0 Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/2f0f94f2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/2f0f94f2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/2f0f94f2 Branch: refs/heads/apache-blur-0.2 Commit: 2f0f94f2e3bffa3a931fe421a1a726669c18fef6 Parents: 052c131 Author: Aaron McCurry <[email protected]> Authored: Sun Feb 23 21:26:38 2014 -0500 Committer: Aaron McCurry <[email protected]> Committed: Sun Feb 23 21:26:38 2014 -0500 ---------------------------------------------------------------------- blur-core/pom.xml | 22 ++++----- .../blur/manager/writer/IndexImporter.java | 13 +++++- blur-gui/pom.xml | 11 ++++- blur-mapred/pom.xml | 22 ++++----- .../lib/BlurOutputFormatMiniClusterTest.java | 4 +- .../mapreduce/lib/BlurOutputFormatTest.java | 5 +- blur-store/pom.xml | 22 ++++----- .../blur/store/hdfs_v2/HdfsKeyValueStore.java | 35 +++++++------- blur-store/src/test/resources/log4j.xml | 48 ++++++++++++++++++++ blur-thrift/pom.xml | 11 ++++- blur-util/pom.xml | 11 ++++- pom.xml | 14 ++++-- 12 files changed, 146 insertions(+), 72 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f0f94f2/blur-core/pom.xml ---------------------------------------------------------------------- diff --git a/blur-core/pom.xml b/blur-core/pom.xml index 98cdcf8..dea4ccc 100644 --- a/blur-core/pom.xml +++ b/blur-core/pom.xml @@ -149,7 +149,9 @@ under the License. <profile> <id>hadoop-1x</id> <activation> - <activeByDefault>true</activeByDefault> + <property> + <name>hadoop1</name> + </property> </activation> <dependencies> <dependency> @@ -161,7 +163,12 @@ under the License. </dependencies> </profile> <profile> - <id>cdh4-mr1</id> + <id>hadoop-2.2</id> + <activation> + <property> + <name>hadoop2</name> + </property> + </activation> <dependencies> <dependency> <groupId>org.apache.hadoop</groupId> @@ -170,17 +177,6 @@ under the License. </dependency> <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> - <version>${hadoop.version}</version> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-test</artifactId> - <version>${hadoop.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-minicluster</artifactId> <version>${hadoop.version}</version> <scope>test</scope> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f0f94f2/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java b/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java index 0510f84..d8e9c44 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java @@ -26,6 +26,7 @@ import java.util.Timer; import java.util.TimerTask; import java.util.TreeSet; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; @@ -61,8 +62,10 @@ public class IndexImporter extends TimerTask implements Closeable { private final Timer _timer; private final String _table; private final String _shard; + private final AtomicBoolean _running = new AtomicBoolean(); public IndexImporter(BlurIndex blurIndex, ShardContext shardContext, TimeUnit refreshUnit, long refreshAmount) { + _running.set(true); _blurIndex = blurIndex; _shardContext = shardContext; _timer = new Timer("IndexImporter [" + shardContext.getShard() + "/" + shardContext.getTableContext().getTable() @@ -75,8 +78,11 @@ public class IndexImporter extends TimerTask implements Closeable { @Override public void close() throws IOException { - _timer.cancel(); - _timer.purge(); + if (_running.get()) { + _running.set(false); + _timer.cancel(); + _timer.purge(); + } } @Override @@ -87,6 +93,9 @@ public class IndexImporter extends TimerTask implements Closeable { FileSystem fileSystem = path.getFileSystem(configuration); SortedSet<FileStatus> listStatus; while (true) { + if (!_running.get()) { + return; + } try { listStatus = sort(fileSystem.listStatus(path, new PathFilter() { @Override http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f0f94f2/blur-gui/pom.xml ---------------------------------------------------------------------- diff --git a/blur-gui/pom.xml b/blur-gui/pom.xml index 137c088..a3ce4df 100644 --- a/blur-gui/pom.xml +++ b/blur-gui/pom.xml @@ -136,11 +136,18 @@ under the License. <profile> <id>hadoop-1x</id> <activation> - <activeByDefault>true</activeByDefault> + <property> + <name>hadoop1</name> + </property> </activation> </profile> <profile> - <id>cdh4-mr1</id> + <id>hadoop-2.2</id> + <activation> + <property> + <name>hadoop2</name> + </property> + </activation> <dependencies> <dependency> <groupId>org.mortbay.jetty</groupId> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f0f94f2/blur-mapred/pom.xml ---------------------------------------------------------------------- diff --git a/blur-mapred/pom.xml b/blur-mapred/pom.xml index 7d50167..04041a3 100644 --- a/blur-mapred/pom.xml +++ b/blur-mapred/pom.xml @@ -131,7 +131,9 @@ under the License. <profile> <id>hadoop-1x</id> <activation> - <activeByDefault>true</activeByDefault> + <property> + <name>hadoop1</name> + </property> </activation> <dependencies> <dependency> @@ -150,7 +152,12 @@ under the License. </dependencies> </profile> <profile> - <id>cdh4-mr1</id> + <id>hadoop-2.2</id> + <activation> + <property> + <name>hadoop2</name> + </property> + </activation> <dependencies> <dependency> <groupId>org.apache.mrunit</groupId> @@ -166,17 +173,6 @@ under the License. </dependency> <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> - <version>${hadoop.version}</version> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-test</artifactId> - <version>${hadoop.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-minicluster</artifactId> <version>${hadoop.version}</version> <scope>test</scope> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f0f94f2/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java ---------------------------------------------------------------------- diff --git a/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java b/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java index f46a6ba..f470b35 100644 --- a/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java +++ b/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatMiniClusterTest.java @@ -46,8 +46,8 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.TestMapReduceLocal.TrackingTextInputFormat; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -141,7 +141,7 @@ public class BlurOutputFormatMiniClusterTest { Job job = new Job(jobConf, "blur index"); job.setJarByClass(BlurOutputFormatMiniClusterTest.class); job.setMapperClass(CsvBlurMapper.class); - job.setInputFormatClass(TrackingTextInputFormat.class); + job.setInputFormatClass(TextInputFormat.class); FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in")); String tableUri = new Path(TEST_ROOT_DIR + "/blur/" + tableName).toString(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f0f94f2/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java ---------------------------------------------------------------------- diff --git a/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java b/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java index 0666368..811dba5 100644 --- a/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java +++ b/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java @@ -43,8 +43,9 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.TestMapReduceLocal.TrackingTextInputFormat; +import org.apache.hadoop.mapreduce.TestMapperReducerCleanup.TrackingTextInputFormat; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.lucene.index.DirectoryReader; import org.junit.AfterClass; import org.junit.Before; @@ -111,7 +112,7 @@ public class BlurOutputFormatTest { Job job = new Job(jobConf, "blur index"); job.setJarByClass(BlurOutputFormatTest.class); job.setMapperClass(CsvBlurMapper.class); - job.setInputFormatClass(TrackingTextInputFormat.class); + job.setInputFormatClass(TextInputFormat.class); FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in")); String tableUri = new Path(TEST_ROOT_DIR + "/out").toString(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f0f94f2/blur-store/pom.xml ---------------------------------------------------------------------- diff --git a/blur-store/pom.xml b/blur-store/pom.xml index 35a9dea..9760d0c 100644 --- a/blur-store/pom.xml +++ b/blur-store/pom.xml @@ -140,7 +140,9 @@ under the License. <profile> <id>hadoop-1x</id> <activation> - <activeByDefault>true</activeByDefault> + <property> + <name>hadoop1</name> + </property> </activation> <dependencies> <dependency> @@ -152,7 +154,12 @@ under the License. </dependencies> </profile> <profile> - <id>cdh4-mr1</id> + <id>hadoop-2.2</id> + <activation> + <property> + <name>hadoop2</name> + </property> + </activation> <dependencies> <dependency> <groupId>org.apache.hadoop</groupId> @@ -161,17 +168,6 @@ under the License. </dependency> <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> - <version>${hadoop.version}</version> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-test</artifactId> - <version>${hadoop.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-minicluster</artifactId> <version>${hadoop.version}</version> <scope>test</scope> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f0f94f2/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/HdfsKeyValueStore.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/HdfsKeyValueStore.java b/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/HdfsKeyValueStore.java index ec469f3..1841140 100644 --- a/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/HdfsKeyValueStore.java +++ b/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/HdfsKeyValueStore.java @@ -21,6 +21,7 @@ import java.io.DataOutput; import java.io.FilterInputStream; import java.io.IOException; import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.util.Arrays; import java.util.Comparator; import java.util.HashSet; @@ -45,7 +46,6 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DFSClient.DFSInputStream; import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.Writable; @@ -55,6 +55,8 @@ import org.apache.lucene.util.BytesRef; public class HdfsKeyValueStore implements Store { + private static final String IN = "in"; + private static final String GET_FILE_LENGTH = "getFileLength"; private static final int DEFAULT_MAX = 64 * 1024 * 1024; private final Log LOG = LogFactory.getLog(HdfsKeyValueStore.class); @@ -370,8 +372,8 @@ public class HdfsKeyValueStore implements Store { private long getFileLength(Path path, FSDataInputStream inputStream) throws IOException { FileStatus fileStatus = _fileSystem.getFileStatus(path); - DFSInputStream dfs = getDFS(inputStream); - return Math.max(dfs.getFileLength(), fileStatus.getLen()); + long dfsLength = getDFSLength(inputStream); + return Math.max(dfsLength, fileStatus.getLen()); } private void syncInternal() throws IOException { @@ -417,25 +419,24 @@ public class HdfsKeyValueStore implements Store { } private SortedSet<FileStatus> getList(Path p) throws IOException { - FileStatus[] listStatus = _fileSystem.listStatus(p); - if (listStatus == null) { - return new TreeSet<FileStatus>(); + if (_fileSystem.exists(p)) { + FileStatus[] listStatus = _fileSystem.listStatus(p); + if (listStatus != null) { + return new TreeSet<FileStatus>(Arrays.asList(listStatus)); + } } - return new TreeSet<FileStatus>(Arrays.asList(listStatus)); + return new TreeSet<FileStatus>(); } - private static DFSInputStream getDFS(FSDataInputStream inputStream) throws IOException { + private long getDFSLength(FSDataInputStream inputStream) throws IOException { try { - Field field = FilterInputStream.class.getDeclaredField("in"); + Field field = FilterInputStream.class.getDeclaredField(IN); field.setAccessible(true); - return (DFSInputStream) field.get(inputStream); - } catch (NoSuchFieldException e) { - throw new IOException(e); - } catch (SecurityException e) { - throw new IOException(e); - } catch (IllegalArgumentException e) { - throw new IOException(e); - } catch (IllegalAccessException e) { + Object dfs = field.get(inputStream); + Method method = dfs.getClass().getMethod(GET_FILE_LENGTH, new Class[] {}); + Object length = method.invoke(dfs, new Object[] {}); + return (Long) length; + } catch (Exception e) { throw new IOException(e); } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f0f94f2/blur-store/src/test/resources/log4j.xml ---------------------------------------------------------------------- diff --git a/blur-store/src/test/resources/log4j.xml b/blur-store/src/test/resources/log4j.xml new file mode 100644 index 0000000..fad9ffa --- /dev/null +++ b/blur-store/src/test/resources/log4j.xml @@ -0,0 +1,48 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> +<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd"> + +<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/";> + <appender name="console" class="org.apache.log4j.ConsoleAppender"> + <param name="Target" value="System.out" /> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%-5p %d{yyyyMMdd_HH:mm:ss:SSS_z} [%t] %c{2}: %m%n" /> + </layout> + </appender> + <logger name="org.apache.hadoop"> + <level value="ERROR" /> + <appender-ref ref="console"/> + </logger> + <logger name="REQUEST_LOG" additivity="false"> + <!-- Make value = "INFO"to enable --> + <level value="ERROR" /> + <appender-ref ref="console"/> + </logger> + + <logger name="RESPONSE_LOG" additivity="false"> + <!-- Make value = "INFO"to enable --> + <level value="ERROR" /> + <appender-ref ref="console"/> + </logger> + <root> + <priority value="info" /> + <appender-ref ref="console" /> + </root> +</log4j:configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f0f94f2/blur-thrift/pom.xml ---------------------------------------------------------------------- diff --git a/blur-thrift/pom.xml b/blur-thrift/pom.xml index c812bc3..cc9a5d4 100644 --- a/blur-thrift/pom.xml +++ b/blur-thrift/pom.xml @@ -108,11 +108,18 @@ under the License. <profile> <id>hadoop-1x</id> <activation> - <activeByDefault>true</activeByDefault> + <property> + <name>hadoop1</name> + </property> </activation> </profile> <profile> - <id>cdh4-mr1</id> + <id>hadoop-2.2</id> + <activation> + <property> + <name>hadoop2</name> + </property> + </activation> <dependencies> <dependency> <groupId>javax.servlet</groupId> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f0f94f2/blur-util/pom.xml ---------------------------------------------------------------------- diff --git a/blur-util/pom.xml b/blur-util/pom.xml index 52dfcc6..faf45c7 100644 --- a/blur-util/pom.xml +++ b/blur-util/pom.xml @@ -192,7 +192,9 @@ under the License. <profile> <id>hadoop-1x</id> <activation> - <activeByDefault>true</activeByDefault> + <property> + <name>hadoop1</name> + </property> </activation> <dependencies> <dependency> @@ -203,7 +205,12 @@ under the License. </dependencies> </profile> <profile> - <id>cdh4-mr1</id> + <id>hadoop-2.2</id> + <activation> + <property> + <name>hadoop2</name> + </property> + </activation> <dependencies> <dependency> <groupId>org.apache.hadoop</groupId> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f0f94f2/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 9d114e1..bbb005a 100644 --- a/pom.xml +++ b/pom.xml @@ -382,17 +382,23 @@ under the License. <profile> <id>hadoop-1x</id> <activation> - <activeByDefault>true</activeByDefault> + <property> + <name>hadoop1</name> + </property> </activation> <properties> <hadoop.version>1.2.1</hadoop.version> </properties> </profile> - <profile> - <id>cdh4-mr1</id> + <id>hadoop-2.2</id> + <activation> + <property> + <name>hadoop2</name> + </property> + </activation> <properties> - <hadoop.version>2.0.0-mr1-cdh4.3.0</hadoop.version> + <hadoop.version>2.2.0</hadoop.version> </properties> </profile> </profiles>
