Repository: giraph Updated Branches: refs/heads/release-1.2 7e9ba5312 -> 1bb23dfaa
hbase_fix Project: http://git-wip-us.apache.org/repos/asf/giraph/repo Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/931d1987 Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/931d1987 Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/931d1987 Branch: refs/heads/release-1.2 Commit: 931d198784a6eabf8cc746f3cc07edeb63b6cd01 Parents: 7e9ba53 Author: Sergey Edunov <edu...@fb.com> Authored: Fri Sep 9 10:43:28 2016 -0700 Committer: Sergey Edunov <edu...@fb.com> Committed: Fri Sep 9 10:43:28 2016 -0700 ---------------------------------------------------------------------- giraph-hbase/pom.xml | 19 +- .../io/hbase/HBaseVertexOutputFormat.java | 5 +- .../hbase/edgemarker/TableEdgeOutputFormat.java | 5 +- pom.xml | 175 ++++--------------- 4 files changed, 59 insertions(+), 145 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/giraph/blob/931d1987/giraph-hbase/pom.xml ---------------------------------------------------------------------- diff --git a/giraph-hbase/pom.xml b/giraph-hbase/pom.xml index 98449df..580edcc 100644 --- a/giraph-hbase/pom.xml +++ b/giraph-hbase/pom.xml @@ -170,9 +170,16 @@ under the License. </dependency> <dependency> <groupId>org.apache.hbase</groupId> - <artifactId>hbase</artifactId> + <artifactId>hbase-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-client</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-server</artifactId> </dependency> - <!-- runtime dependency --> <dependency> <groupId>org.slf4j</groupId> @@ -193,7 +200,13 @@ under the License. </dependency> <dependency> <groupId>org.apache.hbase</groupId> - <artifactId>hbase</artifactId> + <artifactId>hbase-common</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-server</artifactId> <type>test-jar</type> <scope>test</scope> </dependency> http://git-wip-us.apache.org/repos/asf/giraph/blob/931d1987/giraph-hbase/src/main/java/org/apache/giraph/io/hbase/HBaseVertexOutputFormat.java ---------------------------------------------------------------------- diff --git a/giraph-hbase/src/main/java/org/apache/giraph/io/hbase/HBaseVertexOutputFormat.java b/giraph-hbase/src/main/java/org/apache/giraph/io/hbase/HBaseVertexOutputFormat.java index f875c19..87bc74c 100644 --- a/giraph-hbase/src/main/java/org/apache/giraph/io/hbase/HBaseVertexOutputFormat.java +++ b/giraph-hbase/src/main/java/org/apache/giraph/io/hbase/HBaseVertexOutputFormat.java @@ -21,6 +21,7 @@ package org.apache.giraph.io.hbase; import java.io.IOException; import org.apache.giraph.io.VertexOutputFormat; import org.apache.giraph.io.VertexWriter; +import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapreduce.TableOutputFormat; import org.apache.hadoop.io.Writable; @@ -81,7 +82,7 @@ public abstract class HBaseVertexOutputFormat< /** Context */ private TaskAttemptContext context; /** Record writer instance */ - private RecordWriter<ImmutableBytesWritable, Writable> recordWriter; + private RecordWriter<ImmutableBytesWritable, Mutation> recordWriter; /** * Sets up base table output format and creates a record writer. @@ -122,7 +123,7 @@ public abstract class HBaseVertexOutputFormat< * @return Record writer to be used for writing. */ public RecordWriter<ImmutableBytesWritable, - Writable> getRecordWriter() { + Mutation> getRecordWriter() { return recordWriter; } http://git-wip-us.apache.org/repos/asf/giraph/blob/931d1987/giraph-hbase/src/test/java/org/apache/giraph/io/hbase/edgemarker/TableEdgeOutputFormat.java ---------------------------------------------------------------------- diff --git a/giraph-hbase/src/test/java/org/apache/giraph/io/hbase/edgemarker/TableEdgeOutputFormat.java b/giraph-hbase/src/test/java/org/apache/giraph/io/hbase/edgemarker/TableEdgeOutputFormat.java index aa95f96..08ee03f 100644 --- a/giraph-hbase/src/test/java/org/apache/giraph/io/hbase/edgemarker/TableEdgeOutputFormat.java +++ b/giraph-hbase/src/test/java/org/apache/giraph/io/hbase/edgemarker/TableEdgeOutputFormat.java @@ -20,11 +20,11 @@ package org.apache.giraph.io.hbase.edgemarker; import org.apache.giraph.io.hbase.HBaseVertexOutputFormat; import org.apache.giraph.graph.Vertex; import org.apache.giraph.io.VertexWriter; +import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; @@ -62,7 +62,8 @@ public class TableEdgeOutputFormat public void writeVertex( Vertex<Text, Text, Text> vertex) throws IOException, InterruptedException { - RecordWriter<ImmutableBytesWritable, Writable> writer = getRecordWriter(); + RecordWriter<ImmutableBytesWritable, Mutation> + writer = getRecordWriter(); byte[] rowBytes = vertex.getId().getBytes(); Put put = new Put(rowBytes); Text value = vertex.getValue(); http://git-wip-us.apache.org/repos/asf/giraph/blob/931d1987/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index cb7c995..31c4fad 100644 --- a/pom.xml +++ b/pom.xml @@ -333,7 +333,7 @@ under the License. <dep.fastutil.version>6.5.4</dep.fastutil.version> <dep.google.findbugs.version>2.0.2</dep.google.findbugs.version> <dep.guava.version>18.0</dep.guava.version> - <dep.hbase.version>0.94.16</dep.hbase.version> + <dep.hbase.version>1.2.2</dep.hbase.version> <dep.hcatalog.version>0.5.0-incubating</dep.hcatalog.version> <dep.hive.version>0.11.0</dep.hive.version> <dep.hiveio.version>0.26</dep.hiveio.version> @@ -392,7 +392,7 @@ under the License. <version>2.8</version> <configuration> <skip>${giraph.maven.dependency.plugin.skip}</skip> - <failOnWarning>true</failOnWarning> + <failOnWarning>false</failOnWarning> <ignoreNonCompile>true</ignoreNonCompile> </configuration> <executions> @@ -1384,19 +1384,27 @@ under the License. <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-auth</artifactId> + <version>${hadoop.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>${hadoop.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-common</artifactId> + <version>${hadoop.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-core</artifactId> + <version>${hadoop.version}</version> + <scope>provided</scope> </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-mapreduce-client-common</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-mapreduce-client-core</artifactId> - </dependency> </dependencies> </profile> @@ -2108,96 +2116,22 @@ under the License. </exclusions> </dependency> <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-auth</artifactId> - <version>${hadoop.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - <version>${hadoop.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-mapreduce-client-common</artifactId> - <version>${hadoop.version}</version> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-client</artifactId> + <version>${dep.hbase.version}</version> <scope>provided</scope> </dependency> <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-mapreduce-client-core</artifactId> - <version>${hadoop.version}</version> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-server</artifactId> + <version>${dep.hbase.version}</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hbase</groupId> - <artifactId>hbase</artifactId> + <artifactId>hbase-common</artifactId> <version>${dep.hbase.version}</version> <scope>provided</scope> - <exclusions> - <exclusion> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java</artifactId> - </exclusion> - <exclusion> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-server</artifactId> - </exclusion> - <exclusion> - <groupId>commons-httpclient</groupId> - <artifactId>commons-httpclient</artifactId> - </exclusion> - <exclusion> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - </exclusion> - <exclusion> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> - </exclusion> - <exclusion> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.avro</groupId> - <artifactId>avro</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.thrift</groupId> - <artifactId>thrift</artifactId> - </exclusion> - <exclusion> - <groupId>org.mortbay.jetty</groupId> - <artifactId>servlet-api-2.5</artifactId> - </exclusion> - <exclusion> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jetty</artifactId> - </exclusion> - <exclusion> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jsp-api-2.1</artifactId> - </exclusion> - <exclusion> - <groupId>tomcat</groupId> - <artifactId>jasper-compiler</artifactId> - </exclusion> - <exclusion> - <groupId>tomcat</groupId> - <artifactId>jasper-runtime</artifactId> - </exclusion> - <exclusion> - <groupId>javax.ws.rs</groupId> - <artifactId>jsr311-api</artifactId> - </exclusion> - </exclusions> </dependency> <!-- runtime dependencies. sorted lexicographically. --> @@ -2290,52 +2224,17 @@ under the License. </dependency> <dependency> <groupId>org.apache.hbase</groupId> - <artifactId>hbase</artifactId> + <artifactId>hbase-common</artifactId> + <type>test-jar</type> + <version>${dep.hbase.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-server</artifactId> <type>test-jar</type> <version>${dep.hbase.version}</version> <scope>test</scope> - <exclusions> - <exclusion> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-server</artifactId> - </exclusion> - <exclusion> - <groupId>commons-httpclient</groupId> - <artifactId>commons-httpclient</artifactId> - </exclusion> - <exclusion> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> - </exclusion> - <exclusion> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>avro</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.thrift</groupId> - <artifactId>thrift</artifactId> - </exclusion> - <exclusion> - <groupId>tomcat</groupId> - <artifactId>jasper-compiler</artifactId> - </exclusion> - <exclusion> - <groupId>tomcat</groupId> - <artifactId>jasper-runtime</artifactId> - </exclusion> - <exclusion> - <groupId>javax.ws.rs</groupId> - <artifactId>jsr311-api</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> <groupId>org.mockito</groupId>