Updated Branches: refs/heads/trunk 40e201df9 -> 0e52d19f1
GIRAPH-550: HiveVertexReader should call initialize, not users (nitay). Project: http://git-wip-us.apache.org/repos/asf/giraph/repo Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/0e52d19f Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/0e52d19f Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/0e52d19f Branch: refs/heads/trunk Commit: 0e52d19f132be6078526e8ebb3a4db8f0ef40cb1 Parents: 40e201d Author: Nitay Joffe <[email protected]> Authored: Thu Feb 28 15:32:20 2013 -0500 Committer: Nitay Joffe <[email protected]> Committed: Mon Mar 11 16:21:39 2013 -0400 ---------------------------------------------------------------------- CHANGELOG | 2 + checkstyle.xml | 1 - giraph-accumulo/pom.xml | 30 --- giraph-core/pom.xml | 16 -- giraph-examples/pom.xml | 30 --- giraph-hbase/pom.xml | 30 --- giraph-hcatalog/pom.xml | 30 --- giraph-hive/pom.xml | 30 --- .../org/apache/giraph/hive/HiveGiraphRunner.java | 137 +++++++-------- .../giraph/hive/input/edge/AbstractHiveToEdge.java | 38 ++++ .../giraph/hive/input/edge/HiveEdgeReader.java | 33 ++-- .../apache/giraph/hive/input/edge/HiveToEdge.java | 6 +- .../input/vertex/AbstractHiveToVertexEdges.java | 38 ++++ .../input/vertex/AbstractHiveToVertexValue.java | 37 ++++ .../giraph/hive/input/vertex/HiveToVertex.java | 45 ----- .../hive/input/vertex/HiveToVertexEdges.java | 67 +++++++ .../hive/input/vertex/HiveToVertexValue.java | 49 +++++ .../giraph/hive/input/vertex/HiveVertexReader.java | 63 ++++++-- .../giraph/hive/output/AbstractVertexToHive.java | 38 ++++ .../giraph/hive/output/HiveVertexWriter.java | 5 +- .../apache/giraph/hive/output/VertexToHive.java | 3 +- pom.xml | 41 +++++- 22 files changed, 449 insertions(+), 320 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/CHANGELOG ---------------------------------------------------------------------- diff --git a/CHANGELOG b/CHANGELOG index f06627f..ee894ba 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,8 @@ Giraph Change Log Release 0.2.0 - unreleased + GIRAPH-550: HiveVertexReader should call initialize, not users (nitay) + GIRAPH-523: MasterObserver should pass in superstep number (gaurav.menghani via nitay) GIRAPH-556: Race condition in EdgeStore (apresta) http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/checkstyle.xml ---------------------------------------------------------------------- diff --git a/checkstyle.xml b/checkstyle.xml index 10a8b43..3d8a6d4 100644 --- a/checkstyle.xml +++ b/checkstyle.xml @@ -86,7 +86,6 @@ <!-- Checks for common coding problems --> <!-- See http://checkstyle.sf.net/config_coding.html --> - <module name="DoubleCheckedLocking"/> <module name="EmptyStatement"/> <!-- Require hash code override when equals is --> <module name="EqualsHashCode"/> http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-accumulo/pom.xml ---------------------------------------------------------------------- diff --git a/giraph-accumulo/pom.xml b/giraph-accumulo/pom.xml index 1f7e84a..1f229da 100644 --- a/giraph-accumulo/pom.xml +++ b/giraph-accumulo/pom.xml @@ -44,22 +44,6 @@ under the License. <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-checkstyle-plugin</artifactId> - <version>2.9</version> - <configuration> - <configLocation>${top.dir}/checkstyle.xml</configLocation> - <headerLocation>${top.dir}/license-header.txt</headerLocation> - <enableRulesSummary>false</enableRulesSummary> - <failOnError>true</failOnError> - <includeTestSourceDirectory>false</includeTestSourceDirectory> - </configuration> - <executions> - <execution> - <phase>verify</phase> - <goals> - <goal>check</goal> - </goals> - </execution> - </executions> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> @@ -81,20 +65,6 @@ under the License. <plugin> <groupId>org.codehaus.mojo</groupId> <artifactId>findbugs-maven-plugin</artifactId> - <version>2.5.1</version> - <configuration> - <xmlOutput>true</xmlOutput> - <findbugsXmlOutput>false</findbugsXmlOutput> - <excludeFilterFile>${top.dir}/findbugs-exclude.xml</excludeFilterFile> - </configuration> - <executions> - <execution> - <phase>verify</phase> - <goals> - <goal>check</goal> - </goals> - </execution> - </executions> </plugin> </plugins> </build> http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-core/pom.xml ---------------------------------------------------------------------- diff --git a/giraph-core/pom.xml b/giraph-core/pom.xml index 2a500d8..3580d0c 100644 --- a/giraph-core/pom.xml +++ b/giraph-core/pom.xml @@ -64,22 +64,6 @@ under the License. <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-checkstyle-plugin</artifactId> - <version>2.9</version> - <configuration> - <configLocation>checkstyle.xml</configLocation> - <enableRulesSummary>false</enableRulesSummary> - <headerLocation>license-header.txt</headerLocation> - <failOnViolation>true</failOnViolation> - <includeTestSourceDirectory>false</includeTestSourceDirectory> - </configuration> - <executions> - <execution> - <phase>verify</phase> - <goals> - <goal>check</goal> - </goals> - </execution> - </executions> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-examples/pom.xml ---------------------------------------------------------------------- diff --git a/giraph-examples/pom.xml b/giraph-examples/pom.xml index 7a18711..3b6a08c 100644 --- a/giraph-examples/pom.xml +++ b/giraph-examples/pom.xml @@ -46,22 +46,6 @@ under the License. <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-checkstyle-plugin</artifactId> - <version>2.9</version> - <configuration> - <configLocation>checkstyle.xml</configLocation> - <enableRulesSummary>false</enableRulesSummary> - <headerLocation>license-header.txt</headerLocation> - <failOnViolation>true</failOnViolation> - <includeTestSourceDirectory>false</includeTestSourceDirectory> - </configuration> - <executions> - <execution> - <phase>verify</phase> - <goals> - <goal>check</goal> - </goals> - </execution> - </executions> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> @@ -107,20 +91,6 @@ under the License. <plugin> <groupId>org.codehaus.mojo</groupId> <artifactId>findbugs-maven-plugin</artifactId> - <version>2.5.1</version> - <configuration> - <xmlOutput>true</xmlOutput> - <findbugsXmlOutput>false</findbugsXmlOutput> - <excludeFilterFile>${top.dir}/findbugs-exclude.xml</excludeFilterFile> - </configuration> - <executions> - <execution> - <phase>verify</phase> - <goals> - <goal>check</goal> - </goals> - </execution> - </executions> </plugin> </plugins> </build> http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hbase/pom.xml ---------------------------------------------------------------------- diff --git a/giraph-hbase/pom.xml b/giraph-hbase/pom.xml index 8eafff1..e7aa420 100644 --- a/giraph-hbase/pom.xml +++ b/giraph-hbase/pom.xml @@ -44,22 +44,6 @@ under the License. <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-checkstyle-plugin</artifactId> - <version>2.9</version> - <configuration> - <configLocation>${top.dir}/checkstyle.xml</configLocation> - <headerLocation>${top.dir}/license-header.txt</headerLocation> - <enableRulesSummary>false</enableRulesSummary> - <failOnError>true</failOnError> - <includeTestSourceDirectory>false</includeTestSourceDirectory> - </configuration> - <executions> - <execution> - <phase>verify</phase> - <goals> - <goal>check</goal> - </goals> - </execution> - </executions> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> @@ -81,20 +65,6 @@ under the License. <plugin> <groupId>org.codehaus.mojo</groupId> <artifactId>findbugs-maven-plugin</artifactId> - <version>2.5.1</version> - <configuration> - <xmlOutput>true</xmlOutput> - <findbugsXmlOutput>false</findbugsXmlOutput> - <excludeFilterFile>${top.dir}/findbugs-exclude.xml</excludeFilterFile> - </configuration> - <executions> - <execution> - <phase>verify</phase> - <goals> - <goal>check</goal> - </goals> - </execution> - </executions> </plugin> </plugins> </build> http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hcatalog/pom.xml ---------------------------------------------------------------------- diff --git a/giraph-hcatalog/pom.xml b/giraph-hcatalog/pom.xml index 73359e3..ee5710a 100644 --- a/giraph-hcatalog/pom.xml +++ b/giraph-hcatalog/pom.xml @@ -44,22 +44,6 @@ under the License. <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-checkstyle-plugin</artifactId> - <version>2.9</version> - <configuration> - <configLocation>${top.dir}/checkstyle.xml</configLocation> - <headerLocation>${top.dir}/license-header.txt</headerLocation> - <enableRulesSummary>false</enableRulesSummary> - <failOnError>true</failOnError> - <includeTestSourceDirectory>false</includeTestSourceDirectory> - </configuration> - <executions> - <execution> - <phase>verify</phase> - <goals> - <goal>check</goal> - </goals> - </execution> - </executions> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> @@ -81,20 +65,6 @@ under the License. <plugin> <groupId>org.codehaus.mojo</groupId> <artifactId>findbugs-maven-plugin</artifactId> - <version>2.5.1</version> - <configuration> - <xmlOutput>true</xmlOutput> - <findbugsXmlOutput>false</findbugsXmlOutput> - <excludeFilterFile>${top.dir}/findbugs-exclude.xml</excludeFilterFile> - </configuration> - <executions> - <execution> - <phase>verify</phase> - <goals> - <goal>check</goal> - </goals> - </execution> - </executions> </plugin> </plugins> </build> http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/pom.xml ---------------------------------------------------------------------- diff --git a/giraph-hive/pom.xml b/giraph-hive/pom.xml index 84b7c91..cc16ca2 100644 --- a/giraph-hive/pom.xml +++ b/giraph-hive/pom.xml @@ -44,22 +44,6 @@ under the License. <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-checkstyle-plugin</artifactId> - <version>2.9</version> - <configuration> - <configLocation>${top.dir}/checkstyle.xml</configLocation> - <headerLocation>${top.dir}/license-header.txt</headerLocation> - <enableRulesSummary>false</enableRulesSummary> - <failOnError>true</failOnError> - <includeTestSourceDirectory>false</includeTestSourceDirectory> - </configuration> - <executions> - <execution> - <phase>verify</phase> - <goals> - <goal>check</goal> - </goals> - </execution> - </executions> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> @@ -81,20 +65,6 @@ under the License. <plugin> <groupId>org.codehaus.mojo</groupId> <artifactId>findbugs-maven-plugin</artifactId> - <version>2.5.1</version> - <configuration> - <xmlOutput>true</xmlOutput> - <findbugsXmlOutput>false</findbugsXmlOutput> - <excludeFilterFile>${top.dir}/findbugs-exclude.xml</excludeFilterFile> - </configuration> - <executions> - <execution> - <phase>verify</phase> - <goals> - <goal>check</goal> - </goals> - </execution> - </executions> </plugin> </plugins> </build> http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/src/main/java/org/apache/giraph/hive/HiveGiraphRunner.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/HiveGiraphRunner.java b/giraph-hive/src/main/java/org/apache/giraph/hive/HiveGiraphRunner.java index d89bad3..eeea265 100644 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/HiveGiraphRunner.java +++ b/giraph-hive/src/main/java/org/apache/giraph/hive/HiveGiraphRunner.java @@ -30,7 +30,8 @@ import org.apache.giraph.hive.common.HiveProfiles; import org.apache.giraph.hive.input.edge.HiveEdgeInputFormat; import org.apache.giraph.hive.input.edge.HiveEdgeReader; import org.apache.giraph.hive.input.edge.HiveToEdge; -import org.apache.giraph.hive.input.vertex.HiveToVertex; +import org.apache.giraph.hive.input.vertex.HiveToVertexValue; +import org.apache.giraph.hive.input.vertex.HiveToVertexEdges; import org.apache.giraph.hive.input.vertex.HiveVertexInputFormat; import org.apache.giraph.hive.input.vertex.HiveVertexReader; import org.apache.giraph.hive.output.HiveVertexOutputFormat; @@ -77,7 +78,9 @@ public class HiveGiraphRunner implements Tool { private Class<? extends Vertex> vertexClass; /** Vertex creator from hive records. */ - private Class<? extends HiveToVertex> hiveToVertexClass; + private Class<? extends HiveToVertexValue> hiveToVertexClass; + /** Vertex edges creator from hive records. */ + private Class<? extends HiveToVertexEdges> hiveToVertexEdgesClass; /** hive vertex input information */ private final HiveInputDescription hiveVertexInputDescription; @@ -104,58 +107,40 @@ public class HiveGiraphRunner implements Tool { hiveOutputDescription = new HiveOutputDescription(); } - /** - * Get Vertex class used. - * - * @return Vertex class - */ public Class<? extends Vertex> getVertexClass() { return vertexClass; } - /** - * Set Vertex class to use - * - * @param vertexClass Vertex class - */ public void setVertexClass(Class<? extends Vertex> vertexClass) { this.vertexClass = vertexClass; } - /** - * Get hive vertex input description - * - * @return HiveInputDescription - */ public HiveInputDescription getHiveVertexInputDescription() { return hiveVertexInputDescription; } - /** - * Get hive output description - * - * @return HiveOutputDescription - */ public HiveOutputDescription getHiveOutputDescription() { return hiveOutputDescription; } - /** - * Get hive input description - * - * @return HiveInputDescription - */ public HiveInputDescription getHiveEdgeInputDescription() { return hiveEdgeInputDescription; } + public Class<? extends HiveToVertexValue> getHiveToVertexValueClass() { + return hiveToVertexClass; + } + /** - * Get HiveVertexCreator used with HiveVertexInputFormat + * Set HiveVertexCreator used with HiveVertexInputFormat * - * @return HiveVertexCreator + * @param hiveToVertexClass HiveVertexCreator */ - public Class<? extends HiveToVertex> getHiveToVertexClass() { - return hiveToVertexClass; + public void setHiveToVertexValueClass( + Class<? extends HiveToVertexValue> hiveToVertexClass) { + this.hiveToVertexClass = hiveToVertexClass; + conf.setClass(HiveVertexReader.HIVE_TO_VERTEX_KEY, hiveToVertexClass, + HiveToVertexValue.class); } /** @@ -163,37 +148,35 @@ public class HiveGiraphRunner implements Tool { * * @return true if vertex input enabled (HiveVertexCreator is set). */ - public boolean hasVertexInput() { + public boolean hasVertexValueInput() { return hiveToVertexClass != null; } - /** - * Set class used to convert hive records to Vertexes. - * - * @param hiveToVertexValueClass HiveToVertex class - */ - public void setHiveToVertexValueClass( - Class<? extends HiveToVertex> hiveToVertexValueClass) { - setHiveToVertexClass(hiveToVertexValueClass); + public Class<? extends HiveToVertexEdges> getHiveToVertexEdgesClass() { + return hiveToVertexEdgesClass; } /** - * Set HiveVertexCreator used with HiveVertexInputFormat + * Whether we have a class for reading per-vertex edges from Hive. * - * @param hiveToVertexClass HiveVertexCreator + * @return true if user set class for reading vertex edges. */ - public void setHiveToVertexClass( - Class<? extends HiveToVertex> hiveToVertexClass) { - this.hiveToVertexClass = hiveToVertexClass; - conf.setClass(HiveVertexReader.HIVE_TO_VERTEX_KEY, hiveToVertexClass, - HiveToVertex.class); + public boolean hasHiveToVertexEdgesClass() { + return hiveToVertexEdgesClass != null; } /** - * Get HiveEdgeCreator used with HiveEdgeInputFormat + * Set class for reading per-vertex edges from hive. * - * @return HiveEdgeCreator + * @param klass Class to use */ + public void setHiveToVertexEdgesClass( + Class<? extends HiveToVertexEdges> klass) { + this.hiveToVertexEdgesClass = klass; + conf.setClass(HiveVertexReader.HIVE_TO_VERTEX_EDGES_KEY, klass, + HiveToVertexEdges.class); + } + public Class<? extends HiveToEdge> getHiveToEdgeClass() { return hiveToEdgeClass; } @@ -214,15 +197,10 @@ public class HiveGiraphRunner implements Tool { */ public void setHiveToEdgeClass(Class<? extends HiveToEdge> hiveToEdgeClass) { this.hiveToEdgeClass = hiveToEdgeClass; - conf.setClass(HiveEdgeReader.EDGE_CREATOR_KEY, hiveToEdgeClass, + conf.setClass(HiveEdgeReader.HIVE_TO_EDGE_KEY, hiveToEdgeClass, HiveToEdge.class); } - /** - * Get class used to write vertices to Hive. - * - * @return class for writing vertices to Hive - */ public Class<? extends VertexToHive> getVertexToHiveClass() { return vertexToHiveClass; } @@ -386,7 +364,15 @@ public class HiveGiraphRunner implements Tool { String hiveToVertexClassStr = cmdln.getOptionValue("hiveToVertexClass"); if (hiveToVertexClassStr != null) { - hiveToVertexClass = findClass(hiveToVertexClassStr, HiveToVertex.class); + setHiveToVertexValueClass(findClass(hiveToVertexClassStr, + HiveToVertexValue.class)); + } + + String hiveToVertexEdgesClassStr = + cmdln.getOptionValue("hiveToVertexEdgesClass"); + if (hiveToVertexEdgesClassStr != null) { + setHiveToVertexEdgesClass(findClass(hiveToVertexEdgesClassStr, + HiveToVertexEdges.class)); } String hiveToEdgeClassStr = cmdln.getOptionValue("hiveToEdgeClass"); @@ -396,26 +382,26 @@ public class HiveGiraphRunner implements Tool { String vertexToHiveClassStr = cmdln.getOptionValue("vertexToHiveClass"); if (vertexToHiveClassStr != null) { - vertexToHiveClass = findClass(vertexToHiveClassStr, VertexToHive.class); + setVertexToHiveClass(findClass(vertexToHiveClassStr, VertexToHive.class)); } if (cmdln.hasOption("skipOutput")) { skipOutput = true; } -// if (hiveToVertexClass == null && hiveToEdgeClass == null) { -// throw new IllegalArgumentException( -// "Need at least one of Giraph " + -// HiveToVertex.class.getSimpleName() + -// " class name (-hiveToVertexClass) and " + -// HiveToEdge.class.getSimpleName() + -// " class name (-hiveToEdgeClass)"); -// } -// if (vertexToHiveClass == null && !skipOutput) { -// throw new IllegalArgumentException( -// "Need the Giraph " + VertexToHive.class.getSimpleName() + -// " class name (-vertexToHiveClass) to use"); -// } + if (hiveToVertexClass == null && hiveToEdgeClass == null) { + throw new IllegalArgumentException( + "Need at least one of Giraph " + + HiveToVertexValue.class.getSimpleName() + + " class name (-hiveToVertexClass) and " + + HiveToEdge.class.getSimpleName() + + " class name (-hiveToEdgeClass)"); + } + if (vertexToHiveClass == null && !skipOutput) { + throw new IllegalArgumentException( + "Need the Giraph " + VertexToHive.class.getSimpleName() + + " class name (-vertexToHiveClass) to use"); + } String workersStr = cmdln.getOptionValue("workers"); if (workersStr == null) { throw new IllegalArgumentException( @@ -536,7 +522,12 @@ public class HiveGiraphRunner implements Tool { // Vertex input settings if (hiveToVertexClass == null) { options.addOption(null, "hiveToVertexClass", true, - "Giraph " + HiveToVertex.class.getSimpleName() + " class to use"); + "Giraph " + HiveToVertexValue.class.getSimpleName() + " class to use"); + } + if (hiveToVertexEdgesClass == null) { + options.addOption(null, "hiveToVertexEdgesClass", true, + "Giraph " + HiveToVertexEdges.class.getSimpleName() + + " class to use"); } options.addOption("vi", "vertexInputTable", true, "Vertex input table name"); @@ -662,6 +653,10 @@ public class HiveGiraphRunner implements Tool { LOG.info(LOG_PREFIX + "-hiveToVertexClass=" + hiveToVertexClass.getCanonicalName()); } + if (hiveToVertexEdgesClass != null) { + LOG.info(LOG_PREFIX + "-hiveToVertexEdgesClass=" + + hiveToVertexEdgesClass.getCanonicalName()); + } LOG.info(LOG_PREFIX + "-vertexInputFormatClass=" + classes.getVertexInputFormatClass().getCanonicalName()); logInputDesc(hiveVertexInputDescription, "vertex"); http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/AbstractHiveToEdge.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/AbstractHiveToEdge.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/AbstractHiveToEdge.java new file mode 100644 index 0000000..c418085 --- /dev/null +++ b/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/AbstractHiveToEdge.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.giraph.hive.input.edge; + +import org.apache.giraph.conf.ImmutableClassesGiraphConfigurable; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; + +import com.facebook.giraph.hive.HiveTableSchemaAware; + +/** + * Base class for HiveToEdge implementations + * + * @param <I> Vertex ID + * @param <V> Vertex Value + * @param <E> Edge Value + * @param <M> Message Value + */ +public abstract class AbstractHiveToEdge<I extends WritableComparable, + V extends Writable, E extends Writable, M extends Writable> implements + HiveTableSchemaAware, + HiveToEdge<I, E>, + ImmutableClassesGiraphConfigurable<I, V, E, M> { } http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/HiveEdgeReader.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/HiveEdgeReader.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/HiveEdgeReader.java index 0906996..e0c502c 100644 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/HiveEdgeReader.java +++ b/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/HiveEdgeReader.java @@ -18,20 +18,21 @@ package org.apache.giraph.hive.input.edge; -import com.facebook.giraph.hive.HiveRecord; -import com.facebook.giraph.hive.HiveTableSchema; import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration; -import org.apache.giraph.io.EdgeReader; -import org.apache.giraph.utils.ReflectionUtils; import org.apache.giraph.edge.Edge; -import org.apache.giraph.edge.EdgeFactory; import org.apache.giraph.edge.MutableEdge; +import org.apache.giraph.io.EdgeReader; +import org.apache.giraph.utils.ReflectionUtils; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; +import com.facebook.giraph.hive.HiveRecord; +import com.facebook.giraph.hive.HiveTableSchema; +import com.facebook.giraph.hive.HiveTableSchemas; + import java.io.IOException; /** @@ -43,7 +44,7 @@ import java.io.IOException; public class HiveEdgeReader<I extends WritableComparable, E extends Writable> implements EdgeReader<I, E> { /** Configuration key for edge creator class */ - public static final String EDGE_CREATOR_KEY = "giraph.hive.to.edge.class"; + public static final String HIVE_TO_EDGE_KEY = "giraph.hive.to.edge.class"; /** Configuration key for whether to reuse edge */ public static final String REUSE_EDGE_KEY = "giraph.hive.reuse.edge"; @@ -126,13 +127,13 @@ public class HiveEdgeReader<I extends WritableComparable, E extends Writable> * @throws IOException if anything goes wrong reading from Configuration */ private void instantiateHiveToEdgeFromConf() throws IOException { - Class<? extends HiveToEdge> klass = conf.getClass(EDGE_CREATOR_KEY, + Class<? extends HiveToEdge> klass = conf.getClass(HIVE_TO_EDGE_KEY, null, HiveToEdge.class); if (klass == null) { - throw new IOException(EDGE_CREATOR_KEY + " not set in conf"); + throw new IOException(HIVE_TO_EDGE_KEY + " not set in conf"); } hiveToEdge = ReflectionUtils.newInstance(klass, conf); - hiveToEdge.setTableSchema(tableSchema); + HiveTableSchemas.configure(hiveToEdge, tableSchema); } @Override @@ -159,14 +160,12 @@ public class HiveEdgeReader<I extends WritableComparable, E extends Writable> public Edge<I, E> getCurrentEdge() throws IOException, InterruptedException { HiveRecord record = hiveRecordReader.getCurrentValue(); - I targetId = hiveToEdge.getTargetVertexId(record); - E edgeValue = hiveToEdge.getEdgeValue(record); - if (edgeToReuse == null) { - return EdgeFactory.create(targetId, edgeValue); - } else { - edgeToReuse.setTargetVertexId(targetId); - edgeToReuse.setValue(edgeValue); - return edgeToReuse; + MutableEdge<I, E> edge = edgeToReuse; + if (edge == null) { + edge = conf.createMutableEdge(); } + edge.setValue(hiveToEdge.getEdgeValue(record)); + edge.setTargetVertexId(hiveToEdge.getTargetVertexId(record)); + return edge; } } http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/HiveToEdge.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/HiveToEdge.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/HiveToEdge.java index e800321..2205b82 100644 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/HiveToEdge.java +++ b/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/HiveToEdge.java @@ -22,16 +22,14 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import com.facebook.giraph.hive.HiveReadableRecord; -import com.facebook.giraph.hive.HiveTableSchemaAware; /** - * An interface used to create edges from Hive records + * An interface used to create edges from Hive records. * * @param <I> Vertex ID * @param <E> Edge Value */ -public interface HiveToEdge<I extends WritableComparable, E extends Writable> - extends HiveTableSchemaAware { +public interface HiveToEdge<I extends WritableComparable, E extends Writable> { /** * Read source vertex ID from Hive record * http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/AbstractHiveToVertexEdges.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/AbstractHiveToVertexEdges.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/AbstractHiveToVertexEdges.java new file mode 100644 index 0000000..746e9e3 --- /dev/null +++ b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/AbstractHiveToVertexEdges.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.giraph.hive.input.vertex; + +import org.apache.giraph.conf.ImmutableClassesGiraphConfigurable; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; + +import com.facebook.giraph.hive.HiveTableSchemaAware; + +/** + * Base class for HiveToVertexEdges implementations + * + * @param <I> Vertex ID + * @param <V> Vertex Value + * @param <E> Edge Value + * @param <M> Message Value + */ +public abstract class AbstractHiveToVertexEdges<I extends WritableComparable, + V extends Writable, E extends Writable, M extends Writable> implements + HiveTableSchemaAware, + HiveToVertexEdges<I, E>, + ImmutableClassesGiraphConfigurable<I, V, E, M> { } http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/AbstractHiveToVertexValue.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/AbstractHiveToVertexValue.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/AbstractHiveToVertexValue.java new file mode 100644 index 0000000..9636d2a --- /dev/null +++ b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/AbstractHiveToVertexValue.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.giraph.hive.input.vertex; + +import org.apache.giraph.conf.ImmutableClassesGiraphConfigurable; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; + +import com.facebook.giraph.hive.HiveTableSchemaAware; + +/** + * Base class for HiveToVertex implementations + * + * @param <I> Vertex ID + * @param <V> Vertex Value + * @param <E> Edge Value + * @param <M> Message Value + */ +public abstract class AbstractHiveToVertexValue<I extends WritableComparable, + V extends Writable, E extends Writable, M extends Writable> implements + HiveTableSchemaAware, HiveToVertexValue<I, V>, + ImmutableClassesGiraphConfigurable<I, V, E, M> { } http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveToVertex.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveToVertex.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveToVertex.java deleted file mode 100644 index 7c84ff1..0000000 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveToVertex.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.giraph.hive.input.vertex; - -import org.apache.giraph.graph.Vertex; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableComparable; - -import com.facebook.giraph.hive.HiveReadableRecord; -import com.facebook.giraph.hive.HiveTableSchemaAware; - -/** - * Interface for creating vertices from a Hive record. - * Also used for reading vertex values. - * - * @param <I> Vertex ID - * @param <V> Vertex Value - * @param <E> Edge Value - */ -public interface HiveToVertex<I extends WritableComparable, - V extends Writable, E extends Writable> extends HiveTableSchemaAware { - /** - * Fill the Vertex from the HiveRecord given. - * - * @param record HiveRecord to read from. - * @param vertex Vertex to fill. - */ - void fillVertex(HiveReadableRecord record, Vertex<I, V, E, ?> vertex); -} http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveToVertexEdges.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveToVertexEdges.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveToVertexEdges.java new file mode 100644 index 0000000..cf7ea33 --- /dev/null +++ b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveToVertexEdges.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.giraph.hive.input.vertex; + +import org.apache.giraph.edge.Edge; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; + +import com.facebook.giraph.hive.HiveReadableRecord; + +import java.util.Collections; + +/** + * Interface for creating edges for a vertex from a Hive record. + * Used with HiveToVertex if you want to also read edges per vertex, as opposed + * to using {@link org.apache.giraph.hive.input.edge.HiveEdgeInputFormat} + * + * @param <I> Vertex ID + * @param <E> extends Writable + */ +public interface HiveToVertexEdges<I extends WritableComparable, + E extends Writable> { + /** + * Read Vertex's edges from the HiveRecord given. + * + * @param record HiveRecord to read from. + * @return iterable of edges + */ + Iterable<Edge<I, E>> getEdges(HiveReadableRecord record); + + /** + * Default implementation that returns an empty list of edges. + */ + public class Empty implements HiveToVertexEdges { + /** Singleton */ + private static final Empty INSTANCE = new Empty(); + + /** Don't construct */ + private Empty() { } + + /** + * Get singleton instance + * @return Empty + */ + public static Empty get() { return INSTANCE; } + + @Override public Iterable getEdges(HiveReadableRecord record) { + return Collections.emptyList(); + } + } +} http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveToVertexValue.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveToVertexValue.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveToVertexValue.java new file mode 100644 index 0000000..593eb9a --- /dev/null +++ b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveToVertexValue.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.giraph.hive.input.vertex; + +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; + +import com.facebook.giraph.hive.HiveReadableRecord; + +/** + * Interface for creating vertices from a Hive record. + * + * @param <I> Vertex ID + * @param <V> Vertex Value + */ +public interface HiveToVertexValue<I extends WritableComparable, + V extends Writable> { + /** + * Read the Vertex's ID from the HiveRecord given. + * + * @param record HiveRecord to read from. + * @return Vertex ID + */ + I getVertexId(HiveReadableRecord record); + + /** + * Read the Vertex's Value from the HiveRecord given. + * + * @param record HiveRecord to read from. + * @return Vertex Value + */ + V getVertexValue(HiveReadableRecord record); +} http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveVertexReader.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveVertexReader.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveVertexReader.java index b0f348b..2311e72 100644 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveVertexReader.java +++ b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/HiveVertexReader.java @@ -19,9 +19,10 @@ package org.apache.giraph.hive.input.vertex; import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration; +import org.apache.giraph.edge.Edge; +import org.apache.giraph.graph.Vertex; import org.apache.giraph.io.VertexReader; import org.apache.giraph.utils.ReflectionUtils; -import org.apache.giraph.graph.Vertex; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapreduce.InputSplit; @@ -31,6 +32,7 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext; import com.facebook.giraph.hive.HiveRecord; import com.facebook.giraph.hive.HiveTableSchema; import com.facebook.giraph.hive.HiveTableSchemaAware; +import com.facebook.giraph.hive.HiveTableSchemas; import java.io.IOException; @@ -45,8 +47,14 @@ import java.io.IOException; public class HiveVertexReader<I extends WritableComparable, V extends Writable, E extends Writable, M extends Writable> implements VertexReader<I, V, E, M>, HiveTableSchemaAware { - /** Key in Configuration for HiveToVertex class */ - public static final String HIVE_TO_VERTEX_KEY = "giraph.hive.to.vertex.class"; + /** Configuration key for {@link HiveToVertexValue} class */ + public static final String HIVE_TO_VERTEX_KEY = + "giraph.hive.to.vertex.value.class"; + /** Configuration key for {@link HiveToVertexEdges} class */ + public static final String HIVE_TO_VERTEX_EDGES_KEY = + "giraph.hive.to.vertex.edges.class"; + /** Configuration key for whether to reuse vertex */ + public static final String REUSE_VERTEX_KEY = "giraph.hive.reuse.vertex"; /** Underlying Hive RecordReader used */ private RecordReader<WritableComparable, HiveRecord> hiveRecordReader; @@ -57,7 +65,15 @@ public class HiveVertexReader<I extends WritableComparable, private ImmutableClassesGiraphConfiguration<I, V, E, M> conf; /** User class to create vertices from a HiveRecord */ - private HiveToVertex<I, V, E> hiveToVertex; + private HiveToVertexValue<I, V> hiveToVertexValue; + /** User class to create vertex edges from HiveRecord - optional */ + private HiveToVertexEdges<I, E> hiveToVertexEdges; + + /** + * If we are reusing vertices this will be the single vertex to read into. + * Otherwise if it's null we will create a new vertex each time. + */ + private Vertex<I, V, E, M> vertexToReuse = null; /** * Get underlying Hive record reader used. @@ -103,21 +119,40 @@ public class HiveVertexReader<I extends WritableComparable, hiveRecordReader.initialize(inputSplit, context); conf = new ImmutableClassesGiraphConfiguration(context.getConfiguration()); instantiateHiveToVertexFromConf(); + instantiateHiveToVertexEdgesFromConf(); + if (conf.getBoolean(REUSE_VERTEX_KEY, false)) { + vertexToReuse = conf.createVertex(); + } } /** - * Retrieve the user's HiveVertexCreator from our configuration. + * Retrieve the user's HiveToVertex from our configuration. * * @throws IOException if anything goes wrong reading from Configuration. */ private void instantiateHiveToVertexFromConf() throws IOException { - Class<? extends HiveToVertex> klass = conf.getClass(HIVE_TO_VERTEX_KEY, - null, HiveToVertex.class); + Class<? extends HiveToVertexValue> klass = conf.getClass(HIVE_TO_VERTEX_KEY, + null, HiveToVertexValue.class); if (klass == null) { throw new IOException(HIVE_TO_VERTEX_KEY + " not set in conf"); } - hiveToVertex = ReflectionUtils.newInstance(klass, conf); - hiveToVertex.setTableSchema(tableSchema); + hiveToVertexValue = ReflectionUtils.newInstance(klass, conf); + HiveTableSchemas.configure(hiveToVertexValue, tableSchema); + } + + /** + * Retrieve the user's HiveToVertexEdges from our configuration. This class + * is optional. If not specified will just use HiveToVertexEdges.Empty. + */ + private void instantiateHiveToVertexEdgesFromConf() { + Class<? extends HiveToVertexEdges> klass = conf.getClass( + HIVE_TO_VERTEX_EDGES_KEY, null, HiveToVertexEdges.class); + if (klass == null) { + hiveToVertexEdges = HiveToVertexEdges.Empty.get(); + } else { + hiveToVertexEdges = ReflectionUtils.newInstance(klass, conf); + } + HiveTableSchemas.configure(hiveToVertexEdges, tableSchema); } @Override @@ -139,8 +174,14 @@ public class HiveVertexReader<I extends WritableComparable, public final Vertex<I, V, E, M> getCurrentVertex() throws IOException, InterruptedException { HiveRecord hiveRecord = hiveRecordReader.getCurrentValue(); - Vertex vertex = conf.createVertex(); - hiveToVertex.fillVertex(hiveRecord, vertex); + Vertex<I, V, E, M> vertex = vertexToReuse; + if (vertex == null) { + vertex = conf.createVertex(); + } + I id = hiveToVertexValue.getVertexId(hiveRecord); + V value = hiveToVertexValue.getVertexValue(hiveRecord); + Iterable<Edge<I, E>> edges = hiveToVertexEdges.getEdges(hiveRecord); + vertex.initialize(id, value, edges); return vertex; } } http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/src/main/java/org/apache/giraph/hive/output/AbstractVertexToHive.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/output/AbstractVertexToHive.java b/giraph-hive/src/main/java/org/apache/giraph/hive/output/AbstractVertexToHive.java new file mode 100644 index 0000000..f2f2267 --- /dev/null +++ b/giraph-hive/src/main/java/org/apache/giraph/hive/output/AbstractVertexToHive.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.giraph.hive.output; + +import org.apache.giraph.conf.ImmutableClassesGiraphConfigurable; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; + +import com.facebook.giraph.hive.HiveTableSchemaAware; + +/** + * Base class for HiveToVertexEdges implementations + * + * @param <I> Vertex ID + * @param <V> Vertex Value + * @param <E> Edge Value + * @param <M> Message Value + */ +public abstract class AbstractVertexToHive<I extends WritableComparable, + V extends Writable, E extends Writable, M extends Writable> implements + HiveTableSchemaAware, + VertexToHive<I, V, E>, + ImmutableClassesGiraphConfigurable<I, V, E, M> { } http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveVertexWriter.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveVertexWriter.java b/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveVertexWriter.java index 32bae15..47d096b 100644 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveVertexWriter.java +++ b/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveVertexWriter.java @@ -19,9 +19,9 @@ package org.apache.giraph.hive.output; import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration; +import org.apache.giraph.graph.Vertex; import org.apache.giraph.io.VertexWriter; import org.apache.giraph.utils.ReflectionUtils; -import org.apache.giraph.graph.Vertex; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; @@ -31,6 +31,7 @@ import org.apache.log4j.Logger; import com.facebook.giraph.hive.HiveRecord; import com.facebook.giraph.hive.HiveTableSchema; +import com.facebook.giraph.hive.HiveTableSchemas; import com.facebook.giraph.hive.impl.HiveApiRecord; import java.io.IOException; @@ -117,7 +118,7 @@ public class HiveVertexWriter<I extends WritableComparable, V extends Writable, throw new IOException(VERTEX_TO_HIVE_KEY + " not set in conf"); } vertexToHive = ReflectionUtils.newInstance(klass, conf); - vertexToHive.setTableSchema(tableSchema); + HiveTableSchemas.configure(vertexToHive, tableSchema); } @Override http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/giraph-hive/src/main/java/org/apache/giraph/hive/output/VertexToHive.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/output/VertexToHive.java b/giraph-hive/src/main/java/org/apache/giraph/hive/output/VertexToHive.java index 434d804..6d323bd 100644 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/output/VertexToHive.java +++ b/giraph-hive/src/main/java/org/apache/giraph/hive/output/VertexToHive.java @@ -22,7 +22,6 @@ import org.apache.giraph.graph.Vertex; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; -import com.facebook.giraph.hive.HiveTableSchemaAware; import com.facebook.giraph.hive.HiveWritableRecord; /** @@ -33,7 +32,7 @@ import com.facebook.giraph.hive.HiveWritableRecord; * @param <E> Edge Value */ public interface VertexToHive<I extends WritableComparable, V extends Writable, - E extends Writable> extends HiveTableSchemaAware { + E extends Writable> { /** * Fill the HiveRecord from the Vertex given. * http://git-wip-us.apache.org/repos/asf/giraph/blob/0e52d19f/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index a65208d..509831b 100644 --- a/pom.xml +++ b/pom.xml @@ -301,6 +301,27 @@ under the License. </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-checkstyle-plugin</artifactId> + <version>2.10</version> + <configuration> + <configLocation>${top.dir}/checkstyle.xml</configLocation> + <consoleOutput>true</consoleOutput> + <enableRulesSummary>false</enableRulesSummary> + <headerLocation>${top.dir}/license-header.txt</headerLocation> + <failOnViolation>true</failOnViolation> + <includeTestSourceDirectory>false</includeTestSourceDirectory> + </configuration> + <executions> + <execution> + <phase>verify</phase> + <goals> + <goal>check</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> <version>3.0</version> <configuration> @@ -484,6 +505,24 @@ under the License. </configuration> </plugin> <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>findbugs-maven-plugin</artifactId> + <version>2.5.2</version> + <configuration> + <xmlOutput>true</xmlOutput> + <findbugsXmlOutput>false</findbugsXmlOutput> + <excludeFilterFile>${top.dir}/findbugs-exclude.xml</excludeFilterFile> + </configuration> + <executions> + <execution> + <phase>verify</phase> + <goals> + <goal>check</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> <groupId>org.sonatype.plugins</groupId> <artifactId>munge-maven-plugin</artifactId> <version>1.0</version> @@ -837,7 +876,7 @@ under the License. <dependency> <groupId>com.facebook.giraph.hive</groupId> <artifactId>hive-io-experimental</artifactId> - <version>0.1</version> + <version>0.2-SNAPSHOT</version> </dependency> <dependency> <groupId>com.google.guava</groupId>
