Author: jghoman Date: Thu Nov 17 21:37:05 2011 New Revision: 1203379 URL: http://svn.apache.org/viewvc?rev=1203379&view=rev Log: GIRAPH-92. Need outputformat for just vertex ID and value.
Added: incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/IdWithValueTextOutputFormat.java incubator/giraph/trunk/src/test/java/org/apache/giraph/lib/TestIdWithValueTextOutputFormat.java Modified: incubator/giraph/trunk/CHANGELOG incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/AdjacencyListTextVertexOutputFormat.java Modified: incubator/giraph/trunk/CHANGELOG URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/CHANGELOG?rev=1203379&r1=1203378&r2=1203379&view=diff ============================================================================== --- incubator/giraph/trunk/CHANGELOG (original) +++ incubator/giraph/trunk/CHANGELOG Thu Nov 17 21:37:05 2011 @@ -2,6 +2,8 @@ Giraph Change Log Release 0.70.0 - unreleased + GIRAPH-92: Need outputformat for just vertex ID and value. (jghoman) + GIRAPH-86: Simplify boolean expressions in ZooKeeperExt::createExt. (attilacsordas via jghoman) Modified: incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/AdjacencyListTextVertexOutputFormat.java URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/AdjacencyListTextVertexOutputFormat.java?rev=1203379&r1=1203378&r2=1203379&view=diff ============================================================================== --- incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/AdjacencyListTextVertexOutputFormat.java (original) +++ incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/AdjacencyListTextVertexOutputFormat.java Thu Nov 17 21:37:05 2011 @@ -43,7 +43,7 @@ public class AdjacencyListTextVertexOutp static class AdjacencyListVertexWriter<I extends WritableComparable, V extends Writable, E extends Writable> extends TextVertexWriter<I, V, E> { - public static final String LINE_TOKENIZE_VALUE = "adj.list.output.delimiter"; + public static final String LINE_TOKENIZE_VALUE = "output.delimiter"; public static final String LINE_TOKENIZE_VALUE_DEFAULT = "\t"; private String delimiter; Added: incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/IdWithValueTextOutputFormat.java URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/IdWithValueTextOutputFormat.java?rev=1203379&view=auto ============================================================================== --- incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/IdWithValueTextOutputFormat.java (added) +++ incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/IdWithValueTextOutputFormat.java Thu Nov 17 21:37:05 2011 @@ -0,0 +1,76 @@ +package org.apache.giraph.lib; + + +import org.apache.giraph.graph.BasicVertex; +import org.apache.giraph.graph.VertexWriter; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.mapreduce.RecordWriter; +import org.apache.hadoop.mapreduce.TaskAttemptContext; + +import java.io.IOException; + +/** + * Write out Vertices' IDs and values, but not their edges nor edges' values. + * This is a useful output format when the final value of the vertex is + * all that's needed. The boolean configuration parameter reverse.id.and.value + * allows reversing the output of id and value. + * + * @param <I> Vertex index value + * @param <V> Vertex value + * @param <E> Edge value + */ +public class IdWithValueTextOutputFormat <I extends WritableComparable, + V extends Writable, E extends Writable> extends TextVertexOutputFormat<I, V, E>{ + + static class IdWithValueVertexWriter<I extends WritableComparable, V extends + Writable, E extends Writable> extends TextVertexWriter<I, V, E> { + + public static final String LINE_TOKENIZE_VALUE = "output.delimiter"; + public static final String LINE_TOKENIZE_VALUE_DEFAULT = "\t"; + + public static final String REVERSE_ID_AND_VALUE = "reverse.id.and.value"; + public static final boolean REVERSE_ID_AND_VALUE_DEFAULT = false; + + private String delimiter; + + public IdWithValueVertexWriter(RecordWriter<Text, Text> recordWriter) { + super(recordWriter); + } + + @Override + public void writeVertex(BasicVertex<I, V, E, ?> vertex) throws IOException, + InterruptedException { + if (delimiter == null) { + delimiter = getContext().getConfiguration() + .get(LINE_TOKENIZE_VALUE, LINE_TOKENIZE_VALUE_DEFAULT); + } + + String first; + String second; + boolean reverseOutput = getContext().getConfiguration() + .getBoolean(REVERSE_ID_AND_VALUE, REVERSE_ID_AND_VALUE_DEFAULT); + + if (reverseOutput) { + first = vertex.getVertexValue().toString(); + second = vertex.getVertexId().toString(); + } else { + first = vertex.getVertexId().toString(); + second = vertex.getVertexValue().toString(); + } + + Text line = new Text(first + delimiter + second); + + getRecordWriter().write(line, null); + } + } + + @Override + public VertexWriter<I, V, E> createVertexWriter(TaskAttemptContext context) + throws IOException, InterruptedException { + return new IdWithValueVertexWriter<I, V, E> + (textOutputFormat.getRecordWriter(context)); + } + +} Added: incubator/giraph/trunk/src/test/java/org/apache/giraph/lib/TestIdWithValueTextOutputFormat.java URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/src/test/java/org/apache/giraph/lib/TestIdWithValueTextOutputFormat.java?rev=1203379&view=auto ============================================================================== --- incubator/giraph/trunk/src/test/java/org/apache/giraph/lib/TestIdWithValueTextOutputFormat.java (added) +++ incubator/giraph/trunk/src/test/java/org/apache/giraph/lib/TestIdWithValueTextOutputFormat.java Thu Nov 17 21:37:05 2011 @@ -0,0 +1,70 @@ +package org.apache.giraph.lib; + +import junit.framework.TestCase; +import org.apache.giraph.graph.BasicVertex; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.DoubleWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.mapreduce.RecordWriter; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.mockito.Matchers; + +import java.io.IOException; +import java.util.ArrayList; + +import static org.apache.giraph.lib.IdWithValueTextOutputFormat.IdWithValueVertexWriter; +import static org.apache.giraph.lib.IdWithValueTextOutputFormat.IdWithValueVertexWriter.LINE_TOKENIZE_VALUE; +import static org.apache.giraph.lib.IdWithValueTextOutputFormat.IdWithValueVertexWriter.REVERSE_ID_AND_VALUE; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class TestIdWithValueTextOutputFormat extends TestCase { + public void testHappyPath() throws IOException, InterruptedException { + Configuration conf = new Configuration(); + Text expected = new Text("Four Tops\t4.0"); + + IdWithValueTestWorker(conf, expected); + } + + public void testReverseIdAndValue() throws IOException, InterruptedException { + Configuration conf = new Configuration(); + conf.setBoolean(REVERSE_ID_AND_VALUE, true); + Text expected = new Text("4.0\tFour Tops"); + + IdWithValueTestWorker(conf, expected); + } + + public void testWithDifferentDelimiter() throws IOException, + InterruptedException { + Configuration conf = new Configuration(); + conf.set(LINE_TOKENIZE_VALUE, "blah"); + Text expected = new Text("Four Topsblah4.0"); + + IdWithValueTestWorker(conf, expected); + } + + private void IdWithValueTestWorker(Configuration conf, Text expected) + throws IOException, InterruptedException { + TaskAttemptContext tac = mock(TaskAttemptContext.class); + when(tac.getConfiguration()).thenReturn(conf); + + BasicVertex vertex = mock(BasicVertex.class); + when(vertex.getVertexId()).thenReturn(new Text("Four Tops")); + when(vertex.getVertexValue()).thenReturn(new DoubleWritable(4d)); + + // Create empty iterator == no edges + when(vertex.iterator()).thenReturn(new ArrayList<Text>().iterator()); + + RecordWriter<Text, Text> tw = mock(RecordWriter.class); + IdWithValueVertexWriter writer = new IdWithValueVertexWriter(tw); + writer.initialize(tac); + writer.writeVertex(vertex); + + verify(tw).write(expected, null); + verify(vertex, times(0)).iterator(); + verify(vertex, times(0)).getEdgeValue(Matchers.<WritableComparable>any()); + } +}