http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/output/AbstractVertexToHive.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/output/AbstractVertexToHive.java b/giraph-hive/src/main/java/org/apache/giraph/hive/output/AbstractVertexToHive.java deleted file mode 100644 index 4046bb2..0000000 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/output/AbstractVertexToHive.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.giraph.hive.output; - -import org.apache.giraph.hive.common.DefaultConfigurableAndTableSchemaAware; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableComparable; - -/** - * Base class for VertexToHive implementations - * - * @param <I> Vertex ID - * @param <V> Vertex Value - * @param <E> Edge Value - */ -public abstract class AbstractVertexToHive<I extends WritableComparable, - V extends Writable, E extends Writable> - extends DefaultConfigurableAndTableSchemaAware<I, V, E> - implements VertexToHive<I, V, E> { - @Override - public void initialize() { } -}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveRecordSaver.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveRecordSaver.java b/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveRecordSaver.java deleted file mode 100644 index 3477211..0000000 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveRecordSaver.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.giraph.hive.output; - -import com.facebook.hiveio.record.HiveWritableRecord; - -import java.io.IOException; - -/** - * Interface which can save {@link HiveWritableRecord}s. - */ -public interface HiveRecordSaver { - /** - * Save the record. - * - * @param record Record to save. - * @throws IOException - * @throws InterruptedException - */ - void save(HiveWritableRecord record) throws IOException, InterruptedException; -} http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveVertexOutputFormat.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveVertexOutputFormat.java b/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveVertexOutputFormat.java deleted file mode 100644 index 97f61bd..0000000 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveVertexOutputFormat.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.giraph.hive.output; - -import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration; -import org.apache.giraph.io.VertexOutputFormat; -import org.apache.giraph.io.VertexWriter; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.mapreduce.OutputCommitter; -import org.apache.hadoop.mapreduce.RecordWriter; -import org.apache.hadoop.mapreduce.TaskAttemptContext; - -import com.facebook.hiveio.common.HiveTableDesc; -import com.facebook.hiveio.output.HiveApiOutputFormat; -import com.facebook.hiveio.output.HiveOutputDescription; -import com.facebook.hiveio.record.HiveRecordFactory; -import com.facebook.hiveio.record.HiveWritableRecord; -import com.facebook.hiveio.schema.HiveTableSchema; - -import java.io.IOException; - -import static org.apache.giraph.hive.common.GiraphHiveConstants.HIVE_VERTEX_OUTPUT_DATABASE; -import static org.apache.giraph.hive.common.GiraphHiveConstants.HIVE_VERTEX_OUTPUT_PARTITION; -import static org.apache.giraph.hive.common.GiraphHiveConstants.HIVE_VERTEX_OUTPUT_PROFILE_ID; -import static org.apache.giraph.hive.common.GiraphHiveConstants.HIVE_VERTEX_OUTPUT_TABLE; -import static org.apache.giraph.hive.common.HiveUtils.newVertexToHive; -import static org.apache.giraph.hive.common.HiveUtils.parsePartitionValues; - -/** - * VertexOutputFormat using Hive - * - * @param <I> Vertex ID - * @param <V> Vertex Value - * @param <E> Edge Value - */ -public class HiveVertexOutputFormat<I extends WritableComparable, - V extends Writable, E extends Writable> - extends VertexOutputFormat<I, V, E> { - /** Underlying Hive OutputFormat used */ - private final HiveApiOutputFormat hiveOutputFormat; - - /** - * Create vertex output format - */ - public HiveVertexOutputFormat() { - hiveOutputFormat = new HiveApiOutputFormat(); - } - - /** - * Create HiveOutputDescription from Configuration - * - * @return HiveOutputDescription - */ - private HiveOutputDescription makeOutputDesc() { - HiveOutputDescription outputDesc = new HiveOutputDescription(); - HiveTableDesc tableDesc = outputDesc.getTableDesc(); - tableDesc.setDatabaseName(HIVE_VERTEX_OUTPUT_DATABASE.get(getConf())); - tableDesc.setTableName(HIVE_VERTEX_OUTPUT_TABLE.get(getConf())); - outputDesc.setPartitionValues( - parsePartitionValues(HIVE_VERTEX_OUTPUT_PARTITION.get(getConf()))); - return outputDesc; - } - - @Override - public void setConf( - ImmutableClassesGiraphConfiguration<I, V, E> conf) { - super.setConf(conf); - hiveOutputFormat.initialize(makeOutputDesc(), - HIVE_VERTEX_OUTPUT_PROFILE_ID.get(conf), conf); - } - - @Override - public VertexWriter<I, V, E> createVertexWriter(TaskAttemptContext context) - throws IOException, InterruptedException { - RecordWriter<WritableComparable, HiveWritableRecord> baseWriter = - hiveOutputFormat.getRecordWriter(context); - HiveVertexWriter<I, V, E> writer = new HiveVertexWriter<I, V, E>(); - writer.setBaseWriter(baseWriter); - writer.setTableSchema(hiveOutputFormat.getTableSchema(getConf())); - return writer; - } - - @Override - public void checkOutputSpecs(JobContext context) - throws IOException, InterruptedException { - hiveOutputFormat.checkOutputSpecs(context); - HiveTableSchema schema = hiveOutputFormat.getTableSchema(getConf()); - VertexToHive<I, V, E> vertexToHive = newVertexToHive(getConf(), schema); - vertexToHive.checkOutput(makeOutputDesc(), schema, - HiveRecordFactory.newWritableRecord(schema)); - } - - @Override - public OutputCommitter getOutputCommitter(TaskAttemptContext context) - throws IOException, InterruptedException { - return hiveOutputFormat.getOutputCommitter(context); - } -} http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveVertexWriter.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveVertexWriter.java b/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveVertexWriter.java deleted file mode 100644 index 796bc8f..0000000 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveVertexWriter.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.giraph.hive.output; - -import org.apache.giraph.graph.Vertex; -import org.apache.giraph.hive.common.HiveUtils; -import org.apache.giraph.io.VertexWriter; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.mapreduce.RecordWriter; -import org.apache.hadoop.mapreduce.TaskAttemptContext; -import org.apache.log4j.Logger; - -import com.facebook.hiveio.record.HiveRecordFactory; -import com.facebook.hiveio.record.HiveWritableRecord; -import com.facebook.hiveio.schema.HiveTableSchema; - -import java.io.IOException; - -/** - * Vertex writer using Hive. - * - * @param <I> Vertex ID - * @param <V> Vertex Value - * @param <E> Edge Value - */ -public class HiveVertexWriter<I extends WritableComparable, V extends Writable, - E extends Writable> - extends VertexWriter<I, V, E> implements HiveRecordSaver { - /** Logger */ - private static final Logger LOG = Logger.getLogger(HiveVertexWriter.class); - /** Underlying Hive RecordWriter used */ - private RecordWriter<WritableComparable, HiveWritableRecord> hiveRecordWriter; - /** Schema for table in Hive */ - private HiveTableSchema tableSchema; - /** Reusable {@link HiveRecord} */ - private HiveWritableRecord reusableRecord; - /** User class to write vertices from a HiveRecord */ - private VertexToHive<I, V, E> vertexToHive; - - /** - * Get underlying Hive record writer used. - * - * @return RecordWriter for Hive. - */ - public RecordWriter<WritableComparable, HiveWritableRecord> getBaseWriter() { - return hiveRecordWriter; - } - - /** - * Set underlying Hive record writer used. - * - * @param hiveRecordWriter RecordWriter to write to Hive. - */ - public void setBaseWriter( - RecordWriter<WritableComparable, HiveWritableRecord> hiveRecordWriter) { - this.hiveRecordWriter = hiveRecordWriter; - } - - /** - * Get Hive table schema for table being read from. - * - * @return Hive table schema for table - */ - public HiveTableSchema getTableSchema() { - return tableSchema; - } - - /** - * Set Hive schema for table being read from. - * - * @param tableSchema Hive table schema - */ - public void setTableSchema(HiveTableSchema tableSchema) { - this.tableSchema = tableSchema; - reusableRecord = HiveRecordFactory.newWritableRecord(tableSchema); - } - - @Override - public void initialize(TaskAttemptContext context) - throws IOException, InterruptedException { - vertexToHive = HiveUtils.newVertexToHive(getConf(), tableSchema); - vertexToHive.initialize(); - } - - @Override - public void writeVertex(Vertex<I, V, E> vertex) - throws IOException, InterruptedException { - vertexToHive.saveVertex(vertex, reusableRecord, this); - } - - @Override - public void close(TaskAttemptContext context) - throws IOException, InterruptedException { - hiveRecordWriter.close(context); - } - - @Override - public void save(HiveWritableRecord record) throws IOException, - InterruptedException { - hiveRecordWriter.write(NullWritable.get(), record); - } -} http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/output/SimpleVertexToHive.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/output/SimpleVertexToHive.java b/giraph-hive/src/main/java/org/apache/giraph/hive/output/SimpleVertexToHive.java deleted file mode 100644 index f6de715..0000000 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/output/SimpleVertexToHive.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.giraph.hive.output; - -import org.apache.giraph.graph.Vertex; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableComparable; - -import com.facebook.hiveio.record.HiveWritableRecord; - -import java.io.IOException; - -/** - * Simple implementation of {@link VertexToHive} when each {@link Vertex} is - * stored to one row in the output. - * - * @param <I> Vertex ID - * @param <V> Vertex Value - * @param <E> Edge Value - */ -public abstract class SimpleVertexToHive<I extends WritableComparable, - V extends Writable, E extends Writable> extends - AbstractVertexToHive<I, V, E> { - - /** - * Fill the HiveRecord from the Vertex given. - * - * @param vertex Vertex to read from. - * @param record HiveRecord to write to. - */ - public abstract void fillRecord(Vertex<I, V, E> vertex, - HiveWritableRecord record); - - @Override - public final void saveVertex( - Vertex<I, V, E> vertex, - HiveWritableRecord reusableRecord, - HiveRecordSaver recordSaver) throws IOException, InterruptedException { - fillRecord(vertex, reusableRecord); - recordSaver.save(reusableRecord); - } -} http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/output/VertexToHive.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/output/VertexToHive.java b/giraph-hive/src/main/java/org/apache/giraph/hive/output/VertexToHive.java deleted file mode 100644 index bf13ec1..0000000 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/output/VertexToHive.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.giraph.hive.output; - -import org.apache.giraph.graph.Vertex; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableComparable; - -import com.facebook.hiveio.output.HiveOutputDescription; -import com.facebook.hiveio.record.HiveWritableRecord; -import com.facebook.hiveio.schema.HiveTableSchema; - -import java.io.IOException; - -/** - * Interface for writing vertices to Hive. - * - * @param <I> Vertex ID - * @param <V> Vertex Value - * @param <E> Edge Value - */ -public interface VertexToHive<I extends WritableComparable, V extends Writable, - E extends Writable> { - /** - * User initialization before any saveVertex() calls but after Configuration - * and HiveTableSchema are guaranteed to be set. - */ - void initialize(); - - /** - * Check the output is valid. This method provides information to the user as - * early as possible so that they may validate they are using the correct - * input and fail the job early rather than getting into it and waiting a long - * time only to find out something was misconfigured. - * - * @param outputDesc HiveOutputDescription - * @param schema Hive table schema - * @param emptyRecord Example Hive record that you can write to as if you were - * writing a Vertex. This record will check column types. - */ - void checkOutput(HiveOutputDescription outputDesc, HiveTableSchema schema, - HiveWritableRecord emptyRecord); - - /** - * Save vertex to the output. One vertex can be stored to multiple rows in - * the output. - * - * Record you get here is reusable, and the protocol to follow is: - * - fill the reusableRecord with your data - * - call recordSaver.save(reusableRecord) - * - repeat - * If you don't call save() at all then there won't be any output for this - * vertex. - * - * @param vertex Vertex which we want to save. - * @param reusableRecord Record to use for writing data to it. - * @param recordSaver Saver of records - */ - void saveVertex(Vertex<I, V, E> vertex, HiveWritableRecord reusableRecord, - HiveRecordSaver recordSaver) throws IOException, InterruptedException; -} http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/output/examples/HiveOutputIntIntVertex.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/output/examples/HiveOutputIntIntVertex.java b/giraph-hive/src/main/java/org/apache/giraph/hive/output/examples/HiveOutputIntIntVertex.java deleted file mode 100644 index 29af71f..0000000 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/output/examples/HiveOutputIntIntVertex.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.giraph.hive.output.examples; - -import org.apache.giraph.graph.Vertex; -import org.apache.giraph.hive.output.SimpleVertexToHive; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.NullWritable; - -import com.facebook.hiveio.common.HiveType; -import com.facebook.hiveio.output.HiveOutputDescription; -import com.facebook.hiveio.record.HiveWritableRecord; -import com.facebook.hiveio.schema.HiveTableSchema; -import com.google.common.base.Preconditions; - -/** - * VertexToHive that writes Vertexes with integer IDs and integer values - */ -public class HiveOutputIntIntVertex extends SimpleVertexToHive<IntWritable, - IntWritable, NullWritable> { - @Override public void checkOutput(HiveOutputDescription outputDesc, - HiveTableSchema schema, HiveWritableRecord emptyRecord) { - Preconditions.checkArgument(schema.columnType(0) == HiveType.LONG); - Preconditions.checkArgument(schema.columnType(1) == HiveType.LONG); - } - - @Override public void fillRecord( - Vertex<IntWritable, IntWritable, NullWritable> vertex, - HiveWritableRecord record) { - record.set(0, (long) vertex.getId().get()); - record.set(1, (long) vertex.getValue().get()); - } -} http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/output/examples/package-info.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/output/examples/package-info.java b/giraph-hive/src/main/java/org/apache/giraph/hive/output/examples/package-info.java deleted file mode 100644 index 5195e69..0000000 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/output/examples/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Hive output examples. - */ -package org.apache.giraph.hive.output.examples; http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/output/package-info.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/output/package-info.java b/giraph-hive/src/main/java/org/apache/giraph/hive/output/package-info.java deleted file mode 100644 index 65d87e3..0000000 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/output/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Hive output things. - */ -package org.apache.giraph.hive.output; http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/package-info.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/package-info.java b/giraph-hive/src/main/java/org/apache/giraph/hive/package-info.java deleted file mode 100644 index d828d2a..0000000 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/package-info.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * Package of Giraph configuration related things. - */ -package org.apache.giraph.hive; http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/primitives/PrimitiveValueReader.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/primitives/PrimitiveValueReader.java b/giraph-hive/src/main/java/org/apache/giraph/hive/primitives/PrimitiveValueReader.java deleted file mode 100644 index f67db6e..0000000 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/primitives/PrimitiveValueReader.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.giraph.hive.primitives; - -import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration; -import org.apache.giraph.conf.StrConfOption; -import org.apache.giraph.graph.GraphType; -import org.apache.giraph.hive.values.HiveValueReader; -import org.apache.giraph.types.WritableWrapper; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.io.Writable; - -import com.facebook.hiveio.record.HiveReadableRecord; -import com.facebook.hiveio.schema.HiveTableSchema; -import com.google.common.base.Preconditions; - -import static org.apache.giraph.hive.common.HiveUtils.columnIndexOrThrow; -import static org.apache.giraph.types.WritableWrappers.lookup; - -/** - * Reader for Vertex/Edge Values from Hive with known types - * - * @param <W> Vertex/Edge Value - */ -public class PrimitiveValueReader<W extends Writable> - implements HiveValueReader<W> { - /** Hive column index */ - private final int columnIndex; - /** {@link WritableWrapper} for Hive column to Giraph Writable */ - private final WritableWrapper<W, Object> writableWrapper; - - /** - * Constructor - * - * @param columnIndex column index - * @param writableWrapper {@link WritableWrapper} - */ - public PrimitiveValueReader(int columnIndex, - WritableWrapper<W, Object> writableWrapper) { - Preconditions.checkNotNull(writableWrapper); - this.columnIndex = columnIndex; - this.writableWrapper = writableWrapper; - } - - /** - * Create from Configuration with column name and Schema - * - * @param conf Configuration - * @param graphType GraphType - * @param columnOption StrConfOption for column name - * @param schema HiveTableSchema - * @param <W> Graph type - * @return TypedVertexValueReader - */ - public static <W extends Writable> HiveValueReader<W> create( - ImmutableClassesGiraphConfiguration conf, GraphType graphType, - StrConfOption columnOption, HiveTableSchema schema) { - Class<W> valueClass = graphType.get(conf); - if (NullWritable.class.isAssignableFrom(valueClass)) { - return HiveValueReader.Null.get(); - } - int columnIndex = columnIndexOrThrow(schema, conf, columnOption); - Class hiveClass = schema.columnType(columnIndex).javaClass(); - WritableWrapper wrapper = lookup(valueClass, hiveClass); - return new PrimitiveValueReader(columnIndex, wrapper); - } - - @Override - public void readFields(W value, HiveReadableRecord record) { - Object object = record.get(columnIndex); - writableWrapper.wrap(object, value); - } -} http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/primitives/PrimitiveValueWriter.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/primitives/PrimitiveValueWriter.java b/giraph-hive/src/main/java/org/apache/giraph/hive/primitives/PrimitiveValueWriter.java deleted file mode 100644 index 8cd3ea4..0000000 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/primitives/PrimitiveValueWriter.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.giraph.hive.primitives; - -import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration; -import org.apache.giraph.conf.StrConfOption; -import org.apache.giraph.graph.GraphType; -import org.apache.giraph.hive.values.HiveValueWriter; -import org.apache.giraph.types.WritableUnwrapper; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.io.Writable; - -import com.facebook.hiveio.record.HiveWritableRecord; -import com.facebook.hiveio.schema.HiveTableSchema; -import com.google.common.base.Preconditions; - -import static org.apache.giraph.hive.common.HiveUtils.columnIndexOrThrow; -import static org.apache.giraph.types.WritableUnwrappers.lookup; - -/** - * Writer for graph values (IVEMM) from Hive with known types - * - * @param <W> Graph user type (IVEMM) - */ -public class PrimitiveValueWriter<W extends Writable> - implements HiveValueWriter<W> { - /** Hive column index */ - private final int columnIndex; - /** {@link WritableUnwrapper} for Hive column to Giraph Writable */ - private final WritableUnwrapper<W, Object> writableUnwrapper; - - /** - * Constructor - * - * @param columnIndex column index - * @param writableUnwrapper JavaWritableConverter - */ - public PrimitiveValueWriter(int columnIndex, - WritableUnwrapper<W, Object> writableUnwrapper) { - Preconditions.checkNotNull(writableUnwrapper); - this.columnIndex = columnIndex; - this.writableUnwrapper = writableUnwrapper; - } - - /** - * Create from Configuration with column name and Schema - * - * @param <T> Graph Type (IVEMM) - * @param conf Configuration - * @param columnOption StrConfOption for column name - * @param schema HiveTableSchema - * @param graphType GraphType - * @return TypedVertexValueReader - */ - public static <T extends Writable> HiveValueWriter<T> create( - ImmutableClassesGiraphConfiguration conf, StrConfOption columnOption, - HiveTableSchema schema, GraphType graphType) { - Class<T> valueClass = graphType.get(conf); - if (NullWritable.class.isAssignableFrom(valueClass)) { - return HiveValueWriter.Null.get(); - } - int columnIndex = columnIndexOrThrow(schema, conf, columnOption); - Class hiveClass = schema.columnType(columnIndex).javaClass(); - WritableUnwrapper unwrapper = lookup(valueClass, hiveClass); - return new PrimitiveValueWriter(columnIndex, unwrapper); - } - - @Override - public void write(W value, HiveWritableRecord record) { - Object object = writableUnwrapper.unwrap(value); - record.set(columnIndex, object); - } -} http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/primitives/package-info.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/primitives/package-info.java b/giraph-hive/src/main/java/org/apache/giraph/hive/primitives/package-info.java deleted file mode 100644 index 53d50ed..0000000 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/primitives/package-info.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * HiveIO primitive class oriented things. - */ -package org.apache.giraph.hive.primitives; http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/values/HiveValueReader.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/values/HiveValueReader.java b/giraph-hive/src/main/java/org/apache/giraph/hive/values/HiveValueReader.java deleted file mode 100644 index e989e28..0000000 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/values/HiveValueReader.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.giraph.hive.values; - -import org.apache.hadoop.io.Writable; - -import com.facebook.hiveio.record.HiveReadableRecord; - -/** - * Interface for reading Vertex / Edge values from Hive records - * - * @param <T> Value type - */ -public interface HiveValueReader<T extends Writable> { - /** - * Read value from record - * - * @param value graph value to read into - * @param record Hive record - */ - void readFields(T value, HiveReadableRecord record); - - /** - * Null implementation that return NullWritable - * - * @param <W> Writable type - */ - public class Null<W extends Writable> implements HiveValueReader<W> { - /** Singleton */ - private static final Null INSTANCE = new Null(); - - /** - * Get singleton - * - * @return singleton instance - */ - public static Null get() { - return INSTANCE; - } - - @Override - public void readFields(W value, HiveReadableRecord record) { } - } -} http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/values/HiveValueWriter.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/values/HiveValueWriter.java b/giraph-hive/src/main/java/org/apache/giraph/hive/values/HiveValueWriter.java deleted file mode 100644 index 4b624f6..0000000 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/values/HiveValueWriter.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.giraph.hive.values; - -import org.apache.hadoop.io.Writable; - -import com.facebook.hiveio.record.HiveWritableRecord; - -/** - * Interface for writing Vertex / Edge values from Hive records - * - * @param <T> Value type - */ -public interface HiveValueWriter<T extends Writable> { - /** - * Write value to record - * - * @param value the value - * @param record Hive record - */ - void write(T value, HiveWritableRecord record); - - /** - * Null implementation that does nothing - * - * @param <W> Writable type - */ - public static class Null<W extends Writable> implements HiveValueWriter<W> { - /** Singleton */ - private static final Null INSTANCE = new Null(); - - /** - * Get singleton - * - * @return singleton instance - */ - public static Null get() { - return INSTANCE; - } - - @Override - public void write(W value, HiveWritableRecord record) { } - } -} http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/values/package-info.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/values/package-info.java b/giraph-hive/src/main/java/org/apache/giraph/hive/values/package-info.java deleted file mode 100644 index 1e8f497..0000000 --- a/giraph-hive/src/main/java/org/apache/giraph/hive/values/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Hive type (vertex ID, vertex value, edge value, etc) related things. - */ -package org.apache.giraph.hive.values; http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/test/java/org/apache/giraph/hive/GiraphHiveTestBase.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/GiraphHiveTestBase.java b/giraph-hive/src/test/java/org/apache/giraph/hive/GiraphHiveTestBase.java deleted file mode 100644 index 27a7fe0..0000000 --- a/giraph-hive/src/test/java/org/apache/giraph/hive/GiraphHiveTestBase.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.giraph.hive; - -import org.junit.BeforeClass; - -import com.facebook.hiveio.log.LogHelpers; - -public class GiraphHiveTestBase { - @BeforeClass - public static void silenceLoggers() { - LogHelpers.silenceLoggers(); - } -} http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/test/java/org/apache/giraph/hive/Helpers.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/Helpers.java b/giraph-hive/src/test/java/org/apache/giraph/hive/Helpers.java deleted file mode 100644 index d2103a7..0000000 --- a/giraph-hive/src/test/java/org/apache/giraph/hive/Helpers.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.giraph.hive; - -import org.apache.giraph.conf.GiraphConfiguration; -import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration; -import org.apache.giraph.io.internal.WrappedVertexOutputFormat; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapred.HackJobContext; -import org.apache.hadoop.mapred.HackTaskAttemptContext; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.TaskAttemptID; -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.mapreduce.OutputCommitter; -import org.apache.hadoop.mapreduce.TaskAttemptContext; - -import com.google.common.collect.Maps; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Map; - -public class Helpers { - public static InputStream getResource(String name) { - return Helpers.class.getClassLoader().getResourceAsStream(name); - } - - public static Map<Integer, Double> parseIntDoubleResults(Iterable<String> results) { - Map<Integer, Double> values = Maps.newHashMap(); - for (String line : results) { - String[] tokens = line.split("\\s+"); - int id = Integer.valueOf(tokens[0]); - double value = Double.valueOf(tokens[1]); - values.put(id, value); - } - return values; - } - - public static Map<Integer, Integer> parseIntIntResults(Iterable<String> results) { - Map<Integer, Integer> values = Maps.newHashMap(); - for (String line : results) { - String[] tokens = line.split("\\s+"); - int id = Integer.valueOf(tokens[0]); - int value = Integer.valueOf(tokens[1]); - values.put(id, value); - } - return values; - } - - public static void commitJob(GiraphConfiguration conf) - throws IOException, InterruptedException { - ImmutableClassesGiraphConfiguration iconf = new ImmutableClassesGiraphConfiguration(conf); - WrappedVertexOutputFormat outputFormat = iconf.createWrappedVertexOutputFormat(); - JobConf jobConf = new JobConf(conf); - TaskAttemptContext - taskContext = new HackTaskAttemptContext(jobConf, new TaskAttemptID()); - OutputCommitter outputCommitter = outputFormat.getOutputCommitter( - taskContext); - JobContext jobContext = new HackJobContext(jobConf, taskContext.getJobID()); - outputCommitter.commitJob(jobContext); - } - - public static JobContext makeJobContext(Configuration conf) { - JobConf jobConf = new JobConf(conf); - TaskAttemptContext - taskContext = new HackTaskAttemptContext(jobConf, new TaskAttemptID()); - return new HackJobContext(jobConf, taskContext.getJobID()); - } -} http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/test/java/org/apache/giraph/hive/computations/ComputationCountEdges.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/computations/ComputationCountEdges.java b/giraph-hive/src/test/java/org/apache/giraph/hive/computations/ComputationCountEdges.java deleted file mode 100644 index 14ab478..0000000 --- a/giraph-hive/src/test/java/org/apache/giraph/hive/computations/ComputationCountEdges.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.giraph.hive.computations; - -import org.apache.giraph.graph.BasicComputation; -import org.apache.giraph.graph.Vertex; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.NullWritable; - -import java.io.IOException; - -public class ComputationCountEdges extends BasicComputation<IntWritable, - IntWritable, NullWritable, NullWritable> { - @Override - public void compute(Vertex<IntWritable, IntWritable, NullWritable> vertex, - Iterable<NullWritable> messages) throws IOException { - vertex.setValue(new IntWritable(vertex.getNumEdges())); - vertex.voteToHalt(); - } -} - http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/test/java/org/apache/giraph/hive/computations/ComputationSumEdges.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/computations/ComputationSumEdges.java b/giraph-hive/src/test/java/org/apache/giraph/hive/computations/ComputationSumEdges.java deleted file mode 100644 index 613095e..0000000 --- a/giraph-hive/src/test/java/org/apache/giraph/hive/computations/ComputationSumEdges.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.giraph.hive.computations; - -import org.apache.giraph.edge.Edge; -import org.apache.giraph.graph.BasicComputation; -import org.apache.giraph.graph.Vertex; -import org.apache.hadoop.io.DoubleWritable; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.NullWritable; - -import java.io.IOException; - -public class ComputationSumEdges extends BasicComputation<IntWritable, - DoubleWritable, DoubleWritable, NullWritable> { - @Override - public void compute( - Vertex<IntWritable, DoubleWritable, DoubleWritable> vertex, - Iterable<NullWritable> messages) throws IOException { - double sum = 0; - for (Edge<IntWritable, DoubleWritable> edge : vertex.getEdges()) { - sum += edge.getValue().get(); - } - vertex.setValue(new DoubleWritable(sum)); - vertex.voteToHalt(); - } -} http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/test/java/org/apache/giraph/hive/computations/package-info.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/computations/package-info.java b/giraph-hive/src/test/java/org/apache/giraph/hive/computations/package-info.java deleted file mode 100644 index 5569bda..0000000 --- a/giraph-hive/src/test/java/org/apache/giraph/hive/computations/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Computation implementations for Hive tests. - */ -package org.apache.giraph.hive.computations; http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/test/java/org/apache/giraph/hive/input/CheckInputTest.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/input/CheckInputTest.java b/giraph-hive/src/test/java/org/apache/giraph/hive/input/CheckInputTest.java deleted file mode 100644 index f51531a..0000000 --- a/giraph-hive/src/test/java/org/apache/giraph/hive/input/CheckInputTest.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.giraph.hive.input; - -import org.apache.giraph.conf.GiraphConfiguration; -import org.apache.giraph.hive.GiraphHiveTestBase; -import org.apache.giraph.hive.computations.ComputationCountEdges; -import org.apache.giraph.hive.input.edge.HiveEdgeInputFormat; -import org.apache.giraph.hive.input.edge.HiveToEdge; -import org.apache.giraph.hive.input.edge.examples.HiveIntNullEdge; -import org.apache.giraph.hive.input.vertex.HiveToVertex; -import org.apache.giraph.hive.input.vertex.examples.HiveIntNullNullVertex; -import org.apache.giraph.io.formats.IdWithValueTextOutputFormat; -import org.apache.giraph.utils.InternalVertexRunner; -import org.apache.thrift.TException; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import com.facebook.hiveio.common.HiveMetastores; -import com.facebook.hiveio.common.HiveType; -import com.facebook.hiveio.input.HiveInputDescription; -import com.facebook.hiveio.schema.HiveTableSchema; -import com.facebook.hiveio.schema.TestSchema; -import com.facebook.hiveio.testing.LocalHiveServer; - -import java.io.IOException; - -import static junit.framework.Assert.assertNull; -import static org.apache.giraph.hive.common.GiraphHiveConstants.HIVE_EDGE_INPUT; - -public class CheckInputTest extends GiraphHiveTestBase { - private LocalHiveServer hiveServer = new LocalHiveServer("giraph-hive"); - - @Before - public void setUp() throws IOException, TException { - hiveServer.init(); - HiveMetastores.setTestClient(hiveServer.getClient()); - } - - @Test - public void testCheckEdge() throws Exception { - HiveToEdge hiveToEdge = new HiveIntNullEdge(); - HiveInputDescription inputDesc = new HiveInputDescription(); - HiveTableSchema schema = TestSchema.builder() - .addColumn("foo", HiveType.INT) - .addColumn("bar", HiveType.INT) - .build(); - hiveToEdge.checkInput(inputDesc, schema); - - schema = TestSchema.builder() - .addColumn("foo", HiveType.INT) - .addColumn("bar", HiveType.LONG) - .build(); - checkEdgeThrows(hiveToEdge, inputDesc, schema); - } - - private void checkEdgeThrows(HiveToEdge hiveToEdge, - HiveInputDescription inputDesc, HiveTableSchema schema) { - try { - hiveToEdge.checkInput(inputDesc, schema); - } catch (IllegalArgumentException e) { - return; - } - Assert.fail(); - } - - @Test - public void testCheckVertex() throws Exception { - HiveToVertex hiveToVertex = new HiveIntNullNullVertex(); - HiveInputDescription inputDesc = new HiveInputDescription(); - HiveTableSchema schema = TestSchema.builder() - .addColumn("foo", HiveType.INT) - .addColumn("bar", HiveType.LIST) - .build(); - hiveToVertex.checkInput(inputDesc, schema); - - schema = TestSchema.builder() - .addColumn("foo", HiveType.INT) - .addColumn("bar", HiveType.STRING) - .build(); - checkVertexThrows(hiveToVertex, inputDesc, schema); - } - - private void checkVertexThrows(HiveToVertex hiveToVertex, - HiveInputDescription inputDesc, HiveTableSchema schema) { - try { - hiveToVertex.checkInput(inputDesc, schema); - } catch (IllegalArgumentException e) { - return; - } - Assert.fail(); - } - - @Test - public void testCheckJobThrows() throws Exception { - String tableName = "test1"; - hiveServer.createTable("CREATE TABLE " + tableName + - " (i1 BIGINT, i2 INT) " + - " ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'"); - String[] rows = { - "1\t2", - "2\t3", - "2\t4", - "4\t1", - }; - hiveServer.loadData(tableName, rows); - - GiraphConfiguration conf = new GiraphConfiguration(); - HIVE_EDGE_INPUT.setTable(conf, tableName); - HIVE_EDGE_INPUT.setClass(conf, HiveIntNullEdge.class); - conf.setComputationClass(ComputationCountEdges.class); - conf.setEdgeInputFormatClass(HiveEdgeInputFormat.class); - conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class); - assertNull(InternalVertexRunner.run(conf, new String[0], new String[0])); - } -} http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/test/java/org/apache/giraph/hive/input/HiveEdgeInputTest.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/input/HiveEdgeInputTest.java b/giraph-hive/src/test/java/org/apache/giraph/hive/input/HiveEdgeInputTest.java deleted file mode 100644 index bf5af0f..0000000 --- a/giraph-hive/src/test/java/org/apache/giraph/hive/input/HiveEdgeInputTest.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.giraph.hive.input; - -import org.apache.giraph.conf.GiraphConfiguration; -import org.apache.giraph.hive.GiraphHiveTestBase; -import org.apache.giraph.hive.Helpers; -import org.apache.giraph.hive.computations.ComputationCountEdges; -import org.apache.giraph.hive.computations.ComputationSumEdges; -import org.apache.giraph.hive.input.edge.HiveEdgeInputFormat; -import org.apache.giraph.hive.input.edge.examples.HiveIntDoubleEdge; -import org.apache.giraph.hive.input.edge.examples.HiveIntNullEdge; -import org.apache.giraph.io.formats.IdWithValueTextOutputFormat; -import org.apache.giraph.utils.InternalVertexRunner; -import org.apache.thrift.TException; -import org.junit.Before; -import org.junit.Test; - -import com.facebook.hiveio.common.HiveMetastores; -import com.facebook.hiveio.testing.LocalHiveServer; - -import java.io.IOException; -import java.util.Map; - -import static junit.framework.Assert.assertEquals; -import static org.apache.giraph.hive.common.GiraphHiveConstants.HIVE_EDGE_INPUT; - -public class HiveEdgeInputTest extends GiraphHiveTestBase { - private LocalHiveServer hiveServer = new LocalHiveServer("giraph-hive"); - - @Before - public void setUp() throws IOException, TException { - hiveServer.init(); - HiveMetastores.setTestClient(hiveServer.getClient()); - } - - @Test - public void testEdgeInput() throws Exception { - String tableName = "test1"; - hiveServer.createTable("CREATE TABLE " + tableName + - " (i1 INT, i2 INT) " + - " ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'"); - String[] rows = { - "1\t2", - "2\t3", - "2\t4", - "4\t1", - }; - hiveServer.loadData(tableName, rows); - - GiraphConfiguration conf = new GiraphConfiguration(); - HIVE_EDGE_INPUT.setTable(conf, tableName); - HIVE_EDGE_INPUT.setClass(conf, HiveIntNullEdge.class); - conf.setComputationClass(ComputationCountEdges.class); - conf.setEdgeInputFormatClass(HiveEdgeInputFormat.class); - conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class); - Iterable<String> output = InternalVertexRunner.run(conf, new String[0], new String[0]); - - Map<Integer, Integer> data = Helpers.parseIntIntResults(output); - assertEquals(3, data.size()); - assertEquals(1, (int) data.get(1)); - assertEquals(2, (int) data.get(2)); - assertEquals(1, (int) data.get(4)); - } - - @Test - public void testEdgeInputWithPartitions() throws Exception { - String tableName = "test1"; - String partition = "ds='foobar'"; - hiveServer.createTable("CREATE TABLE " + tableName + - " (i1 INT, i2 INT) " + - " PARTITIONED BY (ds STRING) " + - " ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' "); - String[] rows = { - "1\t2", - "2\t3", - "2\t4", - "4\t1", - }; - hiveServer.loadData(tableName, partition, rows); - - GiraphConfiguration conf = new GiraphConfiguration(); - HIVE_EDGE_INPUT.setTable(conf, tableName); - HIVE_EDGE_INPUT.setPartition(conf, partition); - HIVE_EDGE_INPUT.setClass(conf, HiveIntNullEdge.class); - conf.setComputationClass(ComputationCountEdges.class); - conf.setEdgeInputFormatClass(HiveEdgeInputFormat.class); - conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class); - Iterable<String> output = InternalVertexRunner.run(conf, new String[0], new String[0]); - - Map<Integer, Integer> data = Helpers.parseIntIntResults(output); - assertEquals(3, data.size()); - assertEquals(1, (int) data.get(1)); - assertEquals(2, (int) data.get(2)); - assertEquals(1, (int) data.get(4)); - } - - @Test - public void testEdgeInputWithValues() throws Exception { - String tableName = "test1"; - hiveServer.createTable("CREATE TABLE " + tableName + - " (i1 INT, i2 INT, d3 DOUBLE) " + - " ROW FORMAT DELIMITED " + - " FIELDS TERMINATED BY '\t' " + - " COLLECTION ITEMS TERMINATED BY ',' "); - String[] rows = { - "1\t2\t0.22", - "2\t3\t0.33", - "2\t4\t0.44", - "4\t1\t0.11", - }; - hiveServer.loadData(tableName, rows); - - GiraphConfiguration conf = new GiraphConfiguration(); - HIVE_EDGE_INPUT.setTable(conf, tableName); - HIVE_EDGE_INPUT.setClass(conf, HiveIntDoubleEdge.class); - conf.setComputationClass(ComputationSumEdges.class); - conf.setEdgeInputFormatClass(HiveEdgeInputFormat.class); - conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class); - Iterable<String> output = InternalVertexRunner.run(conf, new String[0], new String[0]); - - Map<Integer, Double> data = Helpers.parseIntDoubleResults(output); - assertEquals(3, data.size()); - assertEquals(0.22, data.get(1)); - assertEquals(0.77, data.get(2)); - assertEquals(0.11, data.get(4)); - } -} - http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/test/java/org/apache/giraph/hive/input/HiveVertexInputTest.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/input/HiveVertexInputTest.java b/giraph-hive/src/test/java/org/apache/giraph/hive/input/HiveVertexInputTest.java deleted file mode 100644 index 98ba014..0000000 --- a/giraph-hive/src/test/java/org/apache/giraph/hive/input/HiveVertexInputTest.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.giraph.hive.input; - -import com.facebook.hiveio.common.HiveMetastores; -import com.facebook.hiveio.testing.LocalHiveServer; -import org.apache.giraph.conf.GiraphConfiguration; -import org.apache.giraph.hive.GiraphHiveTestBase; -import org.apache.giraph.hive.Helpers; -import org.apache.giraph.hive.computations.ComputationCountEdges; -import org.apache.giraph.hive.computations.ComputationSumEdges; -import org.apache.giraph.hive.input.vertex.HiveVertexInputFormat; -import org.apache.giraph.hive.input.vertex.examples.HiveIntDoubleDoubleVertex; -import org.apache.giraph.hive.input.vertex.examples.HiveIntIntNullVertex; -import org.apache.giraph.io.formats.IdWithValueTextOutputFormat; -import org.apache.giraph.utils.InternalVertexRunner; -import org.apache.thrift.TException; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.util.Map; - -import static junit.framework.Assert.assertEquals; -import static org.apache.giraph.hive.common.GiraphHiveConstants.HIVE_VERTEX_INPUT; - -public class HiveVertexInputTest extends GiraphHiveTestBase { - private LocalHiveServer hiveServer = new LocalHiveServer("giraph-hive"); - - @Before - public void setUp() throws IOException, TException { - hiveServer.init(); - HiveMetastores.setTestClient(hiveServer.getClient()); - } - - @Test - public void testVertexInput() throws Exception { - String tableName = "test1"; - hiveServer.createTable("CREATE TABLE " + tableName + - " (i1 INT, i2 ARRAY<BIGINT>) " + - " ROW FORMAT DELIMITED " + - " FIELDS TERMINATED BY '\t' " + - " COLLECTION ITEMS TERMINATED BY ','"); - String[] rows = { - "1\t2", - "2\t3,4", - "4\t1", - }; - hiveServer.loadData(tableName, rows); - - GiraphConfiguration conf = new GiraphConfiguration(); - HIVE_VERTEX_INPUT.setTable(conf, tableName); - HIVE_VERTEX_INPUT.setClass(conf, HiveIntIntNullVertex.class); - conf.setComputationClass(ComputationCountEdges.class); - conf.setVertexInputFormatClass(HiveVertexInputFormat.class); - conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class); - Iterable<String> output = InternalVertexRunner.run(conf, new String[0], new String[0]); - - Map<Integer, Integer> data = Helpers.parseIntIntResults(output); - assertEquals(3, data.size()); - assertEquals(1, (int) data.get(1)); - assertEquals(2, (int) data.get(2)); - assertEquals(1, (int) data.get(4)); - } - - @Test - public void testVertexInputWithPartitions() throws Exception { - String tableName = "test1"; - String partition = "ds='foobar'"; - hiveServer.createTable("CREATE TABLE " + tableName + - " (i1 INT, i2 ARRAY<BIGINT>) " + - " PARTITIONED BY (ds STRING) " + - " ROW FORMAT DELIMITED " + - " FIELDS TERMINATED BY '\t' " + - " COLLECTION ITEMS TERMINATED BY ','"); - String[] rows = { - "1\t2", - "2\t3,4", - "4\t1", - }; - hiveServer.loadData(tableName, partition, rows); - - GiraphConfiguration conf = new GiraphConfiguration(); - HIVE_VERTEX_INPUT.setTable(conf, tableName); - HIVE_VERTEX_INPUT.setPartition(conf, partition); - HIVE_VERTEX_INPUT.setClass(conf, HiveIntIntNullVertex.class); - conf.setComputationClass(ComputationCountEdges.class); - conf.setVertexInputFormatClass(HiveVertexInputFormat.class); - conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class); - Iterable<String> output = InternalVertexRunner.run(conf, new String[0], new String[0]); - - Map<Integer, Integer> data = Helpers.parseIntIntResults(output); - assertEquals(3, data.size()); - assertEquals(1, (int) data.get(1)); - assertEquals(2, (int) data.get(2)); - assertEquals(1, (int) data.get(4)); - } - - @Test - public void testValues() throws Exception { - String tableName = "test1"; - hiveServer.createTable("CREATE TABLE " + tableName + - " (i1 INT, d2 DOUBLE, m3 MAP<BIGINT,DOUBLE>) " + - " ROW FORMAT DELIMITED " + - " FIELDS TERMINATED BY '\t' " + - " COLLECTION ITEMS TERMINATED BY ',' " + - " MAP KEYS TERMINATED BY ':' "); - String[] rows = { - "1\t1.11\t2:0.22", - "2\t2.22\t3:0.33,4:0.44", - "4\t4.44\t1:0.11", - }; - hiveServer.loadData(tableName, rows); - - GiraphConfiguration conf = new GiraphConfiguration(); - HIVE_VERTEX_INPUT.setTable(conf, tableName); - HIVE_VERTEX_INPUT.setClass(conf, HiveIntDoubleDoubleVertex.class); - conf.setComputationClass(ComputationSumEdges.class); - conf.setVertexInputFormatClass(HiveVertexInputFormat.class); - conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class); - Iterable<String> output = InternalVertexRunner.run(conf, new String[0], new String[0]); - - Map<Integer, Double> data = Helpers.parseIntDoubleResults(output); - assertEquals(3, data.size()); - assertEquals(0.22, data.get(1)); - assertEquals(0.77, data.get(2)); - assertEquals(0.11, data.get(4)); - } -} http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/test/java/org/apache/giraph/hive/input/package-info.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/input/package-info.java b/giraph-hive/src/test/java/org/apache/giraph/hive/input/package-info.java deleted file mode 100644 index c2327ca..0000000 --- a/giraph-hive/src/test/java/org/apache/giraph/hive/input/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Hive Input test. - */ -package org.apache.giraph.hive.input; http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/test/java/org/apache/giraph/hive/jython/TestHiveJythonComplexTypes.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/jython/TestHiveJythonComplexTypes.java b/giraph-hive/src/test/java/org/apache/giraph/hive/jython/TestHiveJythonComplexTypes.java deleted file mode 100644 index 3a07dc4..0000000 --- a/giraph-hive/src/test/java/org/apache/giraph/hive/jython/TestHiveJythonComplexTypes.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.giraph.hive.jython; - -import org.apache.giraph.conf.GiraphConfiguration; -import org.apache.giraph.graph.Language; -import org.apache.giraph.hive.GiraphHiveTestBase; -import org.apache.giraph.hive.Helpers; -import org.apache.giraph.jython.JythonJob; -import org.apache.giraph.scripting.DeployType; -import org.apache.giraph.scripting.ScriptLoader; -import org.apache.giraph.utils.InternalVertexRunner; -import org.apache.thrift.TException; -import org.junit.Before; -import org.junit.Test; -import org.python.util.PythonInterpreter; - -import com.facebook.hiveio.common.HiveMetastores; -import com.facebook.hiveio.input.HiveInput; -import com.facebook.hiveio.input.HiveInputDescription; -import com.facebook.hiveio.record.HiveReadableRecord; -import com.facebook.hiveio.testing.LocalHiveServer; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; -import junit.framework.Assert; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Iterator; -import java.util.Set; - -import static junit.framework.Assert.assertEquals; -import static junit.framework.Assert.assertFalse; -import static junit.framework.Assert.assertNotNull; -import static junit.framework.Assert.assertTrue; -import static org.apache.giraph.hive.Helpers.getResource; - -public class TestHiveJythonComplexTypes extends GiraphHiveTestBase { - private LocalHiveServer hiveServer = new LocalHiveServer("jython-test"); - - @Before - public void setUp() throws IOException, TException { - hiveServer.init(); - HiveMetastores.setTestClient(hiveServer.getClient()); - } - - @Test - public void testFakeLabelPropagation() throws Exception { - String edgesTable = "flp_edges"; - hiveServer.createTable("CREATE TABLE " + edgesTable + - " (source_id INT, " + - " target_id INT," + - " value FLOAT) " + - " ROW FORMAT DELIMITED " + - " FIELDS TERMINATED BY '\t'"); - String[] edges = new String[] { - "1\t2\t0.2", - "2\t3\t0.3", - "3\t4\t0.4", - "4\t1\t0.1", - }; - hiveServer.loadData(edgesTable, edges); - - String vertexesTable = "flp_vertexes"; - hiveServer.createTable("CREATE TABLE " + vertexesTable + - " (id INT, " + - " value MAP<INT,FLOAT>) " + - " ROW FORMAT DELIMITED " + - " FIELDS TERMINATED BY '\t' " + - " COLLECTION ITEMS TERMINATED BY ',' " + - " MAP KEYS TERMINATED BY ':' "); - String[] vertexes = new String[] { - "1\t11:0.8,12:0.1", - "2\t13:0.3,14:0.2", - "3\t15:0.4,16:0.7", - "4\t17:0.1,18:0.6", - }; - hiveServer.loadData(vertexesTable, vertexes); - - String outputTable = "flp_output"; - hiveServer.createTable("CREATE TABLE " + outputTable + - " (id INT," + - " value MAP<INT,DOUBLE>) " + - " ROW FORMAT DELIMITED " + - " FIELDS TERMINATED BY '\t'"); - - String workerJythonPath = - "org/apache/giraph/jython/fake-label-propagation-worker.py"; - - InputStream launcher = getResource( - "org/apache/giraph/jython/fake-label-propagation-launcher.py"); - assertNotNull(launcher); - InputStream worker = getResource(workerJythonPath); - assertNotNull(worker); - - PythonInterpreter interpreter = new PythonInterpreter(); - - JythonJob jythonJob = - HiveJythonUtils.parseJythonStreams(interpreter, launcher, worker); - - GiraphConfiguration conf = new GiraphConfiguration(); - - ScriptLoader.setScriptsToLoad(conf, workerJythonPath, DeployType.RESOURCE, - Language.JYTHON); - - HiveJythonUtils.writeJythonJobToConf(jythonJob, conf, interpreter); - - InternalVertexRunner.run(conf, new String[0], new String[0]); - - Helpers.commitJob(conf); - - HiveInputDescription inputDesc = new HiveInputDescription(); - inputDesc.getTableDesc().setTableName(outputTable); - - Iterator<HiveReadableRecord> records = HiveInput.readTable(inputDesc).iterator(); - - printRecords(HiveInput.readTable(inputDesc)); - - final int rows = 4; - - Set<Integer>[] expected = new Set[rows+1]; - expected[1] = ImmutableSet.of(11,12,15,16,17,18); - expected[2] = ImmutableSet.of(13,14,17,18,11,12); - expected[3] = ImmutableSet.of(15,16,11,12,13,14); - expected[4] = ImmutableSet.of(17,18,13,14,15,16); - - for (int i = 0; i < rows; ++i) { - assertTrue(records.hasNext()); - HiveReadableRecord record = records.next(); - assertEquals(expected[record.getInt(0)], record.getMap(1).keySet()); - } - - assertFalse(records.hasNext()); - } - - private void printRecords(Iterable<HiveReadableRecord> records) { - for (HiveReadableRecord record : records) { - System.out.println("record: " + record); - } - } -} http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/test/java/org/apache/giraph/hive/jython/TestHiveJythonPrimitives.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/jython/TestHiveJythonPrimitives.java b/giraph-hive/src/test/java/org/apache/giraph/hive/jython/TestHiveJythonPrimitives.java deleted file mode 100644 index d7f6c30..0000000 --- a/giraph-hive/src/test/java/org/apache/giraph/hive/jython/TestHiveJythonPrimitives.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.giraph.hive.jython; - -import org.apache.giraph.conf.GiraphConfiguration; -import org.apache.giraph.graph.Language; -import org.apache.giraph.hive.GiraphHiveTestBase; -import org.apache.giraph.hive.Helpers; -import org.apache.giraph.jython.JythonJob; -import org.apache.giraph.scripting.DeployType; -import org.apache.giraph.scripting.ScriptLoader; -import org.apache.giraph.utils.InternalVertexRunner; -import org.apache.thrift.TException; -import org.junit.Before; -import org.junit.Test; -import org.python.util.PythonInterpreter; - -import com.facebook.hiveio.common.HiveMetastores; -import com.facebook.hiveio.input.HiveInput; -import com.facebook.hiveio.input.HiveInputDescription; -import com.facebook.hiveio.record.HiveReadableRecord; -import com.facebook.hiveio.testing.LocalHiveServer; -import junit.framework.Assert; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Iterator; - -import static junit.framework.Assert.assertFalse; -import static junit.framework.Assert.assertNotNull; -import static junit.framework.Assert.assertTrue; -import static org.apache.giraph.hive.Helpers.getResource; -import static org.junit.Assert.assertEquals; - -public class TestHiveJythonPrimitives extends GiraphHiveTestBase { - private LocalHiveServer hiveServer = new LocalHiveServer("jython-test"); - - @Before - public void setUp() throws IOException, TException { - hiveServer.init(); - HiveMetastores.setTestClient(hiveServer.getClient()); - } - - @Test - public void testCountEdges() throws Exception { - String edgesTable = "count_edges_edge_input"; - hiveServer.createTable("CREATE TABLE " + edgesTable + - " (source_edge_id INT, " + - " target_edge_id INT) " + - " ROW FORMAT DELIMITED " + - " FIELDS TERMINATED BY '\t'"); - String[] edges = new String[] { - "1\t2", - "2\t3", - "2\t4", - "4\t1" - }; - hiveServer.loadData(edgesTable, edges); - - String outputTable = "count_edges_output"; - hiveServer.createTable("CREATE TABLE " + outputTable + - " (vertex_id INT," + - " num_edges INT) " + - " ROW FORMAT DELIMITED " + - " FIELDS TERMINATED BY '\t'"); - - String workerJythonPath = "org/apache/giraph/jython/count-edges.py"; - - InputStream launcher = getResource( - "org/apache/giraph/jython/count-edges-launcher.py"); - assertNotNull(launcher); - InputStream worker = getResource(workerJythonPath); - assertNotNull(worker); - - PythonInterpreter interpreter = new PythonInterpreter(); - - JythonJob jythonJob = - HiveJythonUtils.parseJythonStreams(interpreter, launcher, worker); - - GiraphConfiguration conf = new GiraphConfiguration(); - - ScriptLoader.setScriptsToLoad(conf, workerJythonPath, - DeployType.RESOURCE, Language.JYTHON); - - HiveJythonUtils.writeJythonJobToConf(jythonJob, conf, interpreter); - - InternalVertexRunner.run(conf, new String[0], new String[0]); - - Helpers.commitJob(conf); - - HiveInputDescription inputDesc = new HiveInputDescription(); - inputDesc.getTableDesc().setTableName(outputTable); - - Iterator<HiveReadableRecord> records = HiveInput.readTable(inputDesc).iterator(); - - int expected[] = { -1, 1, 2, -1, 1 }; - - assertTrue(records.hasNext()); - HiveReadableRecord record = records.next(); - Assert.assertEquals(expected[record.getInt(0)], record.getInt(1)); - - assertTrue(records.hasNext()); - record = records.next(); - Assert.assertEquals(expected[record.getInt(0)], record.getInt(1)); - - assertTrue(records.hasNext()); - record = records.next(); - Assert.assertEquals(expected[record.getInt(0)], record.getInt(1)); - - assertFalse(records.hasNext()); - } -} http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/test/java/org/apache/giraph/hive/output/CheckOutputTest.java ---------------------------------------------------------------------- diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/output/CheckOutputTest.java b/giraph-hive/src/test/java/org/apache/giraph/hive/output/CheckOutputTest.java deleted file mode 100644 index ab020a1..0000000 --- a/giraph-hive/src/test/java/org/apache/giraph/hive/output/CheckOutputTest.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.giraph.hive.output; - -import org.apache.giraph.conf.GiraphConfiguration; -import org.apache.giraph.edge.ByteArrayEdges; -import org.apache.giraph.hive.GiraphHiveTestBase; -import org.apache.giraph.hive.common.GiraphHiveConstants; -import org.apache.giraph.hive.computations.ComputationCountEdges; -import org.apache.giraph.hive.output.examples.HiveOutputIntIntVertex; -import org.apache.giraph.io.formats.IntNullTextEdgeInputFormat; -import org.apache.giraph.utils.InternalVertexRunner; -import org.apache.thrift.TException; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import com.facebook.hiveio.common.HiveMetastores; -import com.facebook.hiveio.common.HiveType; -import com.facebook.hiveio.output.HiveOutputDescription; -import com.facebook.hiveio.schema.HiveTableSchema; -import com.facebook.hiveio.schema.TestSchema; -import com.facebook.hiveio.testing.LocalHiveServer; - -import java.io.IOException; - -import static com.facebook.hiveio.record.HiveRecordFactory.newWritableRecord; -import static org.junit.Assert.assertNull; - -public class CheckOutputTest extends GiraphHiveTestBase { - private LocalHiveServer hiveServer = new LocalHiveServer("giraph-hive"); - - @Before - public void setUp() throws IOException, TException { - hiveServer.init(); - HiveMetastores.setTestClient(hiveServer.getClient()); - } - - @Test - public void testCheck() throws Exception { - VertexToHive vertexToHive = new HiveOutputIntIntVertex(); - HiveOutputDescription outputDesc = new HiveOutputDescription(); - HiveTableSchema schema = TestSchema.builder() - .addColumn("foo", HiveType.LONG) - .addColumn("bar", HiveType.LONG) - .build(); - vertexToHive.checkOutput(outputDesc, schema, newWritableRecord(schema)); - - schema = TestSchema.builder() - .addColumn("foo", HiveType.INT) - .addColumn("bar", HiveType.LONG) - .build(); - checkThrows(vertexToHive, outputDesc, schema); - } - - private void checkThrows(VertexToHive vertexToHive, - HiveOutputDescription outputDesc, HiveTableSchema schema) { - try { - vertexToHive.checkOutput(outputDesc, schema, newWritableRecord(schema)); - } catch (IllegalArgumentException e) { - return; - } - Assert.fail(); - } - - @Test - public void testCheckFailsJob() throws Exception { - String tableName = "test1"; - hiveServer.createTable("CREATE TABLE " + tableName + - " (i1 INT, i2 BIGINT) "); - - GiraphConfiguration conf = new GiraphConfiguration(); - String[] edges = new String[] { - "1 2", - "2 3", - "2 4", - "4 1" - }; - - GiraphHiveConstants.HIVE_VERTEX_OUTPUT_TABLE.set(conf, tableName); - GiraphHiveConstants.VERTEX_TO_HIVE_CLASS.set(conf, HiveOutputIntIntVertex.class); - - conf.setComputationClass(ComputationCountEdges.class); - conf.setOutEdgesClass(ByteArrayEdges.class); - conf.setEdgeInputFormatClass(IntNullTextEdgeInputFormat.class); - conf.setVertexOutputFormatClass(HiveVertexOutputFormat.class); - try { - Iterable<String> result = InternalVertexRunner.run(conf, null, edges); - assertNull(result); - } catch (IllegalArgumentException e) { } - } -}
