http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/80faf06d/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/iter/NonCloseableRyaStatementCursorIterator.java ---------------------------------------------------------------------- diff --git a/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/iter/NonCloseableRyaStatementCursorIterator.java b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/iter/NonCloseableRyaStatementCursorIterator.java index 48f0931..ba37ca1 100644 --- a/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/iter/NonCloseableRyaStatementCursorIterator.java +++ b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/iter/NonCloseableRyaStatementCursorIterator.java @@ -1,5 +1,25 @@ package mvm.rya.mongodb.iter; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + import java.util.Iterator; import mvm.rya.api.domain.RyaStatement;
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/80faf06d/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/iter/RyaStatementBindingSetCursorIterator.java ---------------------------------------------------------------------- diff --git a/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/iter/RyaStatementBindingSetCursorIterator.java b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/iter/RyaStatementBindingSetCursorIterator.java index b699d96..ce21ff7 100644 --- a/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/iter/RyaStatementBindingSetCursorIterator.java +++ b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/iter/RyaStatementBindingSetCursorIterator.java @@ -1,5 +1,25 @@ package mvm.rya.mongodb.iter; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + import info.aduna.iteration.CloseableIteration; import java.util.Iterator; http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/80faf06d/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/iter/RyaStatementCursorIterable.java ---------------------------------------------------------------------- diff --git a/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/iter/RyaStatementCursorIterable.java b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/iter/RyaStatementCursorIterable.java index d69ab65..83bd2d4 100644 --- a/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/iter/RyaStatementCursorIterable.java +++ b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/iter/RyaStatementCursorIterable.java @@ -1,5 +1,25 @@ package mvm.rya.mongodb.iter; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + import info.aduna.iteration.CloseableIteration; import java.io.IOException; http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/80faf06d/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/iter/RyaStatementCursorIterator.java ---------------------------------------------------------------------- diff --git a/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/iter/RyaStatementCursorIterator.java b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/iter/RyaStatementCursorIterator.java index 8b2ae3b..8df2c60 100644 --- a/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/iter/RyaStatementCursorIterator.java +++ b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/iter/RyaStatementCursorIterator.java @@ -1,5 +1,25 @@ package mvm.rya.mongodb.iter; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + import info.aduna.iteration.CloseableIteration; import java.util.Iterator; http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/80faf06d/dao/pom.xml ---------------------------------------------------------------------- diff --git a/dao/pom.xml b/dao/pom.xml index 315d406..604b30c 100644 --- a/dao/pom.xml +++ b/dao/pom.xml @@ -1,17 +1,39 @@ <?xml version="1.0" encoding="utf-8"?> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> <modelVersion>4.0.0</modelVersion> <parent> - <groupId>mvm.rya</groupId> - <artifactId>parent</artifactId> + <groupId>org.apache.rya</groupId> + <artifactId>rya-project</artifactId> <version>3.2.10-SNAPSHOT</version> </parent> + <artifactId>rya.dao</artifactId> + <name>Apache Rya DAO Projects</name> + <packaging>pom</packaging> - <name>${project.groupId}.${project.artifactId}</name> + <modules> <module>accumulo.rya</module> - <module>mongodb.rya</module> + <module>mongodb.rya</module> </modules> </project> http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/80faf06d/extras/cloudbase.rya.giraph/pom.xml ---------------------------------------------------------------------- diff --git a/extras/cloudbase.rya.giraph/pom.xml b/extras/cloudbase.rya.giraph/pom.xml deleted file mode 100644 index 2552197..0000000 --- a/extras/cloudbase.rya.giraph/pom.xml +++ /dev/null @@ -1,69 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<project xmlns="http://maven.apache.org/POM/4.0.0"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <parent> - <groupId>mvm.rya</groupId> - <artifactId>rya.extras</artifactId> - <version>3.2.10-SNAPSHOT</version> - </parent> - <modelVersion>4.0.0</modelVersion> - - <artifactId>cloudbase.rya.giraph</artifactId> - - <dependencies> - <dependency> - <groupId>org.apache.giraph</groupId> - <artifactId>giraph</artifactId> - <version>0.2-SNAPSHOT</version> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - </dependency> - <dependency> - <groupId>cloudbase</groupId> - <artifactId>cloudbase-core</artifactId> - </dependency> - <dependency> - <groupId>mvm.rya</groupId> - <artifactId>cloudbase.rya</artifactId> - </dependency> - </dependencies> - <profiles> - <profile> - <id>mr</id> - <build> - <plugins> - <plugin> - <!-- NOTE: We don't need a groupId specification because the group is - org.apache.maven.plugins ...which is assumed by default. --> - <artifactId>maven-assembly-plugin</artifactId> - <dependencies> - <dependency> - <groupId>mvm.cloud</groupId> - <artifactId>hadoop-job-assembly</artifactId> - <version>1.0.0-SNAPSHOT</version> - </dependency> - </dependencies> - <executions> - <execution> - <id>make-assembly</id> - <phase>package</phase> - <goals> - <goal>single</goal> - </goals> - <configuration> - <attach>false</attach> - <descriptors> - <descriptor>assemblies/job.xml</descriptor> - </descriptors> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - </profiles> -</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/80faf06d/extras/cloudbase.rya.giraph/src/main/java/mvm/rya/cloudbase/giraph/format/CloudbaseRyaVertexInputFormat.java ---------------------------------------------------------------------- diff --git a/extras/cloudbase.rya.giraph/src/main/java/mvm/rya/cloudbase/giraph/format/CloudbaseRyaVertexInputFormat.java b/extras/cloudbase.rya.giraph/src/main/java/mvm/rya/cloudbase/giraph/format/CloudbaseRyaVertexInputFormat.java deleted file mode 100644 index 490b64d..0000000 --- a/extras/cloudbase.rya.giraph/src/main/java/mvm/rya/cloudbase/giraph/format/CloudbaseRyaVertexInputFormat.java +++ /dev/null @@ -1,88 +0,0 @@ -package mvm.rya.cloudbase.giraph.format; - -import cloudbase.core.data.Key; -import cloudbase.core.data.Value; -import com.google.common.collect.Maps; -import mvm.rya.api.RdfCloudTripleStoreConstants; -import mvm.rya.api.domain.RyaStatement; -import mvm.rya.api.resolver.RyaContext; -import mvm.rya.api.resolver.triple.TripleRow; -import org.apache.giraph.graph.BspUtils; -import org.apache.giraph.graph.Vertex; -import org.apache.giraph.graph.VertexReader; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapreduce.InputSplit; -import org.apache.hadoop.mapreduce.RecordReader; -import org.apache.hadoop.mapreduce.TaskAttemptContext; - -import java.io.IOException; -import java.util.Map; - -/** - * Date: 7/27/12 - * Time: 1:39 PM - */ -public class CloudbaseRyaVertexInputFormat - extends CloudbaseVertexInputFormat<Text, Text, Text, Text> { - - private Configuration conf; - - public VertexReader<Text, Text, Text, Text> - createVertexReader(InputSplit split, TaskAttemptContext context) - throws IOException { - try { - - return new CloudbaseEdgeVertexReader( - cloudbaseInputFormat.createRecordReader(split, context)) { - }; - } catch (InterruptedException e) { - throw new IOException(e); - } - - } - - /* - Reader takes Key/Value pairs from the underlying input format. - */ - public static class CloudbaseEdgeVertexReader - extends CloudbaseVertexReader<Text, Text, Text, Text> { - - private RyaContext ryaContext = RyaContext.getInstance(); - - public CloudbaseEdgeVertexReader(RecordReader<Key, Value> recordReader) { - super(recordReader); - } - - - public boolean nextVertex() throws IOException, InterruptedException { - return getRecordReader().nextKeyValue(); - } - - /* - Each Key/Value contains the information needed to construct the vertices. - */ - public Vertex<Text, Text, Text, Text> getCurrentVertex() - throws IOException, InterruptedException { - try { - Key key = getRecordReader().getCurrentKey(); - Value value = getRecordReader().getCurrentValue(); - RyaStatement ryaStatement = ryaContext.deserializeTriple(RdfCloudTripleStoreConstants.TABLE_LAYOUT.SPO, - new TripleRow(key.getRow().getBytes(), key.getColumnFamily().getBytes(), - key.getColumnQualifier().getBytes()));//TODO: assume spo for now - Vertex<Text, Text, Text, Text> vertex = - BspUtils.<Text, Text, Text, Text>createVertex( - getContext().getConfiguration()); - Text vertexId = new Text(ryaStatement.getSubject().getData()); //TODO: set Text? - Map<Text, Text> edges = Maps.newHashMap(); - Text edgeId = new Text(ryaStatement.getPredicate().getData()); - edges.put(edgeId, new Text(ryaStatement.getObject().getData())); - vertex.initialize(vertexId, new Text(), edges, null); - - return vertex; - } catch (Exception e) { - throw new IOException(e); - } - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/80faf06d/extras/cloudbase.rya.giraph/src/main/java/mvm/rya/cloudbase/giraph/format/CloudbaseRyaVertexOutputFormat.java ---------------------------------------------------------------------- diff --git a/extras/cloudbase.rya.giraph/src/main/java/mvm/rya/cloudbase/giraph/format/CloudbaseRyaVertexOutputFormat.java b/extras/cloudbase.rya.giraph/src/main/java/mvm/rya/cloudbase/giraph/format/CloudbaseRyaVertexOutputFormat.java deleted file mode 100644 index acdbe51..0000000 --- a/extras/cloudbase.rya.giraph/src/main/java/mvm/rya/cloudbase/giraph/format/CloudbaseRyaVertexOutputFormat.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package mvm.rya.cloudbase.giraph.format; - -import cloudbase.core.data.Mutation; -import cloudbase.core.data.Value; -import mvm.rya.api.RdfCloudTripleStoreConstants; -import mvm.rya.api.domain.RyaType; -import mvm.rya.api.domain.RyaURI; -import mvm.rya.cloudbase.RyaTableMutationsFactory; -import org.apache.giraph.graph.Edge; -import org.apache.giraph.graph.Vertex; -import org.apache.giraph.graph.VertexWriter; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapreduce.RecordWriter; -import org.apache.hadoop.mapreduce.TaskAttemptContext; - -import java.io.IOException; -import java.util.Collection; -import java.util.Map; - -/* - Example subclass for writing vertices back to Cloudbase. - */ -public class CloudbaseRyaVertexOutputFormat - extends CloudbaseVertexOutputFormat<Text, Text, Text> { - - public VertexWriter<Text, Text, Text> - createVertexWriter(TaskAttemptContext context) - throws IOException, InterruptedException { - RecordWriter<Text, Mutation> writer = - cloudbaseOutputFormat.getRecordWriter(context); - String tableName = context.getConfiguration().get(OUTPUT_TABLE); - if (tableName == null) - throw new IOException("Forgot to set table name " + - "using CloudbaseVertexOutputFormat.OUTPUT_TABLE"); - return new CloudbaseEdgeVertexWriter(writer, tableName); - } - - /* - Wraps RecordWriter for writing Mutations back to the configured Cloudbase Table. - */ - public static class CloudbaseEdgeVertexWriter - extends CloudbaseVertexWriter<Text, Text, Text> { - - public static final RyaTableMutationsFactory RYA_TABLE_MUTATIONS_FACTORY = new RyaTableMutationsFactory(); - private final Text CF = new Text("cf"); - private final Text PARENT = new Text("parent"); - private Text tableName; - - public CloudbaseEdgeVertexWriter( - RecordWriter<Text, Mutation> writer, String tableName) { - super(writer); - this.tableName = new Text(tableName); - } - - /* - Write back a mutation that adds a qualifier for 'parent' containing the vertex value - as the cell value. Assume the vertex ID corresponds to a key. - */ - public void writeVertex(Vertex<Text, Text, Text, ?> vertex) - throws IOException, InterruptedException { - RecordWriter<Text, Mutation> writer = getRecordWriter(); - Text subj = vertex.getId(); - Iterable<Edge<Text, Text>> edges = vertex.getEdges(); - for (Edge<Text, Text> edge : edges) { - Text pred = edge.getTargetVertexId(); - Text obj = edge.getValue(); - Map<RdfCloudTripleStoreConstants.TABLE_LAYOUT, Collection<Mutation>> serialize = - RYA_TABLE_MUTATIONS_FACTORY.serialize(new RyaURI(subj.toString()), - new RyaURI(pred.toString()), new RyaType(obj.toString()), null); - Collection<Mutation> mutations = serialize.get(RdfCloudTripleStoreConstants.TABLE_LAYOUT.SPO); - for (Mutation mut : mutations) { - writer.write(tableName, mut); //TODO: Assuming SPO - } - } - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/80faf06d/extras/cloudbase.rya.giraph/src/main/java/mvm/rya/cloudbase/giraph/format/CloudbaseVertexInputFormat.java ---------------------------------------------------------------------- diff --git a/extras/cloudbase.rya.giraph/src/main/java/mvm/rya/cloudbase/giraph/format/CloudbaseVertexInputFormat.java b/extras/cloudbase.rya.giraph/src/main/java/mvm/rya/cloudbase/giraph/format/CloudbaseVertexInputFormat.java deleted file mode 100644 index fcc0c5e..0000000 --- a/extras/cloudbase.rya.giraph/src/main/java/mvm/rya/cloudbase/giraph/format/CloudbaseVertexInputFormat.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package mvm.rya.cloudbase.giraph.format; - -import cloudbase.core.client.mapreduce.CloudbaseInputFormat; -import cloudbase.core.data.Key; -import cloudbase.core.data.Value; -import org.apache.giraph.graph.VertexInputFormat; -import org.apache.giraph.graph.VertexReader; -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.mapreduce.InputSplit; -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.mapreduce.RecordReader; -import org.apache.hadoop.mapreduce.TaskAttemptContext; - -import java.io.IOException; -import java.util.List; - -/** - * Class which wraps the CloudbaseInputFormat. It's designed - * as an extension point to VertexInputFormat subclasses who wish - * to read from Cloudbase Tables. - * - * @param <I> vertex id type - * @param <V> vertex value type - * @param <E> edge type - * @param <M> message type - */ -public abstract class CloudbaseVertexInputFormat< - I extends WritableComparable, - V extends Writable, - E extends Writable, - M extends Writable> - extends VertexInputFormat<I, V, E, M> implements Configurable { - /** - * delegate input format for all cloudbase operations. - */ - protected CloudbaseInputFormat cloudbaseInputFormat = - new CloudbaseInputFormat(); - - /** - * Configured and injected by the job - */ - private Configuration conf; - - @Override - public Configuration getConf() { - return conf; - } - - @Override - public void setConf(Configuration conf) { - this.conf = conf; - } - - /** - * Abstract class which provides a template for instantiating vertices - * from Cloudbase Key/Value pairs. - * - * @param <I> vertex id type - * @param <V> vertex value type - * @param <E> edge type - * @param <M> message type - */ - public abstract static class CloudbaseVertexReader< - I extends WritableComparable, - V extends Writable, E extends Writable, M extends Writable> - implements VertexReader<I, V, E, M> { - - /** - * Used by subclasses to read key/value pairs. - */ - private final RecordReader<Key, Value> reader; - /** - * Context passed to initialize - */ - private TaskAttemptContext context; - - /** - * Constructor used to pass Record Reader instance - * - * @param reader Cloudbase record reader - */ - public CloudbaseVertexReader(RecordReader<Key, Value> reader) { - this.reader = reader; - } - - @Override - public void initialize(InputSplit inputSplit, - TaskAttemptContext context) - throws IOException, InterruptedException { - reader.initialize(inputSplit, context); - this.context = context; - } - - /** - * close - * - * @throws IOException - */ - public void close() throws IOException { - reader.close(); - } - - /** - * getProgress - * - * @return progress - * @throws IOException - * @throws InterruptedException - */ - public float getProgress() throws IOException, InterruptedException { - return reader.getProgress(); - } - - /** - * Get the result record reader - * - * @return Record reader to be used for reading. - */ - protected RecordReader<Key, Value> getRecordReader() { - return reader; - } - - /** - * getContext - * - * @return Context passed to initialize. - */ - protected TaskAttemptContext getContext() { - return context; - } - - } - - /** - * getSplits - * - * @param context Context of the job - * @param numWorkers Number of workers used for this job - * @return tablet splits - * @throws IOException - * @throws InterruptedException - */ - public List<InputSplit> getSplits( - JobContext context, int numWorkers) - throws IOException, InterruptedException { - List<InputSplit> splits = null; - try { - splits = cloudbaseInputFormat.getSplits(context); - } catch (IOException e) { - if (e.getMessage().contains("Input info has not been set")) { - throw new IOException(e.getMessage() + - " Make sure you initialized" + - " CloudbaseInputFormat static setters " + - "before passing the config to GiraphJob."); - } - } - return splits; - } -} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/80faf06d/extras/cloudbase.rya.giraph/src/main/java/mvm/rya/cloudbase/giraph/format/CloudbaseVertexOutputFormat.java ---------------------------------------------------------------------- diff --git a/extras/cloudbase.rya.giraph/src/main/java/mvm/rya/cloudbase/giraph/format/CloudbaseVertexOutputFormat.java b/extras/cloudbase.rya.giraph/src/main/java/mvm/rya/cloudbase/giraph/format/CloudbaseVertexOutputFormat.java deleted file mode 100644 index f88dfe6..0000000 --- a/extras/cloudbase.rya.giraph/src/main/java/mvm/rya/cloudbase/giraph/format/CloudbaseVertexOutputFormat.java +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package mvm.rya.cloudbase.giraph.format; - -import cloudbase.core.client.mapreduce.CloudbaseOutputFormat; -import cloudbase.core.data.Mutation; -import org.apache.giraph.graph.VertexOutputFormat; -import org.apache.giraph.graph.VertexWriter; -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.mapreduce.OutputCommitter; -import org.apache.hadoop.mapreduce.RecordWriter; -import org.apache.hadoop.mapreduce.TaskAttemptContext; - -import java.io.IOException; -/** - * - * Class which wraps the CloudbaseOutputFormat. It's designed - * as an extension point to VertexOutputFormat subclasses who wish - * to write vertices back to an Cloudbase table. - * - * - * @param <I> vertex id type - * @param <V> vertex value type - * @param <E> edge type - */ -public abstract class CloudbaseVertexOutputFormat< - I extends WritableComparable, - V extends Writable, - E extends Writable> - extends VertexOutputFormat<I, V, E> implements Configurable { - - - /** - * Output table parameter - */ - protected static final String OUTPUT_TABLE = "OUTPUT_TABLE"; - - /** - * Cloudbase delegate for table output - */ - protected CloudbaseOutputFormat cloudbaseOutputFormat = - new CloudbaseOutputFormat(); - - - /** - * Used by configured interface - */ - private Configuration conf; - - /** - * - * Main abstraction point for vertex writers to persist back - * to Cloudbase tables. - * - * @param <I> vertex id type - * @param <V> vertex value type - * @param <E> edge type - */ - public abstract static class CloudbaseVertexWriter< - I extends WritableComparable, - V extends Writable, - E extends Writable> - implements VertexWriter<I, V, E> { - - /** - * task attempt context. - */ - private TaskAttemptContext context; - - /** - * Cloudbase record writer - */ - private RecordWriter<Text, Mutation> recordWriter; - - /** - * Constructor for use with subclasses - * - * @param recordWriter cloudbase record writer - */ - public CloudbaseVertexWriter(RecordWriter<Text, Mutation> recordWriter) { - this.recordWriter = recordWriter; - } - - /** - * initialize - * - * @param context Context used to write the vertices. - * @throws IOException - */ - public void initialize(TaskAttemptContext context) throws IOException { - this.context = context; - } - - /** - * close - * - * @param context the context of the task - * @throws IOException - * @throws InterruptedException - */ - public void close(TaskAttemptContext context) - throws IOException, InterruptedException { - recordWriter.close(context); - } - - /** - * Get the table record writer; - * - * @return Record writer to be used for writing. - */ - public RecordWriter<Text, Mutation> getRecordWriter() { - return recordWriter; - } - - /** - * Get the context. - * - * @return Context passed to initialize. - */ - public TaskAttemptContext getContext() { - return context; - } - - } - - @Override - public void setConf(Configuration conf) { - this.conf = conf; - } - - @Override - public Configuration getConf() { - return this.conf; - } - - /** - * - * checkOutputSpecs - * - * @param context information about the job - * @throws IOException - * @throws InterruptedException - */ - public void checkOutputSpecs(JobContext context) - throws IOException, InterruptedException { - try { - cloudbaseOutputFormat.checkOutputSpecs(context); - } catch (IOException e) { - if (e.getMessage().contains("Output info has not been set")) { - throw new IOException(e.getMessage() + " Make sure you initialized" + - " CloudbaseOutputFormat static setters " + - "before passing the config to GiraphJob."); - } - } - } - - /** - * getOutputCommitter - * - * @param context the task context - * @return OutputCommitter - * @throws IOException - * @throws InterruptedException - */ - public OutputCommitter getOutputCommitter(TaskAttemptContext context) - throws IOException, InterruptedException { - return cloudbaseOutputFormat.getOutputCommitter(context); - } -} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/80faf06d/extras/cloudbase.rya.giraph/src/main/java/mvm/rya/cloudbase/giraph/format/PrintVertexOutputFormat.java ---------------------------------------------------------------------- diff --git a/extras/cloudbase.rya.giraph/src/main/java/mvm/rya/cloudbase/giraph/format/PrintVertexOutputFormat.java b/extras/cloudbase.rya.giraph/src/main/java/mvm/rya/cloudbase/giraph/format/PrintVertexOutputFormat.java deleted file mode 100644 index e90ca66..0000000 --- a/extras/cloudbase.rya.giraph/src/main/java/mvm/rya/cloudbase/giraph/format/PrintVertexOutputFormat.java +++ /dev/null @@ -1,94 +0,0 @@ -package mvm.rya.cloudbase.giraph.format; - -import org.apache.giraph.graph.Vertex; -import org.apache.giraph.graph.VertexOutputFormat; -import org.apache.giraph.graph.VertexWriter; -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.mapreduce.OutputCommitter; -import org.apache.hadoop.mapreduce.TaskAttemptContext; - -import java.io.IOException; - -/** - * Date: 7/27/12 - * Time: 2:58 PM - */ -public class PrintVertexOutputFormat< - I extends WritableComparable, - V extends Writable, - E extends Writable> - extends VertexOutputFormat<I, V, E> implements Configurable { - @Override - public void setConf(Configuration entries) { - //To change body of implemented methods use File | Settings | File Templates. - } - - @Override - public Configuration getConf() { - return null; //To change body of implemented methods use File | Settings | File Templates. - } - - @Override - public VertexWriter<I, V, E> createVertexWriter(TaskAttemptContext context) throws IOException, InterruptedException { - return new VertexWriter<I, V, E>() { - @Override - public void initialize(TaskAttemptContext context) throws IOException { - //To change body of implemented methods use File | Settings | File Templates. - } - - @Override - public void writeVertex(Vertex<I, V, E, ?> iveVertex) throws IOException, InterruptedException { - System.out.println(iveVertex); - } - - @Override - public void close(TaskAttemptContext context) throws IOException, InterruptedException { - //To change body of implemented methods use File | Settings | File Templates. - } - }; - } - - @Override - public void checkOutputSpecs(JobContext context) throws IOException, InterruptedException { - //To change body of implemented methods use File | Settings | File Templates. - } - - @Override - public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException, InterruptedException { - return new OutputCommitter() { - @Override - public void setupJob(JobContext jobContext) throws IOException { - //To change body of implemented methods use File | Settings | File Templates. - } - - @Override - public void cleanupJob(JobContext jobContext) throws IOException { - //To change body of implemented methods use File | Settings | File Templates. - } - - @Override - public void setupTask(TaskAttemptContext taskAttemptContext) throws IOException { - //To change body of implemented methods use File | Settings | File Templates. - } - - @Override - public boolean needsTaskCommit(TaskAttemptContext taskAttemptContext) throws IOException { - return false; //To change body of implemented methods use File | Settings | File Templates. - } - - @Override - public void commitTask(TaskAttemptContext taskAttemptContext) throws IOException { - //To change body of implemented methods use File | Settings | File Templates. - } - - @Override - public void abortTask(TaskAttemptContext taskAttemptContext) throws IOException { - //To change body of implemented methods use File | Settings | File Templates. - } - }; - } -} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/80faf06d/extras/cloudbase.rya.giraph/src/test/java/mvm/rya/cloudbase/giraph/format/BspCase.java ---------------------------------------------------------------------- diff --git a/extras/cloudbase.rya.giraph/src/test/java/mvm/rya/cloudbase/giraph/format/BspCase.java b/extras/cloudbase.rya.giraph/src/test/java/mvm/rya/cloudbase/giraph/format/BspCase.java deleted file mode 100644 index fb20dd8..0000000 --- a/extras/cloudbase.rya.giraph/src/test/java/mvm/rya/cloudbase/giraph/format/BspCase.java +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package mvm.rya.cloudbase.giraph.format; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.List; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; - -import org.apache.giraph.examples.GeneratedVertexReader; -import org.apache.giraph.graph.GiraphJob; -import org.apache.giraph.zk.ZooKeeperExt; - -import junit.framework.TestCase; - -/** - * Duplicate copy from main giraph trunk. At least until there - * is a maven test artifact for Giraph. - * - * Extended TestCase for making setting up Bsp testing. - */ -public class BspCase extends TestCase implements Watcher { - /** JobTracker system property */ - private final String jobTracker = - System.getProperty("prop.mapred.job.tracker"); - /** Jar location system property */ - private final String jarLocation = - System.getProperty("prop.jarLocation", ""); - /** Number of actual processes for the BSP application */ - private int numWorkers = 1; - /** ZooKeeper list system property */ - private final String zkList = System.getProperty("prop.zookeeper.list"); - - /** - * Adjust the configuration to the basic test case - */ - public final void setupConfiguration(GiraphJob job) { - Configuration conf = job.getConfiguration(); - conf.set("mapred.jar", getJarLocation()); - - // Allow this test to be run on a real Hadoop setup - if (getJobTracker() != null) { - System.out.println("setup: Sending job to job tracker " + - getJobTracker() + " with jar path " + getJarLocation() - + " for " + getName()); - conf.set("mapred.job.tracker", getJobTracker()); - job.setWorkerConfiguration(getNumWorkers(), - getNumWorkers(), - 100.0f); - } - else { - System.out.println("setup: Using local job runner with " + - "location " + getJarLocation() + " for " - + getName()); - job.setWorkerConfiguration(1, 1, 100.0f); - // Single node testing - conf.setBoolean(GiraphJob.SPLIT_MASTER_WORKER, false); - } - conf.setInt(GiraphJob.POLL_ATTEMPTS, 10); - conf.setInt(GiraphJob.POLL_MSECS, 3*1000); - conf.setInt(GiraphJob.ZOOKEEPER_SERVERLIST_POLL_MSECS, 500); - if (getZooKeeperList() != null) { - job.setZooKeeperConfiguration(getZooKeeperList()); - } - // GeneratedInputSplit will generate 5 vertices - conf.setLong(GeneratedVertexReader.READER_VERTICES, 5); - } - - /** - * Create the test case - * - * @param testName name of the test case - */ - public BspCase(String testName) { - super(testName); - - } - - /** - * Get the number of workers used in the BSP application - * - */ - public int getNumWorkers() { - return numWorkers; - } - - /** - * Get the ZooKeeper list - */ - public String getZooKeeperList() { - return zkList; - } - - /** - * Get the jar location - * - * @return location of the jar file - */ - String getJarLocation() { - return jarLocation; - } - - /** - * Get the job tracker location - * - * @return job tracker location as a string - */ - String getJobTracker() { - return jobTracker; - } - - /** - * Get the single part file status and make sure there is only one part - * - * @param job Job to get the file system from - * @param partDirPath Directory where the single part file should exist - * @return Single part file status - * @throws java.io.IOException - */ - public static FileStatus getSinglePartFileStatus(GiraphJob job, - Path partDirPath) throws IOException { - FileSystem fs = FileSystem.get(job.getConfiguration()); - FileStatus[] statusArray = fs.listStatus(partDirPath); - FileStatus singlePartFileStatus = null; - int partFiles = 0; - for (FileStatus fileStatus : statusArray) { - if (fileStatus.getPath().getName().equals("part-m-00000")) { - singlePartFileStatus = fileStatus; - } - if (fileStatus.getPath().getName().startsWith("part-m-")) { - ++partFiles; - } - } - if (partFiles != 1) { - throw new ArithmeticException( - "getSinglePartFile: Part file count should be 1, but is " + - partFiles); - } - return singlePartFileStatus; - } - - @Override - public void setUp() { - if (jobTracker != null) { - System.out.println("Setting tasks to 3 for " + getName() + - " since JobTracker exists..."); - numWorkers = 3; - } - try { - Configuration conf = new Configuration(); - FileSystem hdfs = FileSystem.get(conf); - // Since local jobs always use the same paths, remove them - Path oldLocalJobPaths = new Path( - GiraphJob.ZOOKEEPER_MANAGER_DIR_DEFAULT); - FileStatus[] fileStatusArr; - try { - fileStatusArr = hdfs.listStatus(oldLocalJobPaths); - for (FileStatus fileStatus : fileStatusArr) { - if (fileStatus.isDir() && - fileStatus.getPath().getName().contains("job_local")) { - System.out.println("Cleaning up local job path " + - fileStatus.getPath().getName()); - hdfs.delete(oldLocalJobPaths, true); - } - } - } catch (FileNotFoundException e) { - // ignore this FileNotFound exception and continue. - } - if (zkList == null) { - return; - } - ZooKeeperExt zooKeeperExt = - new ZooKeeperExt(zkList, 30*1000, this); - List<String> rootChildren = zooKeeperExt.getChildren("/", false); - for (String rootChild : rootChildren) { - if (rootChild.startsWith("_hadoopBsp")) { - List<String> children = - zooKeeperExt.getChildren("/" + rootChild, false); - for (String child: children) { - if (child.contains("job_local_")) { - System.out.println("Cleaning up /_hadoopBsp/" + - child); - zooKeeperExt.deleteExt( - "/_hadoopBsp/" + child, -1, true); - } - } - } - } - zooKeeperExt.close(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Override - public void process(WatchedEvent event) { - // Do nothing - } - - /** - * Helper method to remove an old output directory if it exists, - * and set the output path for any VertexOutputFormat that uses - * FileOutputFormat. - * - * @param job Job to set the output path for - * @throws java.io.IOException - */ - public static void removeAndSetOutput(GiraphJob job, - Path outputPath) throws IOException { - remove(job.getConfiguration(), outputPath); - FileOutputFormat.setOutputPath(job.getInternalJob(), outputPath); - } - - /** - * Helper method to remove a path if it exists. - * - * @param conf Configuration to load FileSystem from - * @param path Path to remove - * @throws java.io.IOException - */ - public static void remove(Configuration conf, Path path) - throws IOException { - FileSystem hdfs = FileSystem.get(conf); - hdfs.delete(path, true); - } - - public static String getCallingMethodName() { - return Thread.currentThread().getStackTrace()[2].getMethodName(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/80faf06d/extras/cloudbase.rya.giraph/src/test/java/mvm/rya/cloudbase/giraph/format/TestCloudbaseVertexFormat.java ---------------------------------------------------------------------- diff --git a/extras/cloudbase.rya.giraph/src/test/java/mvm/rya/cloudbase/giraph/format/TestCloudbaseVertexFormat.java b/extras/cloudbase.rya.giraph/src/test/java/mvm/rya/cloudbase/giraph/format/TestCloudbaseVertexFormat.java deleted file mode 100644 index e420ff6..0000000 --- a/extras/cloudbase.rya.giraph/src/test/java/mvm/rya/cloudbase/giraph/format/TestCloudbaseVertexFormat.java +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package mvm.rya.cloudbase.giraph.format; - -import cloudbase.core.client.BatchWriter; -import cloudbase.core.client.Connector; -import cloudbase.core.client.ZooKeeperInstance; -import cloudbase.core.client.mapreduce.CloudbaseInputFormat; -import cloudbase.core.client.mock.MockInstance; -import cloudbase.core.data.Range; -import cloudbase.core.security.Authorizations; -import junit.framework.Test; -import junit.framework.TestSuite; -import mvm.rya.api.domain.RyaStatement; -import mvm.rya.api.domain.RyaURI; -import mvm.rya.cloudbase.CloudbaseRdfConfiguration; -import mvm.rya.cloudbase.CloudbaseRyaDAO; -import org.apache.giraph.graph.EdgeListVertex; -import org.apache.giraph.graph.GiraphJob; -import org.apache.giraph.lib.TextVertexOutputFormat; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; -import org.apache.log4j.Logger; - -import java.io.File; -import java.io.IOException; -import java.util.Collections; - -/* - Test class for Cloudbase vertex input/output formats. - */ -public class TestCloudbaseVertexFormat extends BspCase { - - private final String TABLE_NAME = "rya_spo"; - private final String INSTANCE_NAME = "stratus"; - private final Text FAMILY = new Text("cf"); - private final Text CHILDREN = new Text("children"); - private final String USER = "root"; - private final byte[] PASSWORD = new byte[]{}; - private final Text OUTPUT_FIELD = new Text("parent"); - - - private final Logger log = Logger.getLogger(TestCloudbaseVertexFormat.class); - - /** - * Create the test case - * - * @param testName name of the test case - */ - public TestCloudbaseVertexFormat(String testName) { - super(testName); - } - - /** - * @return the suite of tests being tested - */ - public static Test suite() { - return new TestSuite(TestCloudbaseVertexFormat.class); - - } - - /* - Write a simple parent-child directed graph to Cloudbase. - Run a job which reads the values - into subclasses that extend CloudbaseVertex I/O formats. - Check the output after the job. - */ - public void testCloudbaseInputOutput() throws Exception { -// if (System.getProperty("prop.mapred.job.tracker") != null) { -// if(log.isInfoEnabled()) -// log.info("testCloudbaseInputOutput: " + -// "Ignore this test if not local mode."); -// return; -// } -// -// System.setProperty("prop.jarLocation", "/temp/cloudbase.rya.giraph-3.0.0-SNAPSHOT.jar"); - File jarTest = new File(System.getProperty("prop.jarLocation")); - if (!jarTest.exists()) { - fail("Could not find Giraph jar at " + - "location specified by 'prop.jarLocation'. " + - "Make sure you built the main Giraph artifact?."); - } - - //Write out vertices and edges out to a mock instance. -// MockInstance mockInstance = new MockInstance(INSTANCE_NAME); - Connector c = new ZooKeeperInstance("stratus", "stratus13:2181").getConnector("root", "password".getBytes()); - CloudbaseRyaDAO ryaDAO = new CloudbaseRyaDAO(); - ryaDAO.setConnector(c); - CloudbaseRdfConfiguration cloudbaseRdfConfiguration = new CloudbaseRdfConfiguration(); -// cloudbaseRdfConfiguration.setTablePrefix("test_"); - ryaDAO.init(); -// c.tableOperations().create(TABLE_NAME); -// BatchWriter bw = c.createBatchWriter(TABLE_NAME, 10000L, 1000L, 4); - - ryaDAO.add(new RyaStatement(new RyaURI("urn:test#1234"), - new RyaURI("urn:test#pred1"), - new RyaURI("urn:test#obj1"))); - ryaDAO.add(new RyaStatement(new RyaURI("urn:test#1234"), - new RyaURI("urn:test#pred2"), - new RyaURI("urn:test#obj2"))); - ryaDAO.add(new RyaStatement(new RyaURI("urn:test#1234"), - new RyaURI("urn:test#pred3"), - new RyaURI("urn:test#obj3"))); - ryaDAO.add(new RyaStatement(new RyaURI("urn:test#1234"), - new RyaURI("urn:test#pred4"), - new RyaURI("urn:test#obj4"))); - ryaDAO.commit(); - -// Mutation m1 = new Mutation(new Text("0001")); -// m1.put(FAMILY, CHILDREN, new Value("0002".getBytes())); -// bw.addMutation(m1); -// -// Mutation m2 = new Mutation(new Text("0002")); -// m2.put(FAMILY, CHILDREN, new Value("0003".getBytes())); -// bw.addMutation(m2); -// if(log.isInfoEnabled()) -// log.info("Writing mutations to Cloudbase table"); -// bw.close(); - - Configuration conf = new Configuration(); -// conf.set(CloudbaseVertexOutputFormat.OUTPUT_TABLE, TABLE_NAME); - - /* - Very important to initialize the formats before - sending configuration to the GiraphJob. Otherwise - the internally constructed Job in GiraphJob will - not have the proper context initialization. - */ - GiraphJob job = new GiraphJob(conf, getCallingMethodName()); - CloudbaseInputFormat.setInputInfo(job.getInternalJob(), USER, "password".getBytes(), - TABLE_NAME, new Authorizations()); -// CloudbaseInputFormat.setMockInstance(job.getInternalJob(), INSTANCE_NAME); - CloudbaseInputFormat.setZooKeeperInstance(job.getInternalJob(), "stratus", "stratus13:2181"); - CloudbaseInputFormat.setRanges(job.getInternalJob(), Collections.singleton(new Range())); - -// CloudbaseOutputFormat.setOutputInfo(job.getInternalJob(), USER, PASSWORD, true, null); -// CloudbaseOutputFormat.setMockInstance(job.getInternalJob(), INSTANCE_NAME); - - setupConfiguration(job); - job.setVertexClass(EdgeNotification.class); - job.setVertexInputFormatClass(CloudbaseRyaVertexInputFormat.class); - job.setVertexOutputFormatClass(PrintVertexOutputFormat.class); - FileOutputFormat.setOutputPath(job.getInternalJob(), new Path("/temp/graphout")); - -// HashSet<Pair<Text, Text>> columnsToFetch = new HashSet<Pair<Text,Text>>(); -// columnsToFetch.add(new Pair<Text, Text>(FAMILY, CHILDREN)); -// CloudbaseInputFormat.fetchColumns(job.getInternalJob(), columnsToFetch); - - if (log.isInfoEnabled()) - log.info("Running edge notification job using Cloudbase input"); - assertTrue(job.run(true)); - -// Scanner scanner = c.createScanner(TABLE_NAME, new Authorizations()); -// scanner.setRange(new Range("0002", "0002")); -// scanner.fetchColumn(FAMILY, OUTPUT_FIELD); -// boolean foundColumn = false; -// -// if(log.isInfoEnabled()) -// log.info("Verify job output persisted correctly."); -// //make sure we found the qualifier. -// assertTrue(scanner.iterator().hasNext()); -// -// -// //now we check to make sure the expected value from the job persisted correctly. -// for(Map.Entry<Key,Value> entry : scanner) { -// Text row = entry.getKey().getRow(); -// assertEquals("0002", row.toString()); -// Value value = entry.getValue(); -// assertEquals("0001", ByteBufferUtil.toString( -// ByteBuffer.wrap(value.get()))); -// foundColumn = true; -// } - } - - /* - Test compute method that sends each edge a notification of its parents. - The test set only has a 1-1 parent-to-child ratio for this unit test. - */ - public static class EdgeNotification - extends EdgeListVertex<Text, Text, Text, Text> { - @Override - public void compute(Iterable<Text> messages) throws IOException { - System.out.println("Edges: " + messages); - for (Text message : messages) { - getValue().set(message); - } - if (getSuperstep() == 0) { -// sendMessageToAllEdges(getId()); - } - voteToHalt(); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/80faf06d/extras/generic.mr/generic.mr.accumulo/pom.xml ---------------------------------------------------------------------- diff --git a/extras/generic.mr/generic.mr.accumulo/pom.xml b/extras/generic.mr/generic.mr.accumulo/pom.xml deleted file mode 100644 index a573c61..0000000 --- a/extras/generic.mr/generic.mr.accumulo/pom.xml +++ /dev/null @@ -1,58 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <parent> - <groupId>mvm.rya</groupId> - <artifactId>generic.mr</artifactId> - <version>3.2.10-SNAPSHOT</version> - </parent> - <modelVersion>4.0.0</modelVersion> - - <artifactId>generic.mr.accumulo</artifactId> - <name>${project.groupId}.${project.artifactId}</name> - <dependencies> - <dependency> - <groupId>mvm.rya</groupId> - <artifactId>generic.mr.api</artifactId> - <version>${project.version}</version> - </dependency> - </dependencies> - - <build> - <plugins> - <plugin> - <groupId>org.codehaus.gmaven</groupId> - <artifactId>gmaven-plugin</artifactId> - </plugin> - </plugins> - </build> - - <profiles> - <profile> - <id>accumulo</id> - <activation> - <activeByDefault>true</activeByDefault> - </activation> - <dependencies> - <dependency> - <groupId>org.apache.accumulo</groupId> - <artifactId>accumulo-core</artifactId> - <optional>true</optional> - </dependency> - </dependencies> - </profile> - <profile> - <id>cloudbase</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <dependencies> - <dependency> - <groupId>com.texeltek</groupId> - <artifactId>accumulo-cloudbase-shim</artifactId> - <optional>true</optional> - </dependency> - </dependencies> - </profile> - </profiles> - -</project> http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/80faf06d/extras/generic.mr/generic.mr.accumulo/src/main/groovy/mvm/rya/generic/mr/accumulo/AccumuloMRInfo.groovy ---------------------------------------------------------------------- diff --git a/extras/generic.mr/generic.mr.accumulo/src/main/groovy/mvm/rya/generic/mr/accumulo/AccumuloMRInfo.groovy b/extras/generic.mr/generic.mr.accumulo/src/main/groovy/mvm/rya/generic/mr/accumulo/AccumuloMRInfo.groovy deleted file mode 100644 index 89d4633..0000000 --- a/extras/generic.mr/generic.mr.accumulo/src/main/groovy/mvm/rya/generic/mr/accumulo/AccumuloMRInfo.groovy +++ /dev/null @@ -1,146 +0,0 @@ -package mvm.rya.generic.mr.accumulo - -import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat -import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat -import org.apache.accumulo.core.data.Key -import org.apache.accumulo.core.data.Mutation -import org.apache.accumulo.core.data.Value -import org.apache.accumulo.core.security.Authorizations -import org.apache.accumulo.core.security.ColumnVisibility -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.io.Text -import org.apache.hadoop.mapreduce.Job -import mvm.rya.generic.mr.api.MRInfo -import org.apache.accumulo.core.client.mock.MockInstance -import org.apache.accumulo.core.client.ZooKeeperInstance - -/** - * Date: 12/3/12 - * Time: 9:00 AM - */ -class AccumuloMRInfo implements MRInfo { - - def Configuration conf - def connector; - - @Override - void initMRJob(Job job, String table, String outtable, String[] auths) { - Configuration conf = job.configuration - String username = conf.get(USERNAME) - String password = conf.get(PASSWORD) - String instance = conf.get(INSTANCE) - String zookeepers = conf.get(ZOOKEEPERS) - String mock = conf.get(MOCK) - - //input - if (Boolean.parseBoolean(mock)) { - AccumuloInputFormat.setMockInstance(conf, instance) - AccumuloOutputFormat.setMockInstance(conf, instance) - } else if (zookeepers != null) { - AccumuloInputFormat.setZooKeeperInstance(conf, instance, zookeepers) - AccumuloOutputFormat.setZooKeeperInstance(conf, instance, zookeepers) - } else { - throw new IllegalArgumentException("Must specify either mock or zookeepers"); - } - - AccumuloInputFormat.setInputInfo(conf, username, password.getBytes(), table, new Authorizations(auths)) - job.setInputFormatClass(AccumuloInputFormat.class); - - // OUTPUT - job.setOutputFormatClass(AccumuloOutputFormat.class); - job.setOutputKeyClass(Text.class); - job.setOutputValueClass(Mutation.class); - AccumuloOutputFormat.setOutputInfo(job, username, password.getBytes(), true, outtable); - } - - @Override - def key(byte[] data) { - Key key = new Key(); - key.readFields(new DataInputStream(new ByteArrayInputStream(data))) - return key - } - - @Override - def key(String row, String cf, String cq, String cv, long timestamp) { - return new Key(row, cf, cq, cv, timestamp) - } - - @Override - def value(byte[] data) { - return new Value(data) - } - - @Override - def columnVisibility(String cv) { - return new ColumnVisibility(cv) - } - - @Override - def mutation(String row, String cf, String cq, String cv, long timestamp, byte[] val) { - Mutation mutation = new Mutation(row); - mutation.put(cf, cq, columnVisibility(cv), timestamp, value(val)) - return mutation - } - - @Override - def instance() { - assert conf != null - - String instance_str = conf.get(INSTANCE) - String zookeepers = conf.get(ZOOKEEPERS) - String mock = conf.get(MOCK) - if (Boolean.parseBoolean(mock)) { - return new MockInstance(instance_str) - } else if (zookeepers != null) { - return new ZooKeeperInstance(instance_str, zookeepers) - } else { - throw new IllegalArgumentException("Must specify either mock or zookeepers"); - } - } - - @Override - def connector(def instance) { - if (connector != null) return connector - - String username = conf.get(USERNAME) - String password = conf.get(PASSWORD) - if (instance == null) - instance = instance() - connector = instance.getConnector(username, password) - return connector - } - - @Override - def void writeMutations(def connector, String tableName, Iterator mutations) { - def bw = connector.createBatchWriter(tableName, 10000l, 10000l, 4); - mutations.each { m -> - bw.addMutation(m) - } - bw.flush() - bw.close() - } - - @Override - def scanner(def connector, String tableName, String[] auths) { - return connector.createScanner(tableName, new Authorizations(auths)) - } - - @Override - def batchScanner(def connector, String tableName, String[] auths, int numThreads) { - return connector.createBatchScanner(tableName, new Authorizations(auths), numThreads) - } - - @Override - def range(def startKey, def endKey) { - assert startKey != null - - if (endKey != null) - return new org.apache.accumulo.core.data.Range(startKey, endKey) - return new org.apache.accumulo.core.data.Range(startKey) - } - - @Override - def authorizations(String[] auths) { - return new Authorizations(auths) - } -} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/80faf06d/extras/generic.mr/generic.mr.accumulo/src/main/resources/META-INF/services/mvm.rya.generic.mr.api.MRInfo ---------------------------------------------------------------------- diff --git a/extras/generic.mr/generic.mr.accumulo/src/main/resources/META-INF/services/mvm.rya.generic.mr.api.MRInfo b/extras/generic.mr/generic.mr.accumulo/src/main/resources/META-INF/services/mvm.rya.generic.mr.api.MRInfo deleted file mode 100644 index 81d47de..0000000 --- a/extras/generic.mr/generic.mr.accumulo/src/main/resources/META-INF/services/mvm.rya.generic.mr.api.MRInfo +++ /dev/null @@ -1 +0,0 @@ -mvm.rya.generic.mr.accumulo.AccumuloMRInfo \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/80faf06d/extras/generic.mr/generic.mr.api/pom.xml ---------------------------------------------------------------------- diff --git a/extras/generic.mr/generic.mr.api/pom.xml b/extras/generic.mr/generic.mr.api/pom.xml deleted file mode 100644 index 9bf5124..0000000 --- a/extras/generic.mr/generic.mr.api/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <parent> - <groupId>mvm.rya</groupId> - <artifactId>generic.mr</artifactId> - <version>3.2.10-SNAPSHOT</version> - </parent> - <modelVersion>4.0.0</modelVersion> - - <artifactId>generic.mr.api</artifactId> - <name>${project.groupId}.${project.artifactId}</name> - <dependencies> - <dependency> - <groupId>org.codehaus.groovy</groupId> - <artifactId>groovy-all</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - </dependency> - </dependencies> - - <build> - <plugins> - <plugin> - <groupId>org.codehaus.gmaven</groupId> - <artifactId>gmaven-plugin</artifactId> - </plugin> - </plugins> - </build> - -</project> http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/80faf06d/extras/generic.mr/generic.mr.api/src/main/groovy/mvm/rya/generic/mr/api/MRInfo.groovy ---------------------------------------------------------------------- diff --git a/extras/generic.mr/generic.mr.api/src/main/groovy/mvm/rya/generic/mr/api/MRInfo.groovy b/extras/generic.mr/generic.mr.api/src/main/groovy/mvm/rya/generic/mr/api/MRInfo.groovy deleted file mode 100644 index bdcc61e..0000000 --- a/extras/generic.mr/generic.mr.api/src/main/groovy/mvm/rya/generic/mr/api/MRInfo.groovy +++ /dev/null @@ -1,43 +0,0 @@ -package mvm.rya.generic.mr.api - -import org.apache.hadoop.conf.Configurable -import org.apache.hadoop.mapreduce.Job - -/** - * Date: 12/3/12 - * Time: 8:56 AM - */ -public interface MRInfo extends Configurable{ - - public static final String USERNAME = "username" - public static final String PASSWORD = "password" - public static final String INSTANCE = "instance" - public static final String ZOOKEEPERS = "zookeepers" - public static final String MOCK = "mock" - - def void initMRJob(Job job, String table, String outtable, String[] auths) - - def key(byte[] data); - - def key(String row, String cf, String cq, String cv, long timestamp); - - def value(byte[] data); - - def columnVisibility(String cv); - - def mutation(String row, String cf, String cq, String cv, long timestamp, byte[] val); - - def instance() - - def connector(def instance) - - def void writeMutations(def connector, String tableName, Iterator mutations) - - def scanner(def connector, String tableName, String[] auths) - - def batchScanner(def connector, String tableName, String[] auths, int numThreads) - - def range(def startKey, def endKey) - - def authorizations(String[] auths) -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/80faf06d/extras/generic.mr/generic.mr.api/src/main/groovy/mvm/rya/generic/mr/api/MRInfoContext.groovy ---------------------------------------------------------------------- diff --git a/extras/generic.mr/generic.mr.api/src/main/groovy/mvm/rya/generic/mr/api/MRInfoContext.groovy b/extras/generic.mr/generic.mr.api/src/main/groovy/mvm/rya/generic/mr/api/MRInfoContext.groovy deleted file mode 100644 index a2b92ec..0000000 --- a/extras/generic.mr/generic.mr.api/src/main/groovy/mvm/rya/generic/mr/api/MRInfoContext.groovy +++ /dev/null @@ -1,28 +0,0 @@ -package mvm.rya.generic.mr.api - -import org.apache.hadoop.conf.Configuration - -/** - * Date: 12/5/12 - * Time: 1:32 PM - */ -class MRInfoContext { - - private static currentMrInfo; - - public static MRInfo currentMRInfo() { - return currentMRInfo(null); - } - - public static MRInfo currentMRInfo(Configuration config) { - if (currentMrInfo == null) { - def iter = ServiceLoader.load(MRInfo.class, Thread.currentThread().getContextClassLoader()).iterator() - if (iter.hasNext()) { - currentMrInfo = iter.next() - if (config != null) currentMrInfo.setConf(config) - } - } - return currentMrInfo - } - -} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/80faf06d/extras/generic.mr/generic.mr.cloudbase/pom.xml ---------------------------------------------------------------------- diff --git a/extras/generic.mr/generic.mr.cloudbase/pom.xml b/extras/generic.mr/generic.mr.cloudbase/pom.xml deleted file mode 100644 index 8aec0fc..0000000 --- a/extras/generic.mr/generic.mr.cloudbase/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <parent> - <groupId>mvm.rya</groupId> - <artifactId>generic.mr</artifactId> - <version>3.2.10-SNAPSHOT</version> - </parent> - <modelVersion>4.0.0</modelVersion> - - <artifactId>generic.mr.cloudbase</artifactId> - <name>${project.groupId}.${project.artifactId}</name> - <dependencies> - <dependency> - <groupId>mvm.rya</groupId> - <artifactId>generic.mr.api</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>cloudbase</groupId> - <artifactId>cloudbase-core</artifactId> - </dependency> - </dependencies> - - <build> - <plugins> - <plugin> - <groupId>org.codehaus.gmaven</groupId> - <artifactId>gmaven-plugin</artifactId> - </plugin> - </plugins> - </build> - -</project> http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/80faf06d/extras/generic.mr/generic.mr.cloudbase/src/main/groovy/mvm/rya/generic/mr/cloudbase/CloudbaseMRInfo.groovy ---------------------------------------------------------------------- diff --git a/extras/generic.mr/generic.mr.cloudbase/src/main/groovy/mvm/rya/generic/mr/cloudbase/CloudbaseMRInfo.groovy b/extras/generic.mr/generic.mr.cloudbase/src/main/groovy/mvm/rya/generic/mr/cloudbase/CloudbaseMRInfo.groovy deleted file mode 100644 index 7608fb7..0000000 --- a/extras/generic.mr/generic.mr.cloudbase/src/main/groovy/mvm/rya/generic/mr/cloudbase/CloudbaseMRInfo.groovy +++ /dev/null @@ -1,146 +0,0 @@ -package mvm.rya.generic.mr.cloudbase - -import cloudbase.core.client.mapreduce.CloudbaseInputFormat -import cloudbase.core.client.mapreduce.CloudbaseOutputFormat -import cloudbase.core.security.Authorizations -import mvm.rya.generic.mr.api.MRInfo -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.io.Text -import org.apache.hadoop.mapreduce.Job -import cloudbase.core.data.Mutation -import cloudbase.core.data.Key -import cloudbase.core.data.Value -import cloudbase.core.security.ColumnVisibility -import cloudbase.core.client.mock.MockInstance -import cloudbase.core.client.ZooKeeperInstance - -/** - * Date: 12/3/12 - * Time: 9:00 AM - */ -class CloudbaseMRInfo implements MRInfo { - - def Configuration conf - def connector; - - @Override - void initMRJob(Job job, String table, String outtable, String[] auths) { - Configuration conf = job.configuration - String username = conf.get(USERNAME) - String password = conf.get(PASSWORD) - String instance = conf.get(INSTANCE) - String zookeepers = conf.get(ZOOKEEPERS) - String mock = conf.get(MOCK) - - //input - if (Boolean.parseBoolean(mock)) { - CloudbaseInputFormat.setMockInstance(job, instance) -// CloudbaseOutputFormat.setMockInstance(conf, instance) - } else if (zookeepers != null) { - CloudbaseInputFormat.setZooKeeperInstance(job, instance, zookeepers) - CloudbaseOutputFormat.setZooKeeperInstance(job, instance, zookeepers) - } else { - throw new IllegalArgumentException("Must specify either mock or zookeepers"); - } - - CloudbaseInputFormat.setInputInfo(job, username, password.getBytes(), table, new Authorizations(auths)) - job.setInputFormatClass(CloudbaseInputFormat.class); - - // OUTPUT - job.setOutputFormatClass(CloudbaseOutputFormat.class); - job.setOutputKeyClass(Text.class); - job.setOutputValueClass(Mutation.class); - CloudbaseOutputFormat.setOutputInfo(job, username, password.getBytes(), true, outtable); - } - - @Override - def key(byte[] data) { - Key key = new Key(); - key.readFields(new DataInputStream(new ByteArrayInputStream(data))) - return key - } - - @Override - def key(String row, String cf, String cq, String cv, long timestamp) { - return new Key(row, cf, cq, cv, timestamp) - } - - @Override - def value(byte[] data) { - return new Value(data) - } - - @Override - def columnVisibility(String cv) { - return new ColumnVisibility(cv) - } - - @Override - def mutation(String row, String cf, String cq, String cv, long timestamp, byte[] val) { - Mutation mutation = new Mutation(row); - mutation.put(cf, cq, columnVisibility(cv), timestamp, value(val)) - return mutation - } - - @Override - def instance() { - assert conf != null - - String instance_str = conf.get(INSTANCE) - String zookeepers = conf.get(ZOOKEEPERS) - String mock = conf.get(MOCK) - if (Boolean.parseBoolean(mock)) { - return new MockInstance(instance_str) - } else if (zookeepers != null) { - return new ZooKeeperInstance(instance_str, zookeepers) - } else { - throw new IllegalArgumentException("Must specify either mock or zookeepers"); - } - } - - @Override - def connector(def instance) { - if (connector != null) return connector - - String username = conf.get(USERNAME) - String password = conf.get(PASSWORD) - if (instance == null) - instance = instance() - connector = instance.getConnector(username, password) - return connector - } - - @Override - def void writeMutations(def connector, String tableName, Iterator mutations) { - def bw = connector.createBatchWriter(tableName, 10000l, 10000l, 4); - mutations.each { m -> - bw.addMutation(m) - } - bw.flush() - bw.close() - } - - @Override - def scanner(def connector, String tableName, String[] auths) { - return connector.createScanner(tableName, new Authorizations(auths)) - } - - @Override - def batchScanner(def connector, String tableName, String[] auths, int numThreads) { - return connector.createBatchScanner(tableName, new Authorizations(auths), numThreads) - } - - @Override - def range(def startKey, def endKey) { - assert startKey != null - - if (endKey != null) - return new cloudbase.core.data.Range(startKey, endKey) - return new cloudbase.core.data.Range(startKey) - } - - @Override - def authorizations(String[] auths) { - return new Authorizations(auths) - } -} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/80faf06d/extras/generic.mr/generic.mr.cloudbase/src/main/resources/META-INF/services/mvm.rya.generic.mr.api.MRInfo ---------------------------------------------------------------------- diff --git a/extras/generic.mr/generic.mr.cloudbase/src/main/resources/META-INF/services/mvm.rya.generic.mr.api.MRInfo b/extras/generic.mr/generic.mr.cloudbase/src/main/resources/META-INF/services/mvm.rya.generic.mr.api.MRInfo deleted file mode 100644 index 728f9dd..0000000 --- a/extras/generic.mr/generic.mr.cloudbase/src/main/resources/META-INF/services/mvm.rya.generic.mr.api.MRInfo +++ /dev/null @@ -1 +0,0 @@ -mvm.rya.generic.mr.cloudbase.CloudbaseMRInfo \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/80faf06d/extras/generic.mr/pom.xml ---------------------------------------------------------------------- diff --git a/extras/generic.mr/pom.xml b/extras/generic.mr/pom.xml deleted file mode 100644 index 99a541e..0000000 --- a/extras/generic.mr/pom.xml +++ /dev/null @@ -1,27 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - - <parent> - <groupId>mvm.rya</groupId> - <artifactId>rya.extras</artifactId> - <version>3.2.10-SNAPSHOT</version> - </parent> - <modelVersion>4.0.0</modelVersion> - - <artifactId>generic.mr</artifactId> - <name>${project.groupId}.${project.artifactId}</name> - <packaging>pom</packaging> - <modules> - <module>generic.mr.api</module> - <module>generic.mr.accumulo</module> - </modules> - - <profiles> - <profile> - <id>cloudbase</id> - <modules> - <module>generic.mr.cloudbase</module> - </modules> - </profile> - </profiles> -</project> http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/80faf06d/extras/indexing/pom.xml ---------------------------------------------------------------------- diff --git a/extras/indexing/pom.xml b/extras/indexing/pom.xml index 5e6c7b7..f484916 100644 --- a/extras/indexing/pom.xml +++ b/extras/indexing/pom.xml @@ -1,92 +1,85 @@ +<?xml version='1.0'?> + +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + <project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> <parent> - <groupId>mvm.rya</groupId> + <groupId>org.apache.rya</groupId> <artifactId>rya.extras</artifactId> <version>3.2.10-SNAPSHOT</version> </parent> - <modelVersion>4.0.0</modelVersion> - <name>${project.groupId}.${project.artifactId}</name> <artifactId>rya.indexing</artifactId> + <name>Apache Rya Secondary Indexing</name> <dependencies> <dependency> - <groupId>mvm.rya</groupId> - <artifactId>rya.sail.impl</artifactId> + <groupId>org.apache.rya</groupId> + <artifactId>rya.sail</artifactId> <exclusions> - <exclusion> - <artifactId>hsqldb</artifactId> - <groupId>hsqldb</groupId> - </exclusion> + <exclusion> + <artifactId>hsqldb</artifactId> + <groupId>hsqldb</groupId> + </exclusion> </exclusions> </dependency> + <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>mvm.rya</groupId> + <groupId>org.apache.rya</groupId> <artifactId>accumulo.rya</artifactId> </dependency> <dependency> - <groupId>mvm.rya</groupId> + <groupId>org.apache.rya</groupId> <artifactId>mongodb.rya</artifactId> </dependency> <dependency> - <groupId>org.apache.lucene</groupId> - <artifactId>lucene-core</artifactId> - <version>3.6.2</version> + <groupId>org.apache.rya</groupId> + <artifactId>rya.prospector</artifactId> </dependency> + + <!-- Free Text Indexing --> <dependency> <groupId>org.apache.lucene</groupId> - <artifactId>lucene-analyzers</artifactId> - <version>3.6.2</version> + <artifactId>lucene-core</artifactId> </dependency> <dependency> <groupId>org.apache.lucene</groupId> <artifactId>lucene-analyzers</artifactId> - <version>3.6.2</version> </dependency> <dependency> <groupId>commons-codec</groupId> <artifactId>commons-codec</artifactId> - <version>1.4</version> - </dependency> - - <!-- I was having issues with hadoop conf, but adding xerces and xalan fixed it --> - - <dependency> - <groupId>xerces</groupId> - <artifactId>xercesImpl</artifactId> - <version>2.9.1</version> - </dependency> - <dependency> - <groupId>xalan</groupId> - <artifactId>xalan</artifactId> - <version>2.7.1</version> </dependency> <!-- Geo Indexing --> <dependency> <groupId>org.locationtech.geomesa</groupId> <artifactId>geomesa-accumulo-datastore</artifactId> - <version>${geomesa.version}</version> - </dependency> - <dependency> - <groupId>mvm.rya</groupId> - <artifactId>rya.prospector</artifactId> </dependency> - <dependency> - <groupId>org.apache.accumulo</groupId> - <artifactId>accumulo-core</artifactId> - </dependency> - <dependency> - <groupId>org.mongodb</groupId> - <artifactId>mongo-java-driver</artifactId> - <version>2.13.0-rc0</version> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> </dependency> </dependencies> <build> @@ -97,36 +90,30 @@ <executions> <execution> <configuration> - <shadedArtifactAttached>true</shadedArtifactAttached> + <shadedArtifactAttached>true</shadedArtifactAttached> <shadedClassifierName>map-reduce</shadedClassifierName> <transformers> <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" /> </transformers> </configuration> </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <executions> - <execution> + <execution> <id>accumulo-server</id> <phase>package</phase> <goals> <goal>shade</goal> </goals> <configuration> - <shadedArtifactAttached>true</shadedArtifactAttached> + <shadedArtifactAttached>true</shadedArtifactAttached> <shadedClassifierName>accumulo-server</shadedClassifierName> <artifactSet> <excludes> - <exclude>org.locationtech.geomesa:*</exclude> - <exclude>scala:*</exclude> - <exclude>org.apache.accumulo:*</exclude> - <exclude>org.apache.thrift:*</exclude> - <exclude>org.apache.hadoop:*</exclude> - <exclude>org.apache.zookeeper:*</exclude> + <exclude>org.locationtech.geomesa:*</exclude> + <exclude>scala:*</exclude> + <exclude>org.apache.accumulo:*</exclude> + <exclude>org.apache.thrift:*</exclude> + <exclude>org.apache.hadoop:*</exclude> + <exclude>org.apache.zookeeper:*</exclude> </excludes> </artifactSet> <transformers> @@ -138,64 +125,4 @@ </plugin> </plugins> </build> - <profiles> - <profile> - <id>accumulo</id> - <activation> - <activeByDefault>true</activeByDefault> - </activation> - <dependencies> - <dependency> - <groupId>org.apache.accumulo</groupId> - <artifactId>accumulo-core</artifactId> - <optional>true</optional> - </dependency> - <dependency> - <groupId>mvm.rya</groupId> - <artifactId>accumulo.iterators</artifactId> - <optional>true</optional> - </dependency> - </dependencies> - </profile> - <profile> - <id>cloudbase</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <dependencies> - <dependency> - <groupId>com.texeltek</groupId> - <artifactId>accumulo-cloudbase-shim</artifactId> - <optional>true</optional> - </dependency> - <dependency> - <groupId>mvm.rya</groupId> - <artifactId>cloudbase.iterators</artifactId> - <optional>true</optional> - </dependency> - </dependencies> - </profile> - <profile> - <id>mr</id> - <activation> - <activeByDefault>true</activeByDefault> - </activation> - </profile> - </profiles> - - - <repositories> - <repository> - <id>cloudera</id> - <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url> - </repository> - <repository> - <id>public.opensahara.com</id> - <url>http://dev.opensahara.com/nexus/content/groups/public/</url> - </repository> - <repository> - <id>geotools</id> - <url>http://download.osgeo.org/webdav/geotools/</url> - </repository> - </repositories> -</project> \ No newline at end of file +</project> http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/80faf06d/extras/indexing/src/main/java/mvm/rya/accumulo/documentIndex/DocIndexIteratorUtil.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/accumulo/documentIndex/DocIndexIteratorUtil.java b/extras/indexing/src/main/java/mvm/rya/accumulo/documentIndex/DocIndexIteratorUtil.java index b6063ca..fefd651 100644 --- a/extras/indexing/src/main/java/mvm/rya/accumulo/documentIndex/DocIndexIteratorUtil.java +++ b/extras/indexing/src/main/java/mvm/rya/accumulo/documentIndex/DocIndexIteratorUtil.java @@ -1,5 +1,25 @@ package mvm.rya.accumulo.documentIndex; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + public class DocIndexIteratorUtil {
