[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[ https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319591#comment-16319591 ] ASF GitHub Bot commented on RYA-303: Github user asfgit commented on the issue: https://github.com/apache/incubator-rya/pull/172 Refer to this link for build results (access rights to CI server needed): https://builds.apache.org/job/incubator-rya-master-with-optionals-pull-requests/647/ > Mongo PCJ indexer support > - > > Key: RYA-303 > URL: https://issues.apache.org/jira/browse/RYA-303 > Project: Rya > Issue Type: Improvement >Reporter: Andrew Smith >Assignee: Andrew Smith > -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[GitHub] incubator-rya issue #172: RYA-303 Mongo PCJ Support
Github user asfgit commented on the issue: https://github.com/apache/incubator-rya/pull/172 Refer to this link for build results (access rights to CI server needed): https://builds.apache.org/job/incubator-rya-master-with-optionals-pull-requests/647/ ---
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[ https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319584#comment-16319584 ] ASF GitHub Bot commented on RYA-303: Github user kchilton2 commented on the issue: https://github.com/apache/incubator-rya/pull/172 It's part of the shell. You need to update it since we've changed what we support via the shell in this review. > Mongo PCJ indexer support > - > > Key: RYA-303 > URL: https://issues.apache.org/jira/browse/RYA-303 > Project: Rya > Issue Type: Improvement >Reporter: Andrew Smith >Assignee: Andrew Smith > -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[GitHub] incubator-rya issue #172: RYA-303 Mongo PCJ Support
Github user kchilton2 commented on the issue: https://github.com/apache/incubator-rya/pull/172 It's part of the shell. You need to update it since we've changed what we support via the shell in this review. ---
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319583#comment-16319583
]
ASF GitHub Bot commented on RYA-303:
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160575832
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
+
+private final MongoCollection pcjCollection;
+private static final PcjVarOrderFactory pcjVarOrderFactory = new
ShiftVarOrderFactory();
+
+/**
+ * Creates a new {@link MongoPcjDocuments}.
+ * @param client - The {@link
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160575832
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
+
+private final MongoCollection pcjCollection;
+private static final PcjVarOrderFactory pcjVarOrderFactory = new
ShiftVarOrderFactory();
+
+/**
+ * Creates a new {@link MongoPcjDocuments}.
+ * @param client - The {@link MongoClient} to use to connect to mongo.
+ * @param ryaInstanceName - The rya instance to connect to.
+ */
+public MongoPcjDocuments(final MongoClient client, final String
ryaInstanceName) {
+
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319582#comment-16319582
]
ASF GitHub Bot commented on RYA-303:
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160575734
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
--- End diff --
What is a PCJ name? Did you introduce that idea to PCJ within this review?
> Mongo PCJ indexer support
> -
>
> Key: RYA-303
> URL: https://issues.apache.org/jira/browse/RYA-303
>
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319579#comment-16319579
]
ASF GitHub Bot commented on RYA-303:
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160575656
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
--- End diff --
I thought the square brackets were to to indicate a value needs to be
filled in. The type uri wouldn't be a list or set.
> Mongo PCJ indexer support
> -
>
> Key: RYA-303
> URL: https://issues.apache.org/jira/browse/RYA-303
> Project: Rya
> Issue Type: Improvement
>Reporter: Andrew Smith
>Assignee: Andrew Smith
>
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160575734
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
--- End diff --
What is a PCJ name? Did you introduce that idea to PCJ within this review?
---
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160575656
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
--- End diff --
I thought the square brackets were to to indicate a value needs to be
filled in. The type uri wouldn't be a list or set.
---
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160575498
--- Diff:
extras/indexing/src/main/java/org/apache/rya/indexing/mongodb/pcj/MongoPcjIndexSetProvider.java
---
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.mongodb.pcj;
+
+import static java.util.Objects.requireNonNull;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.rya.api.instance.RyaDetailsRepository;
+import
org.apache.rya.api.instance.RyaDetailsRepository.RyaDetailsRepositoryException;
+import org.apache.rya.indexing.external.tupleSet.ExternalTupleSet;
+import
org.apache.rya.indexing.pcj.matching.provider.AbstractPcjIndexSetProvider;
+import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.mongo.MongoPcjDocuments;
+import org.apache.rya.indexing.pcj.storage.mongo.MongoPcjStorage;
+import org.apache.rya.mongodb.StatefulMongoDBRdfConfiguration;
+import org.apache.rya.mongodb.instance.MongoRyaInstanceDetailsRepository;
+import org.openrdf.query.MalformedQueryException;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.mongodb.MongoClient;
+
+/**
+ * Implementation of {@link AbstractPcjIndexSetProvider} for MongoDB.
+ */
+public class MongoPcjIndexSetProvider extends AbstractPcjIndexSetProvider {
+/**
+ * Creates a new {@link MongoPcjIndexSetProvider}.
+ * @param conf - The configuration for this provider. (not null)
+ */
+public MongoPcjIndexSetProvider(final StatefulMongoDBRdfConfiguration
conf) {
+super(conf);
+}
+
+/**
+ * Creates a new {@link MongoPcjIndexSetProvider}.
+ * @param conf - The configuration for this provider.
+ * @param indices - The predefined indicies on this provider.
+ * @param client - The {@link MongoClient} used to connect to mongo.
+ */
+public MongoPcjIndexSetProvider(final StatefulMongoDBRdfConfiguration
conf, final List indices) {
+super(conf, indices);
+}
+
+@Override
+protected List getIndices() throws
PcjIndexSetException {
+requireNonNull(conf);
+try {
+final StatefulMongoDBRdfConfiguration mongoConf =
(StatefulMongoDBRdfConfiguration) conf;
+final MongoClient client = mongoConf.getMongoClient();
+final MongoPcjDocuments pcjDocs = new
MongoPcjDocuments(client, mongoConf.getRyaInstanceName());
+List tables = null;
--- End diff --
The mongo side is significantly different from the accumulo side when it
comes to PCJ. A bunch of the stuff that was in there has to do with legacy
compatibility.
---
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319577#comment-16319577
]
ASF GitHub Bot commented on RYA-303:
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160575498
--- Diff:
extras/indexing/src/main/java/org/apache/rya/indexing/mongodb/pcj/MongoPcjIndexSetProvider.java
---
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.mongodb.pcj;
+
+import static java.util.Objects.requireNonNull;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.rya.api.instance.RyaDetailsRepository;
+import
org.apache.rya.api.instance.RyaDetailsRepository.RyaDetailsRepositoryException;
+import org.apache.rya.indexing.external.tupleSet.ExternalTupleSet;
+import
org.apache.rya.indexing.pcj.matching.provider.AbstractPcjIndexSetProvider;
+import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.mongo.MongoPcjDocuments;
+import org.apache.rya.indexing.pcj.storage.mongo.MongoPcjStorage;
+import org.apache.rya.mongodb.StatefulMongoDBRdfConfiguration;
+import org.apache.rya.mongodb.instance.MongoRyaInstanceDetailsRepository;
+import org.openrdf.query.MalformedQueryException;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.mongodb.MongoClient;
+
+/**
+ * Implementation of {@link AbstractPcjIndexSetProvider} for MongoDB.
+ */
+public class MongoPcjIndexSetProvider extends AbstractPcjIndexSetProvider {
+/**
+ * Creates a new {@link MongoPcjIndexSetProvider}.
+ * @param conf - The configuration for this provider. (not null)
+ */
+public MongoPcjIndexSetProvider(final StatefulMongoDBRdfConfiguration
conf) {
+super(conf);
+}
+
+/**
+ * Creates a new {@link MongoPcjIndexSetProvider}.
+ * @param conf - The configuration for this provider.
+ * @param indices - The predefined indicies on this provider.
+ * @param client - The {@link MongoClient} used to connect to mongo.
+ */
+public MongoPcjIndexSetProvider(final StatefulMongoDBRdfConfiguration
conf, final List indices) {
+super(conf, indices);
+}
+
+@Override
+protected List getIndices() throws
PcjIndexSetException {
+requireNonNull(conf);
+try {
+final StatefulMongoDBRdfConfiguration mongoConf =
(StatefulMongoDBRdfConfiguration) conf;
+final MongoClient client = mongoConf.getMongoClient();
+final MongoPcjDocuments pcjDocs = new
MongoPcjDocuments(client, mongoConf.getRyaInstanceName());
+List tables = null;
--- End diff --
The mongo side is significantly different from the accumulo side when it
comes to PCJ. A bunch of the stuff that was in there has to do with legacy
compatibility.
> Mongo PCJ indexer support
> -
>
> Key: RYA-303
> URL: https://issues.apache.org/jira/browse/RYA-303
> Project: Rya
> Issue Type: Improvement
>Reporter: Andrew Smith
>Assignee: Andrew Smith
>
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
[jira] [Commented] (RYA-428) Mongo Shell: Can not load statements file.
[
https://issues.apache.org/jira/browse/RYA-428?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319530#comment-16319530
]
ASF GitHub Bot commented on RYA-428:
Github user asfgit commented on the issue:
https://github.com/apache/incubator-rya/pull/262
Refer to this link for build results (access rights to CI server needed):
https://builds.apache.org/job/incubator-rya-master-with-optionals-pull-requests/646/
> Mongo Shell: Can not load statements file.
> --
>
> Key: RYA-428
> URL: https://issues.apache.org/jira/browse/RYA-428
> Project: Rya
> Issue Type: Bug
>Affects Versions: 3.2.12
> Environment: Executing the shaded shell jar connecting to Mongo DB
> running on my local machine.
>Reporter: Kevin Chilton
>Assignee: Kevin Chilton
> Fix For: 3.2.12
>
> Attachments: test1.nt
>
>
> While using the Rya shell, if you connect to an instance of Mongo DB and try
> to load a statements file using the following command:
> {code}
> rya/localhost:test> load-data --file ~/test1.nt
> {code}
> Then you will see the following output:
> {code}
> Detected RDF Format: N-Triples (mimeTypes=text/plain; ext=nt)
> Command failed java.lang.RuntimeException: Can not load the RDF Statement
> data. Reason: Could not load the statements into Rya.
> {code}
> See the attached test1.nt file for the data that was used while executing
> this test.
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
[GitHub] incubator-rya issue #262: RYA-428 Made the Rya Shell's load statements file ...
Github user asfgit commented on the issue: https://github.com/apache/incubator-rya/pull/262 Refer to this link for build results (access rights to CI server needed): https://builds.apache.org/job/incubator-rya-master-with-optionals-pull-requests/646/ ---
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[ https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319528#comment-16319528 ] ASF GitHub Bot commented on RYA-303: Github user isper3at commented on the issue: https://github.com/apache/incubator-rya/pull/172 what is RyaDetailsFormatter. this is more of a problem of just stuff floating around that doesn't break when functionality gets added > Mongo PCJ indexer support > - > > Key: RYA-303 > URL: https://issues.apache.org/jira/browse/RYA-303 > Project: Rya > Issue Type: Improvement >Reporter: Andrew Smith >Assignee: Andrew Smith > -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user isper3at commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160570620
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
+
+private final MongoCollection pcjCollection;
+private static final PcjVarOrderFactory pcjVarOrderFactory = new
ShiftVarOrderFactory();
+
+/**
+ * Creates a new {@link MongoPcjDocuments}.
+ * @param client - The {@link MongoClient} to use to connect to mongo.
+ * @param ryaInstanceName - The rya instance to connect to.
+ */
+public MongoPcjDocuments(final MongoClient client, final String
ryaInstanceName) {
+
[GitHub] incubator-rya issue #172: RYA-303 Mongo PCJ Support
Github user isper3at commented on the issue: https://github.com/apache/incubator-rya/pull/172 what is RyaDetailsFormatter. this is more of a problem of just stuff floating around that doesn't break when functionality gets added ---
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319527#comment-16319527
]
ASF GitHub Bot commented on RYA-303:
Github user isper3at commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160570620
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
+
+private final MongoCollection pcjCollection;
+private static final PcjVarOrderFactory pcjVarOrderFactory = new
ShiftVarOrderFactory();
+
+/**
+ * Creates a new {@link MongoPcjDocuments}.
+ * @param client - The {@link
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319526#comment-16319526
]
ASF GitHub Bot commented on RYA-303:
Github user isper3at commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160570534
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
+
+private final MongoCollection pcjCollection;
+private static final PcjVarOrderFactory pcjVarOrderFactory = new
ShiftVarOrderFactory();
+
+/**
+ * Creates a new {@link MongoPcjDocuments}.
+ * @param client - The {@link
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user isper3at commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160570534
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
+
+private final MongoCollection pcjCollection;
+private static final PcjVarOrderFactory pcjVarOrderFactory = new
ShiftVarOrderFactory();
+
+/**
+ * Creates a new {@link MongoPcjDocuments}.
+ * @param client - The {@link MongoClient} to use to connect to mongo.
+ * @param ryaInstanceName - The rya instance to connect to.
+ */
+public MongoPcjDocuments(final MongoClient client, final String
ryaInstanceName) {
+
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319525#comment-16319525
]
ASF GitHub Bot commented on RYA-303:
Github user isper3at commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160570420
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
--- End diff --
no, its for the pcj name. the ID is calculated differently
> Mongo PCJ indexer support
> -
>
> Key: RYA-303
> URL: https://issues.apache.org/jira/browse/RYA-303
>
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319524#comment-16319524
]
ASF GitHub Bot commented on RYA-303:
Github user isper3at commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160570327
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
--- End diff --
done
> Mongo PCJ indexer support
> -
>
> Key: RYA-303
> URL: https://issues.apache.org/jira/browse/RYA-303
> Project: Rya
> Issue Type: Improvement
>Reporter: Andrew Smith
>Assignee: Andrew Smith
>
--
This message was sent by Atlassian JIRA
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user isper3at commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160570420
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
--- End diff --
no, its for the pcj name. the ID is calculated differently
---
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319523#comment-16319523
]
ASF GitHub Bot commented on RYA-303:
Github user isper3at commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160570226
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
--- End diff --
done
> Mongo PCJ indexer support
> -
>
> Key: RYA-303
> URL: https://issues.apache.org/jira/browse/RYA-303
> Project: Rya
> Issue Type: Improvement
>Reporter: Andrew Smith
>Assignee: Andrew Smith
>
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user isper3at commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160570327
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
--- End diff --
done
---
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319522#comment-16319522
]
ASF GitHub Bot commented on RYA-303:
Github user isper3at commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160570170
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
--- End diff --
nvm. didn't look right
> Mongo PCJ indexer support
> -
>
> Key: RYA-303
> URL: https://issues.apache.org/jira/browse/RYA-303
> Project: Rya
> Issue Type: Improvement
>Reporter: Andrew Smith
>Assignee: Andrew Smith
>
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user isper3at commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160570226
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
--- End diff --
done
---
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user isper3at commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160570170
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
--- End diff --
nvm. didn't look right
---
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319520#comment-16319520
]
ASF GitHub Bot commented on RYA-303:
Github user isper3at commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160570068
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
--- End diff --
well square brackets denotes list/set so really the docs should not
have them
> Mongo PCJ indexer support
> -
>
> Key: RYA-303
> URL: https://issues.apache.org/jira/browse/RYA-303
> Project: Rya
> Issue Type: Improvement
>Reporter: Andrew Smith
>Assignee: Andrew Smith
>
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user isper3at commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160570068
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
--- End diff --
well square brackets denotes list/set so really the docs should not
have them
---
[jira] [Commented] (RYA-410) Add geo profile dependency management to the rya-project pom file.
[ https://issues.apache.org/jira/browse/RYA-410?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319514#comment-16319514 ] ASF GitHub Bot commented on RYA-410: Github user asfgit commented on the issue: https://github.com/apache/incubator-rya/pull/252 Refer to this link for build results (access rights to CI server needed): https://builds.apache.org/job/incubator-rya-master-with-optionals-pull-requests/645/ > Add geo profile dependency management to the rya-project pom file. > -- > > Key: RYA-410 > URL: https://issues.apache.org/jira/browse/RYA-410 > Project: Rya > Issue Type: Improvement >Reporter: Kevin Chilton >Assignee: Kevin Chilton > > Right now the geo indexing feature of Rya is an optional portion of Rya that > is not built by default because it is not compatible with the Apache license. > However, geo dependencies are declared at the top of the project as if it all > builds depend on them. The top level pom file needs to be updated to include > the geoindexing profile and all artifacts related to that build need to be > moved to that profile's dependency management section. -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[GitHub] incubator-rya issue #252: RYA-410 Add geo profile dependency management to t...
Github user asfgit commented on the issue: https://github.com/apache/incubator-rya/pull/252 Refer to this link for build results (access rights to CI server needed): https://builds.apache.org/job/incubator-rya-master-with-optionals-pull-requests/645/ ---
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[ https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319513#comment-16319513 ] ASF GitHub Bot commented on RYA-303: Github user isper3at commented on a diff in the pull request: https://github.com/apache/incubator-rya/pull/172#discussion_r160569466 --- Diff: extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java --- @@ -0,0 +1,438 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.indexing.pcj.storage.mongo; + +import static com.google.common.base.Preconditions.checkNotNull; +import static java.util.Objects.requireNonNull; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +import org.apache.accumulo.core.security.Authorizations; +import org.apache.rya.api.domain.RyaType; +import org.apache.rya.api.resolver.RdfToRyaConversions; +import org.apache.rya.api.resolver.RyaToRdfConversions; +import org.apache.rya.indexing.pcj.storage.PcjMetadata; +import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator; +import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException; +import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory; +import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory; +import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder; +import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.openrdf.model.URI; +import org.openrdf.model.Value; +import org.openrdf.model.impl.URIImpl; +import org.openrdf.query.BindingSet; +import org.openrdf.query.MalformedQueryException; +import org.openrdf.query.QueryEvaluationException; +import org.openrdf.query.QueryLanguage; +import org.openrdf.query.TupleQuery; +import org.openrdf.query.TupleQueryResult; +import org.openrdf.query.impl.MapBindingSet; +import org.openrdf.repository.RepositoryConnection; +import org.openrdf.repository.RepositoryException; + +import com.mongodb.MongoClient; +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoCollection; +import com.mongodb.util.JSON; + +/** + * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows: + * + * + * + * - PCJ Metadata Doc - --- End diff -- freaking.git dropping more stuff. I need to figure out what I did and why it dropped everything > Mongo PCJ indexer support > - > > Key: RYA-303 > URL: https://issues.apache.org/jira/browse/RYA-303 > Project: Rya > Issue Type: Improvement >Reporter: Andrew Smith >Assignee: Andrew Smith > -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319512#comment-16319512
]
ASF GitHub Bot commented on RYA-303:
Github user isper3at commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160569265
--- Diff:
extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstallIT.java
---
@@ -41,7 +41,7 @@
*/
public class MongoInstallIT extends MongoITBase {
-@Test
+ @Test
--- End diff --
done
> Mongo PCJ indexer support
> -
>
> Key: RYA-303
> URL: https://issues.apache.org/jira/browse/RYA-303
> Project: Rya
> Issue Type: Improvement
>Reporter: Andrew Smith
>Assignee: Andrew Smith
>
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user isper3at commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160569220
--- Diff:
extras/indexing/src/main/java/org/apache/rya/indexing/pcj/matching/provider/AccumuloIndexSetProvider.java
---
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.matching.provider;
+
+import static java.util.Objects.requireNonNull;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.log4j.Logger;
+import
org.apache.rya.accumulo.instance.AccumuloRyaInstanceDetailsRepository;
+import org.apache.rya.api.RdfCloudTripleStoreConfiguration;
+import org.apache.rya.api.instance.RyaDetailsRepository;
+import
org.apache.rya.api.instance.RyaDetailsRepository.RyaDetailsRepositoryException;
+import org.apache.rya.indexing.accumulo.ConfigUtils;
+import org.apache.rya.indexing.external.tupleSet.AccumuloIndexSet;
+import org.apache.rya.indexing.external.tupleSet.ExternalTupleSet;
+import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjTableNameFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjTables;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.sail.SailException;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+/**
+ * Implementation of {@link AbstractPcjIndexSetProvider} for Accumulo.
+ * This provider uses either user specified Accumulo configuration
information or user a specified
+ * List of ExternalTupleSets to populate an internal cache of
ExternalTupleSets. If Accumulo configuration
+ * is provided, the provider connects to an instance of RyaDetails and
populates the cache with
+ * PCJs registered in RyaDetails.
+ */
+public class AccumuloIndexSetProvider extends AbstractPcjIndexSetProvider {
+private static final Logger log =
Logger.getLogger(AccumuloIndexSetProvider.class);
+
+public AccumuloIndexSetProvider(final Configuration conf) {
+super(conf);
+}
+
+public AccumuloIndexSetProvider(final Configuration conf, final
List indices) {
+super(conf, indices);
+}
+
+@Override
+protected List getIndices() throws
PcjIndexSetException {
+requireNonNull(conf);
+try {
+final String tablePrefix =
requireNonNull(conf.get(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX));
+final Connector conn =
requireNonNull(ConfigUtils.getConnector(conf));
+List tables = null;
+
+if (conf instanceof RdfCloudTripleStoreConfiguration) {
+tables = ((RdfCloudTripleStoreConfiguration)
conf).getPcjTables();
+}
+// this maps associates pcj table name with pcj sparql query
+final Map indexTables =
Maps.newLinkedHashMap();
+
+try(final PrecomputedJoinStorage storage = new
AccumuloPcjStorage(conn, tablePrefix)) {
+final PcjTableNameFactory pcjFactory = new
PcjTableNameFactory();
+
+final boolean tablesProvided = tables != null &&
!tables.isEmpty();
+
+if (tablesProvided) {
--- End diff --
I'm not touching accumulo code. this is just a repackage
---
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319509#comment-16319509
]
ASF GitHub Bot commented on RYA-303:
Github user isper3at commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160569137
--- Diff:
extras/indexing/src/main/java/org/apache/rya/indexing/mongodb/pcj/MongoPcjIndexSetProvider.java
---
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.mongodb.pcj;
+
+import static java.util.Objects.requireNonNull;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.rya.api.instance.RyaDetailsRepository;
+import
org.apache.rya.api.instance.RyaDetailsRepository.RyaDetailsRepositoryException;
+import org.apache.rya.indexing.external.tupleSet.ExternalTupleSet;
+import
org.apache.rya.indexing.pcj.matching.provider.AbstractPcjIndexSetProvider;
+import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.mongo.MongoPcjDocuments;
+import org.apache.rya.indexing.pcj.storage.mongo.MongoPcjStorage;
+import org.apache.rya.mongodb.StatefulMongoDBRdfConfiguration;
+import org.apache.rya.mongodb.instance.MongoRyaInstanceDetailsRepository;
+import org.openrdf.query.MalformedQueryException;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.mongodb.MongoClient;
+
+/**
+ * Implementation of {@link AbstractPcjIndexSetProvider} for MongoDB.
+ */
+public class MongoPcjIndexSetProvider extends AbstractPcjIndexSetProvider {
+/**
+ * Creates a new {@link MongoPcjIndexSetProvider}.
+ * @param conf - The configuration for this provider. (not null)
+ */
+public MongoPcjIndexSetProvider(final StatefulMongoDBRdfConfiguration
conf) {
+super(conf);
+}
+
+/**
+ * Creates a new {@link MongoPcjIndexSetProvider}.
+ * @param conf - The configuration for this provider.
+ * @param indices - The predefined indicies on this provider.
+ * @param client - The {@link MongoClient} used to connect to mongo.
+ */
+public MongoPcjIndexSetProvider(final StatefulMongoDBRdfConfiguration
conf, final List indices) {
+super(conf, indices);
+}
+
+@Override
+protected List getIndices() throws
PcjIndexSetException {
+requireNonNull(conf);
+try {
+final StatefulMongoDBRdfConfiguration mongoConf =
(StatefulMongoDBRdfConfiguration) conf;
+final MongoClient client = mongoConf.getMongoClient();
+final MongoPcjDocuments pcjDocs = new
MongoPcjDocuments(client, mongoConf.getRyaInstanceName());
+List tables = null;
--- End diff --
there should really only be the PCJ collection. I may take a look at just
cleaning this up to use the PCJ collection
> Mongo PCJ indexer support
> -
>
> Key: RYA-303
> URL: https://issues.apache.org/jira/browse/RYA-303
> Project: Rya
> Issue Type: Improvement
>Reporter: Andrew Smith
>Assignee: Andrew Smith
>
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user isper3at commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160569137
--- Diff:
extras/indexing/src/main/java/org/apache/rya/indexing/mongodb/pcj/MongoPcjIndexSetProvider.java
---
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.mongodb.pcj;
+
+import static java.util.Objects.requireNonNull;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.rya.api.instance.RyaDetailsRepository;
+import
org.apache.rya.api.instance.RyaDetailsRepository.RyaDetailsRepositoryException;
+import org.apache.rya.indexing.external.tupleSet.ExternalTupleSet;
+import
org.apache.rya.indexing.pcj.matching.provider.AbstractPcjIndexSetProvider;
+import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.mongo.MongoPcjDocuments;
+import org.apache.rya.indexing.pcj.storage.mongo.MongoPcjStorage;
+import org.apache.rya.mongodb.StatefulMongoDBRdfConfiguration;
+import org.apache.rya.mongodb.instance.MongoRyaInstanceDetailsRepository;
+import org.openrdf.query.MalformedQueryException;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.mongodb.MongoClient;
+
+/**
+ * Implementation of {@link AbstractPcjIndexSetProvider} for MongoDB.
+ */
+public class MongoPcjIndexSetProvider extends AbstractPcjIndexSetProvider {
+/**
+ * Creates a new {@link MongoPcjIndexSetProvider}.
+ * @param conf - The configuration for this provider. (not null)
+ */
+public MongoPcjIndexSetProvider(final StatefulMongoDBRdfConfiguration
conf) {
+super(conf);
+}
+
+/**
+ * Creates a new {@link MongoPcjIndexSetProvider}.
+ * @param conf - The configuration for this provider.
+ * @param indices - The predefined indicies on this provider.
+ * @param client - The {@link MongoClient} used to connect to mongo.
+ */
+public MongoPcjIndexSetProvider(final StatefulMongoDBRdfConfiguration
conf, final List indices) {
+super(conf, indices);
+}
+
+@Override
+protected List getIndices() throws
PcjIndexSetException {
+requireNonNull(conf);
+try {
+final StatefulMongoDBRdfConfiguration mongoConf =
(StatefulMongoDBRdfConfiguration) conf;
+final MongoClient client = mongoConf.getMongoClient();
+final MongoPcjDocuments pcjDocs = new
MongoPcjDocuments(client, mongoConf.getRyaInstanceName());
+List tables = null;
--- End diff --
there should really only be the PCJ collection. I may take a look at just
cleaning this up to use the PCJ collection
---
[jira] [Commented] (RYA-410) Add geo profile dependency management to the rya-project pom file.
[ https://issues.apache.org/jira/browse/RYA-410?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319498#comment-16319498 ] ASF GitHub Bot commented on RYA-410: Github user asfgit commented on the issue: https://github.com/apache/incubator-rya/pull/252 Refer to this link for build results (access rights to CI server needed): https://builds.apache.org/job/incubator-rya-master-with-optionals-pull-requests/644/ > Add geo profile dependency management to the rya-project pom file. > -- > > Key: RYA-410 > URL: https://issues.apache.org/jira/browse/RYA-410 > Project: Rya > Issue Type: Improvement >Reporter: Kevin Chilton >Assignee: Kevin Chilton > > Right now the geo indexing feature of Rya is an optional portion of Rya that > is not built by default because it is not compatible with the Apache license. > However, geo dependencies are declared at the top of the project as if it all > builds depend on them. The top level pom file needs to be updated to include > the geoindexing profile and all artifacts related to that build need to be > moved to that profile's dependency management section. -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[GitHub] incubator-rya issue #252: RYA-410 Add geo profile dependency management to t...
Github user asfgit commented on the issue: https://github.com/apache/incubator-rya/pull/252 Refer to this link for build results (access rights to CI server needed): https://builds.apache.org/job/incubator-rya-master-with-optionals-pull-requests/644/ ---
[GitHub] incubator-rya issue #252: RYA-410 Add geo profile dependency management to t...
Github user asfgit commented on the issue: https://github.com/apache/incubator-rya/pull/252 Refer to this link for build results (access rights to CI server needed): https://builds.apache.org/job/incubator-rya-master-with-optionals-pull-requests/643/ ---
[jira] [Commented] (RYA-410) Add geo profile dependency management to the rya-project pom file.
[ https://issues.apache.org/jira/browse/RYA-410?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319479#comment-16319479 ] ASF GitHub Bot commented on RYA-410: Github user asfgit commented on the issue: https://github.com/apache/incubator-rya/pull/252 Refer to this link for build results (access rights to CI server needed): https://builds.apache.org/job/incubator-rya-master-with-optionals-pull-requests/643/ > Add geo profile dependency management to the rya-project pom file. > -- > > Key: RYA-410 > URL: https://issues.apache.org/jira/browse/RYA-410 > Project: Rya > Issue Type: Improvement >Reporter: Kevin Chilton >Assignee: Kevin Chilton > > Right now the geo indexing feature of Rya is an optional portion of Rya that > is not built by default because it is not compatible with the Apache license. > However, geo dependencies are declared at the top of the project as if it all > builds depend on them. The top level pom file needs to be updated to include > the geoindexing profile and all artifacts related to that build need to be > moved to that profile's dependency management section. -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[jira] [Commented] (RYA-410) Add geo profile dependency management to the rya-project pom file.
[ https://issues.apache.org/jira/browse/RYA-410?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319458#comment-16319458 ] ASF GitHub Bot commented on RYA-410: Github user kchilton2 commented on a diff in the pull request: https://github.com/apache/incubator-rya/pull/252#discussion_r160563035 --- Diff: pom.xml --- @@ -144,7 +141,83 @@ under the License. true + + +geoindexing + +1.3.0-m1 +0.9.3 +1.13 + + --- End diff -- I'm not worried about the eclipse problem since as you said, you can trivially tell eclipse to resolve using the geoindexing profile. The transitive dependency problem exists regardless of how dependency management is defined since the management section only defines dependencies that CAN be used, not the dependencies that ARE used. Actual dependencies are required to change that. If we're pulling down different transitive dependencies between geo on and geo off, that's already happening sans this patch. > Add geo profile dependency management to the rya-project pom file. > -- > > Key: RYA-410 > URL: https://issues.apache.org/jira/browse/RYA-410 > Project: Rya > Issue Type: Improvement >Reporter: Kevin Chilton >Assignee: Kevin Chilton > > Right now the geo indexing feature of Rya is an optional portion of Rya that > is not built by default because it is not compatible with the Apache license. > However, geo dependencies are declared at the top of the project as if it all > builds depend on them. The top level pom file needs to be updated to include > the geoindexing profile and all artifacts related to that build need to be > moved to that profile's dependency management section. -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[GitHub] incubator-rya pull request #252: RYA-410 Add geo profile dependency manageme...
Github user kchilton2 commented on a diff in the pull request: https://github.com/apache/incubator-rya/pull/252#discussion_r160563035 --- Diff: pom.xml --- @@ -144,7 +141,83 @@ under the License. true + + +geoindexing + +1.3.0-m1 +0.9.3 +1.13 + + --- End diff -- I'm not worried about the eclipse problem since as you said, you can trivially tell eclipse to resolve using the geoindexing profile. The transitive dependency problem exists regardless of how dependency management is defined since the management section only defines dependencies that CAN be used, not the dependencies that ARE used. Actual dependencies are required to change that. If we're pulling down different transitive dependencies between geo on and geo off, that's already happening sans this patch. ---
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319453#comment-16319453
]
ASF GitHub Bot commented on RYA-303:
Github user isper3at commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160562616
--- Diff:
extras/indexing/src/main/java/org/apache/rya/indexing/mongodb/pcj/MongoPcjIndexSetProvider.java
---
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.mongodb.pcj;
+
+import static java.util.Objects.requireNonNull;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.rya.api.instance.RyaDetailsRepository;
+import
org.apache.rya.api.instance.RyaDetailsRepository.RyaDetailsRepositoryException;
+import org.apache.rya.indexing.external.tupleSet.ExternalTupleSet;
+import
org.apache.rya.indexing.pcj.matching.provider.AbstractPcjIndexSetProvider;
+import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.mongo.MongoPcjDocuments;
+import org.apache.rya.indexing.pcj.storage.mongo.MongoPcjStorage;
+import org.apache.rya.mongodb.StatefulMongoDBRdfConfiguration;
+import org.apache.rya.mongodb.instance.MongoRyaInstanceDetailsRepository;
+import org.openrdf.query.MalformedQueryException;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.mongodb.MongoClient;
+
+/**
+ * Implementation of {@link AbstractPcjIndexSetProvider} for MongoDB.
+ */
+public class MongoPcjIndexSetProvider extends AbstractPcjIndexSetProvider {
+/**
+ * Creates a new {@link MongoPcjIndexSetProvider}.
+ * @param conf - The configuration for this provider. (not null)
+ */
+public MongoPcjIndexSetProvider(final StatefulMongoDBRdfConfiguration
conf) {
+super(conf);
+}
+
+/**
+ * Creates a new {@link MongoPcjIndexSetProvider}.
+ * @param conf - The configuration for this provider.
+ * @param indices - The predefined indicies on this provider.
+ * @param client - The {@link MongoClient} used to connect to mongo.
+ */
+public MongoPcjIndexSetProvider(final StatefulMongoDBRdfConfiguration
conf, final List indices) {
+super(conf, indices);
+}
+
+@Override
+protected List getIndices() throws
PcjIndexSetException {
+requireNonNull(conf);
--- End diff --
probably should
> Mongo PCJ indexer support
> -
>
> Key: RYA-303
> URL: https://issues.apache.org/jira/browse/RYA-303
> Project: Rya
> Issue Type: Improvement
>Reporter: Andrew Smith
>Assignee: Andrew Smith
>
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user isper3at commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160562616
--- Diff:
extras/indexing/src/main/java/org/apache/rya/indexing/mongodb/pcj/MongoPcjIndexSetProvider.java
---
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.mongodb.pcj;
+
+import static java.util.Objects.requireNonNull;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.rya.api.instance.RyaDetailsRepository;
+import
org.apache.rya.api.instance.RyaDetailsRepository.RyaDetailsRepositoryException;
+import org.apache.rya.indexing.external.tupleSet.ExternalTupleSet;
+import
org.apache.rya.indexing.pcj.matching.provider.AbstractPcjIndexSetProvider;
+import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.mongo.MongoPcjDocuments;
+import org.apache.rya.indexing.pcj.storage.mongo.MongoPcjStorage;
+import org.apache.rya.mongodb.StatefulMongoDBRdfConfiguration;
+import org.apache.rya.mongodb.instance.MongoRyaInstanceDetailsRepository;
+import org.openrdf.query.MalformedQueryException;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.mongodb.MongoClient;
+
+/**
+ * Implementation of {@link AbstractPcjIndexSetProvider} for MongoDB.
+ */
+public class MongoPcjIndexSetProvider extends AbstractPcjIndexSetProvider {
+/**
+ * Creates a new {@link MongoPcjIndexSetProvider}.
+ * @param conf - The configuration for this provider. (not null)
+ */
+public MongoPcjIndexSetProvider(final StatefulMongoDBRdfConfiguration
conf) {
+super(conf);
+}
+
+/**
+ * Creates a new {@link MongoPcjIndexSetProvider}.
+ * @param conf - The configuration for this provider.
+ * @param indices - The predefined indicies on this provider.
+ * @param client - The {@link MongoClient} used to connect to mongo.
+ */
+public MongoPcjIndexSetProvider(final StatefulMongoDBRdfConfiguration
conf, final List indices) {
+super(conf, indices);
+}
+
+@Override
+protected List getIndices() throws
PcjIndexSetException {
+requireNonNull(conf);
--- End diff --
probably should
---
[jira] [Commented] (RYA-417) Implement a forward-chaining rules engine (SPIN)
[ https://issues.apache.org/jira/browse/RYA-417?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319411#comment-16319411 ] ASF GitHub Bot commented on RYA-417: Github user jessehatfield commented on the issue: https://github.com/apache/incubator-rya/pull/255 Note that the RYA-417 work sits on top of RYA-416 -- the first two commits here are identical to PR https://github.com/apache/incubator-rya/pull/254 . That means most of the aggregation pipeline logic, so if there are changes that need to be made there it might be better to address them as part of that PR and then I can fold them into this one once resolved. > Implement a forward-chaining rules engine (SPIN) > > > Key: RYA-417 > URL: https://issues.apache.org/jira/browse/RYA-417 > Project: Rya > Issue Type: New Feature >Reporter: Jesse Hatfield >Assignee: Jesse Hatfield > > Implement a forward-chaining reasoner that: > * Runs as a batch process > * Operates on user-defined [SPIN|http://spinrdf.org] rules > * Inserts derived information back into Rya > * Iterates until no new information can be derived -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[GitHub] incubator-rya issue #255: RYA-417 Forward-chaining batch rules engine
Github user jessehatfield commented on the issue: https://github.com/apache/incubator-rya/pull/255 Note that the RYA-417 work sits on top of RYA-416 -- the first two commits here are identical to PR https://github.com/apache/incubator-rya/pull/254 . That means most of the aggregation pipeline logic, so if there are changes that need to be made there it might be better to address them as part of that PR and then I can fold them into this one once resolved. ---
[jira] [Commented] (RYA-417) Implement a forward-chaining rules engine (SPIN)
[
https://issues.apache.org/jira/browse/RYA-417?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319407#comment-16319407
]
ASF GitHub Bot commented on RYA-417:
Github user jessehatfield commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/255#discussion_r160554313
--- Diff:
dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/aggregation/SparqlToPipelineTransformVisitor.java
---
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.mongodb.aggregation;
+
+import java.util.Arrays;
+
+import org.apache.rya.mongodb.StatefulMongoDBRdfConfiguration;
+import org.bson.Document;
+import org.openrdf.query.algebra.Distinct;
+import org.openrdf.query.algebra.Extension;
+import org.openrdf.query.algebra.Filter;
+import org.openrdf.query.algebra.Join;
+import org.openrdf.query.algebra.MultiProjection;
+import org.openrdf.query.algebra.Projection;
+import org.openrdf.query.algebra.Reduced;
+import org.openrdf.query.algebra.StatementPattern;
+import org.openrdf.query.algebra.helpers.QueryModelVisitorBase;
+
+import com.google.common.base.Preconditions;
+import com.mongodb.MongoClient;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+
+/**
+ * Visitor that transforms a SPARQL query tree by replacing as much of the
tree
+ * as possible with one or more {@code AggregationPipelineQueryNode}s.
+ *
+ * Each {@link AggregationPipelineQueryNode} contains a MongoDB aggregation
+ * pipeline which is equivalent to the replaced portion of the original
query.
+ * Evaluating this node executes the pipeline and converts the results into
+ * query solutions. If only part of the query was transformed, the
remaining
+ * query logic (higher up in the query tree) can be applied to those
+ * intermediate solutions as normal.
+ *
+ * In general, processes the tree in bottom-up order: A leaf node
+ * ({@link StatementPattern}) is replaced with a pipeline that matches the
+ * corresponding statements. Then, if the parent node's semantics are
supported
+ * by the visitor, stages are appended to the pipeline and the subtree at
the
+ * parent node is replaced with the extended pipeline. This continues up
the
+ * tree until reaching a node that cannot be transformed, in which case
that
+ * node's child is now a single {@code AggregationPipelineQueryNode} (a
leaf
+ * node) instead of the previous subtree, or until the entire tree has been
+ * subsumed into a single pipeline node.
+ *
+ * Nodes which are transformed into pipeline stages:
+ *
+ * A {@code StatementPattern} node forms the beginning of each
pipeline.
+ * Single-argument operations {@link Projection}, {@link
MultiProjection},
+ * {@link Extension}, {@link Distinct}, and {@link Reduced} will be
transformed
+ * into pipeline stages whenever the child {@link TupleExpr} represents a
+ * pipeline.
+ * A {@link Filter} operation will be appended to the pipeline when its
+ * child {@code TupleExpr} represents a pipeline and the filter condition
is a
+ * type of {@link ValueExpr} understood by {@code
AggregationPipelineQueryNode}.
+ * A {@link Join} operation will be appended to the pipeline when one
child
+ * is a {@code StatementPattern} and the other is an
+ * {@code AggregationPipelineQueryNode}.
+ *
+ */
+public class SparqlToPipelineTransformVisitor extends
QueryModelVisitorBase {
--- End diff --
It doesn't strictly have to be executed first, though it may be better. For
example, if the tree is `Join(,
AggregationPipelineQueryNode))`, then the join iterator will get an iterator of
results from the complex thing on the left, then for each result, execute the
pipeline -- likely not optimal.
The only logic to try to group pipeline-amenable nodes/subqueries together
is in this visitor; there's
[GitHub] incubator-rya pull request #255: RYA-417 Forward-chaining batch rules engine
Github user jessehatfield commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/255#discussion_r160554313
--- Diff:
dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/aggregation/SparqlToPipelineTransformVisitor.java
---
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.mongodb.aggregation;
+
+import java.util.Arrays;
+
+import org.apache.rya.mongodb.StatefulMongoDBRdfConfiguration;
+import org.bson.Document;
+import org.openrdf.query.algebra.Distinct;
+import org.openrdf.query.algebra.Extension;
+import org.openrdf.query.algebra.Filter;
+import org.openrdf.query.algebra.Join;
+import org.openrdf.query.algebra.MultiProjection;
+import org.openrdf.query.algebra.Projection;
+import org.openrdf.query.algebra.Reduced;
+import org.openrdf.query.algebra.StatementPattern;
+import org.openrdf.query.algebra.helpers.QueryModelVisitorBase;
+
+import com.google.common.base.Preconditions;
+import com.mongodb.MongoClient;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+
+/**
+ * Visitor that transforms a SPARQL query tree by replacing as much of the
tree
+ * as possible with one or more {@code AggregationPipelineQueryNode}s.
+ *
+ * Each {@link AggregationPipelineQueryNode} contains a MongoDB aggregation
+ * pipeline which is equivalent to the replaced portion of the original
query.
+ * Evaluating this node executes the pipeline and converts the results into
+ * query solutions. If only part of the query was transformed, the
remaining
+ * query logic (higher up in the query tree) can be applied to those
+ * intermediate solutions as normal.
+ *
+ * In general, processes the tree in bottom-up order: A leaf node
+ * ({@link StatementPattern}) is replaced with a pipeline that matches the
+ * corresponding statements. Then, if the parent node's semantics are
supported
+ * by the visitor, stages are appended to the pipeline and the subtree at
the
+ * parent node is replaced with the extended pipeline. This continues up
the
+ * tree until reaching a node that cannot be transformed, in which case
that
+ * node's child is now a single {@code AggregationPipelineQueryNode} (a
leaf
+ * node) instead of the previous subtree, or until the entire tree has been
+ * subsumed into a single pipeline node.
+ *
+ * Nodes which are transformed into pipeline stages:
+ *
+ * A {@code StatementPattern} node forms the beginning of each
pipeline.
+ * Single-argument operations {@link Projection}, {@link
MultiProjection},
+ * {@link Extension}, {@link Distinct}, and {@link Reduced} will be
transformed
+ * into pipeline stages whenever the child {@link TupleExpr} represents a
+ * pipeline.
+ * A {@link Filter} operation will be appended to the pipeline when its
+ * child {@code TupleExpr} represents a pipeline and the filter condition
is a
+ * type of {@link ValueExpr} understood by {@code
AggregationPipelineQueryNode}.
+ * A {@link Join} operation will be appended to the pipeline when one
child
+ * is a {@code StatementPattern} and the other is an
+ * {@code AggregationPipelineQueryNode}.
+ *
+ */
+public class SparqlToPipelineTransformVisitor extends
QueryModelVisitorBase {
--- End diff --
It doesn't strictly have to be executed first, though it may be better. For
example, if the tree is `Join(,
AggregationPipelineQueryNode))`, then the join iterator will get an iterator of
results from the complex thing on the left, then for each result, execute the
pipeline -- likely not optimal.
The only logic to try to group pipeline-amenable nodes/subqueries together
is in this visitor; there's no restructuring done at a higher level to make it
work better (except to the extent that ordinary query planning steps may happen
to help). For example, this visitor can turn `Join(Join(Join(SP1, SP2), SP3),
SP4)` into a
[jira] [Resolved] (RYA-385) BindingSetDecorator always returns the same hashcode value.
[ https://issues.apache.org/jira/browse/RYA-385?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Kevin Chilton resolved RYA-385. --- Resolution: Fixed > BindingSetDecorator always returns the same hashcode value. > --- > > Key: RYA-385 > URL: https://issues.apache.org/jira/browse/RYA-385 > Project: Rya > Issue Type: Bug >Affects Versions: 3.2.11 >Reporter: Kevin Chilton >Assignee: Kevin Chilton >Priority: Critical > > The hashcode() function's implementation always caches the first hashcode > that was returned by the decorator. It will result in an incorrect hashcode > value if a binding is ever added to the binding set after invoking hashcode() > the first time. -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[jira] [Resolved] (RYA-382) CLI Client
[ https://issues.apache.org/jira/browse/RYA-382?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Kevin Chilton resolved RYA-382. --- Resolution: Done > CLI Client > -- > > Key: RYA-382 > URL: https://issues.apache.org/jira/browse/RYA-382 > Project: Rya > Issue Type: Sub-task >Reporter: Andrew Smith >Assignee: Kevin Chilton > > * Create Query > * Delete Query > * Start/Strop query > * List queries plus state > * configure source/sink info -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[jira] [Resolved] (RYA-378) Kafka Streams topology factory from SPARQL
[ https://issues.apache.org/jira/browse/RYA-378?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Kevin Chilton resolved RYA-378. --- Resolution: Done > Kafka Streams topology factory from SPARQL > --- > > Key: RYA-378 > URL: https://issues.apache.org/jira/browse/RYA-378 > Project: Rya > Issue Type: Sub-task >Reporter: Andrew Smith >Assignee: Kevin Chilton > -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[jira] [Resolved] (RYA-381) Stream Processor - Projection
[ https://issues.apache.org/jira/browse/RYA-381?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Kevin Chilton resolved RYA-381. --- Resolution: Done > Stream Processor - Projection > - > > Key: RYA-381 > URL: https://issues.apache.org/jira/browse/RYA-381 > Project: Rya > Issue Type: Sub-task >Reporter: Andrew Smith >Assignee: Kevin Chilton > -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[jira] [Resolved] (RYA-380) Stream Processor - Statement Patter
[ https://issues.apache.org/jira/browse/RYA-380?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Kevin Chilton resolved RYA-380. --- Resolution: Done > Stream Processor - Statement Patter > --- > > Key: RYA-380 > URL: https://issues.apache.org/jira/browse/RYA-380 > Project: Rya > Issue Type: Sub-task >Reporter: Andrew Smith >Assignee: Kevin Chilton > -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[jira] [Resolved] (RYA-379) Stream Processor - Join
[ https://issues.apache.org/jira/browse/RYA-379?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Kevin Chilton resolved RYA-379. --- Resolution: Done > Stream Processor - Join > --- > > Key: RYA-379 > URL: https://issues.apache.org/jira/browse/RYA-379 > Project: Rya > Issue Type: Sub-task >Reporter: Andrew Smith >Assignee: Kevin Chilton > -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[jira] [Resolved] (RYA-377) Implement single node kafka streams incremental SPARQL processor
[ https://issues.apache.org/jira/browse/RYA-377?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Kevin Chilton resolved RYA-377. --- Resolution: Done > Implement single node kafka streams incremental SPARQL processor > > > Key: RYA-377 > URL: https://issues.apache.org/jira/browse/RYA-377 > Project: Rya > Issue Type: New Feature >Reporter: Andrew Smith >Assignee: Kevin Chilton > -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319372#comment-16319372
]
ASF GitHub Bot commented on RYA-303:
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160548138
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
+
+private final MongoCollection pcjCollection;
+private static final PcjVarOrderFactory pcjVarOrderFactory = new
ShiftVarOrderFactory();
+
+/**
+ * Creates a new {@link MongoPcjDocuments}.
+ * @param client - The {@link
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319373#comment-16319373
]
ASF GitHub Bot commented on RYA-303:
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160550261
--- Diff:
extras/shell/src/main/java/org/apache/rya/shell/RyaAdminCommands.java ---
@@ -320,11 +343,14 @@ public String installWithMongoParameters(
final boolean enableFreeTextIndex,
// TODO RYA-215
-//@CliOption(key = {"enableGeospatialIndex"}, mandatory =
false, help = "Use Geospatial Indexing.", unspecifiedDefaultValue = "false",
specifiedDefaultValue = "true")
-//final boolean enableGeospatialIndex,
+//@CliOption(key = {"enableGeospatialIndex"},
mandatory = false, help = "Use Geospatial Indexing.", unspecifiedDefaultValue =
"false", specifiedDefaultValue = "true")
+//final boolean enableGeospatialIndex,
@CliOption(key = {"enableTemporalIndex"}, mandatory = false,
help = "Use Temporal Indexing.", unspecifiedDefaultValue = "false",
specifiedDefaultValue = "true")
-final boolean enableTemporalIndex) {
+final boolean enableTemporalIndex,
--- End diff --
Since you added entity stuff back in, also add it here?
> Mongo PCJ indexer support
> -
>
> Key: RYA-303
> URL: https://issues.apache.org/jira/browse/RYA-303
> Project: Rya
> Issue Type: Improvement
>Reporter: Andrew Smith
>Assignee: Andrew Smith
>
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319369#comment-16319369
]
ASF GitHub Bot commented on RYA-303:
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160546531
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
+
+private final MongoCollection pcjCollection;
+private static final PcjVarOrderFactory pcjVarOrderFactory = new
ShiftVarOrderFactory();
+
+/**
+ * Creates a new {@link MongoPcjDocuments}.
+ * @param client - The {@link
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319360#comment-16319360
]
ASF GitHub Bot commented on RYA-303:
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160546626
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
+
+private final MongoCollection pcjCollection;
+private static final PcjVarOrderFactory pcjVarOrderFactory = new
ShiftVarOrderFactory();
+
+/**
+ * Creates a new {@link MongoPcjDocuments}.
+ * @param client - The {@link
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319365#comment-16319365
]
ASF GitHub Bot commented on RYA-303:
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160546683
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
+
+private final MongoCollection pcjCollection;
+private static final PcjVarOrderFactory pcjVarOrderFactory = new
ShiftVarOrderFactory();
+
+/**
+ * Creates a new {@link MongoPcjDocuments}.
+ * @param client - The {@link
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319368#comment-16319368
]
ASF GitHub Bot commented on RYA-303:
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160541859
--- Diff:
extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoDeletePCJ.java
---
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.api.client.mongo;
+
+import static java.util.Objects.requireNonNull;
+
+import org.apache.rya.api.client.DeletePCJ;
+import org.apache.rya.api.client.InstanceDoesNotExistException;
+import org.apache.rya.api.client.InstanceExists;
+import org.apache.rya.api.client.RyaClientException;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.mongo.MongoPcjStorage;
+
+import com.mongodb.MongoClient;
+
+/**
+ * A Mongo implementation of {@link DeletePCJ}.
+ */
+public class MongoDeletePCJ implements DeletePCJ {
+private final MongoConnectionDetails connectionDetails;
--- End diff --
This field is not used.
> Mongo PCJ indexer support
> -
>
> Key: RYA-303
> URL: https://issues.apache.org/jira/browse/RYA-303
> Project: Rya
> Issue Type: Improvement
>Reporter: Andrew Smith
>Assignee: Andrew Smith
>
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
[jira] [Commented] (RYA-417) Implement a forward-chaining rules engine (SPIN)
[
https://issues.apache.org/jira/browse/RYA-417?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319376#comment-16319376
]
ASF GitHub Bot commented on RYA-417:
Github user jessehatfield commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/255#discussion_r160550347
--- Diff:
extras/rya.forwardchain/src/main/java/org/apache/rya/forwardchain/rule/AntecedentVisitor.java
---
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.forwardchain.rule;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.openrdf.query.algebra.StatementPattern;
+import org.openrdf.query.algebra.helpers.QueryModelVisitorBase;
+
+/**
+ * Query visitor that identifies all triple patterns represented as
+ * {@link StatementPattern}s in a query, which therefore represent triples
+ * that could potentially contribute to a solution. Considers only the
statement
+ * patterns themselves, i.e. the leaves of the query tree, and does not
consider
+ * other constraints that may restrict the set of triples that may be
relevant.
+ * This means relying on this analysis to determine whether a fact can be
part
+ * of a solution can yield false positives, but not false negatives.
+ */
+class AntecedentVisitor extends QueryModelVisitorBase {
--- End diff --
Oh, I think I missed that -- looks like the only difference is that it
skips over filter conditions. I think we don't want to skip filter conditions,
here, though -- if the rule is `CONSTRUCT { stuff } WHERE { ?x a :T . FILTER
EXISTS {?x :p :A} }` then the rule should be triggered by either `:x a :T` or
`:x :p :A` .
> Implement a forward-chaining rules engine (SPIN)
>
>
> Key: RYA-417
> URL: https://issues.apache.org/jira/browse/RYA-417
> Project: Rya
> Issue Type: New Feature
>Reporter: Jesse Hatfield
>Assignee: Jesse Hatfield
>
> Implement a forward-chaining reasoner that:
> * Runs as a batch process
> * Operates on user-defined [SPIN|http://spinrdf.org] rules
> * Inserts derived information back into Rya
> * Iterates until no new information can be derived
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319375#comment-16319375
]
ASF GitHub Bot commented on RYA-303:
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160550034
--- Diff:
extras/shell/src/main/java/org/apache/rya/shell/util/InstallPrompt.java ---
@@ -91,14 +91,14 @@ public InstallConfiguration
promptInstallConfiguration(final String instanceName
checkState(storageType.isPresent(), "The shell must be
connected to a storage to use the install prompt.");
switch(sharedShellState.getShellState().getStorageType().get()) {
-case ACCUMULO:
-return promptAccumuloConfig(instanceName);
+case ACCUMULO:
+return promptAccumuloConfig(instanceName);
-case MONGO:
-return promptMongoConfig(instanceName);
+case MONGO:
+return promptMongoConfig(instanceName);
-default:
-throw new IllegalStateException("Unsupported storage
type: " + storageType.get());
+default:
+throw new IllegalStateException("Unsupported storage type:
" + storageType.get());
--- End diff --
Messsed up switch white space.
> Mongo PCJ indexer support
> -
>
> Key: RYA-303
> URL: https://issues.apache.org/jira/browse/RYA-303
> Project: Rya
> Issue Type: Improvement
>Reporter: Andrew Smith
>Assignee: Andrew Smith
>
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319366#comment-16319366
]
ASF GitHub Bot commented on RYA-303:
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160547307
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
+
+private final MongoCollection pcjCollection;
+private static final PcjVarOrderFactory pcjVarOrderFactory = new
ShiftVarOrderFactory();
+
+/**
+ * Creates a new {@link MongoPcjDocuments}.
+ * @param client - The {@link
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319363#comment-16319363
]
ASF GitHub Bot commented on RYA-303:
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160546330
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
+
+private final MongoCollection pcjCollection;
+private static final PcjVarOrderFactory pcjVarOrderFactory = new
ShiftVarOrderFactory();
+
+/**
+ * Creates a new {@link MongoPcjDocuments}.
+ * @param client - The {@link
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319370#comment-16319370
]
ASF GitHub Bot commented on RYA-303:
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160550164
--- Diff:
extras/shell/src/main/java/org/apache/rya/shell/util/InstallPrompt.java ---
@@ -224,6 +224,10 @@ private InstallConfiguration promptMongoConfig(final
String instanceName) throws
final boolean useTemporalIndexing = promptBoolean(prompt,
Optional.of(true));
builder.setEnableTemporalIndex( useTemporalIndexing );
+prompt = makeFieldPrompt("Use PCJ Indexing", true);
--- End diff --
Since you added entity stuff back in, also add it here?
> Mongo PCJ indexer support
> -
>
> Key: RYA-303
> URL: https://issues.apache.org/jira/browse/RYA-303
> Project: Rya
> Issue Type: Improvement
>Reporter: Andrew Smith
>Assignee: Andrew Smith
>
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319367#comment-16319367
]
ASF GitHub Bot commented on RYA-303:
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160541792
--- Diff:
extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoCreatePCJ.java
---
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.api.client.mongo;
+
+import static java.util.Objects.requireNonNull;
+
+import java.util.Set;
+
+import org.apache.rya.api.client.CreatePCJ;
+import org.apache.rya.api.client.InstanceDoesNotExistException;
+import org.apache.rya.api.client.InstanceExists;
+import org.apache.rya.api.client.RyaClientException;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.mongo.MongoPcjStorage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Sets;
+import com.mongodb.MongoClient;
+
+/**
+ * A Mongo implementation of {@link CreatePCJ}.
+ */
+public class MongoCreatePCJ implements CreatePCJ {
+private static final Logger log =
LoggerFactory.getLogger(MongoCreatePCJ.class);
+
+private final MongoConnectionDetails connectionDetails;
--- End diff --
This field is not used.
> Mongo PCJ indexer support
> -
>
> Key: RYA-303
> URL: https://issues.apache.org/jira/browse/RYA-303
> Project: Rya
> Issue Type: Improvement
>Reporter: Andrew Smith
>Assignee: Andrew Smith
>
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319374#comment-16319374
]
ASF GitHub Bot commented on RYA-303:
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160549265
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
+
+private final MongoCollection pcjCollection;
+private static final PcjVarOrderFactory pcjVarOrderFactory = new
ShiftVarOrderFactory();
+
+/**
+ * Creates a new {@link MongoPcjDocuments}.
+ * @param client - The {@link
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319362#comment-16319362
]
ASF GitHub Bot commented on RYA-303:
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160544633
--- Diff:
extras/shell/src/main/java/org/apache/rya/shell/RyaAdminCommands.java ---
@@ -104,11 +104,11 @@ public RyaAdminCommands(
INSTALL_CMD})
public boolean areStorageCommandsAvailable() {
switch(state.getShellState().getConnectionState()) {
-case CONNECTED_TO_STORAGE:
-case CONNECTED_TO_INSTANCE:
-return true;
-default:
-return false;
+case CONNECTED_TO_STORAGE:
--- End diff --
white space changed
> Mongo PCJ indexer support
> -
>
> Key: RYA-303
> URL: https://issues.apache.org/jira/browse/RYA-303
> Project: Rya
> Issue Type: Improvement
>Reporter: Andrew Smith
>Assignee: Andrew Smith
>
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319371#comment-16319371
]
ASF GitHub Bot commented on RYA-303:
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160547226
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
+
+private final MongoCollection pcjCollection;
+private static final PcjVarOrderFactory pcjVarOrderFactory = new
ShiftVarOrderFactory();
+
+/**
+ * Creates a new {@link MongoPcjDocuments}.
+ * @param client - The {@link
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[ https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319364#comment-16319364 ] ASF GitHub Bot commented on RYA-303: Github user kchilton2 commented on a diff in the pull request: https://github.com/apache/incubator-rya/pull/172#discussion_r160548641 --- Diff: extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java --- @@ -0,0 +1,438 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.indexing.pcj.storage.mongo; + +import static com.google.common.base.Preconditions.checkNotNull; +import static java.util.Objects.requireNonNull; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +import org.apache.accumulo.core.security.Authorizations; --- End diff -- Do we want to mix Accumulo specific code with the Mongo code, even though it's a holder for Authorizations? > Mongo PCJ indexer support > - > > Key: RYA-303 > URL: https://issues.apache.org/jira/browse/RYA-303 > Project: Rya > Issue Type: Improvement >Reporter: Andrew Smith >Assignee: Andrew Smith > -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160550034
--- Diff:
extras/shell/src/main/java/org/apache/rya/shell/util/InstallPrompt.java ---
@@ -91,14 +91,14 @@ public InstallConfiguration
promptInstallConfiguration(final String instanceName
checkState(storageType.isPresent(), "The shell must be
connected to a storage to use the install prompt.");
switch(sharedShellState.getShellState().getStorageType().get()) {
-case ACCUMULO:
-return promptAccumuloConfig(instanceName);
+case ACCUMULO:
+return promptAccumuloConfig(instanceName);
-case MONGO:
-return promptMongoConfig(instanceName);
+case MONGO:
+return promptMongoConfig(instanceName);
-default:
-throw new IllegalStateException("Unsupported storage
type: " + storageType.get());
+default:
+throw new IllegalStateException("Unsupported storage type:
" + storageType.get());
--- End diff --
Messsed up switch white space.
---
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160548138
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
+
+private final MongoCollection pcjCollection;
+private static final PcjVarOrderFactory pcjVarOrderFactory = new
ShiftVarOrderFactory();
+
+/**
+ * Creates a new {@link MongoPcjDocuments}.
+ * @param client - The {@link MongoClient} to use to connect to mongo.
+ * @param ryaInstanceName - The rya instance to connect to.
+ */
+public MongoPcjDocuments(final MongoClient client, final String
ryaInstanceName) {
+
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160549265
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
+
+private final MongoCollection pcjCollection;
+private static final PcjVarOrderFactory pcjVarOrderFactory = new
ShiftVarOrderFactory();
+
+/**
+ * Creates a new {@link MongoPcjDocuments}.
+ * @param client - The {@link MongoClient} to use to connect to mongo.
+ * @param ryaInstanceName - The rya instance to connect to.
+ */
+public MongoPcjDocuments(final MongoClient client, final String
ryaInstanceName) {
+
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160546531
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
+
+private final MongoCollection pcjCollection;
+private static final PcjVarOrderFactory pcjVarOrderFactory = new
ShiftVarOrderFactory();
+
+/**
+ * Creates a new {@link MongoPcjDocuments}.
+ * @param client - The {@link MongoClient} to use to connect to mongo.
+ * @param ryaInstanceName - The rya instance to connect to.
+ */
+public MongoPcjDocuments(final MongoClient client, final String
ryaInstanceName) {
+
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160550261
--- Diff:
extras/shell/src/main/java/org/apache/rya/shell/RyaAdminCommands.java ---
@@ -320,11 +343,14 @@ public String installWithMongoParameters(
final boolean enableFreeTextIndex,
// TODO RYA-215
-//@CliOption(key = {"enableGeospatialIndex"}, mandatory =
false, help = "Use Geospatial Indexing.", unspecifiedDefaultValue = "false",
specifiedDefaultValue = "true")
-//final boolean enableGeospatialIndex,
+//@CliOption(key = {"enableGeospatialIndex"},
mandatory = false, help = "Use Geospatial Indexing.", unspecifiedDefaultValue =
"false", specifiedDefaultValue = "true")
+//final boolean enableGeospatialIndex,
@CliOption(key = {"enableTemporalIndex"}, mandatory = false,
help = "Use Temporal Indexing.", unspecifiedDefaultValue = "false",
specifiedDefaultValue = "true")
-final boolean enableTemporalIndex) {
+final boolean enableTemporalIndex,
--- End diff --
Since you added entity stuff back in, also add it here?
---
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160546330
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
+
+private final MongoCollection pcjCollection;
+private static final PcjVarOrderFactory pcjVarOrderFactory = new
ShiftVarOrderFactory();
+
+/**
+ * Creates a new {@link MongoPcjDocuments}.
+ * @param client - The {@link MongoClient} to use to connect to mongo.
+ * @param ryaInstanceName - The rya instance to connect to.
+ */
+public MongoPcjDocuments(final MongoClient client, final String
ryaInstanceName) {
+
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160541792
--- Diff:
extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoCreatePCJ.java
---
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.api.client.mongo;
+
+import static java.util.Objects.requireNonNull;
+
+import java.util.Set;
+
+import org.apache.rya.api.client.CreatePCJ;
+import org.apache.rya.api.client.InstanceDoesNotExistException;
+import org.apache.rya.api.client.InstanceExists;
+import org.apache.rya.api.client.RyaClientException;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.mongo.MongoPcjStorage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Sets;
+import com.mongodb.MongoClient;
+
+/**
+ * A Mongo implementation of {@link CreatePCJ}.
+ */
+public class MongoCreatePCJ implements CreatePCJ {
+private static final Logger log =
LoggerFactory.getLogger(MongoCreatePCJ.class);
+
+private final MongoConnectionDetails connectionDetails;
--- End diff --
This field is not used.
---
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160541859
--- Diff:
extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoDeletePCJ.java
---
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.api.client.mongo;
+
+import static java.util.Objects.requireNonNull;
+
+import org.apache.rya.api.client.DeletePCJ;
+import org.apache.rya.api.client.InstanceDoesNotExistException;
+import org.apache.rya.api.client.InstanceExists;
+import org.apache.rya.api.client.RyaClientException;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.mongo.MongoPcjStorage;
+
+import com.mongodb.MongoClient;
+
+/**
+ * A Mongo implementation of {@link DeletePCJ}.
+ */
+public class MongoDeletePCJ implements DeletePCJ {
+private final MongoConnectionDetails connectionDetails;
--- End diff --
This field is not used.
---
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user kchilton2 commented on a diff in the pull request: https://github.com/apache/incubator-rya/pull/172#discussion_r160548641 --- Diff: extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java --- @@ -0,0 +1,438 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.indexing.pcj.storage.mongo; + +import static com.google.common.base.Preconditions.checkNotNull; +import static java.util.Objects.requireNonNull; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +import org.apache.accumulo.core.security.Authorizations; --- End diff -- Do we want to mix Accumulo specific code with the Mongo code, even though it's a holder for Authorizations? ---
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160547226
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
+
+private final MongoCollection pcjCollection;
+private static final PcjVarOrderFactory pcjVarOrderFactory = new
ShiftVarOrderFactory();
+
+/**
+ * Creates a new {@link MongoPcjDocuments}.
+ * @param client - The {@link MongoClient} to use to connect to mongo.
+ * @param ryaInstanceName - The rya instance to connect to.
+ */
+public MongoPcjDocuments(final MongoClient client, final String
ryaInstanceName) {
+
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160550164
--- Diff:
extras/shell/src/main/java/org/apache/rya/shell/util/InstallPrompt.java ---
@@ -224,6 +224,10 @@ private InstallConfiguration promptMongoConfig(final
String instanceName) throws
final boolean useTemporalIndexing = promptBoolean(prompt,
Optional.of(true));
builder.setEnableTemporalIndex( useTemporalIndexing );
+prompt = makeFieldPrompt("Use PCJ Indexing", true);
--- End diff --
Since you added entity stuff back in, also add it here?
---
[GitHub] incubator-rya pull request #255: RYA-417 Forward-chaining batch rules engine
Github user jessehatfield commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/255#discussion_r160550347
--- Diff:
extras/rya.forwardchain/src/main/java/org/apache/rya/forwardchain/rule/AntecedentVisitor.java
---
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.forwardchain.rule;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.openrdf.query.algebra.StatementPattern;
+import org.openrdf.query.algebra.helpers.QueryModelVisitorBase;
+
+/**
+ * Query visitor that identifies all triple patterns represented as
+ * {@link StatementPattern}s in a query, which therefore represent triples
+ * that could potentially contribute to a solution. Considers only the
statement
+ * patterns themselves, i.e. the leaves of the query tree, and does not
consider
+ * other constraints that may restrict the set of triples that may be
relevant.
+ * This means relying on this analysis to determine whether a fact can be
part
+ * of a solution can yield false positives, but not false negatives.
+ */
+class AntecedentVisitor extends QueryModelVisitorBase {
--- End diff --
Oh, I think I missed that -- looks like the only difference is that it
skips over filter conditions. I think we don't want to skip filter conditions,
here, though -- if the rule is `CONSTRUCT { stuff } WHERE { ?x a :T . FILTER
EXISTS {?x :p :A} }` then the rule should be triggered by either `:x a :T` or
`:x :p :A` .
---
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160547307
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
+
+private final MongoCollection pcjCollection;
+private static final PcjVarOrderFactory pcjVarOrderFactory = new
ShiftVarOrderFactory();
+
+/**
+ * Creates a new {@link MongoPcjDocuments}.
+ * @param client - The {@link MongoClient} to use to connect to mongo.
+ * @param ryaInstanceName - The rya instance to connect to.
+ */
+public MongoPcjDocuments(final MongoClient client, final String
ryaInstanceName) {
+
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160544633
--- Diff:
extras/shell/src/main/java/org/apache/rya/shell/RyaAdminCommands.java ---
@@ -104,11 +104,11 @@ public RyaAdminCommands(
INSTALL_CMD})
public boolean areStorageCommandsAvailable() {
switch(state.getShellState().getConnectionState()) {
-case CONNECTED_TO_STORAGE:
-case CONNECTED_TO_INSTANCE:
-return true;
-default:
-return false;
+case CONNECTED_TO_STORAGE:
--- End diff --
white space changed
---
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160546683
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
+
+private final MongoCollection pcjCollection;
+private static final PcjVarOrderFactory pcjVarOrderFactory = new
ShiftVarOrderFactory();
+
+/**
+ * Creates a new {@link MongoPcjDocuments}.
+ * @param client - The {@link MongoClient} to use to connect to mongo.
+ * @param ryaInstanceName - The rya instance to connect to.
+ */
+public MongoPcjDocuments(final MongoClient client, final String
ryaInstanceName) {
+
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160546626
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
+
+private final MongoCollection pcjCollection;
+private static final PcjVarOrderFactory pcjVarOrderFactory = new
ShiftVarOrderFactory();
+
+/**
+ * Creates a new {@link MongoPcjDocuments}.
+ * @param client - The {@link MongoClient} to use to connect to mongo.
+ * @param ryaInstanceName - The rya instance to connect to.
+ */
+public MongoPcjDocuments(final MongoClient client, final String
ryaInstanceName) {
+
[jira] [Commented] (RYA-417) Implement a forward-chaining rules engine (SPIN)
[
https://issues.apache.org/jira/browse/RYA-417?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319357#comment-16319357
]
ASF GitHub Bot commented on RYA-417:
Github user jessehatfield commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/255#discussion_r160549312
--- Diff:
dao/mongodb.rya/src/test/java/org/apache/rya/mongodb/aggregation/SparqlToPipelineTransformVisitorTest.java
---
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.mongodb.aggregation;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.bson.Document;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.openrdf.model.URI;
+import org.openrdf.model.ValueFactory;
+import org.openrdf.model.impl.ValueFactoryImpl;
+import org.openrdf.model.vocabulary.OWL;
+import org.openrdf.model.vocabulary.RDF;
+import org.openrdf.query.algebra.Extension;
+import org.openrdf.query.algebra.ExtensionElem;
+import org.openrdf.query.algebra.Join;
+import org.openrdf.query.algebra.MultiProjection;
+import org.openrdf.query.algebra.Not;
+import org.openrdf.query.algebra.Projection;
+import org.openrdf.query.algebra.ProjectionElem;
+import org.openrdf.query.algebra.ProjectionElemList;
+import org.openrdf.query.algebra.QueryRoot;
+import org.openrdf.query.algebra.StatementPattern;
+import org.openrdf.query.algebra.TupleExpr;
+import org.openrdf.query.algebra.ValueConstant;
+import org.openrdf.query.algebra.Var;
+
+import com.google.common.collect.Sets;
+import com.mongodb.MongoNamespace;
+import com.mongodb.client.MongoCollection;
+
+public class SparqlToPipelineTransformVisitorTest {
+
+private static final ValueFactory VF = ValueFactoryImpl.getInstance();
+
+private static final String LUBM = "urn:lubm";
+private static final URI UNDERGRAD = VF.createURI(LUBM,
"UndergraduateStudent");
+private static final URI PROFESSOR = VF.createURI(LUBM, "Professor");
+private static final URI COURSE = VF.createURI(LUBM, "Course");
+private static final URI TAKES = VF.createURI(LUBM, "takesCourse");
+private static final URI TEACHES = VF.createURI(LUBM, "teachesCourse");
+
+private static Var constant(URI value) {
+return new Var(value.stringValue(), value);
+}
+
+MongoCollection collection;
+
+@Before
+@SuppressWarnings("unchecked")
+public void setUp() {
+collection = Mockito.mock(MongoCollection.class);
+Mockito.when(collection.getNamespace()).thenReturn(new
MongoNamespace("db", "collection"));
+}
+
+@Test
+public void testStatementPattern() throws Exception {
+QueryRoot query = new QueryRoot(new StatementPattern(
+new Var("x"), constant(RDF.TYPE), constant(UNDERGRAD)));
+SparqlToPipelineTransformVisitor visitor = new
SparqlToPipelineTransformVisitor(collection);
+query.visit(visitor);
+Assert.assertTrue(query.getArg() instanceof
AggregationPipelineQueryNode);
+AggregationPipelineQueryNode pipelineNode =
(AggregationPipelineQueryNode) query.getArg();
+Assert.assertEquals(Sets.newHashSet("x"),
pipelineNode.getAssuredBindingNames());
+}
+
+@Test
+public void testJoin() throws Exception {
+QueryRoot query = new QueryRoot(new Join(
+new StatementPattern(new Var("x"), constant(RDF.TYPE),
constant(UNDERGRAD)),
+new StatementPattern(new Var("x"), constant(TAKES), new
Var("course";
+SparqlToPipelineTransformVisitor visitor = new
SparqlToPipelineTransformVisitor(collection);
+query.visit(visitor);
+Assert.assertTrue(query.getArg() instanceof
AggregationPipelineQueryNode);
+
[GitHub] incubator-rya pull request #255: RYA-417 Forward-chaining batch rules engine
Github user jessehatfield commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/255#discussion_r160549312
--- Diff:
dao/mongodb.rya/src/test/java/org/apache/rya/mongodb/aggregation/SparqlToPipelineTransformVisitorTest.java
---
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.mongodb.aggregation;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.bson.Document;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.openrdf.model.URI;
+import org.openrdf.model.ValueFactory;
+import org.openrdf.model.impl.ValueFactoryImpl;
+import org.openrdf.model.vocabulary.OWL;
+import org.openrdf.model.vocabulary.RDF;
+import org.openrdf.query.algebra.Extension;
+import org.openrdf.query.algebra.ExtensionElem;
+import org.openrdf.query.algebra.Join;
+import org.openrdf.query.algebra.MultiProjection;
+import org.openrdf.query.algebra.Not;
+import org.openrdf.query.algebra.Projection;
+import org.openrdf.query.algebra.ProjectionElem;
+import org.openrdf.query.algebra.ProjectionElemList;
+import org.openrdf.query.algebra.QueryRoot;
+import org.openrdf.query.algebra.StatementPattern;
+import org.openrdf.query.algebra.TupleExpr;
+import org.openrdf.query.algebra.ValueConstant;
+import org.openrdf.query.algebra.Var;
+
+import com.google.common.collect.Sets;
+import com.mongodb.MongoNamespace;
+import com.mongodb.client.MongoCollection;
+
+public class SparqlToPipelineTransformVisitorTest {
+
+private static final ValueFactory VF = ValueFactoryImpl.getInstance();
+
+private static final String LUBM = "urn:lubm";
+private static final URI UNDERGRAD = VF.createURI(LUBM,
"UndergraduateStudent");
+private static final URI PROFESSOR = VF.createURI(LUBM, "Professor");
+private static final URI COURSE = VF.createURI(LUBM, "Course");
+private static final URI TAKES = VF.createURI(LUBM, "takesCourse");
+private static final URI TEACHES = VF.createURI(LUBM, "teachesCourse");
+
+private static Var constant(URI value) {
+return new Var(value.stringValue(), value);
+}
+
+MongoCollection collection;
+
+@Before
+@SuppressWarnings("unchecked")
+public void setUp() {
+collection = Mockito.mock(MongoCollection.class);
+Mockito.when(collection.getNamespace()).thenReturn(new
MongoNamespace("db", "collection"));
+}
+
+@Test
+public void testStatementPattern() throws Exception {
+QueryRoot query = new QueryRoot(new StatementPattern(
+new Var("x"), constant(RDF.TYPE), constant(UNDERGRAD)));
+SparqlToPipelineTransformVisitor visitor = new
SparqlToPipelineTransformVisitor(collection);
+query.visit(visitor);
+Assert.assertTrue(query.getArg() instanceof
AggregationPipelineQueryNode);
+AggregationPipelineQueryNode pipelineNode =
(AggregationPipelineQueryNode) query.getArg();
+Assert.assertEquals(Sets.newHashSet("x"),
pipelineNode.getAssuredBindingNames());
+}
+
+@Test
+public void testJoin() throws Exception {
+QueryRoot query = new QueryRoot(new Join(
+new StatementPattern(new Var("x"), constant(RDF.TYPE),
constant(UNDERGRAD)),
+new StatementPattern(new Var("x"), constant(TAKES), new
Var("course";
+SparqlToPipelineTransformVisitor visitor = new
SparqlToPipelineTransformVisitor(collection);
+query.visit(visitor);
+Assert.assertTrue(query.getArg() instanceof
AggregationPipelineQueryNode);
+AggregationPipelineQueryNode pipelineNode =
(AggregationPipelineQueryNode) query.getArg();
+Assert.assertEquals(Sets.newHashSet("x", "course"),
pipelineNode.getAssuredBindingNames());
+}
+
+
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[ https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319333#comment-16319333 ] ASF GitHub Bot commented on RYA-303: Github user kchilton2 commented on the issue: https://github.com/apache/incubator-rya/pull/172 I think RyaDetailsFormatter still needs to be updated for this review. > Mongo PCJ indexer support > - > > Key: RYA-303 > URL: https://issues.apache.org/jira/browse/RYA-303 > Project: Rya > Issue Type: Improvement >Reporter: Andrew Smith >Assignee: Andrew Smith > -- This message was sent by Atlassian JIRA (v6.4.14#64029)
[GitHub] incubator-rya issue #172: RYA-303 Mongo PCJ Support
Github user kchilton2 commented on the issue: https://github.com/apache/incubator-rya/pull/172 I think RyaDetailsFormatter still needs to be updated for this review. ---
[jira] [Commented] (RYA-417) Implement a forward-chaining rules engine (SPIN)
[ https://issues.apache.org/jira/browse/RYA-417?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319330#comment-16319330 ] ASF GitHub Bot commented on RYA-417: Github user jessehatfield commented on a diff in the pull request: https://github.com/apache/incubator-rya/pull/255#discussion_r160545261 --- Diff: dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/aggregation/AggregationPipelineQueryNode.java --- @@ -0,0 +1,862 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.mongodb.aggregation; + +import static org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy.CONTEXT; +import static org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy.DOCUMENT_VISIBILITY; +import static org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy.OBJECT; +import static org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy.OBJECT_HASH; +import static org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy.OBJECT_TYPE; +import static org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy.PREDICATE; +import static org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy.PREDICATE_HASH; +import static org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy.STATEMENT_METADATA; +import static org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy.SUBJECT; +import static org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy.SUBJECT_HASH; +import static org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy.TIMESTAMP; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.NavigableSet; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.function.Function; + +import org.apache.rya.api.domain.RyaStatement; +import org.apache.rya.api.domain.RyaType; +import org.apache.rya.api.domain.RyaURI; +import org.apache.rya.api.domain.StatementMetadata; +import org.apache.rya.api.resolver.RdfToRyaConversions; +import org.apache.rya.mongodb.MongoDbRdfConstants; +import org.apache.rya.mongodb.dao.MongoDBStorageStrategy; +import org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy; +import org.apache.rya.mongodb.document.operators.query.ConditionalOperators; +import org.apache.rya.mongodb.document.visibility.DocumentVisibilityAdapter; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.openrdf.model.Literal; +import org.openrdf.model.Resource; +import org.openrdf.model.URI; +import org.openrdf.model.Value; +import org.openrdf.model.vocabulary.XMLSchema; +import org.openrdf.query.BindingSet; +import org.openrdf.query.QueryEvaluationException; +import org.openrdf.query.algebra.Compare; +import org.openrdf.query.algebra.ExtensionElem; +import org.openrdf.query.algebra.ProjectionElem; +import org.openrdf.query.algebra.ProjectionElemList; +import org.openrdf.query.algebra.StatementPattern; +import org.openrdf.query.algebra.ValueConstant; +import org.openrdf.query.algebra.ValueExpr; +import org.openrdf.query.algebra.Var; +import org.openrdf.query.algebra.evaluation.impl.ExternalSet; + +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; +import com.mongodb.BasicDBObject; +import com.mongodb.DBObject; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.model.Aggregates; +import com.mongodb.client.model.BsonField; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; + +import info.aduna.iteration.CloseableIteration; + +/** + * Represents a portion of a query tree as MongoDB aggregation pipeline. Should + * be built bottom-up:
[GitHub] incubator-rya pull request #255: RYA-417 Forward-chaining batch rules engine
Github user jessehatfield commented on a diff in the pull request: https://github.com/apache/incubator-rya/pull/255#discussion_r160545261 --- Diff: dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/aggregation/AggregationPipelineQueryNode.java --- @@ -0,0 +1,862 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.mongodb.aggregation; + +import static org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy.CONTEXT; +import static org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy.DOCUMENT_VISIBILITY; +import static org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy.OBJECT; +import static org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy.OBJECT_HASH; +import static org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy.OBJECT_TYPE; +import static org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy.PREDICATE; +import static org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy.PREDICATE_HASH; +import static org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy.STATEMENT_METADATA; +import static org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy.SUBJECT; +import static org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy.SUBJECT_HASH; +import static org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy.TIMESTAMP; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.NavigableSet; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.function.Function; + +import org.apache.rya.api.domain.RyaStatement; +import org.apache.rya.api.domain.RyaType; +import org.apache.rya.api.domain.RyaURI; +import org.apache.rya.api.domain.StatementMetadata; +import org.apache.rya.api.resolver.RdfToRyaConversions; +import org.apache.rya.mongodb.MongoDbRdfConstants; +import org.apache.rya.mongodb.dao.MongoDBStorageStrategy; +import org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy; +import org.apache.rya.mongodb.document.operators.query.ConditionalOperators; +import org.apache.rya.mongodb.document.visibility.DocumentVisibilityAdapter; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.openrdf.model.Literal; +import org.openrdf.model.Resource; +import org.openrdf.model.URI; +import org.openrdf.model.Value; +import org.openrdf.model.vocabulary.XMLSchema; +import org.openrdf.query.BindingSet; +import org.openrdf.query.QueryEvaluationException; +import org.openrdf.query.algebra.Compare; +import org.openrdf.query.algebra.ExtensionElem; +import org.openrdf.query.algebra.ProjectionElem; +import org.openrdf.query.algebra.ProjectionElemList; +import org.openrdf.query.algebra.StatementPattern; +import org.openrdf.query.algebra.ValueConstant; +import org.openrdf.query.algebra.ValueExpr; +import org.openrdf.query.algebra.Var; +import org.openrdf.query.algebra.evaluation.impl.ExternalSet; + +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; +import com.mongodb.BasicDBObject; +import com.mongodb.DBObject; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.model.Aggregates; +import com.mongodb.client.model.BsonField; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; + +import info.aduna.iteration.CloseableIteration; + +/** + * Represents a portion of a query tree as MongoDB aggregation pipeline. Should + * be built bottom-up: start with a statement pattern implemented as a $match + * step, then add steps to the pipeline to handle higher levels of the query + * tree. Methods are provided to add certain supported query operations to the + *
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319279#comment-16319279
]
ASF GitHub Bot commented on RYA-303:
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160527700
--- Diff:
extras/indexing/src/main/java/org/apache/rya/indexing/mongodb/pcj/MongoPcjIndexSetProvider.java
---
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.mongodb.pcj;
+
+import static java.util.Objects.requireNonNull;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.rya.api.instance.RyaDetailsRepository;
+import
org.apache.rya.api.instance.RyaDetailsRepository.RyaDetailsRepositoryException;
+import org.apache.rya.indexing.external.tupleSet.ExternalTupleSet;
+import
org.apache.rya.indexing.pcj.matching.provider.AbstractPcjIndexSetProvider;
+import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.mongo.MongoPcjDocuments;
+import org.apache.rya.indexing.pcj.storage.mongo.MongoPcjStorage;
+import org.apache.rya.mongodb.StatefulMongoDBRdfConfiguration;
+import org.apache.rya.mongodb.instance.MongoRyaInstanceDetailsRepository;
+import org.openrdf.query.MalformedQueryException;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.mongodb.MongoClient;
+
+/**
+ * Implementation of {@link AbstractPcjIndexSetProvider} for MongoDB.
+ */
+public class MongoPcjIndexSetProvider extends AbstractPcjIndexSetProvider {
+/**
+ * Creates a new {@link MongoPcjIndexSetProvider}.
+ * @param conf - The configuration for this provider. (not null)
+ */
+public MongoPcjIndexSetProvider(final StatefulMongoDBRdfConfiguration
conf) {
+super(conf);
+}
+
+/**
+ * Creates a new {@link MongoPcjIndexSetProvider}.
+ * @param conf - The configuration for this provider.
+ * @param indices - The predefined indicies on this provider.
+ * @param client - The {@link MongoClient} used to connect to mongo.
+ */
+public MongoPcjIndexSetProvider(final StatefulMongoDBRdfConfiguration
conf, final List indices) {
+super(conf, indices);
+}
+
+@Override
+protected List getIndices() throws
PcjIndexSetException {
+requireNonNull(conf);
--- End diff --
Do the supers not check if this is null?
> Mongo PCJ indexer support
> -
>
> Key: RYA-303
> URL: https://issues.apache.org/jira/browse/RYA-303
> Project: Rya
> Issue Type: Improvement
>Reporter: Andrew Smith
>Assignee: Andrew Smith
>
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319278#comment-16319278
]
ASF GitHub Bot commented on RYA-303:
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160528789
--- Diff:
extras/indexing/src/main/java/org/apache/rya/indexing/mongodb/pcj/MongoPcjIndexSetProvider.java
---
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.mongodb.pcj;
+
+import static java.util.Objects.requireNonNull;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.rya.api.instance.RyaDetailsRepository;
+import
org.apache.rya.api.instance.RyaDetailsRepository.RyaDetailsRepositoryException;
+import org.apache.rya.indexing.external.tupleSet.ExternalTupleSet;
+import
org.apache.rya.indexing.pcj.matching.provider.AbstractPcjIndexSetProvider;
+import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.mongo.MongoPcjDocuments;
+import org.apache.rya.indexing.pcj.storage.mongo.MongoPcjStorage;
+import org.apache.rya.mongodb.StatefulMongoDBRdfConfiguration;
+import org.apache.rya.mongodb.instance.MongoRyaInstanceDetailsRepository;
+import org.openrdf.query.MalformedQueryException;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.mongodb.MongoClient;
+
+/**
+ * Implementation of {@link AbstractPcjIndexSetProvider} for MongoDB.
+ */
+public class MongoPcjIndexSetProvider extends AbstractPcjIndexSetProvider {
+/**
+ * Creates a new {@link MongoPcjIndexSetProvider}.
+ * @param conf - The configuration for this provider. (not null)
+ */
+public MongoPcjIndexSetProvider(final StatefulMongoDBRdfConfiguration
conf) {
+super(conf);
+}
+
+/**
+ * Creates a new {@link MongoPcjIndexSetProvider}.
+ * @param conf - The configuration for this provider.
+ * @param indices - The predefined indicies on this provider.
+ * @param client - The {@link MongoClient} used to connect to mongo.
+ */
+public MongoPcjIndexSetProvider(final StatefulMongoDBRdfConfiguration
conf, final List indices) {
+super(conf, indices);
+}
+
+@Override
+protected List getIndices() throws
PcjIndexSetException {
+requireNonNull(conf);
+try {
+final StatefulMongoDBRdfConfiguration mongoConf =
(StatefulMongoDBRdfConfiguration) conf;
+final MongoClient client = mongoConf.getMongoClient();
+final MongoPcjDocuments pcjDocs = new
MongoPcjDocuments(client, mongoConf.getRyaInstanceName());
+List tables = null;
--- End diff --
Oh, this is legacy interface support and the table name is a leftover from
accumulo. This is how we let you provide a list of collection names that
represent the pcj indices through configuration instead of asking details.
Could you document that is what's going on?
> Mongo PCJ indexer support
> -
>
> Key: RYA-303
> URL: https://issues.apache.org/jira/browse/RYA-303
> Project: Rya
> Issue Type: Improvement
>Reporter: Andrew Smith
>Assignee: Andrew Smith
>
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319286#comment-16319286
]
ASF GitHub Bot commented on RYA-303:
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160538868
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
+private static final String AUTHS_FIELD = "auths";
+private static final String PCJ_NAME = "pcjName";
+
+private final MongoCollection pcjCollection;
+private static final PcjVarOrderFactory pcjVarOrderFactory = new
ShiftVarOrderFactory();
+
+/**
+ * Creates a new {@link MongoPcjDocuments}.
+ * @param client - The {@link
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319287#comment-16319287
]
ASF GitHub Bot commented on RYA-303:
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160537019
--- Diff:
extras/rya.indexing.pcj/src/main/java/org/apache/rya/indexing/pcj/storage/mongo/MongoPcjDocuments.java
---
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.storage.mongo;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MapBindingSet;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
+
+/**
+ * Creates and modifies PCJs in MongoDB. PCJ's are stored as follows:
+ *
+ *
+ *
+ * - PCJ Metadata Doc -
+ * {
+ * _id: [table_name]_METADATA,
+ * sparql: [sparql query to match results],
+ * cardinality: [number of results]
+ * }
+ *
+ * - PCJ Results Doc -
+ * {
+ * pcjName: [table_name],
+ * auths: [auths]
+ * [binding_var1]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * .
+ * .
+ * .
+ * [binding_varn]: {
+ * uri: [type_uri],
+ * value: value
+ * }
+ * }
+ *
+ *
+ */
+public class MongoPcjDocuments {
+public static final String PCJ_COLLECTION_NAME = "pcjs";
+
+// metadata fields
+public static final String CARDINALITY_FIELD = "cardinality";
+public static final String SPARQL_FIELD = "sparql";
+public static final String PCJ_ID = "_id";
+public static final String VAR_ORDER_ID = "varOrders";
+
+// pcj results fields
+private static final String BINDING_VALUE = "value";
+private static final String BINDING_TYPE = "uri";
--- End diff --
uri is kind of a weird name for this. rdfType or something would be better
> Mongo PCJ indexer support
> -
>
> Key: RYA-303
> URL: https://issues.apache.org/jira/browse/RYA-303
> Project: Rya
> Issue Type: Improvement
>Reporter: Andrew Smith
>Assignee:
[GitHub] incubator-rya pull request #172: RYA-303 Mongo PCJ Support
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160532572
--- Diff:
extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstallIT.java
---
@@ -41,7 +41,7 @@
*/
public class MongoInstallIT extends MongoITBase {
-@Test
+ @Test
--- End diff --
tab
---
[jira] [Commented] (RYA-303) Mongo PCJ indexer support
[
https://issues.apache.org/jira/browse/RYA-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16319283#comment-16319283
]
ASF GitHub Bot commented on RYA-303:
Github user kchilton2 commented on a diff in the pull request:
https://github.com/apache/incubator-rya/pull/172#discussion_r160532306
--- Diff:
extras/indexing/src/main/java/org/apache/rya/indexing/pcj/matching/provider/AccumuloIndexSetProvider.java
---
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.matching.provider;
+
+import static java.util.Objects.requireNonNull;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.log4j.Logger;
+import
org.apache.rya.accumulo.instance.AccumuloRyaInstanceDetailsRepository;
+import org.apache.rya.api.RdfCloudTripleStoreConfiguration;
+import org.apache.rya.api.instance.RyaDetailsRepository;
+import
org.apache.rya.api.instance.RyaDetailsRepository.RyaDetailsRepositoryException;
+import org.apache.rya.indexing.accumulo.ConfigUtils;
+import org.apache.rya.indexing.external.tupleSet.AccumuloIndexSet;
+import org.apache.rya.indexing.external.tupleSet.ExternalTupleSet;
+import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage;
+import
org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
+import org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjTableNameFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjTables;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.sail.SailException;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+/**
+ * Implementation of {@link AbstractPcjIndexSetProvider} for Accumulo.
+ * This provider uses either user specified Accumulo configuration
information or user a specified
+ * List of ExternalTupleSets to populate an internal cache of
ExternalTupleSets. If Accumulo configuration
+ * is provided, the provider connects to an instance of RyaDetails and
populates the cache with
+ * PCJs registered in RyaDetails.
+ */
+public class AccumuloIndexSetProvider extends AbstractPcjIndexSetProvider {
+private static final Logger log =
Logger.getLogger(AccumuloIndexSetProvider.class);
+
+public AccumuloIndexSetProvider(final Configuration conf) {
+super(conf);
+}
+
+public AccumuloIndexSetProvider(final Configuration conf, final
List indices) {
+super(conf, indices);
+}
+
+@Override
+protected List getIndices() throws
PcjIndexSetException {
+requireNonNull(conf);
+try {
+final String tablePrefix =
requireNonNull(conf.get(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX));
+final Connector conn =
requireNonNull(ConfigUtils.getConnector(conf));
+List tables = null;
+
+if (conf instanceof RdfCloudTripleStoreConfiguration) {
+tables = ((RdfCloudTripleStoreConfiguration)
conf).getPcjTables();
+}
+// this maps associates pcj table name with pcj sparql query
+final Map indexTables =
Maps.newLinkedHashMap();
+
+try(final PrecomputedJoinStorage storage = new
AccumuloPcjStorage(conn, tablePrefix)) {
+final PcjTableNameFactory pcjFactory = new
PcjTableNameFactory();
+
+final boolean tablesProvided = tables != null
