RYA-101 RYA-102 and RYA-133, fixing issues with secondary indexers
Project: http://git-wip-us.apache.org/repos/asf/incubator-rya/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-rya/commit/8597b78b Tree: http://git-wip-us.apache.org/repos/asf/incubator-rya/tree/8597b78b Diff: http://git-wip-us.apache.org/repos/asf/incubator-rya/diff/8597b78b Branch: refs/heads/master Commit: 8597b78b35cbfc0385bb5e74765926df21149237 Parents: 573aedf Author: David W. Lotts <[email protected]> Authored: Wed Aug 17 15:26:07 2016 -0400 Committer: Aaron Mihalik <[email protected]> Committed: Tue Aug 23 10:50:49 2016 -0400 ---------------------------------------------------------------------- .../mvm/rya/mongodb/MongoConnectorFactory.java | 109 ++++++++++++++++--- .../rya/mongodb/MongoDBRdfConfiguration.java | 6 +- .../java/mvm/rya/mongodb/MongoDBRyaDAO.java | 26 +++-- .../mvm/rya/mongodb/MongoSecondaryIndex.java | 12 ++ .../rya/indexing/FilterFunctionOptimizer.java | 20 +--- .../indexing/mongodb/AbstractMongoIndexer.java | 44 ++++---- .../mongodb/freetext/MongoFreeTextIndexer.java | 9 +- .../indexing/mongodb/geo/MongoGeoIndexer.java | 11 +- .../mongodb/temporal/MongoTemporalIndexer.java | 15 +-- .../mongo/MongoFreeTextIndexerTest.java | 16 +-- .../indexing/mongo/MongoGeoIndexerSfTest.java | 4 +- .../rya/indexing/mongo/MongoGeoIndexerTest.java | 36 +++--- .../mongo/MongoTemporalIndexerTest.java | 4 +- .../src/main/java/MongoRyaDirectExample.java | 19 +++- 14 files changed, 204 insertions(+), 127 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoConnectorFactory.java ---------------------------------------------------------------------- diff --git a/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoConnectorFactory.java b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoConnectorFactory.java index 747ad1e..b91c9df 100644 --- a/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoConnectorFactory.java +++ b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoConnectorFactory.java @@ -21,13 +21,19 @@ package mvm.rya.mongodb; import java.net.UnknownHostException; import java.util.Arrays; +import java.io.IOException; +import org.apache.commons.configuration.ConfigurationRuntimeException; import org.apache.hadoop.conf.Configuration; import com.mongodb.MongoClient; import com.mongodb.MongoCredential; +import com.mongodb.MongoException; import com.mongodb.ServerAddress; +import de.flapdoodle.embed.mongo.distribution.Version; +import de.flapdoodle.embed.mongo.tests.MongodForTestsFactory; + /** * Mongo convention generally allows for a single instance of a {@link MongoClient} * throughout the life cycle of an application. This MongoConnectorFactory lazy @@ -37,30 +43,101 @@ import com.mongodb.ServerAddress; public class MongoConnectorFactory { private static MongoClient mongoClient; + private final static String MSG_INTRO = "Failed to connect to MongoDB: "; + /** * @param conf The {@link Configuration} defining how to construct the MongoClient. * @return A {@link MongoClient}. This client is lazy loaded and the same one * is used throughout the lifecycle of the application. - * @throws NumberFormatException - Thrown if the configured port is not a valid number - * @throws UnknownHostException - The configured host cannot be found. + * @throws IOException - if MongodForTestsFactory constructor has an io exception. + * @throws ConfigurationRuntimeException - Thrown if the configured server, port, user, or others are missing. + * @throws MongoException if can't connect despite conf parameters are given */ - public static synchronized MongoClient getMongoClient(final Configuration conf) throws NumberFormatException, UnknownHostException { - if(mongoClient == null) { - final String host = conf.get(MongoDBRdfConfiguration.MONGO_INSTANCE); - final int port = Integer.parseInt(conf.get(MongoDBRdfConfiguration.MONGO_INSTANCE_PORT)); - final ServerAddress server = new ServerAddress(host, port); - - //check for authentication credentials - if (conf.get(MongoDBRdfConfiguration.MONGO_USER) != null) { - final String username = conf.get(MongoDBRdfConfiguration.MONGO_USER); - final String dbName = conf.get(MongoDBRdfConfiguration.MONGO_DB_NAME); - final char[] pswd = conf.get(MongoDBRdfConfiguration.MONGO_USER_PASSWORD).toCharArray(); - final MongoCredential cred = MongoCredential.createCredential(username, dbName, pswd); - mongoClient = new MongoClient(server, Arrays.asList(cred)); + public static synchronized MongoClient getMongoClient(final Configuration conf) + throws ConfigurationRuntimeException, MongoException { + if (mongoClient == null) { + // The static client has not yet created, is it a test/mock instance, or a service? + if (conf.getBoolean(MongoDBRdfConfiguration.USE_TEST_MONGO, false)) { + createMongoClientForTests(); } else { - mongoClient = new MongoClient(server); + createMongoClientForServer(conf); } } return mongoClient; } + + /** + * Create a local temporary MongoDB instance and client object and assign it to this class's static mongoClient + * @throws MongoException if can't connect + */ + private static void createMongoClientForTests() throws MongoException { + try { + MongodForTestsFactory testsFactory = MongodForTestsFactory.with(Version.Main.PRODUCTION); + mongoClient = testsFactory.newMongo(); + } catch (IOException e) { + // Rethrow as an unchecked error. Since we are in a test mode here, just fail fast. + throw new MongoException(MSG_INTRO+"creating a factory for a test/mock MongoDB instance.",e); + } + } + + /** + * Create a MongoDB client object and assign it to this class's static mongoClient + * @param conf configuration containing connection parameters + * @throws ConfigurationRuntimeException - Thrown if the configured server, port, user, or others are missing. + * @throws MongoException if can't connect despite conf parameters are given + */ + private static void createMongoClientForServer(final Configuration conf) + throws ConfigurationRuntimeException, MongoException { + // Connect to a running Mongo server + final String host = requireNonNull(conf.get(MongoDBRdfConfiguration.MONGO_INSTANCE), MSG_INTRO+"host name is required"); + final int port = requireNonNullInt(conf.get(MongoDBRdfConfiguration.MONGO_INSTANCE_PORT), MSG_INTRO+"Port number is required."); + ServerAddress server; + try { + server = new ServerAddress(host, port); + } catch (UnknownHostException e) { + throw new MongoException(MSG_INTRO + "cannot find host="+host,e); + } + // check for authentication credentials + if (conf.get(MongoDBRdfConfiguration.MONGO_USER) != null) { + final String username = conf.get(MongoDBRdfConfiguration.MONGO_USER); + final String dbName = requireNonNull(conf.get(MongoDBRdfConfiguration.MONGO_DB_NAME), + MSG_INTRO + MongoDBRdfConfiguration.MONGO_DB_NAME + " is null but required configuration if " + + MongoDBRdfConfiguration.MONGO_USER + " is configured."); + final char[] pswd = requireNonNull(conf.get(MongoDBRdfConfiguration.MONGO_USER_PASSWORD), + MSG_INTRO + MongoDBRdfConfiguration.MONGO_USER_PASSWORD + " is null but required configuration if " + + MongoDBRdfConfiguration.MONGO_USER + " is configured.").toCharArray(); + final MongoCredential cred = MongoCredential.createCredential(username, dbName, pswd); + mongoClient = new MongoClient(server, Arrays.asList(cred)); + } else { + // No user was configured: + mongoClient = new MongoClient(server); + } + } + + /** + * Throw exception for un-configured required values. + * + * @param required String to check + * @param message throw configuration exception with this description + * @return unaltered required string + * @throws ConfigurationRuntimeException if required is null + */ + private static String requireNonNull(String required, String message) throws ConfigurationRuntimeException { + if (required == null) + throw new ConfigurationRuntimeException(message); + return required; + } + + /* + * Same as above, check that it is a integer and return the parsed integer. + */ + private static int requireNonNullInt(String required, String message) throws ConfigurationRuntimeException { + if (required == null) + throw new ConfigurationRuntimeException(message); + try { + return Integer.parseInt(required); + } catch (NumberFormatException e) { + throw new ConfigurationRuntimeException(message); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoDBRdfConfiguration.java ---------------------------------------------------------------------- diff --git a/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoDBRdfConfiguration.java b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoDBRdfConfiguration.java index 0c38337..e8e301d 100644 --- a/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoDBRdfConfiguration.java +++ b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoDBRdfConfiguration.java @@ -104,7 +104,7 @@ public class MongoDBRdfConfiguration extends RdfCloudTripleStoreConfiguration { return this.get(MONGO_COLLECTION_PREFIX, "rya") + "_ns"; } - public void setAdditionalIndexers(Class<? extends RyaSecondaryIndexer>... indexers) { + public void setAdditionalIndexers(Class<? extends MongoSecondaryIndex>... indexers) { List<String> strs = Lists.newArrayList(); for (Class<?> ai : indexers){ strs.add(ai.getName()); @@ -113,8 +113,8 @@ public class MongoDBRdfConfiguration extends RdfCloudTripleStoreConfiguration { setStrings(CONF_ADDITIONAL_INDEXERS, strs.toArray(new String[]{})); } - public List<RyaSecondaryIndexer> getAdditionalIndexers() { - return getInstances(CONF_ADDITIONAL_INDEXERS, RyaSecondaryIndexer.class); + public List<MongoSecondaryIndex> getAdditionalIndexers() { + return getInstances(CONF_ADDITIONAL_INDEXERS, MongoSecondaryIndex.class); } public void setMongoClient(MongoClient client){ http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoDBRyaDAO.java ---------------------------------------------------------------------- diff --git a/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoDBRyaDAO.java b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoDBRyaDAO.java index 15537e5..b0fd1db 100644 --- a/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoDBRyaDAO.java +++ b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoDBRyaDAO.java @@ -32,8 +32,10 @@ import org.apache.log4j.Logger; import com.mongodb.DB; import com.mongodb.DBCollection; import com.mongodb.DBObject; +import com.mongodb.DuplicateKeyException; import com.mongodb.InsertOptions; import com.mongodb.MongoClient; +import com.mongodb.MongoException.DuplicateKey; import de.flapdoodle.embed.mongo.tests.MongodForTestsFactory; import mvm.rya.api.RdfCloudTripleStoreConfiguration; @@ -64,7 +66,7 @@ public final class MongoDBRyaDAO implements RyaDAO<MongoDBRdfConfiguration>{ private MongoDBNamespaceManager nameSpaceManager; private MongodForTestsFactory testsFactory; - private List<RyaSecondaryIndexer> secondaryIndexers; + private List<MongoSecondaryIndex> secondaryIndexers; /** * Creates a new {@link MongoDBRyaDAO} @@ -73,14 +75,9 @@ public final class MongoDBRyaDAO implements RyaDAO<MongoDBRdfConfiguration>{ */ public MongoDBRyaDAO(final MongoDBRdfConfiguration conf) throws RyaDAOException, NumberFormatException, UnknownHostException { this.conf = conf; - try { - mongoClient = MongoConnectorFactory.getMongoClient(conf); - conf.setMongoClient(mongoClient); - init(); - } catch (NumberFormatException | UnknownHostException e) { - log.error("Unable to create a connection to mongo.", e); - throw e; - } + mongoClient = MongoConnectorFactory.getMongoClient(conf); + conf.setMongoClient(mongoClient); + init(); } @@ -117,8 +114,9 @@ public final class MongoDBRyaDAO implements RyaDAO<MongoDBRdfConfiguration>{ @Override public void init() throws RyaDAOException { secondaryIndexers = conf.getAdditionalIndexers(); - for(final RyaSecondaryIndexer index: secondaryIndexers) { + for(final MongoSecondaryIndex index: secondaryIndexers) { index.setConf(conf); + index.setClient(mongoClient); } db = mongoClient.getDB(conf.get(MongoDBRdfConfiguration.MONGO_DB_NAME)); @@ -127,6 +125,9 @@ public final class MongoDBRyaDAO implements RyaDAO<MongoDBRdfConfiguration>{ queryEngine = new MongoDBQueryEngine(conf, mongoClient); storageStrategy = new SimpleMongoDBStorageStrategy(); storageStrategy.createIndices(coll); + for(final MongoSecondaryIndex index: secondaryIndexers) { + index.init(); + } } @Override @@ -154,10 +155,13 @@ public final class MongoDBRyaDAO implements RyaDAO<MongoDBRdfConfiguration>{ for(final RyaSecondaryIndexer index: secondaryIndexers) { index.storeStatement(statement); } - } catch (final IOException e) { + } catch (IOException e) { log.error("Unable to add: " + statement.toString()); throw new RyaDAOException(e); } + catch (DuplicateKeyException e){ + log.error("Attempting to load duplicate triple: " + statement.toString()); + } } @Override http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoSecondaryIndex.java ---------------------------------------------------------------------- diff --git a/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoSecondaryIndex.java b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoSecondaryIndex.java new file mode 100644 index 0000000..553a78f --- /dev/null +++ b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoSecondaryIndex.java @@ -0,0 +1,12 @@ +package mvm.rya.mongodb; + +import com.mongodb.MongoClient; + +import mvm.rya.api.persist.index.RyaSecondaryIndexer; + +public interface MongoSecondaryIndex extends RyaSecondaryIndexer{ + public void init(); + + public void setClient(MongoClient client); + +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/extras/indexing/src/main/java/mvm/rya/indexing/FilterFunctionOptimizer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/FilterFunctionOptimizer.java b/extras/indexing/src/main/java/mvm/rya/indexing/FilterFunctionOptimizer.java index ad9fe22..6c0cd9a 100644 --- a/extras/indexing/src/main/java/mvm/rya/indexing/FilterFunctionOptimizer.java +++ b/extras/indexing/src/main/java/mvm/rya/indexing/FilterFunctionOptimizer.java @@ -68,6 +68,7 @@ import mvm.rya.indexing.accumulo.freetext.FreeTextTupleSet; import mvm.rya.indexing.accumulo.geo.GeoMesaGeoIndexer; import mvm.rya.indexing.accumulo.geo.GeoTupleSet; import mvm.rya.indexing.accumulo.temporal.AccumuloTemporalIndexer; +import mvm.rya.indexing.mongodb.AbstractMongoIndexer; import mvm.rya.indexing.mongodb.freetext.MongoFreeTextIndexer; import mvm.rya.indexing.mongodb.geo.MongoGeoIndexer; import mvm.rya.indexing.mongodb.temporal.MongoTemporalIndexer; @@ -99,29 +100,18 @@ public class FilterFunctionOptimizer implements QueryOptimizer, Configurable { this.conf = conf; //reset the init. init = false; - try { init(); - } catch (final NumberFormatException | UnknownHostException e) { - LOG.error("Unable to update to use new config, falling back to the old config.", e); - init = true; - } } - private synchronized void init() throws NumberFormatException, UnknownHostException { + private synchronized void init() { if (!init) { if (ConfigUtils.getUseMongo(conf)) { - try { - final MongoClient mongoClient = MongoConnectorFactory.getMongoClient(conf); - geoIndexer = new MongoGeoIndexer(mongoClient); + geoIndexer = new MongoGeoIndexer(); geoIndexer.setConf(conf); - freeTextIndexer = new MongoFreeTextIndexer(mongoClient); + freeTextIndexer = new MongoFreeTextIndexer(); freeTextIndexer.setConf(conf); - temporalIndexer = new MongoTemporalIndexer(mongoClient); + temporalIndexer = new MongoTemporalIndexer(); temporalIndexer.setConf(conf); - } catch (NumberFormatException | UnknownHostException e) { - LOG.error("Unable to connect to mongo.", e); - throw e; - } } else { geoIndexer = new GeoMesaGeoIndexer(); geoIndexer.setConf(conf); http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/AbstractMongoIndexer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/AbstractMongoIndexer.java b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/AbstractMongoIndexer.java index 078cbd2..daa78e4 100644 --- a/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/AbstractMongoIndexer.java +++ b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/AbstractMongoIndexer.java @@ -1,7 +1,5 @@ package mvm.rya.indexing.mongodb; -import static com.google.common.base.Preconditions.checkNotNull; - /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -42,20 +40,23 @@ import com.mongodb.QueryBuilder; import info.aduna.iteration.CloseableIteration; import mvm.rya.api.domain.RyaStatement; import mvm.rya.api.domain.RyaURI; -import mvm.rya.api.persist.index.RyaSecondaryIndexer; import mvm.rya.api.resolver.RyaToRdfConversions; import mvm.rya.indexing.StatementConstraints; +import mvm.rya.mongodb.MongoConnectorFactory; import mvm.rya.mongodb.MongoDBRdfConfiguration; +import mvm.rya.mongodb.MongoDBRyaDAO; +import mvm.rya.mongodb.MongoSecondaryIndex; /** * Secondary Indexer using MondoDB * @param <T> - The {@link AbstractMongoIndexingStorageStrategy} this indexer uses. */ -public abstract class AbstractMongoIndexer<T extends IndexingMongoDBStorageStrategy> implements RyaSecondaryIndexer { +public abstract class AbstractMongoIndexer<T extends IndexingMongoDBStorageStrategy> implements MongoSecondaryIndex { private static final Logger LOG = Logger.getLogger(AbstractMongoIndexer.class); private boolean isInit = false; protected Configuration conf; + protected MongoDBRyaDAO dao; protected MongoClient mongoClient; protected String dbName; protected DB db; @@ -64,31 +65,32 @@ public abstract class AbstractMongoIndexer<T extends IndexingMongoDBStorageStrat protected T storageStrategy; - /** - * Creates a new {@link AbstractMongoIndexer} with the provided mongo client. - * @param mongoClient The {@link MongoClient} to use with this indexer. - */ - public AbstractMongoIndexer(final MongoClient mongoClient) { - this.mongoClient = checkNotNull(mongoClient); - } - - protected void init() throws NumberFormatException, IOException{ + protected void initCore() { dbName = conf.get(MongoDBRdfConfiguration.MONGO_DB_NAME); db = this.mongoClient.getDB(dbName); collection = db.getCollection(conf.get(MongoDBRdfConfiguration.MONGO_COLLECTION_PREFIX, "rya") + getCollectionName()); } + + public void setClient(MongoClient client){ + this.mongoClient = client; + } + + // TODO this method is only intended to be used in testing + public void initIndexer(final Configuration conf, final MongoClient client) { + setConf(conf); + setClient(client); + if (!isInit) { + init(); + isInit = true; + } + } @Override public void setConf(final Configuration conf) { this.conf = conf; - if (!isInit) { - try { - init(); - isInit = true; - } catch (final NumberFormatException | IOException e) { - LOG.warn("Unable to initialize index. Throwing Runtime Exception. ", e); - throw new RuntimeException(e); - } + if (!isInit){ + setClient(MongoConnectorFactory.getMongoClient(conf)); + init(); } } http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/freetext/MongoFreeTextIndexer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/freetext/MongoFreeTextIndexer.java b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/freetext/MongoFreeTextIndexer.java index 7a65d26..0689feb 100644 --- a/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/freetext/MongoFreeTextIndexer.java +++ b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/freetext/MongoFreeTextIndexer.java @@ -24,7 +24,6 @@ import org.apache.log4j.Logger; import org.openrdf.model.Statement; import org.openrdf.query.QueryEvaluationException; -import com.mongodb.MongoClient; import com.mongodb.QueryBuilder; import info.aduna.iteration.CloseableIteration; @@ -37,13 +36,9 @@ public class MongoFreeTextIndexer extends AbstractMongoIndexer<TextMongoDBStorag private static final String COLLECTION_SUFFIX = "freetext"; private static final Logger logger = Logger.getLogger(MongoFreeTextIndexer.class); - public MongoFreeTextIndexer(final MongoClient mongoClient) { - super(mongoClient); - } - @Override - protected void init() throws IOException{ - super.init(); + public void init() { + initCore(); predicates = ConfigUtils.getFreeTextPredicates(conf); storageStrategy = new TextMongoDBStorageStrategy(); storageStrategy.createIndices(collection); http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/geo/MongoGeoIndexer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/geo/MongoGeoIndexer.java b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/geo/MongoGeoIndexer.java index 303848a..7589f03 100644 --- a/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/geo/MongoGeoIndexer.java +++ b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/geo/MongoGeoIndexer.java @@ -22,14 +22,11 @@ import static mvm.rya.indexing.mongodb.geo.GeoMongoDBStorageStrategy.GeoQueryTyp import static mvm.rya.indexing.mongodb.geo.GeoMongoDBStorageStrategy.GeoQueryType.INTERSECTS; import static mvm.rya.indexing.mongodb.geo.GeoMongoDBStorageStrategy.GeoQueryType.WITHIN; -import java.io.IOException; - import org.apache.log4j.Logger; import org.openrdf.model.Statement; import org.openrdf.query.QueryEvaluationException; import com.mongodb.DBObject; -import com.mongodb.MongoClient; import com.vividsolutions.jts.geom.Geometry; import info.aduna.iteration.CloseableIteration; @@ -44,13 +41,9 @@ public class MongoGeoIndexer extends AbstractMongoIndexer<GeoMongoDBStorageStrat private static final String COLLECTION_SUFFIX = "geo"; private static final Logger logger = Logger.getLogger(MongoGeoIndexer.class); - public MongoGeoIndexer(final MongoClient mongoClient) { - super(mongoClient); - } - @Override - protected void init() throws NumberFormatException, IOException{ - super.init(); + public void init() { + initCore(); predicates = ConfigUtils.getGeoPredicates(conf); storageStrategy = new GeoMongoDBStorageStrategy(Double.valueOf(conf.get(MongoDBRdfConfiguration.MONGO_GEO_MAXDISTANCE, "1e-10"))); storageStrategy.createIndices(collection); http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/temporal/MongoTemporalIndexer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/temporal/MongoTemporalIndexer.java b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/temporal/MongoTemporalIndexer.java index 003d9c9..be991c8 100644 --- a/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/temporal/MongoTemporalIndexer.java +++ b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/temporal/MongoTemporalIndexer.java @@ -22,15 +22,12 @@ import static mvm.rya.indexing.mongodb.temporal.TemporalMongoDBStorageStrategy.I import static mvm.rya.indexing.mongodb.temporal.TemporalMongoDBStorageStrategy.INTERVAL_END; import static mvm.rya.indexing.mongodb.temporal.TemporalMongoDBStorageStrategy.INTERVAL_START; -import java.io.IOException; - import org.apache.log4j.Logger; import org.openrdf.model.Statement; import org.openrdf.query.QueryEvaluationException; import com.google.common.annotations.VisibleForTesting; import com.mongodb.DBCollection; -import com.mongodb.MongoClient; import com.mongodb.QueryBuilder; import info.aduna.iteration.CloseableIteration; @@ -48,17 +45,9 @@ public class MongoTemporalIndexer extends AbstractMongoIndexer<TemporalMongoDBSt private static final String COLLECTION_SUFFIX = "temporal"; private static final Logger LOG = Logger.getLogger(MongoTemporalIndexer.class); - /** - * Creates a new {@link MongoTemporalIndexer} - * @param mongoClient - The {@link MongoClient} used to interact with MongoDB. - */ - public MongoTemporalIndexer(final MongoClient mongoClient) { - super(mongoClient); - } - @Override - protected void init() throws IOException{ - super.init(); + public void init() { + initCore(); predicates = ConfigUtils.getTemporalPredicates(conf); storageStrategy = new TemporalMongoDBStorageStrategy(); storageStrategy.createIndices(collection); http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoFreeTextIndexerTest.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoFreeTextIndexerTest.java b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoFreeTextIndexerTest.java index 91ba955..66ddc33 100644 --- a/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoFreeTextIndexerTest.java +++ b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoFreeTextIndexerTest.java @@ -69,8 +69,8 @@ public class MongoFreeTextIndexerTest { @Test public void testSearch() throws Exception { - try (MongoFreeTextIndexer f = new MongoFreeTextIndexer(mongoClient)) { - f.setConf(conf); + try (MongoFreeTextIndexer f = new MongoFreeTextIndexer()) { + f.initIndexer(conf, mongoClient); final ValueFactory vf = new ValueFactoryImpl(); @@ -93,8 +93,8 @@ public class MongoFreeTextIndexerTest { @Test public void testDelete() throws Exception { - try (MongoFreeTextIndexer f = new MongoFreeTextIndexer(mongoClient)) { - f.setConf(conf); + try (MongoFreeTextIndexer f = new MongoFreeTextIndexer()) { + f.initIndexer(conf, mongoClient); final ValueFactory vf = new ValueFactoryImpl(); @@ -143,8 +143,8 @@ public class MongoFreeTextIndexerTest { public void testRestrictPredicatesSearch() throws Exception { conf.setStrings(ConfigUtils.FREETEXT_PREDICATES_LIST, "pred:1,pred:2"); - try (MongoFreeTextIndexer f = new MongoFreeTextIndexer(mongoClient)) { - f.setConf(conf); + try (MongoFreeTextIndexer f = new MongoFreeTextIndexer()) { + f.initIndexer(conf, mongoClient); // These should not be stored because they are not in the predicate list f.storeStatement(new RyaStatement(new RyaURI("foo:subj1"), new RyaURI(RDFS.LABEL.toString()), new RyaType("invalid"))); @@ -176,8 +176,8 @@ public class MongoFreeTextIndexerTest { @Test public void testContextSearch() throws Exception { - try (MongoFreeTextIndexer f = new MongoFreeTextIndexer(mongoClient)) { - f.setConf(conf); + try (MongoFreeTextIndexer f = new MongoFreeTextIndexer()) { + f.initIndexer(conf, mongoClient); final ValueFactory vf = new ValueFactoryImpl(); final URI subject = new URIImpl("foo:subj"); http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerSfTest.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerSfTest.java b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerSfTest.java index 3368d6e..fe6b51a 100644 --- a/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerSfTest.java +++ b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerSfTest.java @@ -121,8 +121,8 @@ public class MongoGeoIndexerSfTest { final MongodForTestsFactory testsFactory = MongodForTestsFactory.with(Version.Main.PRODUCTION); final MongoClient mongoClient = testsFactory.newMongo(); - g = new MongoGeoIndexer(mongoClient); - g.setConf(conf); + g = new MongoGeoIndexer(); + g.initIndexer(conf, mongoClient); g.storeStatement(statement(A)); g.storeStatement(statement(B)); g.storeStatement(statement(C)); http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerTest.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerTest.java b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerTest.java index 7a2dc8e..7a20deb 100644 --- a/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerTest.java +++ b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerTest.java @@ -85,8 +85,8 @@ public class MongoGeoIndexerTest { @Test public void testRestrictPredicatesSearch() throws Exception { conf.setStrings(ConfigUtils.GEO_PREDICATES_LIST, "pred:1,pred:2"); - try (MongoGeoIndexer f = new MongoGeoIndexer(mongoClient)) { - f.setConf(conf); + try (MongoGeoIndexer f = new MongoGeoIndexer()) { + f.initIndexer(conf, mongoClient); final ValueFactory vf = new ValueFactoryImpl(); @@ -132,8 +132,8 @@ public class MongoGeoIndexerTest { @Test public void testPrimeMeridianSearch() throws Exception { - try (MongoGeoIndexer f = new MongoGeoIndexer(mongoClient)) { - f.setConf(conf); + try (MongoGeoIndexer f = new MongoGeoIndexer()) { + f.initIndexer(conf, mongoClient); final ValueFactory vf = new ValueFactoryImpl(); final Resource subject = vf.createURI("foo:subj"); @@ -176,8 +176,8 @@ public class MongoGeoIndexerTest { @Test public void testDcSearch() throws Exception { // test a ring around dc - try (MongoGeoIndexer f = new MongoGeoIndexer(mongoClient)) { - f.setConf(conf); + try (MongoGeoIndexer f = new MongoGeoIndexer()) { + f.initIndexer(conf, mongoClient); final ValueFactory vf = new ValueFactoryImpl(); final Resource subject = vf.createURI("foo:subj"); @@ -205,8 +205,8 @@ public class MongoGeoIndexerTest { @Test public void testDeleteSearch() throws Exception { // test a ring around dc - try (MongoGeoIndexer f = new MongoGeoIndexer(mongoClient)) { - f.setConf(conf); + try (MongoGeoIndexer f = new MongoGeoIndexer()) { + f.initIndexer(conf, mongoClient); final ValueFactory vf = new ValueFactoryImpl(); final Resource subject = vf.createURI("foo:subj"); @@ -244,8 +244,8 @@ public class MongoGeoIndexerTest { @Test public void testDcSearchWithContext() throws Exception { // test a ring around dc - try (MongoGeoIndexer f = new MongoGeoIndexer(mongoClient)) { - f.setConf(conf); + try (MongoGeoIndexer f = new MongoGeoIndexer()) { + f.initIndexer(conf, mongoClient); final ValueFactory vf = new ValueFactoryImpl(); final Resource subject = vf.createURI("foo:subj"); @@ -273,8 +273,8 @@ public class MongoGeoIndexerTest { @Test public void testDcSearchWithSubject() throws Exception { // test a ring around dc - try (MongoGeoIndexer f = new MongoGeoIndexer(mongoClient)) { - f.setConf(conf); + try (MongoGeoIndexer f = new MongoGeoIndexer()) { + f.initIndexer(conf, mongoClient); final ValueFactory vf = new ValueFactoryImpl(); final Resource subject = vf.createURI("foo:subj"); @@ -301,8 +301,8 @@ public class MongoGeoIndexerTest { @Test public void testDcSearchWithSubjectAndContext() throws Exception { // test a ring around dc - try (MongoGeoIndexer f = new MongoGeoIndexer(mongoClient)) { - f.setConf(conf); + try (MongoGeoIndexer f = new MongoGeoIndexer()) { + f.initIndexer(conf, mongoClient); final ValueFactory vf = new ValueFactoryImpl(); final Resource subject = vf.createURI("foo:subj"); @@ -334,8 +334,8 @@ public class MongoGeoIndexerTest { @Test public void testDcSearchWithPredicate() throws Exception { // test a ring around dc - try (MongoGeoIndexer f = new MongoGeoIndexer(mongoClient)) { - f.setConf(conf); + try (MongoGeoIndexer f = new MongoGeoIndexer()) { + f.initIndexer(conf, mongoClient); final ValueFactory vf = new ValueFactoryImpl(); final Resource subject = vf.createURI("foo:subj"); @@ -364,8 +364,8 @@ public class MongoGeoIndexerTest { // @Test public void testAntiMeridianSearch() throws Exception { // verify that a search works if the bounding box crosses the anti meridian - try (MongoGeoIndexer f = new MongoGeoIndexer(mongoClient)) { - f.setConf(conf); + try (MongoGeoIndexer f = new MongoGeoIndexer()) { + f.initIndexer(conf, mongoClient); final ValueFactory vf = new ValueFactoryImpl(); final Resource context = vf.createURI("foo:context"); http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoTemporalIndexerTest.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoTemporalIndexerTest.java b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoTemporalIndexerTest.java index a8481fe..c612338 100644 --- a/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoTemporalIndexerTest.java +++ b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoTemporalIndexerTest.java @@ -192,8 +192,8 @@ public final class MongoTemporalIndexerTest { final MongodForTestsFactory testsFactory = MongodForTestsFactory.with(Version.Main.PRODUCTION); final MongoClient mongoClient = testsFactory.newMongo(); - tIndexer = new MongoTemporalIndexer(mongoClient); - tIndexer.setConf(conf); + tIndexer = new MongoTemporalIndexer(); + tIndexer.initIndexer(conf, mongoClient); final String dbName = conf.get(MongoDBRdfConfiguration.MONGO_DB_NAME); http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/extras/indexingExample/src/main/java/MongoRyaDirectExample.java ---------------------------------------------------------------------- diff --git a/extras/indexingExample/src/main/java/MongoRyaDirectExample.java b/extras/indexingExample/src/main/java/MongoRyaDirectExample.java index 91a31f9..5dbd260 100644 --- a/extras/indexingExample/src/main/java/MongoRyaDirectExample.java +++ b/extras/indexingExample/src/main/java/MongoRyaDirectExample.java @@ -60,6 +60,10 @@ public class MongoRyaDirectExample { private static final boolean PRINT_QUERIES = true; private static final String MONGO_DB = "rya"; private static final String MONGO_COLL_PREFIX = "rya_"; + private static final boolean USE_MOCK = true; + private static final boolean USE_INFER = true; + private static final String MONGO_INSTANCE_URL = "localhost"; + private static final String MONGO_INSTANCE_PORT = "27017"; public static void main(String[] args) throws Exception { Configuration conf = getConf(); @@ -80,7 +84,10 @@ public class MongoRyaDirectExample { testAddNamespaces(conn); testAddPointAndWithinSearch(conn); testAddAndFreeTextSearchWithPCJ(conn); - testInfer(conn, sail); + // to test out inference, set inference to true in the conf + if (USE_INFER){ + testInfer(conn, sail); + } log.info("TIME: " + (System.currentTimeMillis() - start) / 1000.); } finally { @@ -249,10 +256,16 @@ public class MongoRyaDirectExample { MongoDBRdfConfiguration conf = new MongoDBRdfConfiguration(); conf.set(ConfigUtils.USE_MONGO, "true"); + // User name and password must be filled in: conf.set(MongoDBRdfConfiguration.MONGO_USER, "fill this in"); conf.set(MongoDBRdfConfiguration.MONGO_USER_PASSWORD, "fill this in"); - conf.set(MongoDBRdfConfiguration.USE_TEST_MONGO, "true"); + conf.set(MongoDBRdfConfiguration.USE_TEST_MONGO, Boolean.toString(USE_MOCK)); + if (!USE_MOCK){ + conf.set(MongoDBRdfConfiguration.MONGO_INSTANCE, MONGO_INSTANCE_URL); + conf.set(MongoDBRdfConfiguration.MONGO_INSTANCE_PORT, MONGO_INSTANCE_PORT); + + } conf.set(MongoDBRdfConfiguration.MONGO_DB_NAME, MONGO_DB); conf.set(MongoDBRdfConfiguration.MONGO_COLLECTION_PREFIX, MONGO_COLL_PREFIX); conf.set(ConfigUtils.GEO_PREDICATES_LIST, "http://www.opengis.net/ont/geosparql#asWKT";); @@ -261,6 +274,8 @@ public class MongoRyaDirectExample { conf.setTablePrefix(MONGO_COLL_PREFIX); conf.set(ConfigUtils.GEO_PREDICATES_LIST, GeoConstants.GEO_AS_WKT.stringValue()); conf.set(ConfigUtils.FREETEXT_PREDICATES_LIST, RDFS.LABEL.stringValue()); + conf.set(ConfigUtils.FREETEXT_PREDICATES_LIST, RDFS.LABEL.stringValue()); + conf.set(RdfCloudTripleStoreConfiguration.CONF_INFER, Boolean.toString(USE_INFER)); return conf; }
