RYA-112 Integrate RyaDetails with secondary indexers
Project: http://git-wip-us.apache.org/repos/asf/incubator-rya/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-rya/commit/39a46dad Tree: http://git-wip-us.apache.org/repos/asf/incubator-rya/tree/39a46dad Diff: http://git-wip-us.apache.org/repos/asf/incubator-rya/diff/39a46dad Branch: refs/heads/develop Commit: 39a46dad6202bec8901add29c4fe3d24330af454 Parents: e4ef9a9 Author: David W. Lotts <david.lo...@parsons.com> Authored: Thu Jul 28 00:19:17 2016 -0400 Committer: Aaron Mihalik <miha...@alum.mit.edu> Committed: Tue Aug 23 10:41:37 2016 -0400 ---------------------------------------------------------------------- .gitignore | 1 + .../api/persist/index/RyaSecondaryIndexer.java | 8 +- .../mvm/rya/indexing/accumulo/ConfigUtils.java | 87 ++++++++------------ .../accumulo/entity/AccumuloDocIdIndexer.java | 40 ++++----- .../accumulo/entity/EntityCentricIndex.java | 19 ++++- .../freetext/AccumuloFreeTextIndexer.java | 52 ++++++++++-- .../accumulo/geo/GeoMesaGeoIndexer.java | 15 +++- .../temporal/AccumuloTemporalIndexer.java | 6 +- .../external/tupleSet/AccumuloIndexSet.java | 10 +-- .../mongodb/freetext/MongoFreeTextIndexer.java | 3 +- .../indexing/mongodb/geo/MongoGeoIndexer.java | 3 +- .../mongodb/temporal/MongoTemporalIndexer.java | 3 +- .../rya/indexing/pcj/matching/PCJOptimizer.java | 6 +- .../mvm/rya/sail/config/RyaSailFactory.java | 16 +++- .../accumulo/entity/AccumuloDocIndexerTest.java | 62 ++++++-------- .../freetext/AccumuloFreeTextIndexerTest.java | 26 +++--- .../indexing/accumulo/geo/GeoIndexerSfTest.java | 13 ++- .../indexing/accumulo/geo/GeoIndexerTest.java | 14 ++-- .../temporal/AccumuloTemporalIndexerTest.java | 48 ++++------- .../external/AccumuloPcjIntegrationTest.java | 4 +- .../external/PcjIntegrationTestingUtil.java | 2 + .../AccumuloIndexSetColumnVisibilityTest.java | 2 +- .../mongo/MongoFreeTextIndexerTest.java | 10 +-- .../indexing/mongo/MongoGeoIndexerSfTest.java | 6 +- .../rya/indexing/mongo/MongoGeoIndexerTest.java | 8 +- .../mongo/MongoTemporalIndexerTest.java | 15 +--- .../src/main/java/EntityDirectExample.java | 3 +- .../src/main/java/MongoRyaDirectExample.java | 8 +- .../src/main/java/RyaDirectExample.java | 7 +- 29 files changed, 260 insertions(+), 237 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 8ce329e..cb7f0ff 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ local.properties .classpath .settings/ .loadpath +.recommenders/ # External tool builders .externalToolBuilders/ http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/common/rya.api/src/main/java/mvm/rya/api/persist/index/RyaSecondaryIndexer.java ---------------------------------------------------------------------- diff --git a/common/rya.api/src/main/java/mvm/rya/api/persist/index/RyaSecondaryIndexer.java b/common/rya.api/src/main/java/mvm/rya/api/persist/index/RyaSecondaryIndexer.java index e92adef..4047670 100644 --- a/common/rya.api/src/main/java/mvm/rya/api/persist/index/RyaSecondaryIndexer.java +++ b/common/rya.api/src/main/java/mvm/rya/api/persist/index/RyaSecondaryIndexer.java @@ -34,7 +34,13 @@ import mvm.rya.api.domain.RyaURI; public interface RyaSecondaryIndexer extends Closeable, Flushable, Configurable { - public String getTableName(); + /** + * Returns the table name if the implementation supports it. + * Note that some indexers use multiple tables, this only returns one. + * TODO recommend that we deprecate this method because it's a leaky interface. + * @return table name as a string. + */ + public String getTableName(); public void storeStatements(Collection<RyaStatement> statements) throws IOException; http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/ConfigUtils.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/ConfigUtils.java b/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/ConfigUtils.java index c15ba1b..7540de4 100644 --- a/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/ConfigUtils.java +++ b/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/ConfigUtils.java @@ -19,27 +19,12 @@ package mvm.rya.indexing.accumulo; * under the License. */ - +import static java.util.Objects.requireNonNull; import java.util.HashSet; import java.util.List; import java.util.Set; -import mvm.rya.accumulo.AccumuloRdfConfiguration; -import mvm.rya.api.RdfCloudTripleStoreConfiguration; -import mvm.rya.indexing.FilterFunctionOptimizer; -import mvm.rya.indexing.accumulo.entity.EntityCentricIndex; -import mvm.rya.indexing.accumulo.entity.EntityOptimizer; -import mvm.rya.indexing.accumulo.freetext.AccumuloFreeTextIndexer; -import mvm.rya.indexing.accumulo.freetext.LuceneTokenizer; -import mvm.rya.indexing.accumulo.freetext.Tokenizer; -import mvm.rya.indexing.accumulo.geo.GeoMesaGeoIndexer; -import mvm.rya.indexing.accumulo.temporal.AccumuloTemporalIndexer; -import mvm.rya.indexing.external.PrecomputedJoinIndexer; -import mvm.rya.indexing.mongodb.freetext.MongoFreeTextIndexer; -import mvm.rya.indexing.mongodb.geo.MongoGeoIndexer; -import mvm.rya.indexing.pcj.matching.PCJOptimizer; - import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.BatchScanner; @@ -54,7 +39,6 @@ import org.apache.accumulo.core.client.ZooKeeperInstance; import org.apache.accumulo.core.client.admin.TableOperations; import org.apache.accumulo.core.client.mock.MockInstance; import org.apache.accumulo.core.security.Authorizations; -import org.apache.commons.lang.Validate; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.util.ReflectionUtils; @@ -65,8 +49,27 @@ import org.openrdf.model.impl.URIImpl; import com.google.common.base.Optional; import com.google.common.collect.Lists; +import mvm.rya.accumulo.AccumuloRdfConfiguration; +import mvm.rya.api.RdfCloudTripleStoreConfiguration; +import mvm.rya.api.instance.RyaDetails; +import mvm.rya.indexing.FilterFunctionOptimizer; +import mvm.rya.indexing.accumulo.entity.EntityCentricIndex; +import mvm.rya.indexing.accumulo.entity.EntityOptimizer; +import mvm.rya.indexing.accumulo.freetext.AccumuloFreeTextIndexer; +import mvm.rya.indexing.accumulo.freetext.LuceneTokenizer; +import mvm.rya.indexing.accumulo.freetext.Tokenizer; +import mvm.rya.indexing.accumulo.geo.GeoMesaGeoIndexer; +import mvm.rya.indexing.accumulo.temporal.AccumuloTemporalIndexer; +import mvm.rya.indexing.external.PrecomputedJoinIndexer; +import mvm.rya.indexing.mongodb.freetext.MongoFreeTextIndexer; +import mvm.rya.indexing.mongodb.geo.MongoGeoIndexer; +import mvm.rya.indexing.pcj.matching.PCJOptimizer; + /** * A set of configuration utils to read a Hadoop {@link Configuration} object and create Cloudbase/Accumulo objects. + * Soon will deprecate this class. Use installer for the set methods, use {@link RyaDetails} for the get methods. + * New code must separate parameters that are set at Rya install time from that which is specific to the client. + * Also Accumulo index tables are pushed down to the implementation and not configured in conf. */ public class ConfigUtils { private static final Logger logger = Logger.getLogger(ConfigUtils.class); @@ -84,12 +87,7 @@ public class ConfigUtils { public static final String FREE_TEXT_QUERY_TERM_LIMIT = "sc.freetext.querytermlimit"; - public static final String FREE_TEXT_DOC_TABLENAME = "sc.freetext.doctable"; - public static final String FREE_TEXT_TERM_TABLENAME = "sc.freetext.termtable"; - public static final String GEO_TABLENAME = "sc.geo.table"; public static final String GEO_NUM_PARTITIONS = "sc.geo.numPartitions"; - public static final String TEMPORAL_TABLENAME = "sc.temporal.index"; - public static final String ENTITY_TABLENAME = "sc.entity.index"; public static final String USE_GEO = "sc.use_geo"; public static final String USE_FREETEXT = "sc.use_freetext"; @@ -142,7 +140,7 @@ public class ConfigUtils { */ private static String getStringCheckSet(final Configuration conf, final String key) { final String value = conf.get(key); - Validate.notNull(value, key + " not set"); + requireNonNull(value, key + " not set"); return value; } @@ -165,42 +163,25 @@ public class ConfigUtils { return false; } - private static String getIndexTableName(final Configuration conf, final String indexTableNameConf, final String altSuffix){ - String value = conf.get(indexTableNameConf); - if (value == null){ - final String defaultTableName = conf.get(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX); - Validate.notNull(defaultTableName, indexTableNameConf + " not set and " + RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX + " not set. Cannot generate table name."); - value = conf.get(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX) + altSuffix; - } - return value; - } - - public static String getFreeTextDocTablename(final Configuration conf) { - return getIndexTableName(conf, FREE_TEXT_DOC_TABLENAME, "freetext"); - } - - public static String getFreeTextTermTablename(final Configuration conf) { - return getIndexTableName(conf, FREE_TEXT_TERM_TABLENAME, "freetext_term"); + /** + * Lookup the table name prefix in the conf and throw an error if it is null. + * Future, get table prefix from RyaDetails -- the Rya instance name + * -- also getting info from the RyaDetails should happen within RyaSailFactory and not ConfigUtils. + * @param conf Rya configuration map where it extracts the prefix (instance name) + * @return index table prefix corresponding to this Rya instance + */ + public static String getTablePrefix(final Configuration conf) { + final String tablePrefix; + tablePrefix = conf.get(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX); + requireNonNull(tablePrefix, "Configuration key: " + RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX + + " not set. Cannot generate table name."); + return tablePrefix; } public static int getFreeTextTermLimit(final Configuration conf) { return conf.getInt(FREE_TEXT_QUERY_TERM_LIMIT, 100); } - public static String getGeoTablename(final Configuration conf) { - return getIndexTableName(conf, GEO_TABLENAME, "geo"); - } - - public static String getTemporalTableName(final Configuration conf) { - return getIndexTableName(conf, TEMPORAL_TABLENAME, "temporal"); - } - - - public static String getEntityTableName(final Configuration conf) { - return getIndexTableName(conf, ENTITY_TABLENAME, "entity"); - } - - public static Set<URI> getFreeTextPredicates(final Configuration conf) { return getPredicates(conf, FREETEXT_PREDICATES_LIST); } http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/entity/AccumuloDocIdIndexer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/entity/AccumuloDocIdIndexer.java b/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/entity/AccumuloDocIdIndexer.java index 73378cf..4c1a3ad 100644 --- a/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/entity/AccumuloDocIdIndexer.java +++ b/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/entity/AccumuloDocIdIndexer.java @@ -398,9 +398,9 @@ public class AccumuloDocIdIndexer implements DocIdIndexer { return currentSolutionBs; } - private BatchScanner runQuery(StarQuery query, Collection<Range> ranges) throws QueryEvaluationException { + private BatchScanner runQuery(StarQuery query, Collection<Range> ranges) throws QueryEvaluationException { - try { + try { if (ranges.size() == 0) { String rangeText = query.getCommonVarValue(); Range r; @@ -412,31 +412,25 @@ public class AccumuloDocIdIndexer implements DocIdIndexer { ranges = Collections.singleton(r); } - Connector accCon = ConfigUtils.getConnector(conf); - IteratorSetting is = new IteratorSetting(30, "fii", DocumentIndexIntersectingIterator.class); + Connector accCon = ConfigUtils.getConnector(conf); + IteratorSetting is = new IteratorSetting(30, "fii", DocumentIndexIntersectingIterator.class); + + DocumentIndexIntersectingIterator.setColumnFamilies(is, query.getColumnCond()); - DocumentIndexIntersectingIterator.setColumnFamilies(is, query.getColumnCond()); + if (query.hasContext()) { + DocumentIndexIntersectingIterator.setContext(is, query.getContextURI()); + } + bs = accCon.createBatchScanner(EntityCentricIndex.getTableName(conf), + new Authorizations(conf.get(ConfigUtils.CLOUDBASE_AUTHS)), 15); + bs.addScanIterator(is); + bs.setRanges(ranges); - if(query.hasContext()) { - DocumentIndexIntersectingIterator.setContext(is, query.getContextURI()); + return bs; + + } catch (TableNotFoundException | AccumuloException | AccumuloSecurityException e) { + throw new QueryEvaluationException(e); } - bs = accCon.createBatchScanner(ConfigUtils.getEntityTableName(conf), - new Authorizations(conf.get(ConfigUtils.CLOUDBASE_AUTHS)), 15); - bs.addScanIterator(is); - bs.setRanges(ranges); - - return bs; - - } catch (TableNotFoundException e) { - e.printStackTrace(); - } catch (AccumuloException e) { - e.printStackTrace(); - } catch (AccumuloSecurityException e) { - e.printStackTrace(); } - throw new QueryEvaluationException(); - } - @Override public void close() throws IOException { http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/entity/EntityCentricIndex.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/entity/EntityCentricIndex.java b/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/entity/EntityCentricIndex.java index cd2d516..9a9daa5 100644 --- a/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/entity/EntityCentricIndex.java +++ b/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/entity/EntityCentricIndex.java @@ -73,11 +73,9 @@ public class EntityCentricIndex extends AbstractAccumuloIndexer { private BatchWriter writer; private boolean isInit = false; - public static final String CONF_TABLE_SUFFIX = "ac.indexer.eci.tablename"; - private void initInternal() throws AccumuloException, AccumuloSecurityException, TableNotFoundException, IOException, TableExistsException { - ConfigUtils.createTableIfNotExists(conf, ConfigUtils.getEntityTableName(conf)); + ConfigUtils.createTableIfNotExists(conf, getTableName()); } @Override @@ -116,9 +114,22 @@ public class EntityCentricIndex extends AbstractAccumuloIndexer { } } + /** + * Get the Accumulo table used by this index. + * @return table used by instances of this index + */ @Override public String getTableName() { - return ConfigUtils.getEntityTableName(conf); + return getTableName(conf); + } + + /** + * Get the Accumulo table that will be used by this index. + * @param conf + * @return table name guaranteed to be used by instances of this index + */ + public static String getTableName(Configuration conf) { + return ConfigUtils.getTablePrefix(conf) + TABLE_SUFFIX; } @Override http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/freetext/AccumuloFreeTextIndexer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/freetext/AccumuloFreeTextIndexer.java b/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/freetext/AccumuloFreeTextIndexer.java index fe70d82..a15ab45 100644 --- a/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/freetext/AccumuloFreeTextIndexer.java +++ b/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/freetext/AccumuloFreeTextIndexer.java @@ -26,6 +26,8 @@ import static mvm.rya.indexing.accumulo.freetext.query.ASTNodeUtils.getNodeItera import java.io.IOException; import java.nio.charset.CharacterCodingException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -185,6 +187,10 @@ import mvm.rya.indexing.accumulo.freetext.query.TokenMgrError; * <pre> */ public class AccumuloFreeTextIndexer extends AbstractAccumuloIndexer implements FreeTextIndexer { + private static final String TABLE_SUFFIX_TERM = "freetext_term"; + + private static final String TABLE_SUFFFIX_DOC = "freetext"; + private static final Logger logger = Logger.getLogger(AccumuloFreeTextIndexer.class); private static final boolean IS_TERM_TABLE_TOKEN_DELETION_ENABLED = true; @@ -212,8 +218,8 @@ public class AccumuloFreeTextIndexer extends AbstractAccumuloIndexer implements private void initInternal() throws AccumuloException, AccumuloSecurityException, TableNotFoundException, TableExistsException { - String doctable = ConfigUtils.getFreeTextDocTablename(conf); - String termtable = ConfigUtils.getFreeTextTermTablename(conf); + String doctable = getFreeTextDocTablename(conf); + String termtable = getFreeTextTermTablename(conf); docTableNumPartitions = ConfigUtils.getFreeTextDocNumPartitions(conf); int termTableNumPartitions = ConfigUtils.getFreeTextTermNumPartitions(conf); @@ -397,7 +403,7 @@ public class AccumuloFreeTextIndexer extends AbstractAccumuloIndexer implements } private Set<String> unrollWildcard(String string, boolean reverse) throws IOException { - Scanner termTableScan = getScanner(ConfigUtils.getFreeTextTermTablename(conf)); + Scanner termTableScan = getScanner(getFreeTextTermTablename(conf)); Set<String> unrolledTerms = new HashSet<String>(); @@ -482,7 +488,7 @@ public class AccumuloFreeTextIndexer extends AbstractAccumuloIndexer implements @Override public CloseableIteration<Statement, QueryEvaluationException> queryText(String query, StatementConstraints contraints) throws IOException { - Scanner docTableScan = getScanner(ConfigUtils.getFreeTextDocTablename(conf)); + Scanner docTableScan = getScanner(getFreeTextDocTablename(conf)); // test the query to see if it's parses correctly. SimpleNode root = parseQuery(query); @@ -598,10 +604,42 @@ public class AccumuloFreeTextIndexer extends AbstractAccumuloIndexer implements return root; } - + /** + * Get Free Text Document index table's name + * Use the two table version of this below. This one is required by base class. + */ @Override public String getTableName() { - return ConfigUtils.getFreeTextDocTablename(conf); + return getFreeTextDocTablename(conf); + } + + /** + * Get all the tables used by this index. + * @param conf configuration map + * @return an unmodifiable list of all the table names. + */ + public static List<String> getTableNames(Configuration conf) { + return Collections.unmodifiableList( Arrays.asList( + getFreeTextDocTablename(conf), + getFreeTextTermTablename(conf) )); + } + + /** + * Get the Document index's table name. + * @param conf + * @return the Free Text Document index table's name + */ + public static String getFreeTextDocTablename(Configuration conf) { + return mvm.rya.indexing.accumulo.ConfigUtils.getTablePrefix(conf) + TABLE_SUFFFIX_DOC; + } + + /** + * Get the Term index's table name. + * @param conf + * @return the Free Text Term index table's name + */ + public static String getFreeTextTermTablename(Configuration conf) { + return mvm.rya.indexing.accumulo.ConfigUtils.getTablePrefix(conf) + TABLE_SUFFIX_TERM; } private void deleteStatement(Statement statement) throws IOException { @@ -682,7 +720,7 @@ public class AccumuloFreeTextIndexer extends AbstractAccumuloIndexer implements */ private boolean doesTermExistInOtherDocs(String term, int currentDocId, Text docIdText) { try { - String freeTextDocTableName = ConfigUtils.getFreeTextDocTablename(conf); + String freeTextDocTableName = getFreeTextDocTablename(conf); Scanner scanner = getScanner(freeTextDocTableName); String t = StringUtils.removeEnd(term, "*").toLowerCase(); http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/geo/GeoMesaGeoIndexer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/geo/GeoMesaGeoIndexer.java b/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/geo/GeoMesaGeoIndexer.java index becd893..0c8c898 100644 --- a/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/geo/GeoMesaGeoIndexer.java +++ b/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/geo/GeoMesaGeoIndexer.java @@ -124,6 +124,8 @@ import mvm.rya.indexing.accumulo.ConfigUtils; */ public class GeoMesaGeoIndexer extends AbstractAccumuloIndexer implements GeoIndexer { + private static final String TABLE_SUFFIX = "geo"; + private static final Logger logger = Logger.getLogger(GeoMesaGeoIndexer.class); private static final String FEATURE_NAME = "RDF"; @@ -190,7 +192,7 @@ public class GeoMesaGeoIndexer extends AbstractAccumuloIndexer implements GeoInd final String user = ConfigUtils.getUsername(conf); final String password = ConfigUtils.getPassword(conf); final String auths = ConfigUtils.getAuthorizations(conf).toString(); - final String tableName = ConfigUtils.getGeoTablename(conf); + final String tableName = getTableName(conf); final int numParitions = ConfigUtils.getGeoNumPartitions(conf); final String featureSchemaFormat = "%~#s%" + numParitions + "#r%" + FEATURE_NAME @@ -435,7 +437,16 @@ public class GeoMesaGeoIndexer extends AbstractAccumuloIndexer implements GeoInd @Override public String getTableName() { - return ConfigUtils.getGeoTablename(conf); + return getTableName(conf); + } + + /** + * Get the Accumulo table that will be used by this index. + * @param conf + * @return table name guaranteed to be used by instances of this index + */ + public static String getTableName(Configuration conf) { + return ConfigUtils.getTablePrefix(conf) + TABLE_SUFFIX; } private void deleteStatements(final Collection<RyaStatement> ryaStatements) throws IOException { http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/temporal/AccumuloTemporalIndexer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/temporal/AccumuloTemporalIndexer.java b/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/temporal/AccumuloTemporalIndexer.java index be29d3c..35a9982 100644 --- a/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/temporal/AccumuloTemporalIndexer.java +++ b/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/temporal/AccumuloTemporalIndexer.java @@ -77,6 +77,8 @@ import mvm.rya.indexing.accumulo.ConfigUtils; public class AccumuloTemporalIndexer extends AbstractAccumuloIndexer implements TemporalIndexer { + private static final String TABLE_SUFFIX = "temporal"; + private static final Logger logger = Logger.getLogger(AccumuloTemporalIndexer.class); private static final String CF_INTERVAL = "interval"; @@ -103,7 +105,7 @@ public class AccumuloTemporalIndexer extends AbstractAccumuloIndexer implements private void initInternal() throws AccumuloException, AccumuloSecurityException, TableNotFoundException, TableExistsException { - temporalIndexTableName = ConfigUtils.getTemporalTableName(conf); + temporalIndexTableName = getTableName(); // Create one index table on first run. ConfigUtils.createTableIfNotExists(conf, temporalIndexTableName); @@ -878,7 +880,7 @@ public class AccumuloTemporalIndexer extends AbstractAccumuloIndexer implements @Override public String getTableName() { - return ConfigUtils.getTemporalTableName(conf); + return ConfigUtils.getTablePrefix(conf) + TABLE_SUFFIX; } private void deleteStatement(final Statement statement) throws IOException, IllegalArgumentException { http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/AccumuloIndexSet.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/AccumuloIndexSet.java b/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/AccumuloIndexSet.java index a45ba2b..8b56018 100644 --- a/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/AccumuloIndexSet.java +++ b/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/AccumuloIndexSet.java @@ -53,6 +53,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; import org.apache.rya.indexing.pcj.storage.PcjException; import org.apache.rya.indexing.pcj.storage.PcjMetadata; +import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException; import org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjSerializer; import org.apache.rya.indexing.pcj.storage.accumulo.BindingSetConverter.BindingSetConversionException; import org.apache.rya.indexing.pcj.storage.accumulo.PcjTables; @@ -142,10 +143,11 @@ public class AccumuloIndexSet extends ExternalTupleSet implements * @throws TableNotFoundException * @throws AccumuloSecurityException * @throws AccumuloException + * @throws PCJStorageException */ public AccumuloIndexSet(String sparql, Configuration conf, String tablename) throws MalformedQueryException, SailException, - QueryEvaluationException, TableNotFoundException, AccumuloException, AccumuloSecurityException { + QueryEvaluationException, TableNotFoundException, AccumuloException, AccumuloSecurityException, PCJStorageException { this.tablename = tablename; this.accCon = ConfigUtils.getConnector(conf); this.auths = getAuthorizations(conf); @@ -163,11 +165,7 @@ public class AccumuloIndexSet extends ExternalTupleSet implements } setProjectionExpr(projection.get()); Set<VariableOrder> orders = null; - try { - orders = pcj.getPcjMetadata(accCon, tablename).getVarOrders(); - } catch (final PcjException e) { - e.printStackTrace(); - } + orders = pcj.getPcjMetadata(accCon, tablename).getVarOrders(); varOrder = Lists.newArrayList(); for (final VariableOrder var : orders) { http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/freetext/MongoFreeTextIndexer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/freetext/MongoFreeTextIndexer.java b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/freetext/MongoFreeTextIndexer.java index f8dc820..7a65d26 100644 --- a/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/freetext/MongoFreeTextIndexer.java +++ b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/freetext/MongoFreeTextIndexer.java @@ -34,6 +34,7 @@ import mvm.rya.indexing.accumulo.ConfigUtils; import mvm.rya.indexing.mongodb.AbstractMongoIndexer; public class MongoFreeTextIndexer extends AbstractMongoIndexer<TextMongoDBStorageStrategy> implements FreeTextIndexer { + private static final String COLLECTION_SUFFIX = "freetext"; private static final Logger logger = Logger.getLogger(MongoFreeTextIndexer.class); public MongoFreeTextIndexer(final MongoClient mongoClient) { @@ -57,6 +58,6 @@ public class MongoFreeTextIndexer extends AbstractMongoIndexer<TextMongoDBStorag @Override public String getCollectionName() { - return ConfigUtils.getFreeTextDocTablename(conf); + return ConfigUtils.getTablePrefix(conf) + COLLECTION_SUFFIX; } } http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/geo/MongoGeoIndexer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/geo/MongoGeoIndexer.java b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/geo/MongoGeoIndexer.java index 6076fb2..303848a 100644 --- a/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/geo/MongoGeoIndexer.java +++ b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/geo/MongoGeoIndexer.java @@ -41,6 +41,7 @@ import mvm.rya.indexing.mongodb.geo.GeoMongoDBStorageStrategy.GeoQuery; import mvm.rya.mongodb.MongoDBRdfConfiguration; public class MongoGeoIndexer extends AbstractMongoIndexer<GeoMongoDBStorageStrategy> implements GeoIndexer { + private static final String COLLECTION_SUFFIX = "geo"; private static final Logger logger = Logger.getLogger(MongoGeoIndexer.class); public MongoGeoIndexer(final MongoClient mongoClient) { @@ -113,6 +114,6 @@ public class MongoGeoIndexer extends AbstractMongoIndexer<GeoMongoDBStorageStrat @Override public String getCollectionName() { - return ConfigUtils.getGeoTablename(conf); + return ConfigUtils.getTablePrefix(conf) + COLLECTION_SUFFIX; } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/temporal/MongoTemporalIndexer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/temporal/MongoTemporalIndexer.java b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/temporal/MongoTemporalIndexer.java index 6b34ab8..003d9c9 100644 --- a/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/temporal/MongoTemporalIndexer.java +++ b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/temporal/MongoTemporalIndexer.java @@ -45,6 +45,7 @@ import mvm.rya.indexing.mongodb.AbstractMongoIndexer; * Indexes MongoDB based on time instants or intervals. */ public class MongoTemporalIndexer extends AbstractMongoIndexer<TemporalMongoDBStorageStrategy> implements TemporalIndexer { + private static final String COLLECTION_SUFFIX = "temporal"; private static final Logger LOG = Logger.getLogger(MongoTemporalIndexer.class); /** @@ -150,7 +151,7 @@ public class MongoTemporalIndexer extends AbstractMongoIndexer<TemporalMongoDBSt @Override public String getCollectionName() { - return ConfigUtils.getTemporalTableName(conf); + return ConfigUtils.getTablePrefix(conf) + COLLECTION_SUFFIX; } @VisibleForTesting http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexing/src/main/java/mvm/rya/indexing/pcj/matching/PCJOptimizer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/pcj/matching/PCJOptimizer.java b/extras/indexing/src/main/java/mvm/rya/indexing/pcj/matching/PCJOptimizer.java index a163e0b..3ace3c7 100644 --- a/extras/indexing/src/main/java/mvm/rya/indexing/pcj/matching/PCJOptimizer.java +++ b/extras/indexing/src/main/java/mvm/rya/indexing/pcj/matching/PCJOptimizer.java @@ -122,7 +122,7 @@ public class PCJOptimizer implements QueryOptimizer, Configurable { | QueryEvaluationException | TableNotFoundException | AccumuloException | AccumuloSecurityException | PcjException e) { - e.printStackTrace(); + throw new Error(e); } init = true; } @@ -334,9 +334,11 @@ public class PCJOptimizer implements QueryOptimizer, Configurable { } else { //if no tables are provided by user, get ids for rya instance id, create table name, //and associate table name with sparql + // TODO: storage.listPcjs() returns tablenames, not PCJ-IDs. + // See mvm.rya.indexing.external.accumulo.AccumuloPcjStorage.dropPcj(String) List<String> ids = storage.listPcjs(); for(String id: ids) { - indexTables.put(pcjFactory.makeTableName(tablePrefix, id), storage.getPcjMetadata(id).getSparql()); + indexTables.put(id, storage.getPcjMetadata(id).getSparql()); } } http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexing/src/main/java/mvm/rya/sail/config/RyaSailFactory.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/sail/config/RyaSailFactory.java b/extras/indexing/src/main/java/mvm/rya/sail/config/RyaSailFactory.java index ef1f59f..75daec3 100644 --- a/extras/indexing/src/main/java/mvm/rya/sail/config/RyaSailFactory.java +++ b/extras/indexing/src/main/java/mvm/rya/sail/config/RyaSailFactory.java @@ -20,6 +20,7 @@ package mvm.rya.sail.config; */ import java.net.UnknownHostException; +import java.util.Objects; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; @@ -35,7 +36,6 @@ import com.google.common.collect.Lists; import com.mongodb.MongoClient; import com.mongodb.MongoCredential; import com.mongodb.ServerAddress; - import mvm.rya.accumulo.AccumuloRdfConfiguration; import mvm.rya.accumulo.AccumuloRyaDAO; import mvm.rya.accumulo.instance.AccumuloRyaInstanceDetailsRepository; @@ -66,16 +66,26 @@ public class RyaSailFactory { final RyaDAO<?> dao; final RdfCloudTripleStoreConfiguration rdfConfig; - final String user = config.get(ConfigUtils.CLOUDBASE_USER); - final String pswd = config.get(ConfigUtils.CLOUDBASE_PASSWORD); + final String user; + final String pswd; + // XXX Should(?) be MongoDBRdfConfiguration.MONGO_COLLECTION_PREFIX inside the if below. RYA-135 final String ryaInstance = config.get(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX); + Objects.requireNonNull(ryaInstance, "RyaInstance or table prefix is missing from configuration."+RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX); if(ConfigUtils.getUseMongo(config)) { rdfConfig = new MongoDBRdfConfiguration(config); + user = rdfConfig.get(MongoDBRdfConfiguration.MONGO_USER); + pswd = rdfConfig.get(MongoDBRdfConfiguration.MONGO_USER_PASSWORD); + Objects.requireNonNull(user, "MongoDB user name is missing from configuration."+MongoDBRdfConfiguration.MONGO_USER); + Objects.requireNonNull(pswd, "MongoDB user password is missing from configuration."+MongoDBRdfConfiguration.MONGO_USER_PASSWORD); final MongoClient client = updateMongoConfig((MongoDBRdfConfiguration) rdfConfig, user, pswd, ryaInstance); dao = getMongoDAO((MongoDBRdfConfiguration)rdfConfig, client); } else { rdfConfig = new AccumuloRdfConfiguration(config); + user = rdfConfig.get(ConfigUtils.CLOUDBASE_USER); + pswd = rdfConfig.get(ConfigUtils.CLOUDBASE_PASSWORD); + Objects.requireNonNull(user, "Accumulo user name is missing from configuration."+ConfigUtils.CLOUDBASE_USER); + Objects.requireNonNull(pswd, "Accumulo user password is missing from configuration."+ConfigUtils.CLOUDBASE_PASSWORD); rdfConfig.setTableLayoutStrategy( new TablePrefixLayoutStrategy(ryaInstance) ); updateAccumuloConfig((AccumuloRdfConfiguration) rdfConfig, user, pswd, ryaInstance); dao = getAccumuloDAO((AccumuloRdfConfiguration)rdfConfig); http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexing/src/test/java/mvm/rya/indexing/accumulo/entity/AccumuloDocIndexerTest.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/accumulo/entity/AccumuloDocIndexerTest.java b/extras/indexing/src/test/java/mvm/rya/indexing/accumulo/entity/AccumuloDocIndexerTest.java index 6237697..a36e808 100644 --- a/extras/indexing/src/test/java/mvm/rya/indexing/accumulo/entity/AccumuloDocIndexerTest.java +++ b/extras/indexing/src/test/java/mvm/rya/indexing/accumulo/entity/AccumuloDocIndexerTest.java @@ -38,9 +38,13 @@ import mvm.rya.indexing.accumulo.ConfigUtils; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.admin.TableOperations; import org.apache.accumulo.core.client.mock.MockInstance; +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.data.Mutation; import org.apache.hadoop.conf.Configuration; +import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -61,48 +65,51 @@ import com.google.common.collect.Lists; public class AccumuloDocIndexerTest { - + private MockInstance mockInstance; private Connector accCon; AccumuloRdfConfiguration conf = new AccumuloRdfConfiguration(); ValueFactory vf = new ValueFactoryImpl(); - private static final String tableName = "EntityCentric_entity"; + private String tableName; @Before public void init() throws Exception { - - accCon = new MockInstance("instance").getConnector("root", "".getBytes()); - - if(accCon.tableOperations().exists(tableName)) { - accCon.tableOperations().delete(tableName); - } - - accCon.tableOperations().create(tableName); - - + final String INSTANCE = "instance"; Configuration config = new Configuration(); config.set(ConfigUtils.CLOUDBASE_AUTHS, "U"); - config.set(ConfigUtils.CLOUDBASE_INSTANCE, "instance"); + config.set(ConfigUtils.CLOUDBASE_INSTANCE, INSTANCE); config.set(ConfigUtils.CLOUDBASE_USER, "root"); config.set(ConfigUtils.CLOUDBASE_PASSWORD, ""); conf = new AccumuloRdfConfiguration(config); conf.set(ConfigUtils.USE_MOCK_INSTANCE, "true"); conf.setAdditionalIndexers(EntityCentricIndex.class); + conf.setTablePrefix("EntityCentric_"); + tableName = EntityCentricIndex.getTableName(conf); + // Access the accumulo instance. If you assign a name, it persists statically, but otherwise, can't get it by name. + accCon = new MockInstance(INSTANCE).getConnector("root", new PasswordToken("")); + if(accCon.tableOperations().exists(tableName)) { + throw new Exception("New mock accumulo already has a table! Should be deleted in AfterTest."); + } + // This should happen in the index initialization, but some tests need it before: + accCon.tableOperations().create(tableName); } - - - - + + @After + public void afterTest() throws Exception { + if (accCon.tableOperations().exists(tableName)) { + accCon.tableOperations().delete(tableName); + } + } + @Test public void testNoContext1() throws Exception { BatchWriter bw = null; RyaTableMutationsFactory rtm = new RyaTableMutationsFactory(RyaTripleContext.getInstance(conf)); - conf.setTablePrefix("EntityCentric_"); - + bw = accCon.createBatchWriter(tableName, 500L * 1024L * 1024L, Long.MAX_VALUE, 30); @@ -240,8 +247,6 @@ public class AccumuloDocIndexerTest { bw = accCon.createBatchWriter(tableName, 500L * 1024L * 1024L, Long.MAX_VALUE, 30); - conf.setTablePrefix("EntityCentric_"); - for (int i = 0; i < 30; i++) { @@ -360,8 +365,6 @@ public class AccumuloDocIndexerTest { public void testNoContextCommonVarBs() throws Exception { BatchWriter bw = null; - conf.setTablePrefix("EntityCentric_"); - RyaTableMutationsFactory rtm = new RyaTableMutationsFactory(RyaTripleContext.getInstance(conf)); @@ -476,8 +479,6 @@ public class AccumuloDocIndexerTest { public void testNoContextUnCommonVarBs() throws Exception { BatchWriter bw = null; - conf.setTablePrefix("EntityCentric_"); - RyaTableMutationsFactory rtm = new RyaTableMutationsFactory(RyaTripleContext.getInstance(conf)); @@ -590,7 +591,6 @@ public class AccumuloDocIndexerTest { public void testNoContextCommonVarBs2() throws Exception { BatchWriter bw = null; - conf.setTablePrefix("EntityCentric_"); RyaTableMutationsFactory rtm = new RyaTableMutationsFactory(RyaTripleContext.getInstance(conf)); @@ -705,8 +705,6 @@ public class AccumuloDocIndexerTest { public void testNoContextUnCommonVarBs2() throws Exception { BatchWriter bw = null; - conf.setTablePrefix("EntityCentric_"); - RyaTableMutationsFactory rtm = new RyaTableMutationsFactory(RyaTripleContext.getInstance(conf)); @@ -822,7 +820,6 @@ public class AccumuloDocIndexerTest { public void testContext2() throws Exception { BatchWriter bw = null; - conf.setTablePrefix("EntityCentric_"); RyaTableMutationsFactory rtm = new RyaTableMutationsFactory(RyaTripleContext.getInstance(conf)); @@ -1000,7 +997,6 @@ public class AccumuloDocIndexerTest { public void testContextUnCommonVarBs2() throws Exception { BatchWriter bw = null; - conf.setTablePrefix("EntityCentric_"); RyaTableMutationsFactory rtm = new RyaTableMutationsFactory(RyaTripleContext.getInstance(conf)); @@ -1175,7 +1171,6 @@ public class AccumuloDocIndexerTest { public void testContext1() throws Exception { BatchWriter bw = null; - conf.setTablePrefix("EntityCentric_"); RyaTableMutationsFactory rtm = new RyaTableMutationsFactory(RyaTripleContext.getInstance(conf)); @@ -1356,8 +1351,6 @@ public class AccumuloDocIndexerTest { public void testContextUnCommonVarBs1() throws Exception { BatchWriter bw = null; - conf.setTablePrefix("EntityCentric_"); - RyaTableMutationsFactory rtm = new RyaTableMutationsFactory(RyaTripleContext.getInstance(conf)); @@ -1540,7 +1533,6 @@ public class AccumuloDocIndexerTest { public void testContextCommonVarBs1() throws Exception { BatchWriter bw = null; - conf.setTablePrefix("EntityCentric_"); RyaTableMutationsFactory rtm = new RyaTableMutationsFactory(RyaTripleContext.getInstance(conf)); @@ -1720,7 +1712,6 @@ public class AccumuloDocIndexerTest { public void testContextCommonAndUnCommonVarBs1() throws Exception { BatchWriter bw = null; - conf.setTablePrefix("EntityCentric_"); RyaTableMutationsFactory rtm = new RyaTableMutationsFactory(RyaTripleContext.getInstance(conf)); @@ -1927,7 +1918,6 @@ public class AccumuloDocIndexerTest { public void testContextConstantCommonVar() throws Exception { BatchWriter bw = null; - conf.setTablePrefix("EntityCentric_"); RyaTableMutationsFactory rtm = new RyaTableMutationsFactory(RyaTripleContext.getInstance(conf)); http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexing/src/test/java/mvm/rya/indexing/accumulo/freetext/AccumuloFreeTextIndexerTest.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/accumulo/freetext/AccumuloFreeTextIndexerTest.java b/extras/indexing/src/test/java/mvm/rya/indexing/accumulo/freetext/AccumuloFreeTextIndexerTest.java index 8a19b3a..afea47e 100644 --- a/extras/indexing/src/test/java/mvm/rya/indexing/accumulo/freetext/AccumuloFreeTextIndexerTest.java +++ b/extras/indexing/src/test/java/mvm/rya/indexing/accumulo/freetext/AccumuloFreeTextIndexerTest.java @@ -1,6 +1,7 @@ package mvm.rya.indexing.accumulo.freetext; import java.util.HashSet; +import java.util.List; import java.util.Map.Entry; import java.util.Set; @@ -13,6 +14,7 @@ import org.apache.accumulo.core.client.admin.TableOperations; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.security.Authorizations; import org.apache.hadoop.conf.Configuration; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.openrdf.model.Statement; @@ -47,7 +49,7 @@ import com.google.common.collect.Sets; import info.aduna.iteration.CloseableIteration; -import junit.framework.Assert; +import mvm.rya.accumulo.AccumuloRdfConfiguration; import mvm.rya.api.domain.RyaStatement; import mvm.rya.api.domain.RyaType; import mvm.rya.api.domain.RyaURI; @@ -59,32 +61,32 @@ import mvm.rya.indexing.accumulo.ConfigUtils; public class AccumuloFreeTextIndexerTest { private static final StatementConstraints EMPTY_CONSTRAINTS = new StatementConstraints(); - Configuration conf; + private AccumuloRdfConfiguration conf; @Before public void before() throws Exception { - String tableName = "triplestore_freetext"; - String termTableName = "triplestore_freetext_term"; - conf = new Configuration(); + conf = new AccumuloRdfConfiguration(); conf.setBoolean(ConfigUtils.USE_MOCK_INSTANCE, true); conf.set(ConfigUtils.CLOUDBASE_USER, "USERNAME"); conf.set(ConfigUtils.CLOUDBASE_PASSWORD, "PASS"); - conf.set(ConfigUtils.FREE_TEXT_DOC_TABLENAME, tableName); - conf.set(ConfigUtils.FREE_TEXT_TERM_TABLENAME, termTableName); conf.set(ConfigUtils.CLOUDBASE_AUTHS, "U"); conf.setClass(ConfigUtils.TOKENIZER_CLASS, SimpleTokenizer.class, Tokenizer.class); - - createTable(conf, tableName); - createTable(conf, termTableName); + conf.setTablePrefix("triplestore_"); + + // If a table exists from last time, delete it. + List<String> tableNames = AccumuloFreeTextIndexer.getTableNames(conf); + for (String name : tableNames) { + destroyTable(conf, name); + } + // Tables are created in each test with setConf(conf) } - private static void createTable(Configuration conf, String tablename) throws AccumuloException, AccumuloSecurityException, + private static void destroyTable(Configuration conf, String tablename) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, TableExistsException { TableOperations tableOps = ConfigUtils.getConnector(conf).tableOperations(); if (tableOps.exists(tablename)) { tableOps.delete(tablename); } - tableOps.create(tablename); } @Test http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexing/src/test/java/mvm/rya/indexing/accumulo/geo/GeoIndexerSfTest.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/accumulo/geo/GeoIndexerSfTest.java b/extras/indexing/src/test/java/mvm/rya/indexing/accumulo/geo/GeoIndexerSfTest.java index 0f19a4c..81be7d4 100644 --- a/extras/indexing/src/test/java/mvm/rya/indexing/accumulo/geo/GeoIndexerSfTest.java +++ b/extras/indexing/src/test/java/mvm/rya/indexing/accumulo/geo/GeoIndexerSfTest.java @@ -26,20 +26,18 @@ import info.aduna.iteration.CloseableIteration; import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.UUID; import mvm.rya.accumulo.AccumuloRdfConfiguration; +import mvm.rya.api.RdfCloudTripleStoreConfiguration; import mvm.rya.api.domain.RyaStatement; import mvm.rya.api.resolver.RdfToRyaConversions; import mvm.rya.api.resolver.RyaToRdfConversions; -import mvm.rya.indexing.GeoIndexer; import mvm.rya.indexing.StatementConstraints; import mvm.rya.indexing.accumulo.ConfigUtils; import mvm.rya.indexing.accumulo.geo.GeoConstants; import mvm.rya.indexing.accumulo.geo.GeoMesaGeoIndexer; import org.apache.accumulo.core.client.admin.TableOperations; -import org.apache.hadoop.conf.Configuration; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -67,7 +65,7 @@ import com.vividsolutions.jts.geom.impl.PackedCoordinateSequence; * Tests all of the "simple functions" of the geoindexer. */ public class GeoIndexerSfTest { - private static Configuration conf; + private static AccumuloRdfConfiguration conf; private static GeometryFactory gf = new GeometryFactory(new PrecisionModel(), 4326); private static GeoMesaGeoIndexer g; @@ -111,13 +109,12 @@ public class GeoIndexerSfTest { @Before public void before() throws Exception { - System.out.println(UUID.randomUUID().toString()); - String tableName = "triplestore_geospacial"; - conf = new Configuration(); + conf = new AccumuloRdfConfiguration(); + conf.setTablePrefix("triplestore_"); + String tableName = GeoMesaGeoIndexer.getTableName(conf); conf.setBoolean(ConfigUtils.USE_MOCK_INSTANCE, true); conf.set(ConfigUtils.CLOUDBASE_USER, "USERNAME"); conf.set(ConfigUtils.CLOUDBASE_PASSWORD, "PASS"); - conf.set(ConfigUtils.GEO_TABLENAME, tableName); conf.set(ConfigUtils.CLOUDBASE_AUTHS, "U"); TableOperations tops = ConfigUtils.getConnector(conf).tableOperations(); http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexing/src/test/java/mvm/rya/indexing/accumulo/geo/GeoIndexerTest.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/accumulo/geo/GeoIndexerTest.java b/extras/indexing/src/test/java/mvm/rya/indexing/accumulo/geo/GeoIndexerTest.java index 14cd267..94c265b 100644 --- a/extras/indexing/src/test/java/mvm/rya/indexing/accumulo/geo/GeoIndexerTest.java +++ b/extras/indexing/src/test/java/mvm/rya/indexing/accumulo/geo/GeoIndexerTest.java @@ -26,10 +26,7 @@ import static mvm.rya.api.resolver.RdfToRyaConversions.convertStatement; import java.util.Collections; import java.util.HashSet; import java.util.Set; -import java.util.UUID; - import org.apache.accumulo.core.client.admin.TableOperations; -import org.apache.hadoop.conf.Configuration; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -52,6 +49,8 @@ import com.vividsolutions.jts.geom.PrecisionModel; import com.vividsolutions.jts.geom.impl.PackedCoordinateSequence; import info.aduna.iteration.CloseableIteration; +import mvm.rya.accumulo.AccumuloRdfConfiguration; +import mvm.rya.api.RdfCloudTripleStoreConfiguration; import mvm.rya.indexing.StatementConstraints; import mvm.rya.indexing.accumulo.ConfigUtils; @@ -59,18 +58,17 @@ public class GeoIndexerTest { private static final StatementConstraints EMPTY_CONSTRAINTS = new StatementConstraints(); - Configuration conf; + private AccumuloRdfConfiguration conf; GeometryFactory gf = new GeometryFactory(new PrecisionModel(), 4326); @Before public void before() throws Exception { - System.out.println(UUID.randomUUID().toString()); - String tableName = "triplestore_geospacial"; - conf = new Configuration(); + conf = new AccumuloRdfConfiguration(); + conf.setTablePrefix("triplestore_"); + String tableName = GeoMesaGeoIndexer.getTableName(conf); conf.setBoolean(ConfigUtils.USE_MOCK_INSTANCE, true); conf.set(ConfigUtils.CLOUDBASE_USER, "USERNAME"); conf.set(ConfigUtils.CLOUDBASE_PASSWORD, "PASS"); - conf.set(ConfigUtils.GEO_TABLENAME, tableName); conf.set(ConfigUtils.CLOUDBASE_AUTHS, "U"); TableOperations tops = ConfigUtils.getConnector(conf).tableOperations(); http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexing/src/test/java/mvm/rya/indexing/accumulo/temporal/AccumuloTemporalIndexerTest.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/accumulo/temporal/AccumuloTemporalIndexerTest.java b/extras/indexing/src/test/java/mvm/rya/indexing/accumulo/temporal/AccumuloTemporalIndexerTest.java index d2dfc44..bfaa73d 100644 --- a/extras/indexing/src/test/java/mvm/rya/indexing/accumulo/temporal/AccumuloTemporalIndexerTest.java +++ b/extras/indexing/src/test/java/mvm/rya/indexing/accumulo/temporal/AccumuloTemporalIndexerTest.java @@ -21,9 +21,8 @@ package mvm.rya.indexing.accumulo.temporal; import static mvm.rya.api.resolver.RdfToRyaConversions.convertStatement; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - +import static org.junit.Assert.*; +import org.junit.Assert; import java.io.IOException; import java.io.PrintStream; import java.security.NoSuchAlgorithmException; @@ -36,8 +35,6 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; -import java.util.concurrent.atomic.AtomicLong; - import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.Scanner; @@ -67,7 +64,7 @@ import org.openrdf.query.QueryEvaluationException; import com.beust.jcommander.internal.Lists; import info.aduna.iteration.CloseableIteration; -import junit.framework.Assert; +import mvm.rya.api.RdfCloudTripleStoreConfiguration; import mvm.rya.api.domain.RyaStatement; import mvm.rya.indexing.StatementConstraints; import mvm.rya.indexing.StatementSerializer; @@ -95,6 +92,9 @@ import mvm.rya.indexing.accumulo.ConfigUtils; * Instance {hasBeginning, hasEnd} given Interval * And a few more. * + * The temporal predicates are from these ontologies: + * http://linkedevents.org/ontology + * http://motools.sourceforge.net/event/event.html */ public final class AccumuloTemporalIndexerTest { // Configuration properties, this is reset per test in setup. @@ -108,14 +108,8 @@ public final class AccumuloTemporalIndexerTest { private static final String STAT_COUNT = "count"; private static final String STAT_KEYHASH = "keyhash"; private static final String STAT_VALUEHASH = "valuehash"; - private static final String TEST_TEMPORAL_INDEX_TABLE_NAME = "testTemporalIndex"; private static final StatementConstraints EMPTY_CONSTRAINTS = new StatementConstraints(); - // Recreate table name for each test instance in this JVM. - String uniquePerTestTemporalIndexTableName = TEST_TEMPORAL_INDEX_TABLE_NAME + String.format("%05d", nextTableSuffixAtomic.getAndIncrement()); - // start at 0, for uniqueness between jvm's consider AtomicLong(new Random().nextLong()) - private static final AtomicLong nextTableSuffixAtomic = new AtomicLong(); - // Assign this in setUpBeforeClass, store them in each test. // setup() deletes table before each test. static final Statement spo_B00_E01; @@ -213,18 +207,6 @@ public final class AccumuloTemporalIndexerTest { } /** - * Create a table for test after deleting it. - */ - private static void createTable(Configuration conf, String tablename) - throws AccumuloException, AccumuloSecurityException, TableNotFoundException, TableExistsException { - TableOperations tableOps = ConfigUtils.getConnector(conf).tableOperations(); - if (tableOps.exists(tablename)) { - tableOps.delete(tablename); - } - tableOps.create(tablename); - } - - /** * @throws java.lang.Exception */ @AfterClass @@ -237,17 +219,15 @@ public final class AccumuloTemporalIndexerTest { @Before public void setUp() throws Exception { conf = new Configuration(); + conf.set(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX, "triplestore_"); conf.setBoolean(ConfigUtils.USE_MOCK_INSTANCE, true); - conf.set(ConfigUtils.TEMPORAL_TABLENAME, uniquePerTestTemporalIndexTableName); - // This is from http://linkedevents.org/ontology + // The temporal predicates are from http://linkedevents.org/ontology // and http://motools.sourceforge.net/event/event.html conf.setStrings(ConfigUtils.TEMPORAL_PREDICATES_LIST, "" + URI_PROPERTY_AT_TIME + "," + URI_PROPERTY_CIRCA + "," + URI_PROPERTY_EVENT_TIME); - // delete and create table - createTable(conf, uniquePerTestTemporalIndexTableName); tIndexer = new AccumuloTemporalIndexer(); tIndexer.setConf(conf); } @@ -257,11 +237,12 @@ public final class AccumuloTemporalIndexerTest { */ @After public void tearDown() throws Exception { + String indexTableName = tIndexer.getTableName(); tIndexer.close(); TableOperations tableOps = ConfigUtils.getConnector(conf).tableOperations(); - if (tableOps.exists(uniquePerTestTemporalIndexTableName)) - tableOps.delete(uniquePerTestTemporalIndexTableName); + if (tableOps.exists(indexTableName)) + tableOps.delete(indexTableName); } /** @@ -334,7 +315,7 @@ public final class AccumuloTemporalIndexerTest { tIndexer.flush(); int rowsStoredActual = printTables("junit testing: Temporal entities stored in testStoreStatement", null, null); - Assert.assertEquals("Number of rows stored.", rowsStoredExpected*4, rowsStoredActual); // 4 index entries per statement + assertEquals("Number of rows stored.", rowsStoredExpected*4, rowsStoredActual); // 4 index entries per statement } @@ -995,10 +976,11 @@ public final class AccumuloTemporalIndexerTest { int rowsPrinted = 0; long keyHasher = 0; long valueHasher = 0; - out.println("Reading : " + this.uniquePerTestTemporalIndexTableName); + final String indexTableName = tIndexer.getTableName(); + out.println("Reading : " + indexTableName); out.format(FORMAT, "--Row--", "--ColumnFamily--", "--ColumnQualifier--", "--Value--"); - Scanner s = ConfigUtils.getConnector(conf).createScanner(this.uniquePerTestTemporalIndexTableName, Authorizations.EMPTY); + Scanner s = ConfigUtils.getConnector(conf).createScanner(indexTableName, Authorizations.EMPTY); for (Entry<Key, org.apache.accumulo.core.data.Value> entry : s) { rowsPrinted++; Key k = entry.getKey(); http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexing/src/test/java/mvm/rya/indexing/external/AccumuloPcjIntegrationTest.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/external/AccumuloPcjIntegrationTest.java b/extras/indexing/src/test/java/mvm/rya/indexing/external/AccumuloPcjIntegrationTest.java index 917bfa0..4d686fc 100644 --- a/extras/indexing/src/test/java/mvm/rya/indexing/external/AccumuloPcjIntegrationTest.java +++ b/extras/indexing/src/test/java/mvm/rya/indexing/external/AccumuloPcjIntegrationTest.java @@ -31,6 +31,7 @@ import mvm.rya.api.RdfCloudTripleStoreConfiguration; import mvm.rya.api.persist.RyaDAOException; import mvm.rya.indexing.IndexPlanValidator.IndexPlanValidator; import mvm.rya.indexing.accumulo.ConfigUtils; +import mvm.rya.indexing.external.PrecomputedJoinIndexerConfig.PrecomputedJoinStorageType; import mvm.rya.indexing.external.tupleSet.AccumuloIndexSet; import mvm.rya.indexing.external.tupleSet.ExternalTupleSet; import mvm.rya.indexing.pcj.matching.PCJOptimizer; @@ -1435,11 +1436,12 @@ public class AccumuloPcjIntegrationTest { private static Configuration getConf() { final AccumuloRdfConfiguration conf = new AccumuloRdfConfiguration(); conf.setBoolean(ConfigUtils.USE_MOCK_INSTANCE, true); - conf.set(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX, "rya_"); + conf.setTablePrefix("rya_"); conf.set(ConfigUtils.CLOUDBASE_USER, "root"); conf.set(ConfigUtils.CLOUDBASE_PASSWORD, ""); conf.set(ConfigUtils.CLOUDBASE_INSTANCE, "instance"); conf.set(ConfigUtils.CLOUDBASE_AUTHS, ""); + conf.set(PrecomputedJoinIndexerConfig.PCJ_STORAGE_TYPE,PrecomputedJoinStorageType.ACCUMULO.name()); return conf; } http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexing/src/test/java/mvm/rya/indexing/external/PcjIntegrationTestingUtil.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/external/PcjIntegrationTestingUtil.java b/extras/indexing/src/test/java/mvm/rya/indexing/external/PcjIntegrationTestingUtil.java index c11c7e2..5cc3f9a 100644 --- a/extras/indexing/src/test/java/mvm/rya/indexing/external/PcjIntegrationTestingUtil.java +++ b/extras/indexing/src/test/java/mvm/rya/indexing/external/PcjIntegrationTestingUtil.java @@ -65,6 +65,7 @@ import com.google.common.collect.Sets; import mvm.rya.accumulo.AccumuloRdfConfiguration; import mvm.rya.api.persist.RyaDAOException; import mvm.rya.indexing.accumulo.ConfigUtils; +import mvm.rya.indexing.external.PrecomputedJoinIndexerConfig.PrecomputedJoinStorageType; import mvm.rya.indexing.external.tupleSet.ExternalTupleSet; import mvm.rya.rdftriplestore.inference.InferenceEngineException; import mvm.rya.sail.config.RyaSailFactory; @@ -101,6 +102,7 @@ public class PcjIntegrationTestingUtil { final AccumuloRdfConfiguration pcjConf = new AccumuloRdfConfiguration(); pcjConf.set(ConfigUtils.USE_PCJ, "true"); + pcjConf.set(PrecomputedJoinIndexerConfig.PCJ_STORAGE_TYPE,PrecomputedJoinStorageType.ACCUMULO.name()); populateTestConfig(instance, tablePrefix, pcjConf); final Sail pcjSail = RyaSailFactory.getInstance(pcjConf); http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexing/src/test/java/mvm/rya/indexing/external/tupleSet/AccumuloIndexSetColumnVisibilityTest.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/external/tupleSet/AccumuloIndexSetColumnVisibilityTest.java b/extras/indexing/src/test/java/mvm/rya/indexing/external/tupleSet/AccumuloIndexSetColumnVisibilityTest.java index aef08a4..28c8389 100644 --- a/extras/indexing/src/test/java/mvm/rya/indexing/external/tupleSet/AccumuloIndexSetColumnVisibilityTest.java +++ b/extras/indexing/src/test/java/mvm/rya/indexing/external/tupleSet/AccumuloIndexSetColumnVisibilityTest.java @@ -223,7 +223,7 @@ public class AccumuloIndexSetColumnVisibilityTest { private static Configuration getConf() { final AccumuloRdfConfiguration conf = new AccumuloRdfConfiguration(); - conf.set(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX, "rya_"); + conf.setTablePrefix("rya_"); conf.set(ConfigUtils.CLOUDBASE_USER, "root"); conf.set(ConfigUtils.CLOUDBASE_PASSWORD, "password"); conf.set(ConfigUtils.CLOUDBASE_INSTANCE, instance); http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoFreeTextIndexerTest.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoFreeTextIndexerTest.java b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoFreeTextIndexerTest.java index eac5353..91ba955 100644 --- a/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoFreeTextIndexerTest.java +++ b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoFreeTextIndexerTest.java @@ -21,7 +21,7 @@ package mvm.rya.indexing.mongo; import java.util.HashSet; import java.util.Set; -import org.apache.hadoop.conf.Configuration; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.openrdf.model.Statement; @@ -38,7 +38,7 @@ import com.mongodb.MongoClient; import de.flapdoodle.embed.mongo.distribution.Version; import de.flapdoodle.embed.mongo.tests.MongodForTestsFactory; import info.aduna.iteration.CloseableIteration; -import junit.framework.Assert; +import mvm.rya.accumulo.AccumuloRdfConfiguration; import mvm.rya.api.domain.RyaStatement; import mvm.rya.api.domain.RyaType; import mvm.rya.api.domain.RyaURI; @@ -52,17 +52,17 @@ import mvm.rya.mongodb.MongoDBRdfConfiguration; public class MongoFreeTextIndexerTest { private static final StatementConstraints EMPTY_CONSTRAINTS = new StatementConstraints(); - Configuration conf; + AccumuloRdfConfiguration conf; MongoClient mongoClient; @Before public void before() throws Exception { - conf = new Configuration(); + conf = new AccumuloRdfConfiguration(); conf.set(ConfigUtils.USE_MONGO, "true"); conf.set(MongoDBRdfConfiguration.USE_TEST_MONGO, "true"); conf.set(MongoDBRdfConfiguration.MONGO_DB_NAME, "test"); conf.set(MongoDBRdfConfiguration.MONGO_COLLECTION_PREFIX, "rya_"); - conf.set(ConfigUtils.FREE_TEXT_DOC_TABLENAME, "freetext"); + conf.setTablePrefix("another_"); final MongodForTestsFactory testsFactory = MongodForTestsFactory.with(Version.Main.PRODUCTION); mongoClient = testsFactory.newMongo(); } http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerSfTest.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerSfTest.java b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerSfTest.java index 1220370..3368d6e 100644 --- a/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerSfTest.java +++ b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerSfTest.java @@ -65,7 +65,7 @@ import mvm.rya.mongodb.MongoDBRdfConfiguration; * Tests all of the "simple functions" of the geoindexer. */ public class MongoGeoIndexerSfTest { - private static Configuration conf; + private MongoDBRdfConfiguration conf; private static GeometryFactory gf = new GeometryFactory(new PrecisionModel(), 4326); private static MongoGeoIndexer g; @@ -110,14 +110,14 @@ public class MongoGeoIndexerSfTest { @Before public void before() throws Exception { System.out.println(UUID.randomUUID().toString()); - conf = new Configuration(); + conf = new MongoDBRdfConfiguration(); conf.set(ConfigUtils.USE_MONGO, "true"); conf.set(MongoDBRdfConfiguration.USE_TEST_MONGO, "true"); conf.set(MongoDBRdfConfiguration.MONGO_DB_NAME, "test"); conf.set(MongoDBRdfConfiguration.MONGO_COLLECTION_PREFIX, "rya_"); conf.set(ConfigUtils.GEO_PREDICATES_LIST, "http://www.opengis.net/ont/geosparql#asWKT";); conf.set(ConfigUtils.USE_GEO, "true"); - conf.set(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX, "rya_"); + conf.setTablePrefix("rya_"); final MongodForTestsFactory testsFactory = MongodForTestsFactory.with(Version.Main.PRODUCTION); final MongoClient mongoClient = testsFactory.newMongo(); http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerTest.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerTest.java b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerTest.java index cdbf36a..7a2dc8e 100644 --- a/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerTest.java +++ b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerTest.java @@ -53,7 +53,6 @@ import com.vividsolutions.jts.geom.impl.PackedCoordinateSequence; import de.flapdoodle.embed.mongo.distribution.Version; import de.flapdoodle.embed.mongo.tests.MongodForTestsFactory; import info.aduna.iteration.CloseableIteration; -import mvm.rya.api.RdfCloudTripleStoreConfiguration; import mvm.rya.indexing.StatementConstraints; import mvm.rya.indexing.accumulo.ConfigUtils; import mvm.rya.indexing.accumulo.geo.GeoConstants; @@ -64,21 +63,20 @@ public class MongoGeoIndexerTest { private static final StatementConstraints EMPTY_CONSTRAINTS = new StatementConstraints(); - Configuration conf; + MongoDBRdfConfiguration conf; MongoClient mongoClient; GeometryFactory gf = new GeometryFactory(new PrecisionModel(), 4326); @Before public void before() throws Exception { - conf = new Configuration(); + conf = new MongoDBRdfConfiguration(); conf.set(ConfigUtils.USE_MONGO, "true"); conf.set(MongoDBRdfConfiguration.USE_TEST_MONGO, "true"); conf.set(MongoDBRdfConfiguration.MONGO_DB_NAME, "test"); conf.set(MongoDBRdfConfiguration.MONGO_COLLECTION_PREFIX, "rya_"); conf.set(ConfigUtils.GEO_PREDICATES_LIST, "http://www.opengis.net/ont/geosparql#asWKT";); conf.set(ConfigUtils.USE_GEO, "true"); - conf.set(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX, "rya_"); - conf.set(ConfigUtils.GEO_TABLENAME, "geospacial"); + conf.setTablePrefix("rya_"); final MongodForTestsFactory testsFactory = MongodForTestsFactory.with(Version.Main.PRODUCTION); mongoClient = testsFactory.newMongo(); http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoTemporalIndexerTest.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoTemporalIndexerTest.java b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoTemporalIndexerTest.java index c8e1289..a8481fe 100644 --- a/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoTemporalIndexerTest.java +++ b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoTemporalIndexerTest.java @@ -29,8 +29,6 @@ import java.io.IOException; import java.security.NoSuchAlgorithmException; import java.util.Arrays; import java.util.HashSet; -import java.util.concurrent.atomic.AtomicLong; - import org.apache.accumulo.core.client.TableExistsException; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.hadoop.conf.Configuration; @@ -85,7 +83,7 @@ import mvm.rya.mongodb.MongoDBRdfConfiguration; * */ public final class MongoTemporalIndexerTest { - Configuration conf; + MongoDBRdfConfiguration conf; MongoTemporalIndexer tIndexer; DBCollection collection; @@ -93,14 +91,8 @@ public final class MongoTemporalIndexerTest { private static final String URI_PROPERTY_CIRCA = "Property:circa"; private static final String URI_PROPERTY_AT_TIME = "Property:atTime"; private static final String STAT_VALUEHASH = "valuehash"; - private static final String TEST_TEMPORAL_INDEX_TABLE_NAME = "testTemporalIndex"; private static final StatementConstraints EMPTY_CONSTRAINTS = new StatementConstraints(); - // Recreate table name for each test instance in this JVM. - String uniquePerTestTemporalIndexTableName = TEST_TEMPORAL_INDEX_TABLE_NAME + String.format("%05d", nextTableSuffixAtomic.getAndIncrement()); - // start at 0, for uniqueness between jvm's consider AtomicLong(new Random().nextLong()) - private static final AtomicLong nextTableSuffixAtomic = new AtomicLong(); - // Assign this in setUpBeforeClass, store them in each test. // setup() deletes table before each test. static final Statement spo_B00_E01; @@ -184,12 +176,13 @@ public final class MongoTemporalIndexerTest { @Before public void before() throws Exception { - conf = new Configuration(); + conf = new MongoDBRdfConfiguration(); conf.set(ConfigUtils.USE_MONGO, "true"); conf.set(MongoDBRdfConfiguration.USE_TEST_MONGO, "true"); conf.set(MongoDBRdfConfiguration.MONGO_DB_NAME, "test"); conf.set(MongoDBRdfConfiguration.MONGO_COLLECTION_PREFIX, "rya_"); - conf.set(ConfigUtils.TEMPORAL_TABLENAME, uniquePerTestTemporalIndexTableName); + conf.setTablePrefix("isthisused_"); + // This is from http://linkedevents.org/ontology // and http://motools.sourceforge.net/event/event.html conf.setStrings(ConfigUtils.TEMPORAL_PREDICATES_LIST, "" http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexingExample/src/main/java/EntityDirectExample.java ---------------------------------------------------------------------- diff --git a/extras/indexingExample/src/main/java/EntityDirectExample.java b/extras/indexingExample/src/main/java/EntityDirectExample.java index d8e74e1..6efd933 100644 --- a/extras/indexingExample/src/main/java/EntityDirectExample.java +++ b/extras/indexingExample/src/main/java/EntityDirectExample.java @@ -243,8 +243,7 @@ public class EntityDirectExample { conf.setBoolean(ConfigUtils.USE_MOCK_INSTANCE, USE_MOCK_INSTANCE); conf.set(ConfigUtils.USE_ENTITY, "true"); - conf.set(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX, RYA_TABLE_PREFIX); - conf.set(ConfigUtils.ENTITY_TABLENAME, RYA_TABLE_PREFIX + "entity"); + conf.setTablePrefix(RYA_TABLE_PREFIX); conf.set(ConfigUtils.CLOUDBASE_USER, "root"); conf.set(ConfigUtils.CLOUDBASE_PASSWORD, ""); conf.set(ConfigUtils.CLOUDBASE_INSTANCE, INSTANCE); http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexingExample/src/main/java/MongoRyaDirectExample.java ---------------------------------------------------------------------- diff --git a/extras/indexingExample/src/main/java/MongoRyaDirectExample.java b/extras/indexingExample/src/main/java/MongoRyaDirectExample.java index c1ff790..9c746c8 100644 --- a/extras/indexingExample/src/main/java/MongoRyaDirectExample.java +++ b/extras/indexingExample/src/main/java/MongoRyaDirectExample.java @@ -43,7 +43,6 @@ import org.openrdf.repository.sail.SailRepository; import org.openrdf.repository.sail.SailRepositoryConnection; import org.openrdf.sail.Sail; -import mvm.rya.api.RdfCloudTripleStoreConfiguration; import mvm.rya.indexing.accumulo.ConfigUtils; import mvm.rya.indexing.accumulo.geo.GeoConstants; import mvm.rya.mongodb.MongoDBRdfConfiguration; @@ -249,15 +248,18 @@ public class MongoRyaDirectExample { private static Configuration getConf() { - Configuration conf = new Configuration(); + MongoDBRdfConfiguration conf = new MongoDBRdfConfiguration(); conf.set(ConfigUtils.USE_MONGO, "true"); + // User name and password must be filled in: + conf.set(MongoDBRdfConfiguration.MONGO_USER, "fill this in"); + conf.set(MongoDBRdfConfiguration.MONGO_USER_PASSWORD, "fill this in"); conf.set(MongoDBRdfConfiguration.USE_TEST_MONGO, "true"); conf.set(MongoDBRdfConfiguration.MONGO_DB_NAME, MONGO_DB); conf.set(MongoDBRdfConfiguration.MONGO_COLLECTION_PREFIX, MONGO_COLL_PREFIX); conf.set(ConfigUtils.GEO_PREDICATES_LIST, "http://www.opengis.net/ont/geosparql#asWKT";); conf.set(ConfigUtils.USE_GEO, "true"); conf.set(ConfigUtils.USE_FREETEXT, "true"); - conf.set(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX, MONGO_COLL_PREFIX); + conf.setTablePrefix(MONGO_COLL_PREFIX); conf.set(ConfigUtils.GEO_PREDICATES_LIST, GeoConstants.GEO_AS_WKT.stringValue()); conf.set(ConfigUtils.FREETEXT_PREDICATES_LIST, RDFS.LABEL.stringValue()); return conf; http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/39a46dad/extras/indexingExample/src/main/java/RyaDirectExample.java ---------------------------------------------------------------------- diff --git a/extras/indexingExample/src/main/java/RyaDirectExample.java b/extras/indexingExample/src/main/java/RyaDirectExample.java index d912060..cb69f51 100644 --- a/extras/indexingExample/src/main/java/RyaDirectExample.java +++ b/extras/indexingExample/src/main/java/RyaDirectExample.java @@ -25,6 +25,8 @@ import mvm.rya.api.RdfCloudTripleStoreConfiguration; import mvm.rya.api.persist.RyaDAOException; import mvm.rya.indexing.accumulo.ConfigUtils; import mvm.rya.indexing.accumulo.geo.GeoConstants; +import mvm.rya.indexing.external.PrecomputedJoinIndexerConfig; +import mvm.rya.indexing.external.PrecomputedJoinIndexerConfig.PrecomputedJoinStorageType; import mvm.rya.rdftriplestore.inference.InferenceEngineException; import mvm.rya.sail.config.RyaSailFactory; @@ -89,7 +91,6 @@ public class RyaDirectExample { try { log.info("Connecting to Indexing Sail Repository."); - final Sail extSail = RyaSailFactory.getInstance(conf); repository = new SailRepository(extSail); repository.initialize(); @@ -156,8 +157,8 @@ public class RyaDirectExample { conf.set(ConfigUtils.USE_GEO, "true"); conf.set(ConfigUtils.USE_FREETEXT, "true"); conf.set(ConfigUtils.USE_TEMPORAL, "true"); - conf.set(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX, - RYA_TABLE_PREFIX); + conf.set(PrecomputedJoinIndexerConfig.PCJ_STORAGE_TYPE, PrecomputedJoinStorageType.ACCUMULO.name()); + conf.set(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX, RYA_TABLE_PREFIX); conf.set(ConfigUtils.CLOUDBASE_USER, "root"); conf.set(ConfigUtils.CLOUDBASE_PASSWORD, ""); conf.set(ConfigUtils.CLOUDBASE_INSTANCE, INSTANCE);
