Repository: incubator-rya Updated Branches: refs/heads/master 60090ad52 -> 36f271b69
RYA-304 Fixed rya.merger integration tests. Closes #173. Project: http://git-wip-us.apache.org/repos/asf/incubator-rya/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-rya/commit/36f271b6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-rya/tree/36f271b6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-rya/diff/36f271b6 Branch: refs/heads/master Commit: 36f271b695a0a9ca01b681bee354c9301002c758 Parents: 60090ad Author: eric.white <[email protected]> Authored: Thu Jun 22 11:35:03 2017 -0400 Committer: Caleb Meier <[email protected]> Committed: Thu Jun 22 12:47:58 2017 -0700 ---------------------------------------------------------------------- .../mr/merge/mappers/BaseCopyToolMapper.java | 11 ++-- .../mr/merge/util/AccumuloInstanceDriver.java | 13 +++-- .../rya/accumulo/mr/merge/util/TimeUtils.java | 3 +- .../rya/accumulo/mr/merge/RulesetCopyIT.java | 55 +++++++++++--------- .../driver/AccumuloDualInstanceDriver.java | 3 +- 5 files changed, 43 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/36f271b6/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/mappers/BaseCopyToolMapper.java ---------------------------------------------------------------------- diff --git a/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/mappers/BaseCopyToolMapper.java b/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/mappers/BaseCopyToolMapper.java index 96f0ce6..1e73d1f 100644 --- a/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/mappers/BaseCopyToolMapper.java +++ b/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/mappers/BaseCopyToolMapper.java @@ -37,11 +37,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.Mapper; - -import com.google.common.base.Joiner; -import com.google.common.base.Splitter; -import com.google.common.collect.Lists; - +import org.apache.log4j.Logger; import org.apache.rya.accumulo.AccumuloRdfConfiguration; import org.apache.rya.accumulo.AccumuloRyaDAO; import org.apache.rya.accumulo.mr.MRUtils; @@ -50,7 +46,10 @@ import org.apache.rya.accumulo.mr.merge.MergeTool; import org.apache.rya.accumulo.mr.merge.util.AccumuloRyaUtils; import org.apache.rya.api.persist.RyaDAOException; import org.apache.rya.api.resolver.RyaTripleContext; -import twitter4j.Logger; + +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.collect.Lists; /** * The base {@link Mapper} for the copy tool which initializes the mapper for use. The mapper will take all http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/36f271b6/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/util/AccumuloInstanceDriver.java ---------------------------------------------------------------------- diff --git a/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/util/AccumuloInstanceDriver.java b/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/util/AccumuloInstanceDriver.java index 9d67e38..d3dde34 100644 --- a/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/util/AccumuloInstanceDriver.java +++ b/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/util/AccumuloInstanceDriver.java @@ -42,18 +42,17 @@ import org.apache.accumulo.core.security.TablePermission; import org.apache.accumulo.minicluster.MiniAccumuloCluster; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.SystemUtils; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import com.google.common.io.Files; - +import org.apache.log4j.Logger; import org.apache.rya.accumulo.AccumuloRdfConfiguration; import org.apache.rya.accumulo.AccumuloRyaDAO; import org.apache.rya.accumulo.mr.MRUtils; import org.apache.rya.accumulo.mr.merge.MergeTool; import org.apache.rya.api.RdfCloudTripleStoreConstants; import org.apache.rya.api.persist.RyaDAOException; -import twitter4j.Logger; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.io.Files; /** * Handles running a single {@link MiniAccumuloCluster} or a single {@link MockInstance} for an instance. @@ -151,10 +150,10 @@ public class AccumuloInstanceDriver { * @throws Exception */ public void setUp() throws Exception { + setUpConfig(); setUpInstance(); setUpTables(); setUpDao(); - setUpConfig(); } /** http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/36f271b6/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/util/TimeUtils.java ---------------------------------------------------------------------- diff --git a/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/util/TimeUtils.java b/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/util/TimeUtils.java index db760e5..5c3c72e 100644 --- a/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/util/TimeUtils.java +++ b/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/util/TimeUtils.java @@ -33,13 +33,12 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.net.ntp.NTPUDPClient; import org.apache.commons.net.ntp.TimeInfo; +import org.apache.log4j.Logger; import org.codehaus.plexus.util.StringUtils; import org.mortbay.jetty.HttpMethods; import com.google.common.net.HttpHeaders; -import twitter4j.Logger; - /** * Utility methods for time. */ http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/36f271b6/extras/rya.merger/src/test/java/org/apache/rya/accumulo/mr/merge/RulesetCopyIT.java ---------------------------------------------------------------------- diff --git a/extras/rya.merger/src/test/java/org/apache/rya/accumulo/mr/merge/RulesetCopyIT.java b/extras/rya.merger/src/test/java/org/apache/rya/accumulo/mr/merge/RulesetCopyIT.java index ec3f673..b85ee4e 100644 --- a/extras/rya.merger/src/test/java/org/apache/rya/accumulo/mr/merge/RulesetCopyIT.java +++ b/extras/rya.merger/src/test/java/org/apache/rya/accumulo/mr/merge/RulesetCopyIT.java @@ -31,6 +31,23 @@ import java.util.Set; import org.apache.accumulo.core.client.Connector; import org.apache.hadoop.conf.Configuration; import org.apache.log4j.Logger; +import org.apache.rya.accumulo.AccumuloRdfConfiguration; +import org.apache.rya.accumulo.AccumuloRyaDAO; +import org.apache.rya.accumulo.mr.MRUtils; +import org.apache.rya.accumulo.mr.merge.common.InstanceType; +import org.apache.rya.accumulo.mr.merge.demo.util.DemoUtilities; +import org.apache.rya.accumulo.mr.merge.demo.util.DemoUtilities.LoggingDetail; +import org.apache.rya.accumulo.mr.merge.driver.AccumuloDualInstanceDriver; +import org.apache.rya.accumulo.mr.merge.util.AccumuloRyaUtils; +import org.apache.rya.accumulo.mr.merge.util.TestUtils; +import org.apache.rya.api.RdfCloudTripleStoreConfiguration; +import org.apache.rya.api.domain.RyaStatement; +import org.apache.rya.api.domain.RyaType; +import org.apache.rya.api.domain.RyaURI; +import org.apache.rya.api.persist.RyaDAOException; +import org.apache.rya.api.resolver.RyaToRdfConversions; +import org.apache.rya.indexing.accumulo.ConfigUtils; +import org.apache.rya.sail.config.RyaSailFactory; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -56,21 +73,6 @@ import org.openrdf.sail.Sail; import info.aduna.iteration.CloseableIteration; import junit.framework.Assert; -import org.apache.rya.accumulo.AccumuloRdfConfiguration; -import org.apache.rya.accumulo.AccumuloRyaDAO; -import org.apache.rya.accumulo.mr.MRUtils; -import org.apache.rya.accumulo.mr.merge.common.InstanceType; -import org.apache.rya.accumulo.mr.merge.driver.AccumuloDualInstanceDriver; -import org.apache.rya.accumulo.mr.merge.util.AccumuloRyaUtils; -import org.apache.rya.accumulo.mr.merge.util.TestUtils; -import org.apache.rya.api.RdfCloudTripleStoreConfiguration; -import org.apache.rya.api.domain.RyaStatement; -import org.apache.rya.api.domain.RyaType; -import org.apache.rya.api.domain.RyaURI; -import org.apache.rya.api.persist.RyaDAOException; -import org.apache.rya.api.resolver.RyaToRdfConversions; -import org.apache.rya.indexing.accumulo.ConfigUtils; -import org.apache.rya.sail.config.RyaSailFactory; public class RulesetCopyIT { private static final Logger log = Logger.getLogger(RulesetCopyIT.class); @@ -144,32 +146,33 @@ public class RulesetCopyIT { @BeforeClass public static void setUpPerClass() throws Exception { + DemoUtilities.setupLogging(LoggingDetail.LIGHT); accumuloDualInstanceDriver = new AccumuloDualInstanceDriver(IS_MOCK, true, true, false, false); accumuloDualInstanceDriver.setUpInstances(); } @Before public void setUpPerTest() throws Exception { - parentConfig = accumuloDualInstanceDriver.getParentConfig(); - childConfig = accumuloDualInstanceDriver.getChildConfig(); - accumuloDualInstanceDriver.setUpTables(); accumuloDualInstanceDriver.setUpConfigs(); + accumuloDualInstanceDriver.setUpTables(); accumuloDualInstanceDriver.setUpDaos(); + parentConfig = accumuloDualInstanceDriver.getParentConfig(); + childConfig = accumuloDualInstanceDriver.getChildConfig(); parentDao = accumuloDualInstanceDriver.getParentDao(); } @After public void tearDownPerTest() throws Exception { log.info("tearDownPerTest(): tearing down now."); - accumuloDualInstanceDriver.tearDownDaos(); accumuloDualInstanceDriver.tearDownTables(); + accumuloDualInstanceDriver.tearDownDaos(); + if (rulesetTool != null) { + rulesetTool.shutdown(); + } } @AfterClass public static void tearDownPerClass() throws Exception { - if (rulesetTool != null) { - rulesetTool.shutdown(); - } accumuloDualInstanceDriver.tearDown(); } @@ -185,6 +188,9 @@ public class RulesetCopyIT { AccumuloRyaUtils.printTablePretty(table, parentConfig, false); } + parentConfig.set(RdfCloudTripleStoreConfiguration.CONF_INFER, Boolean.toString(infer)); + childConfig.set(RdfCloudTripleStoreConfiguration.CONF_INFER, Boolean.toString(infer)); + rulesetTool = new CopyTool(); rulesetTool.setupAndRun(new String[] { makeArgument(MRUtils.AC_MOCK_PROP, Boolean.toString(IS_MOCK)), @@ -210,8 +216,8 @@ public class RulesetCopyIT { }); final Configuration toolConfig = rulesetTool.getConf(); - childConfig.set(MRUtils.AC_ZK_PROP, toolConfig.get(MRUtils.AC_ZK_PROP + CHILD_SUFFIX)); - MergeTool.setDuplicateKeys(childConfig); + final String zooKeepers = toolConfig.get(MRUtils.AC_ZK_PROP + CHILD_SUFFIX); + MergeTool.setDuplicateKeysForProperty(childConfig, MRUtils.AC_ZK_PROP, zooKeepers); log.info("Finished running tool."); @@ -272,7 +278,6 @@ public class RulesetCopyIT { try { final Sail extSail = RyaSailFactory.getInstance(conf); repository = new SailRepository(extSail); - repository.initialize(); conn = repository.getConnection(); final ResultHandler handler = new ResultHandler(); final TupleQuery tq = conn.prepareTupleQuery(QueryLanguage.SPARQL, query); http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/36f271b6/extras/rya.merger/src/test/java/org/apache/rya/accumulo/mr/merge/driver/AccumuloDualInstanceDriver.java ---------------------------------------------------------------------- diff --git a/extras/rya.merger/src/test/java/org/apache/rya/accumulo/mr/merge/driver/AccumuloDualInstanceDriver.java b/extras/rya.merger/src/test/java/org/apache/rya/accumulo/mr/merge/driver/AccumuloDualInstanceDriver.java index c5a5a29..0cd4942 100644 --- a/extras/rya.merger/src/test/java/org/apache/rya/accumulo/mr/merge/driver/AccumuloDualInstanceDriver.java +++ b/extras/rya.merger/src/test/java/org/apache/rya/accumulo/mr/merge/driver/AccumuloDualInstanceDriver.java @@ -34,13 +34,12 @@ import org.apache.accumulo.core.client.mock.MockInstance; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.accumulo.minicluster.MiniAccumuloCluster; - +import org.apache.log4j.Logger; import org.apache.rya.accumulo.AccumuloRdfConfiguration; import org.apache.rya.accumulo.AccumuloRyaDAO; import org.apache.rya.accumulo.mr.merge.util.AccumuloInstanceDriver; import org.apache.rya.api.domain.RyaStatement; import org.apache.rya.api.persist.RyaDAOException; -import twitter4j.Logger; /** * Handles running a {@link MiniAccumuloCluster} or a {@link MockInstance} for a parent and child instance for testing.
