DRILL-6341: Fixed failing tests for mongodb storage plugin by upgrading MongoDB version.
closes #1222 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/24193b1b Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/24193b1b Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/24193b1b Branch: refs/heads/master Commit: 24193b1b038a6315681a65c76a67034b64f71fc5 Parents: 2f275d1 Author: Timothy Farkas <[email protected]> Authored: Tue Apr 17 13:29:14 2018 -0700 Committer: Vitalii Diravka <[email protected]> Committed: Sun Apr 29 23:20:55 2018 +0300 ---------------------------------------------------------------------- contrib/storage-mongo/pom.xml | 2 +- .../exec/store/mongo/MongoTestConstants.java | 6 +- .../drill/exec/store/mongo/MongoTestSuit.java | 68 ++++++++++++++------ .../exec/store/mongo/TestTableGenerator.java | 2 +- 4 files changed, 55 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/24193b1b/contrib/storage-mongo/pom.xml ---------------------------------------------------------------------- diff --git a/contrib/storage-mongo/pom.xml b/contrib/storage-mongo/pom.xml index 221524d..80cbf3e 100644 --- a/contrib/storage-mongo/pom.xml +++ b/contrib/storage-mongo/pom.xml @@ -66,7 +66,7 @@ <dependency> <groupId>de.flapdoodle.embed</groupId> <artifactId>de.flapdoodle.embed.mongo</artifactId> - <version>1.50.5</version> + <version>2.0.3</version> <scope>test</scope> </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/drill/blob/24193b1b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestConstants.java ---------------------------------------------------------------------- diff --git a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestConstants.java b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestConstants.java index 2320d53..5aeaf9f 100644 --- a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestConstants.java +++ b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestConstants.java @@ -21,7 +21,9 @@ public interface MongoTestConstants { public static final String LOCALHOST = "localhost"; // TODO: DRILL-3934: add some randomization to this as it fails when running concurrent builds - public static final int CONFIG_SERVER_PORT = 27019; + int CONFIG_SERVER_1_PORT = 61114; + int CONFIG_SERVER_2_PORT = 61215; + int CONFIG_SERVER_3_PORT = 61316; public static final int MONGOD_1_PORT = 27020; public static final int MONGOD_2_PORT = 27021; public static final int MONGOD_3_PORT = 27022; @@ -44,10 +46,12 @@ public interface MongoTestConstants { public static final String EMP_DATA = "emp.json"; public static final String SCHEMA_CHANGE_DATA = "schema_change_int_to_string.json"; + String STORAGE_ENGINE = "wiredTiger"; String DATATYPE_DB = "datatype"; String DATATYPE_COLLECTION = "types"; String DATATYPE_DATA = "datatype-oid.json"; + String CONFIG_REPLICA_SET = "config_replicas"; public static final String REPLICA_SET_1_NAME = "shard_1_replicas"; public static final String REPLICA_SET_2_NAME = "shard_2_replicas"; http://git-wip-us.apache.org/repos/asf/drill/blob/24193b1b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestSuit.java ---------------------------------------------------------------------- diff --git a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestSuit.java b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestSuit.java index 7e2782c..b3f0bd1 100644 --- a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestSuit.java +++ b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestSuit.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import com.google.common.collect.Lists; import org.bson.Document; import org.bson.conversions.Bson; import org.junit.AfterClass; @@ -88,9 +89,9 @@ public class MongoTestSuit implements MongoTestConstants { private static void setup() throws Exception { // creating configServers List<IMongodConfig> configServers = new ArrayList<>(1); - IMongodConfig configIMongodConfig = crateConfigServerConfig( - CONFIG_SERVER_PORT, true); - configServers.add(configIMongodConfig); + configServers.add(crateConfigServerConfig(CONFIG_SERVER_1_PORT)); + configServers.add(crateConfigServerConfig(CONFIG_SERVER_2_PORT)); + configServers.add(crateConfigServerConfig(CONFIG_SERVER_3_PORT)); // creating replicaSets Map<String, List<IMongodConfig>> replicaSets = new HashMap<>(); @@ -110,11 +111,12 @@ public class MongoTestSuit implements MongoTestConstants { replicaSet2.add(crateIMongodConfig(MONGOD_6_PORT, false, REPLICA_SET_2_NAME)); replicaSets.put(REPLICA_SET_2_NAME, replicaSet2); + replicaSets.put(CONFIG_REPLICA_SET, configServers); // create mongos IMongosConfig mongosConfig = createIMongosConfig(); mongosTestFactory = new MongosSystemForTestFactory(mongosConfig, - replicaSets, configServers, EMPLOYEE_DB, EMPINFO_COLLECTION, + replicaSets, Lists.newArrayList(), EMPLOYEE_DB, EMPINFO_COLLECTION, "employee_id"); try { mongosTestFactory.start(); @@ -128,42 +130,63 @@ public class MongoTestSuit implements MongoTestConstants { createDbAndCollections(DATATYPE_DB, DATATYPE_COLLECTION, "_id"); } - private static IMongodConfig crateConfigServerConfig(int configServerPort, - boolean flag) throws UnknownHostException, IOException { - IMongoCmdOptions cmdOptions = new MongoCmdOptionsBuilder().useNoJournal(false).verbose(false) - .build(); + private static IMongodConfig crateConfigServerConfig(int configServerPort) throws UnknownHostException, IOException { + IMongoCmdOptions cmdOptions = new MongoCmdOptionsBuilder() + .useNoPrealloc(false) + .useSmallFiles(false) + .useNoJournal(false) + .useStorageEngine(STORAGE_ENGINE) + .verbose(false) + .build(); + + Storage replication = new Storage(null, CONFIG_REPLICA_SET, 0); IMongodConfig mongodConfig = new MongodConfigBuilder() - .version(Version.Main.PRODUCTION) + .version(Version.Main.V3_4) .net(new Net(LOCALHOST, configServerPort, Network.localhostIsIPv6())) - .configServer(flag).cmdOptions(cmdOptions).build(); + .replication(replication) + .shardServer(false) + .configServer(true).cmdOptions(cmdOptions).build(); return mongodConfig; } private static IMongodConfig crateIMongodConfig(int mongodPort, boolean flag, String replicaName) throws UnknownHostException, IOException { - IMongoCmdOptions cmdOptions = new MongoCmdOptionsBuilder().useNoJournal(false).verbose(false) - .build(); + IMongoCmdOptions cmdOptions = new MongoCmdOptionsBuilder() + .useNoPrealloc(false) + .useSmallFiles(false) + .useNoJournal(false) + .useStorageEngine(STORAGE_ENGINE) + .verbose(false) + .build(); Storage replication = new Storage(null, replicaName, 0); IMongodConfig mongodConfig = new MongodConfigBuilder() - .version(Version.Main.PRODUCTION) + .version(Version.Main.V3_4) + .shardServer(true) .net(new Net(LOCALHOST, mongodPort, Network.localhostIsIPv6())) .configServer(flag).replication(replication).cmdOptions(cmdOptions) .build(); + return mongodConfig; } private static IMongosConfig createIMongosConfig() throws UnknownHostException, IOException { - IMongoCmdOptions cmdOptions = new MongoCmdOptionsBuilder().useNoJournal(false).verbose(false) - .build(); + IMongoCmdOptions cmdOptions = new MongoCmdOptionsBuilder() + .useNoPrealloc(false) + .useSmallFiles(false) + .useNoJournal(false) + .useStorageEngine(STORAGE_ENGINE) + .verbose(false) + .build(); IMongosConfig mongosConfig = new MongosConfigBuilder() - .version(Version.Main.PRODUCTION) + .version(Version.Main.V3_4) .net(new Net(LOCALHOST, MONGOS_PORT, Network.localhostIsIPv6())) - .configDB(LOCALHOST + ":" + CONFIG_SERVER_PORT) + .replicaSet(CONFIG_REPLICA_SET) + .configDB(LOCALHOST + ":" + CONFIG_SERVER_1_PORT) .cmdOptions(cmdOptions).build(); return mongosConfig; } @@ -191,7 +214,7 @@ public class MongoTestSuit implements MongoTestConstants { .enableAuth(authEnabled).build(); IMongodConfig mongodConfig = new MongodConfigBuilder() - .version(Version.Main.PRODUCTION) + .version(Version.Main.V3_4) .net(new Net(LOCALHOST, MONGOS_PORT, Network.localhostIsIPv6())) .cmdOptions(cmdOptions).build(); @@ -247,8 +270,13 @@ public class MongoTestSuit implements MongoTestConstants { db.createCollection(collectionName); mongoCollection = db.getCollection(collectionName); } - IndexOptions indexOptions = new IndexOptions().unique(true) - .background(false).name(indexFieldName); + + if (indexFieldName.equals("_id")) { + // Mongo 3.4 and later already makes an index for a field named _id + return; + } + + IndexOptions indexOptions = new IndexOptions().unique(true).background(false).name(indexFieldName); Bson keys = Indexes.ascending(indexFieldName); mongoCollection.createIndex(keys, indexOptions); } http://git-wip-us.apache.org/repos/asf/drill/blob/24193b1b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestTableGenerator.java ---------------------------------------------------------------------- diff --git a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestTableGenerator.java b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestTableGenerator.java index 37b975a..c0b1e0a 100644 --- a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestTableGenerator.java +++ b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestTableGenerator.java @@ -53,7 +53,7 @@ public class TestTableGenerator implements MongoTestConstants { logger.info("Started importing file {} into collection {} ", jsonFile, collection); IMongoImportConfig mongoImportConfig = new MongoImportConfigBuilder() - .version(Version.Main.PRODUCTION) + .version(Version.Main.V3_4) .net(new Net(MONGOS_PORT, Network.localhostIsIPv6())).db(dbName) .collection(collection).upsert(upsert).dropCollection(drop) .jsonArray(jsonArray).importFile(jsonFile).build();
