This is an automated email from the ASF dual-hosted git repository. volodymyr pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git
commit 4818c5431f53c9e01d9ef2a446aff99be423e248 Author: Volodymyr Vysotskyi <[email protected]> AuthorDate: Fri Mar 6 20:33:00 2020 +0200 DRILL-7628: Fix Mongo tests broken after the fix for DRILL-7547 closes #2015 --- contrib/storage-mongo/pom.xml | 2 +- .../exec/store/mongo/DrillMongoConstants.java | 36 ++++++++------- .../drill/exec/store/mongo/MongoStoragePlugin.java | 26 +++++------ .../drill/exec/store/mongo/MongoTestConstants.java | 1 + .../drill/exec/store/mongo/MongoTestSuite.java | 51 ++++++++++++++++------ ...TestMongoStoragePluginUsesCredentialsStore.java | 9 ++-- 6 files changed, 80 insertions(+), 45 deletions(-) diff --git a/contrib/storage-mongo/pom.xml b/contrib/storage-mongo/pom.xml index 6c4aa44..27f15c3 100644 --- a/contrib/storage-mongo/pom.xml +++ b/contrib/storage-mongo/pom.xml @@ -31,7 +31,7 @@ <name>contrib/mongo-storage-plugin</name> <properties> - <mongo.TestSuite>**/MongoTestSuit.class</mongo.TestSuite> + <mongo.TestSuite>**/MongoTestSuite.class</mongo.TestSuite> </properties> <dependencies> diff --git a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/DrillMongoConstants.java b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/DrillMongoConstants.java index 5c6f68e..2ce8a2c 100644 --- a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/DrillMongoConstants.java +++ b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/DrillMongoConstants.java @@ -19,33 +19,39 @@ package org.apache.drill.exec.store.mongo; public interface DrillMongoConstants { - public static final String SYS_STORE_PROVIDER_MONGO_URL = "drill.exec.sys.store.provider.mongo.url"; + String SYS_STORE_PROVIDER_MONGO_URL = "drill.exec.sys.store.provider.mongo.url"; - public static final String ID = "_id"; + String ID = "_id"; - public static final String SHARDS = "shards"; + String SHARDS = "shards"; - public static final String NS = "ns"; + String NS = "ns"; - public static final String SHARD = "shard"; + String SHARD = "shard"; - public static final String HOST = "host"; + String HOST = "host"; - public static final String CHUNKS = "chunks"; + String CHUNKS = "chunks"; - public static final String SIZE = "size"; + String SIZE = "size"; - public static final String COUNT = "count"; + String COUNT = "count"; - public static final String CONFIG = "config"; + String CONFIG = "config"; - public static final String MIN = "min"; + String MIN = "min"; - public static final String MAX = "max"; + String MAX = "max"; - public static final String PARTITIONED = "partitioned"; + String PARTITIONED = "partitioned"; - public static final String PRIMARY = "primary"; + String PRIMARY = "primary"; - public static final String DATABASES = "databases"; + String DATABASES = "databases"; + + String STORE_CONFIG_PREFIX = "drill.exec.store."; + + String USERNAME_CONFIG_SUFFIX = ".username"; + + String PASSWORD_CONFIG_SUFFIX = ".password"; } diff --git a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoStoragePlugin.java b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoStoragePlugin.java index 2dfba8c..1ed2026 100644 --- a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoStoragePlugin.java +++ b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoStoragePlugin.java @@ -45,7 +45,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.net.URLEncoder; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Set; @@ -60,9 +59,9 @@ public class MongoStoragePlugin extends AbstractStoragePlugin { private final MongoClientURI clientURI; public MongoStoragePlugin( - MongoStoragePluginConfig mongoConfig, - DrillbitContext context, - String name) throws IOException, ExecutionSetupException { + MongoStoragePluginConfig mongoConfig, + DrillbitContext context, + String name) throws ExecutionSetupException { super(context, name); this.mongoConfig = mongoConfig; String connection = addCredentialsFromCredentialsProvider(this.mongoConfig.getConnection(), name); @@ -73,20 +72,22 @@ public class MongoStoragePlugin extends AbstractStoragePlugin { this.schemaFactory = new MongoSchemaFactory(this, name); } - private static final String addCredentialsFromCredentialsProvider(String connection, String name) { + private static String addCredentialsFromCredentialsProvider(String connection, String name) { MongoClientURI parsed = new MongoClientURI(connection); if (parsed.getCredentials() == null) { Configuration configuration = new Configuration(); try { // The default connection has the name "mongo" but multiple connections can be added; // each will need their own credentials. - char[] usernameChars = configuration.getPassword("drill.exec.store." + name + ".username"); - char[] passwordChars = configuration.getPassword("drill.exec.store." + name + ".password"); + char[] usernameChars = configuration.getPassword( + DrillMongoConstants.STORE_CONFIG_PREFIX + name + DrillMongoConstants.USERNAME_CONFIG_SUFFIX); + char[] passwordChars = configuration.getPassword( + DrillMongoConstants.STORE_CONFIG_PREFIX + name + DrillMongoConstants.PASSWORD_CONFIG_SUFFIX); if (usernameChars != null && passwordChars != null) { String username = URLEncoder.encode(new String(usernameChars), "UTF-8"); String password = URLEncoder.encode(new String(passwordChars), "UTF-8"); - String updatedUrl = connection.replaceFirst("://", "://" + username + ":" + password + "@"); - return updatedUrl.toString(); + return connection.replaceFirst("://", + String.format("://%s:%s@", username, password)); } } catch (IOException e) { logger.error("Error fetching mongodb username and password from configuration", e); @@ -123,7 +124,7 @@ public class MongoStoragePlugin extends AbstractStoragePlugin { } - private class AddressCloser implements + private static class AddressCloser implements RemovalListener<MongoCnxnKey, MongoClient> { @Override public synchronized void onRemoval( @@ -155,8 +156,7 @@ public class MongoStoragePlugin extends AbstractStoragePlugin { MongoClient client = addressClientMap.getIfPresent(key); if (client == null) { if (credential != null) { - List<MongoCredential> credentialList = Arrays.asList(credential); - client = new MongoClient(addresses, credentialList, clientURI.getOptions()); + client = new MongoClient(addresses, credential, clientURI.getOptions()); } else { client = new MongoClient(addresses, clientURI.getOptions()); } @@ -168,7 +168,7 @@ public class MongoStoragePlugin extends AbstractStoragePlugin { } @Override - public void close() throws Exception { + public void close() { addressClientMap.invalidateAll(); } diff --git a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestConstants.java b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestConstants.java index 6e69447..a6d9d2f 100644 --- a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestConstants.java +++ b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestConstants.java @@ -35,6 +35,7 @@ public interface MongoTestConstants { int MONGOS_PORT = 27017; String EMPLOYEE_DB = "employee"; + String AUTHENTICATION_DB = "admin"; String DONUTS_DB = "donuts"; String DONUTS_COLLECTION = "donuts"; diff --git a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestSuite.java b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestSuite.java index 6f8b165..7797cdd 100644 --- a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestSuite.java +++ b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestSuite.java @@ -17,6 +17,7 @@ */ package org.apache.drill.exec.store.mongo; +import com.mongodb.BasicDBObject; import com.mongodb.MongoClient; import com.mongodb.ServerAddress; import com.mongodb.client.MongoCollection; @@ -44,6 +45,7 @@ import org.apache.drill.categories.MongoStorageTest; import org.apache.drill.categories.SlowTest; import org.apache.drill.shaded.guava.com.google.common.collect.Lists; import org.apache.drill.test.BaseTest; +import org.apache.hadoop.conf.Configuration; import org.bson.Document; import org.bson.conversions.Bson; import org.junit.AfterClass; @@ -56,23 +58,26 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.net.URLEncoder; import java.util.ArrayList; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; @RunWith(Suite.class) -@SuiteClasses({ TestMongoFilterPushDown.class, TestMongoProjectPushDown.class, - TestMongoQueries.class, TestMongoChunkAssignment.class }) +@SuiteClasses({TestMongoFilterPushDown.class, TestMongoProjectPushDown.class, + TestMongoQueries.class, TestMongoChunkAssignment.class, + TestMongoStoragePluginUsesCredentialsStore.class}) @Category({SlowTest.class, MongoStorageTest.class}) public class MongoTestSuite extends BaseTest implements MongoTestConstants { private static final Logger logger = LoggerFactory.getLogger(MongoTestSuite.class); protected static MongoClient mongoClient; - private static boolean distMode = Boolean.valueOf(System.getProperty("drill.mongo.tests.shardMode", "false")); - private static boolean authEnabled = Boolean.valueOf(System.getProperty("drill.mongo.tests.authEnabled", "false")); + private static boolean distMode = Boolean.parseBoolean(System.getProperty("drill.mongo.tests.shardMode", "false")); + private static boolean authEnabled = Boolean.parseBoolean(System.getProperty("drill.mongo.tests.authEnabled", "false")); private static volatile String connectionURL = null; private static volatile AtomicInteger initCount = new AtomicInteger(0); @@ -138,13 +143,12 @@ public class MongoTestSuite extends BaseTest implements MongoTestConstants { Storage replication = new Storage(null, CONFIG_REPLICA_SET, 0); - IMongodConfig mongodConfig = new MongodConfigBuilder() + return new MongodConfigBuilder() .version(Version.Main.V3_4) .net(new Net(LOCALHOST, configServerPort, Network.localhostIsIPv6())) .replication(replication) .shardServer(false) .configServer(true).cmdOptions(cmdOptions).build(); - return mongodConfig; } private static IMongodConfig crateIMongodConfig(int mongodPort, boolean flag, String replicaName) @@ -158,14 +162,13 @@ public class MongoTestSuite extends BaseTest implements MongoTestConstants { .build(); Storage replication = new Storage(null, replicaName, 0); - IMongodConfig mongodConfig = new MongodConfigBuilder() + + return new MongodConfigBuilder() .version(Version.Main.V3_4) - .shardServer(true) + .shardServer(true) .net(new Net(LOCALHOST, mongodPort, Network.localhostIsIPv6())) .configServer(flag).replication(replication).cmdOptions(cmdOptions) .build(); - - return mongodConfig; } private static IMongosConfig createIMongosConfig() throws IOException { @@ -177,13 +180,12 @@ public class MongoTestSuite extends BaseTest implements MongoTestConstants { .verbose(false) .build(); - IMongosConfig mongosConfig = new MongosConfigBuilder() + return new MongosConfigBuilder() .version(Version.Main.V3_4) .net(new Net(LOCALHOST, MONGOS_PORT, Network.localhostIsIPv6())) .replicaSet(CONFIG_REPLICA_SET) .configDB(LOCALHOST + ":" + CONFIG_SERVER_1_PORT) .cmdOptions(cmdOptions).build(); - return mongosConfig; } private static void cleanup() { @@ -220,6 +222,8 @@ public class MongoTestSuite extends BaseTest implements MongoTestConstants { mongodConfig); mongod = mongodExecutable.start(); mongoClient = new MongoClient(new ServerAddress(LOCALHOST, MONGOS_PORT)); + + createMongoUser(); createDbAndCollections(EMPLOYEE_DB, EMPINFO_COLLECTION, "employee_id"); createDbAndCollections(EMPLOYEE_DB, SCHEMA_CHANGE_COLLECTION, "field_2"); createDbAndCollections(EMPLOYEE_DB, EMPTY_COLLECTION, "field_2"); @@ -278,8 +282,29 @@ public class MongoTestSuite extends BaseTest implements MongoTestConstants { mongoCollection.createIndex(keys, indexOptions); } + private static void createMongoUser() throws IOException { + Configuration configuration = new Configuration(); + String storeName = "mongo"; + char[] usernameChars = configuration.getPassword(DrillMongoConstants.STORE_CONFIG_PREFIX + storeName + DrillMongoConstants.USERNAME_CONFIG_SUFFIX); + char[] passwordChars = configuration.getPassword(DrillMongoConstants.STORE_CONFIG_PREFIX + storeName + DrillMongoConstants.PASSWORD_CONFIG_SUFFIX); + if (usernameChars != null && passwordChars != null) { + String username = URLEncoder.encode(new String(usernameChars), "UTF-8"); + String password = URLEncoder.encode(new String(passwordChars), "UTF-8"); + + BasicDBObject createUserCommand = new BasicDBObject("createUser", username) + .append("pwd", password) + .append("roles", + Collections.singletonList( + new BasicDBObject("role", "readWrite") + .append("db", AUTHENTICATION_DB))); + + MongoDatabase db = mongoClient.getDatabase(AUTHENTICATION_DB); + db.runCommand(createUserCommand); + } + } + @AfterClass - public static void tearDownCluster() throws Exception { + public static void tearDownCluster() { synchronized (MongoTestSuite.class) { if (initCount.decrementAndGet() == 0) { try { diff --git a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestMongoStoragePluginUsesCredentialsStore.java b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestMongoStoragePluginUsesCredentialsStore.java index 58d9bf2..f5da553 100644 --- a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestMongoStoragePluginUsesCredentialsStore.java +++ b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestMongoStoragePluginUsesCredentialsStore.java @@ -18,17 +18,20 @@ package org.apache.drill.exec.store.mongo; import com.mongodb.MongoCredential; +import org.apache.drill.categories.MongoStorageTest; import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.test.BaseTest; import org.junit.Test; +import org.junit.experimental.categories.Category; -import java.io.IOException; import java.util.List; import static org.junit.Assert.assertEquals; -public class TestMongoStoragePluginUsesCredentialsStore { +@Category({MongoStorageTest.class}) +public class TestMongoStoragePluginUsesCredentialsStore extends BaseTest { - private void test(String expectedUserName, String expectedPassword, String connection, String name) throws IOException, ExecutionSetupException { + private void test(String expectedUserName, String expectedPassword, String connection, String name) throws ExecutionSetupException { MongoStoragePlugin plugin = new MongoStoragePlugin(new MongoStoragePluginConfig( connection), null, name); List<MongoCredential> creds = plugin.getClient().getCredentialsList();
