This is an automated email from the ASF dual-hosted git repository. drazzib pushed a commit to branch GORA-647-test-containers-mongo in repository https://gitbox.apache.org/repos/asf/gora.git
commit 1d7b76548e085a6ad104c7eff646c7fdde7c1a3c Author: Damien Raude-Morvan <[email protected]> AuthorDate: Sat Mar 21 23:39:15 2020 +0100 GORA-647 Refactor GoraMongodbAuthenticationTestDriver * Use MongoDB docker container feature to initialize superuser (via env) * Pass --auth and --setParameter authenticationMechanisms as commands * Use mongo shell CLI - inside container - to execute script --- .../GoraMongodbAuthenticationTestDriver.java | 194 ++++++--------------- .../authentications/PLAIN_AuthenticationTest.java | 6 +- .../SCRAM_SHA_1_AuthenticationTest.java | 9 +- 3 files changed, 56 insertions(+), 153 deletions(-) diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/GoraMongodbAuthenticationTestDriver.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/GoraMongodbAuthenticationTestDriver.java index 38e8616..b96fa7f 100644 --- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/GoraMongodbAuthenticationTestDriver.java +++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/GoraMongodbAuthenticationTestDriver.java @@ -17,91 +17,55 @@ */ package org.apache.gora.mongodb.authentications; -import de.flapdoodle.embed.mongo.*; -import de.flapdoodle.embed.mongo.config.*; -import de.flapdoodle.embed.mongo.distribution.Version; -import de.flapdoodle.embed.process.config.IRuntimeConfig; -import de.flapdoodle.embed.process.config.io.ProcessOutput; -import de.flapdoodle.embed.process.io.IStreamProcessor; -import de.flapdoodle.embed.process.io.LogWatchStreamProcessor; -import de.flapdoodle.embed.process.io.NamedOutputStreamProcessor; -import de.flapdoodle.embed.process.runtime.Network; +import com.mongodb.ServerAddress; import org.apache.gora.GoraTestDriver; +import org.apache.gora.mongodb.MongoContainer; import org.apache.gora.mongodb.store.MongoStore; import org.apache.gora.mongodb.store.MongoStoreParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testcontainers.containers.Container; -import java.io.File; import java.io.IOException; -import java.io.PrintWriter; -import java.util.Collections; -import java.util.HashSet; +import java.time.Duration; -import static de.flapdoodle.embed.process.io.Processors.console; -import static de.flapdoodle.embed.process.io.Processors.namedConsole; -import static java.util.Arrays.asList; import static org.apache.commons.lang3.StringUtils.isEmpty; /** * Driver to set up an embedded MongoDB database instance for use in our * unit tests. * This class is specially written to automate authentication mechanisms. - * We use embedded mongodb which is available from - * https://github.com/flapdoodle-oss/embedmongo.flapdoodle.de */ class GoraMongodbAuthenticationTestDriver extends GoraTestDriver { private static final Logger log = LoggerFactory.getLogger(GoraMongodbAuthenticationTestDriver.class); - private static final int INIT_TIMEOUT_MS = 30000; - private static final String USER_ADDED_TOKEN = "Successfully added user"; - private ThreadLocal<Boolean> started = new ThreadLocal<>(); - private int port; - private MongodExecutable _mongodExe; - private MongodProcess _mongod; - private MongodStarter runtime; - private IMongodConfig mongodConfig; - private String adminUsername = "madhawa"; - private String adminPassword = "123"; - private Version.Main useVersion; - private String authMechanisms; - private boolean auth = false; - - GoraMongodbAuthenticationTestDriver(String authMechanisms, Version.Main useVersion) throws IOException { + private MongoContainer _container; + private final String adminUsername = "madhawa"; + private final String adminPassword = "123"; + private final String useVersion; + private final String authMechanisms; + + GoraMongodbAuthenticationTestDriver(String authMechanisms, String useVersion) { super(MongoStore.class); this.authMechanisms = authMechanisms; this.useVersion = useVersion; - started.set(false); - if (!this.authMechanisms.equals("MONGODB-CR")) { - auth = true; - } - } private void doStart() throws Exception { - IRuntimeConfig runtimeConfig = new RuntimeConfigBuilder() - .defaultsWithLogger(Command.MongoD, log) - .processOutput(ProcessOutput.getDefaultInstanceSilent()) - .build(); - runtime = MongodStarter.getInstance(runtimeConfig); try { - log.info("Starting the mongo server without authentications"); + log.info("Starting the embedded Mongodb server"); startWithAuth(); - log.info("Adding admin user"); - addAdmin(); - if (this.authMechanisms.equals("SCRAM-SHA-1")) { + if (authMechanisms.equals("SCRAM-SHA-1")) { setSCRAM_SHA_1Credentials(); } - if (this.authMechanisms.equals("MONGODB-CR")) { - setMongoDB_CRCredentials(); - tearDownClass(); - auth = true; - startWithAuth(); - addAdmin(); - } // Store Mongo server "host:port" in Hadoop configuration // so that MongoStore will be able to get it latter - conf.set(MongoStoreParameters.PROP_MONGO_SERVERS, "127.0.0.1:" + port); + ServerAddress address = _container.getServerAddress(); + int port = address.getPort(); + String host = address.getHost(); + String mongoServersProp = String.format("%s:%d", host, port); + + conf.set(MongoStoreParameters.PROP_MONGO_SERVERS, mongoServersProp); conf.set(MongoStoreParameters.PROP_MONGO_DB, "admin"); - conf.set(MongoStoreParameters.PROP_MONGO_AUTHENTICATION_TYPE, this.authMechanisms); + conf.set(MongoStoreParameters.PROP_MONGO_AUTHENTICATION_TYPE, authMechanisms); conf.set(MongoStoreParameters.PROP_MONGO_LOGIN, adminUsername); conf.set(MongoStoreParameters.PROP_MONGO_SECRET, adminPassword); } catch (Exception e) { @@ -112,11 +76,8 @@ class GoraMongodbAuthenticationTestDriver extends GoraTestDriver { private void startWithAuth() throws IOException { try { - if(!started.get()) { - prepareExecutable(); - _mongod = _mongodExe.start(); - started.set(true); - } + prepareExecutable(); + _container.start(); } catch (Exception e) { log.error("Error starting embedded Mongodb server... tearing down test driver."); tearDownClass(); @@ -124,96 +85,51 @@ class GoraMongodbAuthenticationTestDriver extends GoraTestDriver { } private void prepareExecutable() throws IOException { - final MongoCmdOptionsBuilder cmdBuilder = new MongoCmdOptionsBuilder(); - cmdBuilder.enableAuth(auth); - final IMongoCmdOptions cmdOptions = cmdBuilder.build(); - MongodConfigBuilder builder = new MongodConfigBuilder() - .version(useVersion) - .cmdOptions(cmdOptions) - .net(new Net(port, Network.localhostIsIPv6())); - if (auth) { - builder.setParameter("authenticationMechanisms", authMechanisms); - } - - mongodConfig = builder.build(); - _mongodExe = runtime.prepare(mongodConfig); - } - - private void addAdmin() throws IOException, InterruptedException { - final String scriptText = "db.createUser(\n" + - " {\n" + - " user: \"madhawa\",\n" + - " pwd: \"123\",\n" + - " roles: [ { role: \"root\", db: \"admin\" } ]\n" + - " }\n" + - ");"; - runScriptAndWait(scriptText, USER_ADDED_TOKEN, new String[]{"couldn't add user", "failed to load", "login failed"}, "admin", null, null); - } - - private void setMongoDB_CRCredentials() throws Exception { - final String scriptText1 = "var schema = db.system.version.findOne({\"_id\" : \"authSchema\"});\nschema.currentVersion = 3;\ndb.system.version.save(schema);\n"; - // final String scriptText1 = "db.system.version.remove({});\ndb.system.version.insert({ \"_id\" : \"authSchema\", \"currentVersion\" : 3 });"; - runScriptAndWait(scriptText1, "Successfully added authSchema", null, "admin", adminUsername, adminPassword); + _container = new MongoContainer(useVersion); + // https://hub.docker.com/_/mongo + // These variables, used in conjunction, create a new user and set that user's password. + // This user is created in the admin authentication database + // and given the role of root, which is a "superuser" role. + _container.withEnv("MONGO_INITDB_ROOT_USERNAME", adminUsername); + _container.withEnv("MONGO_INITDB_ROOT_PASSWORD", adminPassword); + + // To enable authentication, MongoDB will have to restart itself + // so wait for at least 5 sec + _container.withMinimumRunningDuration(Duration.ofSeconds(5)); + + // https://docs.mongodb.com/manual/tutorial/enable-authentication/ + // https://docs.mongodb.com/manual/reference/parameters/#param.authenticationMechanisms + _container.withCommand("--auth", "--setParameter", "authenticationMechanisms=" + authMechanisms); } private void setSCRAM_SHA_1Credentials() throws Exception { final String scriptText1 = "db.adminCommand({authSchemaUpgrade: 1});\n"; - runScriptAndWait(scriptText1, "Successfully added authSchema", null, "admin", adminUsername, adminPassword); + runScriptAndWait(scriptText1, "admin", adminUsername, adminPassword); } - private void runScriptAndWait(String scriptText, String token, String[] failures, String dbName, String username, String password) throws IOException { - IStreamProcessor mongoOutput; - if (!isEmpty(token)) { - mongoOutput = new LogWatchStreamProcessor( - token, - (failures != null) ? new HashSet<>(asList(failures)) : Collections.emptySet(), - namedConsole("[mongo shell output]")); - } else { - mongoOutput = new NamedOutputStreamProcessor("[mongo shell output]", console()); - } - IRuntimeConfig runtimeConfig = new RuntimeConfigBuilder() - .defaults(Command.Mongo) - .processOutput(new ProcessOutput( - mongoOutput, - namedConsole("[mongo shell error]"), - console())) - .build(); - MongoShellStarter starter = MongoShellStarter.getInstance(runtimeConfig); - - final File scriptFile = writeTmpScriptFile(scriptText); - final MongoShellConfigBuilder builder = new MongoShellConfigBuilder(); - if (!isEmpty(dbName)) { - builder.dbName(dbName); - } + private void runScriptAndWait(String scriptText, String dbName, String username, String password) + throws InterruptedException, IOException { + final StringBuilder builder = new StringBuilder("mongo --quiet"); if (!isEmpty(username)) { - builder.username(username); + builder.append(" --username ").append(username); } if (!isEmpty(password)) { - builder.password(password); + builder.append(" --password ").append(password); } - starter.prepare(builder - .scriptName(scriptFile.getAbsolutePath()) - .version(mongodConfig.version()) - .net(mongodConfig.net()) - .build()).start(); - if (mongoOutput instanceof LogWatchStreamProcessor) { - ((LogWatchStreamProcessor) mongoOutput).waitForResult(INIT_TIMEOUT_MS); + if (!isEmpty(dbName)) { + builder.append(" ").append(dbName); } - } + builder.append(" --eval '").append(scriptText).append("'"); - private File writeTmpScriptFile(String scriptText) throws IOException { - File scriptFile = File.createTempFile("tempfile", ".js"); - scriptFile.deleteOnExit(); - PrintWriter writer = new PrintWriter(scriptFile, "UTF-8"); - writer.write(scriptText); - writer.close(); - return scriptFile; + Container.ExecResult res = _container.execInContainer("/bin/bash", "-c", builder.toString()); + if (!isEmpty(res.getStderr())) { + log.error("Unable to run script on Mongodb server {}: {}", scriptText, res.getStderr()); + throw new IOException(res.getStderr()); + } } @Override public void setUpClass() throws Exception { - port = Network.getFreeServerPort(); - log.info("Starting embedded Mongodb server on {} port.", port); doStart(); } @@ -223,12 +139,6 @@ class GoraMongodbAuthenticationTestDriver extends GoraTestDriver { @Override public void tearDownClass() { log.info("Shutting down mongodb server..."); - if(started.get()) { - _mongod.stop(); - _mongodExe.stop(); - started.set(false); - } + _container.stop(); } - - } diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/PLAIN_AuthenticationTest.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/PLAIN_AuthenticationTest.java index efd14fc..a44f70d 100644 --- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/PLAIN_AuthenticationTest.java +++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/PLAIN_AuthenticationTest.java @@ -17,21 +17,19 @@ */ package org.apache.gora.mongodb.authentications; -import de.flapdoodle.embed.mongo.distribution.Version; -import org.apache.gora.mongodb.GoraMongodbTestDriver; import org.apache.gora.mongodb.store.TestMongoStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Perform {@link TestMongoStore} tests on MongoDB 3.2.x server with Plain Authentication mechanism. + * Perform {@link TestMongoStore} tests on MongoDB 3.6.x server with Plain Authentication mechanism. */ public class PLAIN_AuthenticationTest extends TestMongoStore { private static Logger log = LoggerFactory .getLogger(PLAIN_AuthenticationTest.class); static { try { - setTestDriver(new GoraMongodbAuthenticationTestDriver("PLAIN", Version.Main.V3_4)); + setTestDriver(new GoraMongodbAuthenticationTestDriver("PLAIN", "3.6")); } catch (Exception e) { log.error("MongoDb Test Driver initialization failed. "+ e.getMessage()); } diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/SCRAM_SHA_1_AuthenticationTest.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/SCRAM_SHA_1_AuthenticationTest.java index c7fd045..c8e95fb 100644 --- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/SCRAM_SHA_1_AuthenticationTest.java +++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/SCRAM_SHA_1_AuthenticationTest.java @@ -17,20 +17,15 @@ */ package org.apache.gora.mongodb.authentications; -import de.flapdoodle.embed.mongo.distribution.Version; import org.apache.gora.mongodb.store.TestMongoStore; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; - /** - * Perform {@link TestMongoStore} tests on MongoDB 3.2.x server with SCRAM-SHA-1 Authentication mechanism + * Perform {@link TestMongoStore} tests on MongoDB 3.6.x server with SCRAM-SHA-1 Authentication mechanism */ public class SCRAM_SHA_1_AuthenticationTest extends TestMongoStore { static { try { - setTestDriver(new GoraMongodbAuthenticationTestDriver("SCRAM-SHA-1", Version.Main.V3_4)); + setTestDriver(new GoraMongodbAuthenticationTestDriver("SCRAM-SHA-1", "3.6")); } catch (Exception e) { log.error("MongoDb Test Driver initialization failed. "+ e.getMessage()); }
