This is an automated email from the ASF dual-hosted git repository. btellier pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 6e597252351c0d5986f97be41ece6cd501a2b737 Author: Benoit Tellier <btell...@linagora.com> AuthorDate: Fri Apr 24 20:12:05 2020 +0700 JAMES-3138 Integration tests for consistency tasks --- .../james/backends/cassandra/TestingSession.java | 2 +- .../org/apache/james/modules/MailboxProbeImpl.java | 6 + .../rabbitmq/ConsistencyTasksIntegrationTest.java | 314 +++++++++++++++++++++ 3 files changed, 321 insertions(+), 1 deletion(-) diff --git a/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/TestingSession.java b/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/TestingSession.java index b229b46..cbb7ae7 100644 --- a/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/TestingSession.java +++ b/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/TestingSession.java @@ -35,7 +35,7 @@ public class TestingSession implements Session { private final Session delegate; private volatile Scenario scenario; - TestingSession(Session delegate) { + public TestingSession(Session delegate) { this.delegate = delegate; this.scenario = Scenario.NOTHING; } diff --git a/server/container/guice/mailbox/src/main/java/org/apache/james/modules/MailboxProbeImpl.java b/server/container/guice/mailbox/src/main/java/org/apache/james/modules/MailboxProbeImpl.java index 97e0c16..1d5936a 100644 --- a/server/container/guice/mailbox/src/main/java/org/apache/james/modules/MailboxProbeImpl.java +++ b/server/container/guice/mailbox/src/main/java/org/apache/james/modules/MailboxProbeImpl.java @@ -36,6 +36,7 @@ import org.apache.james.mailbox.SubscriptionManager; import org.apache.james.mailbox.exception.MailboxException; import org.apache.james.mailbox.exception.MailboxNotFoundException; import org.apache.james.mailbox.model.ComposedMessageId; +import org.apache.james.mailbox.model.MailboxCounters; import org.apache.james.mailbox.model.MailboxId; import org.apache.james.mailbox.model.MailboxMetaData; import org.apache.james.mailbox.model.MailboxPath; @@ -146,6 +147,11 @@ public class MailboxProbeImpl implements GuiceProbe, MailboxProbe { } } + public MailboxCounters retrieveCounters(MailboxPath path) throws MailboxException { + MailboxSession systemSession = mailboxManager.createSystemSession(path.getUser()); + return mailboxManager.getMailbox(path, systemSession).getMailboxCounters(systemSession); + } + @Override public ComposedMessageId appendMessage(String username, MailboxPath mailboxPath, InputStream message, Date internalDate, boolean isRecent, Flags flags) throws MailboxException { diff --git a/server/protocols/webadmin-integration-test/distributed-webadmin-integration-test/src/test/java/org/apache/james/webadmin/integration/rabbitmq/ConsistencyTasksIntegrationTest.java b/server/protocols/webadmin-integration-test/distributed-webadmin-integration-test/src/test/java/org/apache/james/webadmin/integration/rabbitmq/ConsistencyTasksIntegrationTest.java new file mode 100644 index 0000000..f821ced --- /dev/null +++ b/server/protocols/webadmin-integration-test/distributed-webadmin-integration-test/src/test/java/org/apache/james/webadmin/integration/rabbitmq/ConsistencyTasksIntegrationTest.java @@ -0,0 +1,314 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.webadmin.integration.rabbitmq; + +import static io.restassured.RestAssured.when; +import static io.restassured.RestAssured.with; +import static org.apache.james.backends.cassandra.Scenario.Builder.awaitOn; +import static org.apache.james.backends.cassandra.Scenario.Builder.fail; +import static org.apache.james.jmap.JMAPTestingConstants.BOB; +import static org.apache.james.jmap.JMAPTestingConstants.BOB_PASSWORD; +import static org.apache.james.webadmin.Constants.SEPARATOR; +import static org.assertj.core.api.Assertions.assertThat; +import static org.hamcrest.CoreMatchers.hasItems; +import static org.hamcrest.Matchers.is; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Date; +import java.util.Optional; + +import javax.mail.Flags; + +import org.apache.james.CassandraExtension; +import org.apache.james.CassandraRabbitMQJamesServerMain; +import org.apache.james.DockerElasticSearchExtension; +import org.apache.james.GuiceJamesServer; +import org.apache.james.JamesServerBuilder; +import org.apache.james.JamesServerExtension; +import org.apache.james.backends.cassandra.Scenario; +import org.apache.james.backends.cassandra.Scenario.Barrier; +import org.apache.james.backends.cassandra.TestingSession; +import org.apache.james.backends.cassandra.init.SessionWithInitializedTablesFactory; +import org.apache.james.junit.categories.BasicFeature; +import org.apache.james.mailbox.events.RetryBackoffConfiguration; +import org.apache.james.mailbox.exception.MailboxException; +import org.apache.james.mailbox.model.MailboxPath; +import org.apache.james.mailbox.model.QuotaRoot; +import org.apache.james.modules.AwsS3BlobStoreExtension; +import org.apache.james.modules.MailboxProbeImpl; +import org.apache.james.modules.QuotaProbesImpl; +import org.apache.james.modules.RabbitMQExtension; +import org.apache.james.probe.DataProbe; +import org.apache.james.utils.DataProbeImpl; +import org.apache.james.utils.GuiceProbe; +import org.apache.james.utils.WebAdminGuiceProbe; +import org.apache.james.webadmin.WebAdminUtils; +import org.apache.james.webadmin.integration.WebadminIntegrationTestModule; +import org.apache.james.webadmin.routes.AliasRoutes; +import org.apache.james.webadmin.routes.CassandraMappingsRoutes; +import org.apache.james.webadmin.routes.TasksRoutes; +import org.eclipse.jetty.http.HttpStatus; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.RepeatedTest; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import com.datastax.driver.core.Session; +import com.google.inject.AbstractModule; +import com.google.inject.Inject; +import com.google.inject.Provides; +import com.google.inject.Singleton; +import com.google.inject.multibindings.Multibinder; + +import io.restassured.RestAssured; +import io.restassured.http.ContentType; + +@Tag(BasicFeature.TAG) +class ConsistencyTasksIntegrationTest { + + private static class TestingSessionProbe implements GuiceProbe { + private final TestingSession testingSession; + + @Inject + private TestingSessionProbe(TestingSession testingSession) { + this.testingSession = testingSession; + } + + public TestingSession getTestingSession() { + return testingSession; + } + } + + private static class TestingSessionModule extends AbstractModule { + @Override + protected void configure() { + Multibinder.newSetBinder(binder(), GuiceProbe.class) + .addBinding() + .to(TestingSessionProbe.class); + + bind(Session.class).to(TestingSession.class); + } + + @Provides + @Singleton + TestingSession provideSession(SessionWithInitializedTablesFactory factory) { + return new TestingSession(factory.get()); + } + } + + @RegisterExtension + static JamesServerExtension testExtension = new JamesServerBuilder() + .extension(new DockerElasticSearchExtension()) + .extension(new CassandraExtension()) + .extension(new AwsS3BlobStoreExtension()) + .extension(new RabbitMQExtension()) + .server(configuration -> GuiceJamesServer.forConfiguration(configuration) + .combineWith(CassandraRabbitMQJamesServerMain.MODULES) + .overrideWith(new WebadminIntegrationTestModule()) + // Enforce a single eventBus retry. Required as Current Quotas are handled by the eventBus. + .overrideWith(binder -> binder.bind(RetryBackoffConfiguration.class) + .toInstance(RetryBackoffConfiguration.builder() + .maxRetries(1) + .firstBackoff(Duration.ofMillis(2)) + .jitterFactor(0.5) + .build())) + .overrideWith(new TestingSessionModule())) + .build(); + + private static final String VERSION = "/cassandra/version"; + private static final String UPGRADE_VERSION = VERSION + "/upgrade"; + private static final String UPGRADE_TO_LATEST_VERSION = UPGRADE_VERSION + "/latest"; + private static final String DOMAIN = "domain"; + private static final String USERNAME = "username@" + DOMAIN; + private static final String ALIAS_1 = "alias1@" + DOMAIN; + private static final String ALIAS_2 = "alias2@" + DOMAIN; + private static final boolean IS_RECENT = true; + + private DataProbe dataProbe; + + @BeforeEach + void setUp(GuiceJamesServer guiceJamesServer) throws Exception { + dataProbe = guiceJamesServer.getProbe(DataProbeImpl.class); + dataProbe.addDomain(DOMAIN); + WebAdminGuiceProbe webAdminGuiceProbe = guiceJamesServer.getProbe(WebAdminGuiceProbe.class); + + RestAssured.requestSpecification = WebAdminUtils.buildRequestSpecification(webAdminGuiceProbe.getWebAdminPort()) + .build(); + } + + @Test + void shouldSolveCassandraMappingInconsistency(GuiceJamesServer server) { + server.getProbe(TestingSessionProbe.class) + .getTestingSession().registerScenario(fail() + .times(1) + .whenQueryStartsWith("INSERT INTO mappings_sources")); + + with() + .put(AliasRoutes.ROOT_PATH + SEPARATOR + USERNAME + "/sources/" + ALIAS_1); + with() + .put(AliasRoutes.ROOT_PATH + SEPARATOR + USERNAME + "/sources/" + ALIAS_2); + + String taskId = with() + .queryParam("action", "SolveInconsistencies") + .post(CassandraMappingsRoutes.ROOT_PATH) + .jsonPath() + .get("taskId"); + + with() + .basePath(TasksRoutes.BASE) + .get(taskId + "/await"); + + when() + .get(AliasRoutes.ROOT_PATH + SEPARATOR + USERNAME) + .then() + .contentType(ContentType.JSON) + .statusCode(HttpStatus.OK_200) + .body("source", hasItems(ALIAS_1, ALIAS_2)); + } + + @Test + void shouldSolveMailboxesInconsistency(GuiceJamesServer server) { + MailboxProbeImpl probe = server.getProbe(MailboxProbeImpl.class); + + server.getProbe(TestingSessionProbe.class) + .getTestingSession().registerScenario(fail() + .times(6) // Insertion in the DAO is retried 5 times once it failed + .whenQueryStartsWith("INSERT INTO mailbox (id,name,uidvalidity,mailboxbase)")); + + try { + probe.createMailbox(MailboxPath.inbox(BOB)); + } catch (Exception e) { + // Failure is expected + } + + // schema version 6 or higher required to run solve mailbox inconsistencies task + String taskId = with().post(UPGRADE_TO_LATEST_VERSION) + .jsonPath() + .get("taskId"); + + with() + .get("/tasks/" + taskId + "/await") + .then() + .body("status", is("completed")); + + taskId = with() + .header("I-KNOW-WHAT-I-M-DOING", "ALL-SERVICES-ARE-OFFLINE") + .queryParam("task", "SolveInconsistencies") + .post("/mailboxes") + .jsonPath() + .get("taskId"); + + with() + .basePath(TasksRoutes.BASE) + .get(taskId + "/await"); + + // The mailbox is removed as it is not in the mailboxDAO source of truth. + assertThat(probe.listUserMailboxes(BOB.asString())) + .isEmpty(); + } + + @Test + void shouldRecomputeMailboxCounters(GuiceJamesServer server) throws MailboxException { + MailboxProbeImpl probe = server.getProbe(MailboxProbeImpl.class); + MailboxPath inbox = MailboxPath.inbox(BOB); + probe.createMailbox(inbox); + + server.getProbe(TestingSessionProbe.class) + .getTestingSession().registerScenario(fail() + .times(1) + .whenQueryStartsWith("INSERT INTO messageCounter (nextUid,mailboxId)")); + + try { + probe.appendMessage(BOB.asString(), inbox, + new ByteArrayInputStream("Subject: test\r\n\r\ntestmail".getBytes(StandardCharsets.UTF_8)), new Date(), false, new Flags(Flags.Flag.SEEN)); + } catch (Exception e) { + // Expected to fail + } + + String taskId = with() + .basePath("/mailboxes") + .queryParam("task", "RecomputeMailboxCounters") + .post() + .jsonPath() + .get("taskId"); + + with() + .basePath(TasksRoutes.BASE) + .get(taskId + "/await"); + + assertThat(probe.retrieveCounters(inbox).getCount()).isEqualTo(1); + } + + @Test + void shouldRecomputeQuotas(GuiceJamesServer server) throws Exception { + dataProbe.fluent() + .addDomain(BOB.getDomainPart().get().asString()) + .addUser(BOB.asString(), BOB_PASSWORD); + MailboxProbeImpl probe = server.getProbe(MailboxProbeImpl.class); + MailboxPath inbox = MailboxPath.inbox(BOB); + probe.createMailbox(inbox); + + Barrier barrier1 = new Barrier(); + Barrier barrier2 = new Barrier(); + String updatedQuotaQueryString = "UPDATE currentQuota SET messageCount=messageCount+?,storage=storage+? WHERE quotaRoot=?;"; + server.getProbe(TestingSessionProbe.class) + .getTestingSession().registerScenario(Scenario.combine( + awaitOn(barrier1) // Event bus first execution + .thenFail() + .times(1) + .whenQueryStartsWith(updatedQuotaQueryString), + awaitOn(barrier2) // scenari for event bus retry + .thenFail() + .times(1) + .whenQueryStartsWith(updatedQuotaQueryString))); + + probe.appendMessage(BOB.asString(), inbox, + new ByteArrayInputStream("Subject: test\r\n\r\ntestmail".getBytes(StandardCharsets.UTF_8)), new Date(), + !IS_RECENT, new Flags(Flags.Flag.SEEN)); + + // Await first execution + barrier1.awaitCaller(); + barrier1.releaseCaller(); + // Await event bus retry + barrier2.awaitCaller(); + barrier2.releaseCaller(); + + String taskId = with() + .basePath("/quota/users") + .queryParam("task", "RecomputeCurrentQuotas") + .post() + .jsonPath() + .get("taskId"); + + with() + .basePath(TasksRoutes.BASE) + .get(taskId + "/await"); + + QuotaProbesImpl quotaProbe = server.getProbe(QuotaProbesImpl.class); + assertThat( + quotaProbe.getMessageCountQuota(QuotaRoot.quotaRoot("#private&" + BOB.asString(), Optional.empty())) + .getUsed() + .asLong()) + .isEqualTo(1); + } +} \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: server-dev-unsubscr...@james.apache.org For additional commands, e-mail: server-dev-h...@james.apache.org