JAMES-2541 Implement RabbitMQ mail queue : mail envelope only
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/5f68dd5b Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/5f68dd5b Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/5f68dd5b Branch: refs/heads/master Commit: 5f68dd5b4314d3db0a8e0e4f1fb19f66057a63f7 Parents: 61b0031 Author: Benoit Tellier <[email protected]> Authored: Wed Sep 5 11:49:25 2018 +0700 Committer: Benoit Tellier <[email protected]> Committed: Mon Sep 10 17:19:38 2018 +0700 ---------------------------------------------------------------------- server/queue/queue-rabbitmq/pom.xml | 25 ++++ .../apache/james/queue/rabbitmq/MailDTO.java | 67 +++++++++++ .../james/queue/rabbitmq/RabbitClient.java | 74 ++++++++++++ .../james/queue/rabbitmq/RabbitMQMailQueue.java | 114 ++++++++++++++++++- .../rabbitmq/RabbitMQMailQueueFactory.java | 32 +----- .../queue/rabbitmq/RabbitMQMailQueueTest.java | 110 ++++++++++++++++++ 6 files changed, 391 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/james-project/blob/5f68dd5b/server/queue/queue-rabbitmq/pom.xml ---------------------------------------------------------------------- diff --git a/server/queue/queue-rabbitmq/pom.xml b/server/queue/queue-rabbitmq/pom.xml index 045c786..2339392 100644 --- a/server/queue/queue-rabbitmq/pom.xml +++ b/server/queue/queue-rabbitmq/pom.xml @@ -39,6 +39,10 @@ <dependencies> <dependency> <groupId>${james.groupId}</groupId> + <artifactId>james-server-core</artifactId> + </dependency> + <dependency> + <groupId>${james.groupId}</groupId> <artifactId>james-server-queue-api</artifactId> </dependency> <dependency> @@ -61,6 +65,18 @@ <artifactId>jackson-databind</artifactId> </dependency> <dependency> + <groupId>com.fasterxml.jackson.datatype</groupId> + <artifactId>jackson-datatype-guava</artifactId> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.datatype</groupId> + <artifactId>jackson-datatype-jdk8</artifactId> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.datatype</groupId> + <artifactId>jackson-datatype-jsr310</artifactId> + </dependency> + <dependency> <groupId>com.github.fge</groupId> <artifactId>throwing-lambdas</artifactId> </dependency> @@ -73,6 +89,10 @@ <artifactId>guava</artifactId> </dependency> <dependency> + <groupId>com.nurkiewicz.asyncretry</groupId> + <artifactId>asyncretry</artifactId> + </dependency> + <dependency> <groupId>com.rabbitmq</groupId> <artifactId>amqp-client</artifactId> </dependency> @@ -134,6 +154,11 @@ <artifactId>slf4j-api</artifactId> </dependency> <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>ch.qos.logback</groupId> <artifactId>logback-classic</artifactId> <scope>test</scope> http://git-wip-us.apache.org/repos/asf/james-project/blob/5f68dd5b/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/MailDTO.java ---------------------------------------------------------------------- diff --git a/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/MailDTO.java b/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/MailDTO.java new file mode 100644 index 0000000..3022154 --- /dev/null +++ b/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/MailDTO.java @@ -0,0 +1,67 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.queue.rabbitmq; + +import java.util.Collection; + +import org.apache.james.core.MailAddress; +import org.apache.mailet.Mail; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.github.steveash.guavate.Guavate; +import com.google.common.collect.ImmutableList; + +class MailDTO { + + static MailDTO fromMail(Mail mail) { + return new MailDTO( + mail.getRecipients().stream() + .map(MailAddress::asString) + .collect(Guavate.toImmutableList()), + mail.getName(), + mail.getSender().asString()); + } + + private final ImmutableList<String> recipients; + private final String name; + private final String sender; + + @JsonCreator + private MailDTO(@JsonProperty("recipients") ImmutableList<String> recipients, + @JsonProperty("name") String name, + @JsonProperty("sender") String sender) { + this.recipients = recipients; + this.name = name; + this.sender = sender; + } + + Collection<String> getRecipients() { + return recipients; + } + + String getName() { + return name; + } + + String getSender() { + return sender; + } +} http://git-wip-us.apache.org/repos/asf/james-project/blob/5f68dd5b/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/RabbitClient.java ---------------------------------------------------------------------- diff --git a/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/RabbitClient.java b/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/RabbitClient.java new file mode 100644 index 0000000..7439956 --- /dev/null +++ b/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/RabbitClient.java @@ -0,0 +1,74 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.queue.rabbitmq; + +import java.io.IOException; +import java.util.Optional; + +import org.apache.james.queue.api.MailQueue; + +import com.google.common.collect.ImmutableMap; +import com.rabbitmq.client.AMQP; +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.GetResponse; + +class RabbitClient { + + private static final boolean AUTO_ACK = true; + private static final boolean AUTO_DELETE = true; + private static final boolean DURABLE = true; + private static final boolean EXCLUSIVE = true; + private static final boolean MULTIPLE = true; + private static final ImmutableMap<String, Object> NO_ARGUMENTS = ImmutableMap.of(); + private static final String ROUTING_KEY = ""; + + private final Channel channel; + + RabbitClient(Channel channel) { + this.channel = channel; + } + + RabbitMQMailQueue attemptQueueCreation(MailQueueName name) { + try { + channel.exchangeDeclare(name.toRabbitExchangeName(), "direct", DURABLE); + channel.queueDeclare(name.toRabbitWorkQueueName(), DURABLE, !EXCLUSIVE, !AUTO_DELETE, NO_ARGUMENTS); + channel.queueBind(name.toRabbitWorkQueueName(), name.toRabbitExchangeName(), ROUTING_KEY); + } catch (IOException e) { + throw new RuntimeException(e); + } + return new RabbitMQMailQueue(name, this); + } + + void publish(MailQueueName name, byte[] message) throws MailQueue.MailQueueException { + try { + channel.basicPublish(name.toRabbitExchangeName(), ROUTING_KEY, new AMQP.BasicProperties(), message); + } catch (IOException e) { + throw new MailQueue.MailQueueException("Unable to publish to RabbitMQ", e); + } + } + + void ack(long deliveryTag) throws IOException { + channel.basicAck(deliveryTag, !MULTIPLE); + } + + Optional<GetResponse> poll(MailQueueName name) throws IOException { + return Optional.ofNullable(channel.basicGet(name.toRabbitWorkQueueName(), !AUTO_ACK)); + } +} http://git-wip-us.apache.org/repos/asf/james-project/blob/5f68dd5b/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/RabbitMQMailQueue.java ---------------------------------------------------------------------- diff --git a/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/RabbitMQMailQueue.java b/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/RabbitMQMailQueue.java index 3058444..68a3d67 100644 --- a/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/RabbitMQMailQueue.java +++ b/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/RabbitMQMailQueue.java @@ -19,16 +19,72 @@ package org.apache.james.queue.rabbitmq; +import java.io.IOException; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import org.apache.james.core.MailAddress; import org.apache.james.queue.api.MailQueue; +import org.apache.james.server.core.MailImpl; import org.apache.mailet.Mail; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.guava.GuavaModule; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.github.fge.lambdas.Throwing; +import com.github.steveash.guavate.Guavate; +import com.nurkiewicz.asyncretry.AsyncRetryExecutor; +import com.rabbitmq.client.GetResponse; public class RabbitMQMailQueue implements MailQueue { + private static final Logger LOGGER = LoggerFactory.getLogger(RabbitMQMailQueue.class); + + private static class NoMailYetException extends RuntimeException { + } + + private static class RabbitMQMailQueueItem implements MailQueueItem { + private final RabbitClient rabbitClient; + private final long deliveryTag; + private final Mail mail; + + private RabbitMQMailQueueItem(RabbitClient rabbitClient, long deliveryTag, Mail mail) { + this.rabbitClient = rabbitClient; + this.deliveryTag = deliveryTag; + this.mail = mail; + } + + @Override + public Mail getMail() { + return mail; + } + + @Override + public void done(boolean success) throws MailQueueException { + try { + rabbitClient.ack(deliveryTag); + } catch (IOException e) { + throw new MailQueueException("Failed to ACK " + mail.getName() + " with delivery tag " + deliveryTag, e); + } + } + } + + private static final int TEN_MS = 10; + private final MailQueueName name; + private final RabbitClient rabbitClient; + private final ObjectMapper objectMapper; - public RabbitMQMailQueue(MailQueueName name) { + RabbitMQMailQueue(MailQueueName name, RabbitClient rabbitClient) { this.name = name; + this.rabbitClient = rabbitClient; + this.objectMapper = new ObjectMapper() + .registerModule(new Jdk8Module()) + .registerModule(new JavaTimeModule()) + .registerModule(new GuavaModule()); } @Override @@ -38,16 +94,66 @@ public class RabbitMQMailQueue implements MailQueue { @Override public void enQueue(Mail mail, long delay, TimeUnit unit) throws MailQueueException { - + if (delay > 0) { + LOGGER.info("Ignored delay upon enqueue of {} : {} {}.", mail.getName(), delay, unit); + } + enQueue(mail); } @Override public void enQueue(Mail mail) throws MailQueueException { + MailDTO mailDTO = MailDTO.fromMail(mail); + byte[] message = getMessageBytes(mailDTO); + rabbitClient.publish(name, message); + } + private byte[] getMessageBytes(MailDTO mailDTO) throws MailQueueException { + try { + return objectMapper.writeValueAsBytes(mailDTO); + } catch (JsonProcessingException e) { + throw new MailQueueException("Unable to serialize message", e); + } } + @Override - public MailQueueItem deQueue() throws MailQueueException, InterruptedException { - return null; + public MailQueueItem deQueue() throws MailQueueException { + GetResponse getResponse = pollChannel(); + MailDTO mailDTO = toDTO(getResponse); + Mail mail = toMail(mailDTO); + return new RabbitMQMailQueueItem(rabbitClient, getResponse.getEnvelope().getDeliveryTag(), mail); + } + + private MailDTO toDTO(GetResponse getResponse) throws MailQueueException { + try { + return objectMapper.readValue(getResponse.getBody(), MailDTO.class); + } catch (IOException e) { + throw new MailQueueException("Failed to parse DTO", e); + } + } + + private GetResponse pollChannel() { + return new AsyncRetryExecutor(Executors.newSingleThreadScheduledExecutor()) + .withFixedRate() + .withMinDelay(TEN_MS) + .retryOn(NoMailYetException.class) + .getWithRetry(this::singleChannelRead) + .join(); + } + + private GetResponse singleChannelRead() throws IOException { + return rabbitClient.poll(name) + .filter(getResponse -> getResponse.getBody() != null) + .orElseThrow(NoMailYetException::new); + } + + private Mail toMail(MailDTO dto) { + return new MailImpl( + dto.getName(), + MailAddress.getMailSender(dto.getSender()), + dto.getRecipients() + .stream() + .map(Throwing.<String, MailAddress>function(MailAddress::new).sneakyThrow()) + .collect(Guavate.toImmutableList())); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/james-project/blob/5f68dd5b/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/RabbitMQMailQueueFactory.java ---------------------------------------------------------------------- diff --git a/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/RabbitMQMailQueueFactory.java b/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/RabbitMQMailQueueFactory.java index 30b0555..1e541aa 100644 --- a/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/RabbitMQMailQueueFactory.java +++ b/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/RabbitMQMailQueueFactory.java @@ -29,26 +29,16 @@ import org.apache.james.queue.api.MailQueueFactory; import com.github.steveash.guavate.Guavate; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableMap; -import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; - public class RabbitMQMailQueueFactory implements MailQueueFactory<RabbitMQMailQueue> { - - private static final String ROUTING_KEY = ""; - private static final boolean RABBIT_OPTION_DURABLE = true; - private static final boolean RABBIT_OPTION_EXCLUSIVE = true; - private static final boolean RABBIT_OPTION_AUTO_DELETE = true; - private static final ImmutableMap<String, Object> RABBIT_OPTION_NO_ARGUMENTS = ImmutableMap.of(); - - private final Channel channel; + private final RabbitClient rabbitClient; private final RabbitMQManagementApi mqManagementApi; @VisibleForTesting @Inject RabbitMQMailQueueFactory(Connection connection, RabbitMQManagementApi mqManagementApi) throws IOException { - this.channel = connection.createChannel(); + this.rabbitClient = new RabbitClient(connection.createChannel()); this.mqManagementApi = mqManagementApi; } @@ -61,32 +51,20 @@ public class RabbitMQMailQueueFactory implements MailQueueFactory<RabbitMQMailQu public RabbitMQMailQueue createQueue(String name) { MailQueueName mailQueueName = MailQueueName.fromString(name); return getQueue(mailQueueName) - .orElseGet(() -> attemptQueueCreation(mailQueueName)); + .orElseGet(() -> rabbitClient.attemptQueueCreation(mailQueueName)); } @Override public Set<RabbitMQMailQueue> listCreatedMailQueues() { return mqManagementApi.listCreatedMailQueueNames() - .map(RabbitMQMailQueue::new) + .map(name -> new RabbitMQMailQueue(name, rabbitClient)) .collect(Guavate.toImmutableSet()); } private Optional<RabbitMQMailQueue> getQueue(MailQueueName name) { return mqManagementApi.listCreatedMailQueueNames() .filter(name::equals) - .map(RabbitMQMailQueue::new) + .map(queueName -> new RabbitMQMailQueue(queueName, rabbitClient)) .findFirst(); } - - private RabbitMQMailQueue attemptQueueCreation(MailQueueName name) { - try { - channel.exchangeDeclare(name.toRabbitExchangeName().asString(), "direct", RABBIT_OPTION_DURABLE); - channel.queueDeclare(name.toWorkQueueName().asString(), RABBIT_OPTION_DURABLE, !RABBIT_OPTION_EXCLUSIVE, !RABBIT_OPTION_AUTO_DELETE, RABBIT_OPTION_NO_ARGUMENTS); - channel.queueBind(name.toWorkQueueName().asString(), name.toRabbitExchangeName().asString(), ROUTING_KEY); - } catch (IOException e) { - throw new RuntimeException(e); - } - return new RabbitMQMailQueue(name); - } - } http://git-wip-us.apache.org/repos/asf/james-project/blob/5f68dd5b/server/queue/queue-rabbitmq/src/test/java/org/apache/james/queue/rabbitmq/RabbitMQMailQueueTest.java ---------------------------------------------------------------------- diff --git a/server/queue/queue-rabbitmq/src/test/java/org/apache/james/queue/rabbitmq/RabbitMQMailQueueTest.java b/server/queue/queue-rabbitmq/src/test/java/org/apache/james/queue/rabbitmq/RabbitMQMailQueueTest.java new file mode 100644 index 0000000..9573ab8 --- /dev/null +++ b/server/queue/queue-rabbitmq/src/test/java/org/apache/james/queue/rabbitmq/RabbitMQMailQueueTest.java @@ -0,0 +1,110 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.queue.rabbitmq; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.concurrent.TimeoutException; + +import org.apache.http.client.utils.URIBuilder; +import org.apache.james.queue.api.MailQueue; +import org.apache.james.queue.api.MailQueueContract; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(DockerRabbitMQExtension.class) +public class RabbitMQMailQueueTest implements MailQueueContract { + + private RabbitMQMailQueueFactory mailQueueFactory; + + @BeforeEach + void setup(DockerRabbitMQ rabbitMQ) throws IOException, TimeoutException, URISyntaxException { + + URI rabbitManagementUri = new URIBuilder() + .setScheme("http") + .setHost(rabbitMQ.getHostIp()) + .setPort(rabbitMQ.getAdminPort()) + .build(); + mailQueueFactory = new RabbitMQMailQueueFactory( + rabbitMQ.connectionFactory().newConnection(), + new RabbitMQManagementApi(rabbitManagementUri, new RabbitMQManagementCredentials("guest", "guest".toCharArray()))); + } + + @Override + public MailQueue getMailQueue() { + return mailQueueFactory.createQueue("spool"); + } + + @Disabled + @Override + public void queueShouldPreserveMimeMessage() { + + } + + @Disabled + @Override + public void queueShouldPreserveMailAttribute() { + + } + + @Disabled + @Override + public void queueShouldPreserveErrorMessage() { + + } + + @Disabled + @Override + public void queueShouldPreserveState() { + + } + + @Disabled + @Override + public void queueShouldPreserveRemoteAddress() { + + } + + @Disabled + @Override + public void queueShouldPreserveRemoteHost() { + + } + + @Disabled + @Override + public void queueShouldPreserveLastUpdated() { + + } + + @Disabled + @Override + public void queueShouldPreservePerRecipientHeaders() { + + } + + @Disabled + @Override + public void queueShouldPreserveNonStringMailAttribute() { + + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
