JAMES-2341 Correct SpamAssassin learning Note that for now a static user is being used. This is changed later on.
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/697a5326 Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/697a5326 Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/697a5326 Branch: refs/heads/master Commit: 697a5326ed17676c3b4f8e65deac4bb3026bc651 Parents: b92f3e9 Author: benwa <[email protected]> Authored: Tue Mar 6 09:37:32 2018 +0700 Committer: Antoine Duprat <[email protected]> Committed: Thu Mar 8 10:36:34 2018 +0100 ---------------------------------------------------------------------- .../mailbox/spamassassin/SpamAssassin.java | 10 +++- .../spamassassin/SpamAssassinListener.java | 6 +++ .../src/test/resources/spamassassin.properties | 2 + server/container/guice/memory-guice/pom.xml | 4 ++ .../org/apache/james/MemoryJamesServerMain.java | 4 +- .../modules/mailbox/MemoryMailboxModule.java | 2 + .../src/test/resources/mailetcontainer.xml | 8 +++ .../src/test/resources/spamassassin.properties | 2 + server/container/util/pom.xml | 4 ++ .../james/util/scanner/SpamAssassinInvoker.java | 19 +++++-- .../util/scanner/SpamAssassinExtension.java | 55 +++++++++----------- .../util/scanner/SpamAssassinInvokerTest.java | 4 +- 12 files changed, 82 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/james-project/blob/697a5326/mailbox/plugin/spamassassin/src/main/java/org/apache/james/mailbox/spamassassin/SpamAssassin.java ---------------------------------------------------------------------- diff --git a/mailbox/plugin/spamassassin/src/main/java/org/apache/james/mailbox/spamassassin/SpamAssassin.java b/mailbox/plugin/spamassassin/src/main/java/org/apache/james/mailbox/spamassassin/SpamAssassin.java index 0dfb30e..864cba0 100644 --- a/mailbox/plugin/spamassassin/src/main/java/org/apache/james/mailbox/spamassassin/SpamAssassin.java +++ b/mailbox/plugin/spamassassin/src/main/java/org/apache/james/mailbox/spamassassin/SpamAssassin.java @@ -23,6 +23,11 @@ import java.util.List; import javax.inject.Inject; +import org.apache.james.util.Host; +import org.apache.james.util.scanner.SpamAssassinInvoker; + +import com.github.fge.lambdas.Throwing; + public class SpamAssassin { private final SpamAssassinConfiguration spamAssassinConfiguration; @@ -34,7 +39,10 @@ public class SpamAssassin { public void learnSpam(List<InputStream> messages) { if (spamAssassinConfiguration.isEnable()) { - // Will call SpamAssassinInvoker + Host host = spamAssassinConfiguration.getHost().get(); + SpamAssassinInvoker invoker = new SpamAssassinInvoker(host.getHostName(), host.getPort()); + messages + .forEach(Throwing.consumer(message -> invoker.learnAsSpam(message))); } } } http://git-wip-us.apache.org/repos/asf/james-project/blob/697a5326/mailbox/plugin/spamassassin/src/main/java/org/apache/james/mailbox/spamassassin/SpamAssassinListener.java ---------------------------------------------------------------------- diff --git a/mailbox/plugin/spamassassin/src/main/java/org/apache/james/mailbox/spamassassin/SpamAssassinListener.java b/mailbox/plugin/spamassassin/src/main/java/org/apache/james/mailbox/spamassassin/SpamAssassinListener.java index d0888f6..bccf22d 100644 --- a/mailbox/plugin/spamassassin/src/main/java/org/apache/james/mailbox/spamassassin/SpamAssassinListener.java +++ b/mailbox/plugin/spamassassin/src/main/java/org/apache/james/mailbox/spamassassin/SpamAssassinListener.java @@ -26,6 +26,8 @@ import org.apache.james.mailbox.Role; import org.apache.james.mailbox.store.event.EventFactory; import org.apache.james.mailbox.store.event.SpamEventListener; import org.apache.james.mailbox.store.mail.model.Message; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.github.fge.lambdas.Throwing; import com.github.steveash.guavate.Guavate; @@ -34,6 +36,8 @@ import com.google.common.collect.ImmutableList; public class SpamAssassinListener implements SpamEventListener { + private static final Logger LOGGER = LoggerFactory.getLogger(SpamAssassinListener.class); + private final SpamAssassin spamAssassin; @Inject @@ -53,9 +57,11 @@ public class SpamAssassinListener implements SpamEventListener { @Override public void event(Event event) { + LOGGER.debug("Event {} received in listener.", event); if (event instanceof EventFactory.AddedImpl) { EventFactory.AddedImpl addedToMailboxEvent = (EventFactory.AddedImpl) event; if (isEventOnSpamMailbox(addedToMailboxEvent)) { + LOGGER.debug("Spam event detected"); ImmutableList<InputStream> messages = addedToMailboxEvent.getAvailableMessages() .values() .stream() http://git-wip-us.apache.org/repos/asf/james-project/blob/697a5326/server/container/guice/cassandra-guice/src/test/resources/spamassassin.properties ---------------------------------------------------------------------- diff --git a/server/container/guice/cassandra-guice/src/test/resources/spamassassin.properties b/server/container/guice/cassandra-guice/src/test/resources/spamassassin.properties new file mode 100644 index 0000000..134ed31 --- /dev/null +++ b/server/container/guice/cassandra-guice/src/test/resources/spamassassin.properties @@ -0,0 +1,2 @@ +spamassassin.host=localhost +spamassassin.host=783 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/james-project/blob/697a5326/server/container/guice/memory-guice/pom.xml ---------------------------------------------------------------------- diff --git a/server/container/guice/memory-guice/pom.xml b/server/container/guice/memory-guice/pom.xml index b3f4c83..b839cfb 100644 --- a/server/container/guice/memory-guice/pom.xml +++ b/server/container/guice/memory-guice/pom.xml @@ -88,6 +88,10 @@ </dependency> <dependency> <groupId>${project.groupId}</groupId> + <artifactId>james-server-guice-mailbox-plugin-spamassassin</artifactId> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> <artifactId>james-server-guice-managedsieve</artifactId> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/james-project/blob/697a5326/server/container/guice/memory-guice/src/main/java/org/apache/james/MemoryJamesServerMain.java ---------------------------------------------------------------------- diff --git a/server/container/guice/memory-guice/src/main/java/org/apache/james/MemoryJamesServerMain.java b/server/container/guice/memory-guice/src/main/java/org/apache/james/MemoryJamesServerMain.java index 3170cf4..b7b3a1b 100644 --- a/server/container/guice/memory-guice/src/main/java/org/apache/james/MemoryJamesServerMain.java +++ b/server/container/guice/memory-guice/src/main/java/org/apache/james/MemoryJamesServerMain.java @@ -41,6 +41,7 @@ import org.apache.james.modules.server.MemoryMailQueueModule; import org.apache.james.modules.server.RawPostDequeueDecoratorModule; import org.apache.james.modules.server.SwaggerRoutesModule; import org.apache.james.modules.server.WebAdminServerModule; +import org.apache.james.modules.spamassassin.SpamAssassinListenerModule; import com.google.inject.Module; import com.google.inject.util.Modules; @@ -61,7 +62,8 @@ public class MemoryJamesServerMain { new ManageSieveServerModule(), new POP3ServerModule(), new ProtocolHandlerModule(), - new SMTPServerModule()); + new SMTPServerModule(), + new SpamAssassinListenerModule()); public static final Module JMAP = Modules.combine( new MemoryDataJmapModule(), http://git-wip-us.apache.org/repos/asf/james-project/blob/697a5326/server/container/guice/memory-guice/src/main/java/org/apache/james/modules/mailbox/MemoryMailboxModule.java ---------------------------------------------------------------------- diff --git a/server/container/guice/memory-guice/src/main/java/org/apache/james/modules/mailbox/MemoryMailboxModule.java b/server/container/guice/memory-guice/src/main/java/org/apache/james/modules/mailbox/MemoryMailboxModule.java index de9b544..34041f5 100644 --- a/server/container/guice/memory-guice/src/main/java/org/apache/james/modules/mailbox/MemoryMailboxModule.java +++ b/server/container/guice/memory-guice/src/main/java/org/apache/james/modules/mailbox/MemoryMailboxModule.java @@ -49,6 +49,7 @@ import org.apache.james.mailbox.store.JVMMailboxPathLocker; import org.apache.james.mailbox.store.MailboxSessionMapperFactory; import org.apache.james.mailbox.store.StoreAttachmentManager; import org.apache.james.mailbox.store.StoreBlobManager; +import org.apache.james.mailbox.store.StoreMailboxManager; import org.apache.james.mailbox.store.StoreMessageIdManager; import org.apache.james.mailbox.store.StoreRightManager; import org.apache.james.mailbox.store.StoreSubscriptionManager; @@ -96,6 +97,7 @@ public class MemoryMailboxModule extends AbstractModule { bind(Authenticator.class).to(UserRepositoryAuthenticator.class); bind(Authorizator.class).to(UserRepositoryAuthorizator.class); bind(MailboxManager.class).to(InMemoryMailboxManager.class); + bind(StoreMailboxManager.class).to(InMemoryMailboxManager.class); bind(MessageIdManager.class).to(StoreMessageIdManager.class); bind(AttachmentManager.class).to(StoreAttachmentManager.class); http://git-wip-us.apache.org/repos/asf/james-project/blob/697a5326/server/container/guice/memory-guice/src/test/resources/mailetcontainer.xml ---------------------------------------------------------------------- diff --git a/server/container/guice/memory-guice/src/test/resources/mailetcontainer.xml b/server/container/guice/memory-guice/src/test/resources/mailetcontainer.xml index 887dab3..71de139 100644 --- a/server/container/guice/memory-guice/src/test/resources/mailetcontainer.xml +++ b/server/container/guice/memory-guice/src/test/resources/mailetcontainer.xml @@ -56,6 +56,14 @@ </mailet> <mailet match="All" class="RecipientRewriteTable" /> <mailet match="RecipientIsLocal" class="org.apache.james.jmap.mailet.VacationMailet"/> + <mailet match="RecipientIsLocal" class="SpamAssassin"> + <spamdHost>localhost</spamdHost> + <spamdPort>783</spamdPort> + </mailet> + <mailet match="IsMarkedAsSpam" class="ToRecipientFolder"> + <folder>Spam</folder> + <consume>true</consume> + </mailet> <mailet match="RecipientIsLocal" class="LocalDelivery"/> <mailet match="HostIsLocal" class="ToProcessor"> <processor>local-address-error</processor> http://git-wip-us.apache.org/repos/asf/james-project/blob/697a5326/server/container/guice/memory-guice/src/test/resources/spamassassin.properties ---------------------------------------------------------------------- diff --git a/server/container/guice/memory-guice/src/test/resources/spamassassin.properties b/server/container/guice/memory-guice/src/test/resources/spamassassin.properties new file mode 100644 index 0000000..134ed31 --- /dev/null +++ b/server/container/guice/memory-guice/src/test/resources/spamassassin.properties @@ -0,0 +1,2 @@ +spamassassin.host=localhost +spamassassin.host=783 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/james-project/blob/697a5326/server/container/util/pom.xml ---------------------------------------------------------------------- diff --git a/server/container/util/pom.xml b/server/container/util/pom.xml index 1128afd..c602027 100644 --- a/server/container/util/pom.xml +++ b/server/container/util/pom.xml @@ -52,6 +52,10 @@ <artifactId>javax.mail</artifactId> </dependency> <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> + <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/james-project/blob/697a5326/server/container/util/src/main/java/org/apache/james/util/scanner/SpamAssassinInvoker.java ---------------------------------------------------------------------- diff --git a/server/container/util/src/main/java/org/apache/james/util/scanner/SpamAssassinInvoker.java b/server/container/util/src/main/java/org/apache/james/util/scanner/SpamAssassinInvoker.java index 95063da..3e6f856 100644 --- a/server/container/util/src/main/java/org/apache/james/util/scanner/SpamAssassinInvoker.java +++ b/server/container/util/src/main/java/org/apache/james/util/scanner/SpamAssassinInvoker.java @@ -33,6 +33,8 @@ import javax.mail.MessagingException; import javax.mail.internet.MimeMessage; import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.base.Splitter; import com.google.common.collect.Lists; @@ -42,6 +44,7 @@ import com.google.common.collect.Lists; * href="SpamAssassin.org">SpamAssassin.org</a> for info on configuration. */ public class SpamAssassinInvoker { + private static final Logger LOGGER = LoggerFactory.getLogger(SpamAssassinInvoker.class); /** The mail attribute under which the status get stored */ public static final String STATUS_MAIL_ATTRIBUTE_NAME = "org.apache.james.spamassassin.status"; @@ -88,6 +91,8 @@ public class SpamAssassinInvoker { writer.write("CHECK SPAMC/1.2"); writer.write(CRLF); + writer.write("User: [email protected]"); + writer.write(CRLF); writer.write(CRLF); writer.flush(); @@ -131,6 +136,7 @@ public class SpamAssassinInvoker { try { return Boolean.valueOf(string); } catch (Exception e) { + LOGGER.warn("Fail parsing spamassassin answer: " + string); return false; } } @@ -153,8 +159,13 @@ public class SpamAssassinInvoker { PrintWriter writer = new PrintWriter(out); BufferedReader in = new BufferedReader(new InputStreamReader(socket.getInputStream()))) { + byte[] byteArray = IOUtils.toByteArray(message); writer.write("TELL SPAMC/1.2"); writer.write(CRLF); + writer.write("Content-length: " + byteArray.length); + writer.write(CRLF); + writer.write("User: [email protected]"); + writer.write(CRLF); writer.write("Message-class: spam"); writer.write(CRLF); writer.write("Set: local, remote"); @@ -162,14 +173,12 @@ public class SpamAssassinInvoker { writer.write(CRLF); writer.flush(); - IOUtils.copy(message, out); + out.write(byteArray); out.flush(); socket.shutdownOutput(); return in.lines() - .filter(this::hasBeenSet) - .findAny() - .isPresent(); + .anyMatch(this::hasBeenSet); } catch (UnknownHostException e) { throw new MessagingException("Error communicating with spamd. Unknown host: " + spamdHost); } catch (IOException e) { @@ -178,6 +187,6 @@ public class SpamAssassinInvoker { } private boolean hasBeenSet(String line) { - return line.startsWith("DidSet"); + return line.startsWith("DidSet: local"); } } http://git-wip-us.apache.org/repos/asf/james-project/blob/697a5326/server/container/util/src/test/java/org/apache/james/util/scanner/SpamAssassinExtension.java ---------------------------------------------------------------------- diff --git a/server/container/util/src/test/java/org/apache/james/util/scanner/SpamAssassinExtension.java b/server/container/util/src/test/java/org/apache/james/util/scanner/SpamAssassinExtension.java index c185be6..e161804 100644 --- a/server/container/util/src/test/java/org/apache/james/util/scanner/SpamAssassinExtension.java +++ b/server/container/util/src/test/java/org/apache/james/util/scanner/SpamAssassinExtension.java @@ -19,19 +19,14 @@ package org.apache.james.util.scanner; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.function.Function; +import java.util.Locale; import java.util.stream.Stream; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.extension.AfterAllCallback; import org.junit.jupiter.api.extension.BeforeAllCallback; import org.junit.jupiter.api.extension.ExtensionContext; @@ -72,7 +67,11 @@ public class SpamAssassinExtension implements BeforeAllCallback, AfterAllCallbac public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { return spamAssassin; } - + + public SpamAssassin getSpamAssassin() { + return spamAssassin; + } + public static class SpamAssassin { private static final int SPAMASSASSIN_PORT = 783; @@ -95,38 +94,28 @@ public class SpamAssassinExtension implements BeforeAllCallback, AfterAllCallbac return bindingPort; } - public void train() throws IOException, URISyntaxException { - train(Paths.get(ClassLoader.getSystemResource("spamassassin_db/spam").toURI()), TrainingKind.SPAM); - train(Paths.get(ClassLoader.getSystemResource("spamassassin_db/ham").toURI()), TrainingKind.HAM); + public void train(String user) throws IOException, URISyntaxException { + train(user, Paths.get(ClassLoader.getSystemResource("spamassassin_db/spam").toURI()), TrainingKind.SPAM); + train(user, Paths.get(ClassLoader.getSystemResource("spamassassin_db/ham").toURI()), TrainingKind.HAM); } - private void train(Path folder, TrainingKind trainingKind) throws URISyntaxException, IOException { + private void train(String user, Path folder, TrainingKind trainingKind) throws URISyntaxException, IOException { + spamAssassinContainer.getDockerClient().copyArchiveToContainerCmd(spamAssassinContainer.getContainerId()) + .withHostResource(folder.toAbsolutePath().toString()) + .withRemotePath("/root") + .exec(); try (Stream<Path> paths = Files.walk(folder)) { paths .filter(Files::isRegularFile) .map(Path::toFile) - .map(Throwing.function(FileInputStream::new)) - .map(readAsPair()) - .map(closeStream()) - .map(Pair::getRight) - .forEach(Throwing.consumer(message -> + .forEach(Throwing.consumer(file -> { spamAssassinContainer.execInContainer("sa-learn", - trainingKind.saLearnExtensionName(), - message))); + trainingKind.saLearnExtensionName(), "-u", user, + "/root/" + trainingKind.name().toLowerCase(Locale.US) + "/" + file.getName()); + })); } } - private Function<InputStream, Pair<InputStream, String>> readAsPair() { - return Throwing.function(inputStream -> Pair.of(inputStream, IOUtils.toString(inputStream, StandardCharsets.UTF_8))); - } - - private Function<Pair<InputStream, String>, Pair<InputStream, String>> closeStream() { - return Throwing.function(pair -> { - pair.getLeft().close(); - return pair; - }); - } - private static enum TrainingKind { SPAM("--spam"), HAM("--ham"); @@ -140,6 +129,14 @@ public class SpamAssassinExtension implements BeforeAllCallback, AfterAllCallbac return saLearnExtensionName; } } + + public void sync(String user) throws UnsupportedOperationException, IOException, InterruptedException { + spamAssassinContainer.execInContainer("sa-learn", "--sync", "-u", user); + } + + public void dump(String user) throws UnsupportedOperationException, IOException, InterruptedException { + spamAssassinContainer.execInContainer("sa-learn", "--dump", "magic", "-u", user); + } } } http://git-wip-us.apache.org/repos/asf/james-project/blob/697a5326/server/container/util/src/test/java/org/apache/james/util/scanner/SpamAssassinInvokerTest.java ---------------------------------------------------------------------- diff --git a/server/container/util/src/test/java/org/apache/james/util/scanner/SpamAssassinInvokerTest.java b/server/container/util/src/test/java/org/apache/james/util/scanner/SpamAssassinInvokerTest.java index fdf3b83..ffa796a 100644 --- a/server/container/util/src/test/java/org/apache/james/util/scanner/SpamAssassinInvokerTest.java +++ b/server/container/util/src/test/java/org/apache/james/util/scanner/SpamAssassinInvokerTest.java @@ -70,7 +70,7 @@ public class SpamAssassinInvokerTest { @Test public void scanMailShouldMarkHasSpamWhenKnownHasSpam() throws Exception { - spamAssassin.train(); + spamAssassin.train("user"); MimeMessage mimeMessage = MimeMessageUtil.mimeMessageFromStream( ClassLoader.getSystemResourceAsStream("spamassassin_db/spam/spam1")); @@ -91,7 +91,7 @@ public class SpamAssassinInvokerTest { } @Test - public void scanMailShouldMarkHasSpamWhenMessageAlreadyLearnedAsSpam() throws Exception { + public void scanMailShouldMarkAsSpamWhenMessageAlreadyLearnedAsSpam() throws Exception { MimeMessage mimeMessage = MimeMessageUtil.mimeMessageFromStream( ClassLoader.getSystemResourceAsStream("spamassassin_db/spam/spam1")); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
