JAMES-2429 Implement basic Dlp matcher
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/a8586df3 Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/a8586df3 Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/a8586df3 Branch: refs/heads/master Commit: a8586df34d82dd50e9ea387e6b0becdc34fbae66 Parents: 19caa77 Author: Matthieu Baechler <[email protected]> Authored: Wed Jun 6 08:55:24 2018 +0200 Committer: benwa <[email protected]> Committed: Tue Jun 19 16:53:02 2018 +0700 ---------------------------------------------------------------------- .../apache/mailet/base/MailAddressFixture.java | 5 + .../james/dlp/api/DLPConfigurationLoader.java | 28 + .../james/dlp/api/DLPConfigurationStore.java | 5 +- .../james/transport/matchers/dlp/Dlp.java | 78 +++ .../transport/matchers/dlp/DlpDomainRules.java | 300 +++++++++++ .../transport/matchers/dlp/DlpRulesLoader.java | 57 ++ .../matchers/dlp/DlpDomainRulesTest.java | 60 +++ .../james/transport/matchers/dlp/DlpTest.java | 514 +++++++++++++++++++ 8 files changed, 1043 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/james-project/blob/a8586df3/mailet/base/src/test/java/org/apache/mailet/base/MailAddressFixture.java ---------------------------------------------------------------------- diff --git a/mailet/base/src/test/java/org/apache/mailet/base/MailAddressFixture.java b/mailet/base/src/test/java/org/apache/mailet/base/MailAddressFixture.java index 51bd487..4efe093 100644 --- a/mailet/base/src/test/java/org/apache/mailet/base/MailAddressFixture.java +++ b/mailet/base/src/test/java/org/apache/mailet/base/MailAddressFixture.java @@ -21,6 +21,7 @@ package org.apache.mailet.base; import javax.mail.internet.AddressException; +import org.apache.james.core.Domain; import org.apache.james.core.MailAddress; public class MailAddressFixture { @@ -28,6 +29,10 @@ public class MailAddressFixture { public static final String JAMES_APACHE_ORG = "james.apache.org"; public static final String JAMES2_APACHE_ORG = "james2.apache.org"; + public static final Domain JAMES_LOCAL_DOMAIN = Domain.of(JAMES_LOCAL); + public static final Domain JAMES_APACHE_ORG_DOMAIN = Domain.of(JAMES_APACHE_ORG); + public static final Domain JAMES2_APACHE_ORG_DOMAIN = Domain.of(JAMES2_APACHE_ORG); + public static final MailAddress SENDER = createMailAddress("sender@" + JAMES_LOCAL); public static final MailAddress RECIPIENT1 = createMailAddress("recipient1@" + JAMES_LOCAL); public static final MailAddress RECIPIENT2 = createMailAddress("recipient2@" + JAMES_LOCAL); http://git-wip-us.apache.org/repos/asf/james-project/blob/a8586df3/server/data/data-api/src/main/java/org/apache/james/dlp/api/DLPConfigurationLoader.java ---------------------------------------------------------------------- diff --git a/server/data/data-api/src/main/java/org/apache/james/dlp/api/DLPConfigurationLoader.java b/server/data/data-api/src/main/java/org/apache/james/dlp/api/DLPConfigurationLoader.java new file mode 100644 index 0000000..e2d27f5 --- /dev/null +++ b/server/data/data-api/src/main/java/org/apache/james/dlp/api/DLPConfigurationLoader.java @@ -0,0 +1,28 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.dlp.api; + +import java.util.stream.Stream; + +import org.apache.james.core.Domain; + +public interface DLPConfigurationLoader { + Stream<DLPConfigurationItem> list(Domain domain); +} http://git-wip-us.apache.org/repos/asf/james-project/blob/a8586df3/server/data/data-api/src/main/java/org/apache/james/dlp/api/DLPConfigurationStore.java ---------------------------------------------------------------------- diff --git a/server/data/data-api/src/main/java/org/apache/james/dlp/api/DLPConfigurationStore.java b/server/data/data-api/src/main/java/org/apache/james/dlp/api/DLPConfigurationStore.java index 457341e..4272f49 100644 --- a/server/data/data-api/src/main/java/org/apache/james/dlp/api/DLPConfigurationStore.java +++ b/server/data/data-api/src/main/java/org/apache/james/dlp/api/DLPConfigurationStore.java @@ -20,15 +20,12 @@ package org.apache.james.dlp.api; import java.util.List; -import java.util.stream.Stream; import org.apache.james.core.Domain; import com.google.common.collect.ImmutableList; -public interface DLPConfigurationStore { - - Stream<DLPConfigurationItem> list(Domain domain); +public interface DLPConfigurationStore extends DLPConfigurationLoader { void store(Domain domain, List<DLPConfigurationItem> rule); http://git-wip-us.apache.org/repos/asf/james-project/blob/a8586df3/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/Dlp.java ---------------------------------------------------------------------- diff --git a/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/Dlp.java b/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/Dlp.java new file mode 100644 index 0000000..cb7fc72 --- /dev/null +++ b/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/Dlp.java @@ -0,0 +1,78 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.transport.matchers.dlp; + +import java.util.Collection; +import java.util.Optional; + +import javax.inject.Inject; +import javax.mail.MessagingException; + +import org.apache.james.core.MailAddress; +import org.apache.james.dlp.api.DLPConfigurationItem; +import org.apache.mailet.Mail; +import org.apache.mailet.base.GenericMatcher; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; + +public class Dlp extends GenericMatcher { + + public static final String DLP_MATCHED_RULE = "DlpMatchedRule"; + + private final DlpRulesLoader rulesLoader; + + @Inject + @VisibleForTesting + Dlp(DlpRulesLoader rulesLoader) { + this.rulesLoader = rulesLoader; + } + + @Override + public Collection<MailAddress> match(Mail mail) throws MessagingException { + Optional<DLPConfigurationItem.Id> firstMatchingRuleId = findFirstMatchingRule(mail); + + if (firstMatchingRuleId.isPresent()) { + DLPConfigurationItem.Id ruleId = firstMatchingRuleId.get(); + setRuleIdAsMailAttribute(mail, ruleId); + return mail.getRecipients(); + } + return ImmutableList.of(); + } + + private void setRuleIdAsMailAttribute(Mail mail, DLPConfigurationItem.Id ruleId) { + mail.setAttribute(DLP_MATCHED_RULE, ruleId.asString()); + } + + private Optional<DLPConfigurationItem.Id> findFirstMatchingRule(Mail mail) { + return Optional + .ofNullable(mail.getSender()) + .flatMap(sender -> matchingRule(sender, mail)); + } + + private Optional<DLPConfigurationItem.Id> matchingRule(MailAddress address, Mail mail) { + return rulesLoader.load(address.getDomain()).match(mail); + } + + @Override + public String getMatcherInfo() { + return "Data Leak Prevention Matcher"; + } +} http://git-wip-us.apache.org/repos/asf/james-project/blob/a8586df3/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/DlpDomainRules.java ---------------------------------------------------------------------- diff --git a/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/DlpDomainRules.java b/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/DlpDomainRules.java new file mode 100644 index 0000000..f44a60e --- /dev/null +++ b/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/DlpDomainRules.java @@ -0,0 +1,300 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.transport.matchers.dlp; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; +import java.util.Optional; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import javax.mail.Address; +import javax.mail.BodyPart; +import javax.mail.Message; +import javax.mail.MessagingException; +import javax.mail.Multipart; +import javax.mail.internet.MimeMessage; + +import org.apache.james.core.MailAddress; +import org.apache.james.dlp.api.DLPConfigurationItem; +import org.apache.james.dlp.api.DLPConfigurationItem.Targets; +import org.apache.james.javax.MultipartUtil; +import org.apache.james.mime4j.util.MimeUtil; +import org.apache.james.util.OptionalUtils; +import org.apache.mailet.Mail; + +import com.github.fge.lambdas.Throwing; +import com.github.fge.lambdas.predicates.ThrowingPredicate; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMultimap; + +public class DlpDomainRules { + + @VisibleForTesting static DlpDomainRules matchNothing() { + return DlpDomainRules.of(new Rule(DLPConfigurationItem.Id.of("always false"), (mail) -> false)); + } + + @VisibleForTesting static DlpDomainRules matchAll() { + return DlpDomainRules.of(new Rule(DLPConfigurationItem.Id.of("always true"), (mail) -> true)); + } + + private static DlpDomainRules of(Rule rule) { + return new DlpDomainRules(ImmutableList.of(rule)); + } + + public static DlpDomainRulesBuilder builder() { + return new DlpDomainRulesBuilder(); + } + + static class Rule { + + interface MatcherFunction extends ThrowingPredicate<Mail> { } + + + private static Stream<String> asStringStream(Address[] addresses) { + return Arrays.stream(addresses).map(Rule::asString); + } + + private static String asString(Address address) { + return MimeUtil.unscrambleHeaderValue(address.toString()); + } + + private static class ContentMatcher implements Rule.MatcherFunction { + + private final Pattern pattern; + + private ContentMatcher(Pattern pattern) { + this.pattern = pattern; + } + + @Override + public boolean doTest(Mail mail) throws MessagingException, IOException { + return Stream + .concat(getMessageSubjects(mail), getMessageBodies(mail.getMessage())) + .anyMatch(pattern.asPredicate()); + } + + private Stream<String> getMessageSubjects(Mail mail) throws MessagingException { + MimeMessage message = mail.getMessage(); + if (message != null) { + return OptionalUtils.toStream( + Optional.ofNullable(message.getSubject())); + } + return Stream.of(); + } + + private Stream<String> getMessageBodies(Message message) throws MessagingException, IOException { + if (message != null) { + return getMessageBodiesFromContent(message.getContent()); + } + return Stream.of(); + } + + private Stream<String> getMessageBodiesFromContent(Object content) throws IOException, MessagingException { + if (content instanceof String) { + return Stream.of((String) content); + } + if (content instanceof Message) { + Message message = (Message) content; + return getMessageBodiesFromContent(message.getContent()); + } + if (content instanceof Multipart) { + return MultipartUtil.retrieveBodyParts((Multipart) content) + .stream() + .map(Throwing.function(BodyPart::getContent).sneakyThrow()) + .flatMap(Throwing.function(this::getMessageBodiesFromContent).sneakyThrow()); + } + return Stream.of(); + } + } + + private static class RecipientsMatcher implements Rule.MatcherFunction { + + private final Pattern pattern; + + private RecipientsMatcher(Pattern pattern) { + this.pattern = pattern; + } + + @Override + public boolean doTest(Mail mail) throws MessagingException, IOException { + return listRecipientsAsString(mail).anyMatch(pattern.asPredicate()); + } + + private Stream<String> listRecipientsAsString(Mail mail) throws MessagingException { + return Stream.concat(listEnvelopRecipients(mail), listHeaderRecipients(mail)); + } + + private Stream<String> listEnvelopRecipients(Mail mail) { + return mail.getRecipients().stream().map(MailAddress::asString); + } + + private Stream<String> listHeaderRecipients(Mail mail) throws MessagingException { + return Optional.ofNullable(mail.getMessage()) + .flatMap(Throwing.function(m -> Optional.ofNullable(m.getAllRecipients()))) + .map(Rule::asStringStream) + .orElse(Stream.of()); + } + + } + + private static class SenderMatcher implements Rule.MatcherFunction { + + private final Pattern pattern; + + private SenderMatcher(Pattern pattern) { + this.pattern = pattern; + } + + @Override + public boolean doTest(Mail mail) throws MessagingException { + return listSenders(mail).anyMatch(pattern.asPredicate()); + } + + private Stream<String> listSenders(Mail mail) throws MessagingException { + return Stream.concat(listEnvelopSender(mail), listFromHeaders(mail)); + } + + private Stream<String> listEnvelopSender(Mail mail) { + return OptionalUtils.toStream(Optional.ofNullable(mail.getSender()).map(MailAddress::asString)); + } + + private Stream<String> listFromHeaders(Mail mail) throws MessagingException { + MimeMessage message = mail.getMessage(); + if (message != null) { + return asStringStream(message.getFrom()); + } + return Stream.of(); + } + + } + + private final DLPConfigurationItem.Id id; + private final MatcherFunction matcher; + + public Rule(DLPConfigurationItem.Id id, MatcherFunction matcher) { + this.id = id; + this.matcher = matcher; + } + + public DLPConfigurationItem.Id id() { + return id; + } + + public boolean match(Mail mail) { + return matcher.test(mail); + } + + @Override + public boolean equals(Object o) { + if (o instanceof Rule) { + Rule other = (Rule) o; + return Objects.equals(id, other.id) && + Objects.equals(matcher, other.matcher); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(id, matcher); + } + + } + + public static class DlpDomainRulesBuilder { + + private final ImmutableMultimap.Builder<Targets.Type, Rule> rules; + + private DlpDomainRulesBuilder() { + rules = ImmutableMultimap.builder(); + } + + public DlpDomainRulesBuilder recipientRule(DLPConfigurationItem.Id id, Pattern pattern) { + return rule(Targets.Type.Recipient, id, pattern); + } + + public DlpDomainRulesBuilder senderRule(DLPConfigurationItem.Id id, Pattern pattern) { + return rule(Targets.Type.Sender, id, pattern); + } + + public DlpDomainRulesBuilder contentRule(DLPConfigurationItem.Id id, Pattern pattern) { + return rule(Targets.Type.Content, id, pattern); + } + + public DlpDomainRulesBuilder rule(Targets.Type type, DLPConfigurationItem.Id id, Pattern regexp) { + rules.put(type, toRule(type, id, regexp)); + return this; + } + + private Rule toRule(Targets.Type type, DLPConfigurationItem.Id id, Pattern pattern) { + switch (type) { + case Sender: + return new Rule(id, new Rule.SenderMatcher(pattern)); + case Content: + return new Rule(id, new Rule.ContentMatcher(pattern)); + case Recipient: + return new Rule(id, new Rule.RecipientsMatcher(pattern)); + default: + throw new IllegalArgumentException("unexpected value"); + } + } + + public DlpDomainRules build() { + ImmutableMultimap<Targets.Type, Rule> rules = this.rules.build(); + Preconditions.checkState(!containsDuplicateIds(rules), "Rules should not contain duplicated `id`"); + return new DlpDomainRules(rules.values()); + } + + private boolean containsDuplicateIds(ImmutableMultimap<Targets.Type, Rule> rules) { + return + Stream.of(Targets.Type.values()) + .map(rules::get) + .anyMatch(this::containsDuplicateIds); + } + + private boolean containsDuplicateIds(ImmutableCollection<Rule> rules) { + long distinctIdCount = rules.stream() + .map(Rule::id) + .distinct() + .count(); + return distinctIdCount != rules.size(); + } + + } + + private final ImmutableCollection<Rule> rules; + + private DlpDomainRules(ImmutableCollection<Rule> rules) { + this.rules = rules; + } + + public Optional<DLPConfigurationItem.Id> match(Mail mail) { + return rules.stream() + .filter(rule -> rule.match(mail)) + .map(Rule::id) + .findFirst(); + } + +} http://git-wip-us.apache.org/repos/asf/james-project/blob/a8586df3/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/DlpRulesLoader.java ---------------------------------------------------------------------- diff --git a/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/DlpRulesLoader.java b/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/DlpRulesLoader.java new file mode 100644 index 0000000..13430d9 --- /dev/null +++ b/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/DlpRulesLoader.java @@ -0,0 +1,57 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.transport.matchers.dlp; + +import java.util.stream.Stream; + +import javax.inject.Inject; + +import org.apache.james.core.Domain; +import org.apache.james.dlp.api.DLPConfigurationItem; +import org.apache.james.dlp.api.DLPConfigurationStore; + +public interface DlpRulesLoader { + + DlpDomainRules load(Domain domain); + + class Impl implements DlpRulesLoader { + + private final DLPConfigurationStore configurationStore; + + @Inject + public Impl(DLPConfigurationStore configurationStore) { + this.configurationStore = configurationStore; + } + + @Override + public DlpDomainRules load(Domain domain) { + return toRules(configurationStore.list(domain)); + } + + private DlpDomainRules toRules(Stream<DLPConfigurationItem> items) { + DlpDomainRules.DlpDomainRulesBuilder builder = DlpDomainRules.builder(); + items.forEach(item -> + item.getTargets().list().forEach(type -> + builder.rule(type, item.getId(), item.getRegexp()) + )); + return builder.build(); + } + } +} http://git-wip-us.apache.org/repos/asf/james-project/blob/a8586df3/server/mailet/mailets/src/test/java/org/apache/james/transport/matchers/dlp/DlpDomainRulesTest.java ---------------------------------------------------------------------- diff --git a/server/mailet/mailets/src/test/java/org/apache/james/transport/matchers/dlp/DlpDomainRulesTest.java b/server/mailet/mailets/src/test/java/org/apache/james/transport/matchers/dlp/DlpDomainRulesTest.java new file mode 100644 index 0000000..155795f --- /dev/null +++ b/server/mailet/mailets/src/test/java/org/apache/james/transport/matchers/dlp/DlpDomainRulesTest.java @@ -0,0 +1,60 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.transport.matchers.dlp; + +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.regex.Pattern; + +import org.apache.james.dlp.api.DLPConfigurationItem.Id; +import org.junit.jupiter.api.Test; + +class DlpDomainRulesTest { + + private static final Pattern PATTERN_1 = Pattern.compile("1"); + private static final Pattern PATTERN_2 = Pattern.compile("2"); + + @Test + void builderShouldThrowWhenDuplicateIds() { + assertThatThrownBy(() -> DlpDomainRules.builder() + .senderRule(Id.of("1"), PATTERN_1) + .senderRule(Id.of("1"), PATTERN_2) + .build()) + .isInstanceOf(IllegalStateException.class); + } + + @Test + void builderShouldNotThrowWhenDuplicateIdsOnDifferentTypes() { + assertThatCode(() -> DlpDomainRules.builder() + .senderRule(Id.of("1"), PATTERN_1) + .contentRule(Id.of("1"), PATTERN_2) + .build()) + .doesNotThrowAnyException(); + } + + + @Test + void builderShouldNotThrowWhenEmpty() { + assertThatCode(() -> DlpDomainRules.builder().build()) + .doesNotThrowAnyException(); + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/james-project/blob/a8586df3/server/mailet/mailets/src/test/java/org/apache/james/transport/matchers/dlp/DlpTest.java ---------------------------------------------------------------------- diff --git a/server/mailet/mailets/src/test/java/org/apache/james/transport/matchers/dlp/DlpTest.java b/server/mailet/mailets/src/test/java/org/apache/james/transport/matchers/dlp/DlpTest.java new file mode 100644 index 0000000..12aa394 --- /dev/null +++ b/server/mailet/mailets/src/test/java/org/apache/james/transport/matchers/dlp/DlpTest.java @@ -0,0 +1,514 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.transport.matchers.dlp; + +import static org.apache.mailet.base.MailAddressFixture.ANY_AT_JAMES; +import static org.apache.mailet.base.MailAddressFixture.JAMES_APACHE_ORG; +import static org.apache.mailet.base.MailAddressFixture.JAMES_APACHE_ORG_DOMAIN; +import static org.apache.mailet.base.MailAddressFixture.OTHER_AT_JAMES; +import static org.apache.mailet.base.MailAddressFixture.RECIPIENT1; +import static org.apache.mailet.base.MailAddressFixture.RECIPIENT2; +import static org.apache.mailet.base.MailAddressFixture.RECIPIENT3; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import java.util.regex.Pattern; + +import org.apache.james.core.Domain; +import org.apache.james.core.builder.MimeMessageBuilder; +import org.apache.james.dlp.api.DLPConfigurationItem.Id; +import org.apache.mailet.base.test.FakeMail; +import org.junit.jupiter.api.Test; + +class DlpTest { + + private static final DlpRulesLoader MATCH_ALL_FOR_ALL_DOMAINS = (Domain domain) -> DlpDomainRules.matchAll(); + private static final DlpRulesLoader MATCH_NOTHING_FOR_ALL_DOMAINS = (Domain domain) -> DlpDomainRules.matchNothing(); + + private static DlpRulesLoader asRulesLoaderFor(Domain domain, DlpDomainRules rules) { + return (Domain d) -> Optional + .of(d) + .filter(domain::equals) + .map(ignore -> rules) + .orElse(DlpDomainRules.matchNothing()); + } + + @Test + void matchShouldReturnEmptyWhenNoRecipient() throws Exception { + Dlp dlp = new Dlp(MATCH_ALL_FOR_ALL_DOMAINS); + + FakeMail mail = FakeMail.builder().sender(RECIPIENT1).build(); + + assertThat(dlp.match(mail)).isEmpty(); + } + + @Test + void matchShouldReturnEmptyWhenNoSender() throws Exception { + Dlp dlp = new Dlp(MATCH_ALL_FOR_ALL_DOMAINS); + + FakeMail mail = FakeMail.builder().recipient(RECIPIENT1).build(); + + assertThat(dlp.match(mail)).isEmpty(); + } + + @Test + void matchShouldThrowOnNullMail() { + Dlp dlp = new Dlp(MATCH_ALL_FOR_ALL_DOMAINS); + + assertThatThrownBy(() -> dlp.match(null)).isInstanceOf(NullPointerException.class); + } + + @Test + void matchShouldReturnEmptyWhenNoRuleMatch() throws Exception { + Dlp dlp = new Dlp(MATCH_NOTHING_FOR_ALL_DOMAINS); + + FakeMail mail = FakeMail.builder() + .sender(ANY_AT_JAMES) + .recipient(RECIPIENT1) + .recipient(RECIPIENT2) + .build(); + + assertThat(dlp.match(mail)).isEmpty(); + } + + @Test + void matchSenderShouldReturnRecipientsWhenEnvelopSenderMatches() throws Exception { + Dlp dlp = new Dlp( + asRulesLoaderFor( + JAMES_APACHE_ORG_DOMAIN, + DlpDomainRules.builder().senderRule(Id.of("match sender"), Pattern.compile(ANY_AT_JAMES.asString())).build())); + + FakeMail mail = FakeMail.builder().sender(ANY_AT_JAMES).recipient(RECIPIENT1).build(); + + assertThat(dlp.match(mail)).contains(RECIPIENT1); + } + + @Test + void matchSenderShouldReturnRecipientsWhenFromHeaderMatches() throws Exception { + Dlp dlp = new Dlp( + asRulesLoaderFor( + JAMES_APACHE_ORG_DOMAIN, + DlpDomainRules.builder().senderRule(Id.of("match sender"), Pattern.compile(ANY_AT_JAMES.asString())).build())); + + FakeMail mail = FakeMail + .builder() + .sender(OTHER_AT_JAMES) + .recipient(RECIPIENT1) + .mimeMessage(MimeMessageBuilder + .mimeMessageBuilder() + .addFrom(ANY_AT_JAMES.toInternetAddress())) + .build(); + + assertThat(dlp.match(mail)).contains(RECIPIENT1); + } + + @Test + void matchShouldReturnRecipientsWhenEnvelopRecipientsMatches() throws Exception { + Dlp dlp = new Dlp( + asRulesLoaderFor( + JAMES_APACHE_ORG_DOMAIN, + DlpDomainRules.builder().recipientRule(Id.of("match recipient"), Pattern.compile(RECIPIENT1.asString())).build())); + + FakeMail mail = FakeMail.builder() + .sender(ANY_AT_JAMES) + .recipient(RECIPIENT1) + .recipient(RECIPIENT2) + .build(); + + assertThat(dlp.match(mail)).contains(RECIPIENT1, RECIPIENT2); + } + + @Test + void matchShouldReturnRecipientsWhenToHeaderMatches() throws Exception { + Dlp dlp = new Dlp( + asRulesLoaderFor( + JAMES_APACHE_ORG_DOMAIN, + DlpDomainRules.builder().recipientRule(Id.of("match recipient"), Pattern.compile(RECIPIENT2.asString())).build())); + + FakeMail mail = FakeMail + .builder() + .sender(OTHER_AT_JAMES) + .recipient(RECIPIENT1) + .mimeMessage(MimeMessageBuilder + .mimeMessageBuilder() + .addToRecipient(RECIPIENT2.toInternetAddress())) + .build(); + + assertThat(dlp.match(mail)).contains(RECIPIENT1); + } + + @Test + void matchShouldReturnRecipientsWhenCcHeaderMatches() throws Exception { + Dlp dlp = new Dlp( + asRulesLoaderFor( + JAMES_APACHE_ORG_DOMAIN, + DlpDomainRules.builder().recipientRule(Id.of("match recipient"), Pattern.compile(RECIPIENT2.asString())).build())); + + FakeMail mail = FakeMail + .builder() + .sender(OTHER_AT_JAMES) + .recipient(RECIPIENT1) + .mimeMessage(MimeMessageBuilder + .mimeMessageBuilder() + .addCcRecipient(RECIPIENT2.toInternetAddress())) + .build(); + + assertThat(dlp.match(mail)).contains(RECIPIENT1); + } + + @Test + void matchShouldReturnRecipientsWhenBccHeaderMatches() throws Exception { + Dlp dlp = new Dlp( + asRulesLoaderFor( + JAMES_APACHE_ORG_DOMAIN, + DlpDomainRules.builder().recipientRule(Id.of("match recipient"), Pattern.compile(RECIPIENT2.asString())).build())); + + FakeMail mail = FakeMail + .builder() + .sender(OTHER_AT_JAMES) + .recipient(RECIPIENT1) + .mimeMessage(MimeMessageBuilder + .mimeMessageBuilder() + .addBccRecipient(RECIPIENT2.toInternetAddress())) + .build(); + + assertThat(dlp.match(mail)).contains(RECIPIENT1); + } + + @Test + void matchShouldReturnRecipientsWhenSubjectHeaderMatches() throws Exception { + Dlp dlp = new Dlp( + asRulesLoaderFor( + JAMES_APACHE_ORG_DOMAIN, + DlpDomainRules.builder().contentRule(Id.of("match subject"), Pattern.compile("pony")).build())); + + FakeMail mail = FakeMail + .builder() + .sender(OTHER_AT_JAMES) + .recipient(RECIPIENT1) + .mimeMessage(MimeMessageBuilder + .mimeMessageBuilder() + .setSubject("I just bought a pony")) + .build(); + + assertThat(dlp.match(mail)).contains(RECIPIENT1); + } + + @Test + void matchShouldReturnRecipientsWhenMessageBodyMatches() throws Exception { + Dlp dlp = new Dlp( + asRulesLoaderFor( + JAMES_APACHE_ORG_DOMAIN, + DlpDomainRules.builder().contentRule(Id.of("match content"), Pattern.compile("horse")).build())); + + FakeMail mail = FakeMail + .builder() + .sender(OTHER_AT_JAMES) + .recipient(RECIPIENT1) + .mimeMessage(MimeMessageBuilder + .mimeMessageBuilder() + .setSubject("I just bought a pony") + .setText("It's actually a horse, not a pony")) + .build(); + + assertThat(dlp.match(mail)).contains(RECIPIENT1); + } + + @Test + void matchShouldReturnRecipientsWhenMessageBodyMatchesWithNoSubject() throws Exception { + Dlp dlp = new Dlp( + asRulesLoaderFor( + JAMES_APACHE_ORG_DOMAIN, + DlpDomainRules.builder().contentRule(Id.of("match content"), Pattern.compile("horse")).build())); + + FakeMail mail = FakeMail + .builder() + .sender(OTHER_AT_JAMES) + .recipient(RECIPIENT1) + .mimeMessage(MimeMessageBuilder + .mimeMessageBuilder() + .setText("It's actually a horse, not a pony")) + .build(); + + assertThat(dlp.match(mail)).contains(RECIPIENT1); + } + + @Test + void matchShouldReturnRecipientsWhenMessageMultipartBodyMatches() throws Exception { + Dlp dlp = new Dlp( + asRulesLoaderFor( + JAMES_APACHE_ORG_DOMAIN, + DlpDomainRules.builder().contentRule(Id.of("match content"), Pattern.compile("horse")).build())); + + FakeMail mail = FakeMail + .builder() + .sender(OTHER_AT_JAMES) + .recipient(RECIPIENT1) + .mimeMessage(MimeMessageBuilder + .mimeMessageBuilder() + .setSubject("I just bought a pony") + .setMultipartWithBodyParts( + MimeMessageBuilder.bodyPartBuilder() + .data("It's actually a donkey, not a pony"), + MimeMessageBuilder.bodyPartBuilder() + .data("What??? No it's a horse!!!"))) + .build(); + + assertThat(dlp.match(mail)).contains(RECIPIENT1); + } + + @Test + void matchShouldReturnRecipientsWhenEmbeddedMessageContentMatches() throws Exception { + Dlp dlp = new Dlp( + asRulesLoaderFor( + JAMES_APACHE_ORG_DOMAIN, + DlpDomainRules.builder().contentRule(Id.of("match content"), Pattern.compile("horse")).build())); + + FakeMail mail = FakeMail + .builder() + .sender(OTHER_AT_JAMES) + .recipient(RECIPIENT1) + .mimeMessage(MimeMessageBuilder + .mimeMessageBuilder() + .setSubject("I just bought a pony") + .setContent( + MimeMessageBuilder.multipartBuilder() + .addBody( + MimeMessageBuilder.bodyPartBuilder() + .data("It's actually a donkey, not a pony")) + .addBody( + MimeMessageBuilder.mimeMessageBuilder() + .setSender(RECIPIENT2.asString()) + .setSubject("Embedded message with truth") + .setText("What??? No it's a horse!!!")))) + .build(); + + assertThat(dlp.match(mail)).contains(RECIPIENT1); + } + + @Test + void matchShouldReturnEmptyWhenEmbeddedSenderMatchesInSubMessage() throws Exception { + Dlp dlp = new Dlp( + asRulesLoaderFor( + JAMES_APACHE_ORG_DOMAIN, + DlpDomainRules.builder() + .senderRule(Id.of("match content"), Pattern.compile(RECIPIENT2.asString())) + .build())); + + FakeMail mail = FakeMail + .builder() + .sender(OTHER_AT_JAMES) + .recipient(RECIPIENT1) + .mimeMessage(MimeMessageBuilder + .mimeMessageBuilder() + .setSubject("I just bought a pony") + .setSender(RECIPIENT1.asString()) + .setContent( + MimeMessageBuilder.multipartBuilder() + .addBody( + MimeMessageBuilder.bodyPartBuilder() + .data("It's actually a donkey, not a pony")) + .addBody( + MimeMessageBuilder.mimeMessageBuilder() + .setSender(RECIPIENT2.asString()) + .setSubject("Embedded message with truth") + .setText("What??? No it's a horse!!!")))) + .build(); + + assertThat(dlp.match(mail)).isEmpty(); + } + + @Test + void matchShouldReturnEmptyWhenEmbeddedRecipientMatchesInSubMessage() throws Exception { + Dlp dlp = new Dlp( + asRulesLoaderFor( + JAMES_APACHE_ORG_DOMAIN, + DlpDomainRules.builder() + .recipientRule(Id.of("match content"), Pattern.compile(RECIPIENT2.asString())) + .build())); + + FakeMail mail = FakeMail + .builder() + .sender(OTHER_AT_JAMES) + .recipient(RECIPIENT1) + .mimeMessage(MimeMessageBuilder + .mimeMessageBuilder() + .setSubject("I just bought a pony") + .setSender(RECIPIENT1.asString()) + .setContent( + MimeMessageBuilder.multipartBuilder() + .addBody( + MimeMessageBuilder.bodyPartBuilder() + .data("It's actually a donkey, not a pony")) + .addBody( + MimeMessageBuilder.mimeMessageBuilder() + .addToRecipient(RECIPIENT1.asString()) + .setSubject("Embedded message with truth") + .setText("What??? No it's a horse!!!")))) + .build(); + + assertThat(dlp.match(mail)).isEmpty(); + } + + @Test + void matchShouldReturnRecipientsWhenEncodedSubjectMatchesContentRule() throws Exception { + Dlp dlp = new Dlp( + asRulesLoaderFor( + JAMES_APACHE_ORG_DOMAIN, + DlpDomainRules.builder() + .contentRule(Id.of("match content"), Pattern.compile("poné")) + .build())); + + FakeMail mail = FakeMail + .builder() + .sender(OTHER_AT_JAMES) + .recipient(RECIPIENT1) + .mimeMessage(MimeMessageBuilder + .mimeMessageBuilder() + .setSubject("=?ISO-8859-1?Q?I_just_bought_a_pon=E9?=") + .setSender(RECIPIENT1.asString()) + .setText("Meaningless text")) + .build(); + + assertThat(dlp.match(mail)).containsOnly(RECIPIENT1); + } + + @Test + void matchShouldReturnRecipientsWhenEncodedTextMatchesContentRule() throws Exception { + Dlp dlp = new Dlp( + asRulesLoaderFor( + JAMES_APACHE_ORG_DOMAIN, + DlpDomainRules.builder() + .contentRule(Id.of("match content"), Pattern.compile("poné")) + .build())); + + FakeMail mail = FakeMail + .builder() + .sender(OTHER_AT_JAMES) + .recipient(RECIPIENT1) + .mimeMessage(MimeMessageBuilder + .mimeMessageBuilder() + .setSubject("you know what ?") + .setSender(RECIPIENT1.asString()) + .setText("I bought a poné", "text/plain; charset=" + StandardCharsets.ISO_8859_1.name())) + .build(); + + assertThat(dlp.match(mail)).containsOnly(RECIPIENT1); + } + + @Test + void matchShouldReturnRecipientsWhenRulesMatchesAMailboxRecipient() throws Exception { + Dlp dlp = new Dlp( + asRulesLoaderFor( + JAMES_APACHE_ORG_DOMAIN, + DlpDomainRules.builder() + .recipientRule(Id.of("id1"), Pattern.compile(RECIPIENT1.asString())) + .build())); + + MimeMessageBuilder meaninglessText = MimeMessageBuilder + .mimeMessageBuilder() + .addToRecipient("Name <" + RECIPIENT1.asString() + " >") + .setSubject("=?ISO-8859-1?Q?I_just_bought_a_pon=E9?=") + .setText("Meaningless text"); + + FakeMail mail = FakeMail + .builder() + .sender(OTHER_AT_JAMES) + .recipient(RECIPIENT2) + .mimeMessage(meaninglessText) + .build(); + + assertThat(dlp.match(mail)).containsOnly(RECIPIENT2); + } + + @Test + void matchShouldReturnRecipientsWhenRulesMatchesAQuotedPrintableRecipient() throws Exception { + Dlp dlp = new Dlp( + asRulesLoaderFor( + JAMES_APACHE_ORG_DOMAIN, + DlpDomainRules.builder() + .recipientRule(Id.of("id1"), Pattern.compile("Benoît")) + .build())); + + MimeMessageBuilder meaninglessText = MimeMessageBuilder + .mimeMessageBuilder() + .addToRecipient("=?ISO-8859-1?Q?Beno=EEt_TELLIER?=") + .setSubject("=?ISO-8859-1?Q?I_just_bought_a_pon=E9?=") + .setText("Meaningless text"); + + FakeMail mail = FakeMail + .builder() + .sender(OTHER_AT_JAMES) + .recipient(RECIPIENT2) + .mimeMessage(meaninglessText) + .build(); + + assertThat(dlp.match(mail)).containsOnly(RECIPIENT2); + } + + @Test + void matchShouldReturnRecipientsWhenRulesMatchesAQuotedPrintableSender() throws Exception { + Dlp dlp = new Dlp( + asRulesLoaderFor( + JAMES_APACHE_ORG_DOMAIN, + DlpDomainRules.builder() + .senderRule(Id.of("id1"), Pattern.compile("Benoît")) + .build())); + + MimeMessageBuilder meaninglessText = MimeMessageBuilder + .mimeMessageBuilder() + .addFrom("=?ISO-8859-1?Q?Beno=EEt_TELLIER?=") + .setSubject("=?ISO-8859-1?Q?I_just_bought_a_pon=E9?=") + .setText("Meaningless text"); + + FakeMail mail = FakeMail + .builder() + .sender(OTHER_AT_JAMES) + .recipient(RECIPIENT2) + .mimeMessage(meaninglessText) + .build(); + + assertThat(dlp.match(mail)).containsOnly(RECIPIENT2); + } + + @Test + void matchShouldAttachMatchingRuleNameToMail() throws Exception { + Dlp dlp = new Dlp( + asRulesLoaderFor( + JAMES_APACHE_ORG_DOMAIN, + DlpDomainRules.builder() + .recipientRule(Id.of("should not match recipient"), Pattern.compile(RECIPIENT3.asString())) + .senderRule(Id.of("should match sender"), Pattern.compile(JAMES_APACHE_ORG)) + .build())); + + FakeMail mail = FakeMail.builder() + .sender(ANY_AT_JAMES) + .recipient(RECIPIENT1) + .recipient(RECIPIENT2) + .build(); + + dlp.match(mail); + + assertThat(mail.getAttribute("DlpMatchedRule")).isEqualTo("should match sender"); + } + +} \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
