Repository: allura Updated Branches: refs/heads/master f597b2e67 -> 6742f0f15
[#8192] StopForumSpam filter, and chained spam filters Project: http://git-wip-us.apache.org/repos/asf/allura/repo Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/407559b0 Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/407559b0 Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/407559b0 Branch: refs/heads/master Commit: 407559b02d77284b839e3ea9a26ef48dc1193c32 Parents: f597b2e Author: Dave Brondsema <d...@brondsema.net> Authored: Thu Mar 1 15:33:12 2018 -0500 Committer: Kenton Taylor <ktay...@slashdotmedia.com> Committed: Wed Mar 7 14:38:33 2018 -0500 ---------------------------------------------------------------------- Allura/allura/lib/app_globals.py | 3 +- Allura/allura/lib/spam/__init__.py | 51 +++++++++++++--- Allura/allura/lib/spam/akismetfilter.py | 9 +-- Allura/allura/lib/spam/mollomfilter.py | 9 +-- Allura/allura/lib/spam/stopforumspamfilter.py | 63 ++++++++++++++++++++ .../allura/tests/unit/spam/test_spam_filter.py | 30 +++++++++- .../tests/unit/spam/test_stopforumspam.py | 51 ++++++++++++++++ Allura/development.ini | 5 +- Allura/setup.py | 1 + 9 files changed, 197 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/allura/blob/407559b0/Allura/allura/lib/app_globals.py ---------------------------------------------------------------------- diff --git a/Allura/allura/lib/app_globals.py b/Allura/allura/lib/app_globals.py index 5453055..fc06acc 100644 --- a/Allura/allura/lib/app_globals.py +++ b/Allura/allura/lib/app_globals.py @@ -256,7 +256,8 @@ class Globals(object): value = ep.load() except Exception: log.exception('Could not load entry point [%s] %s', section_name, ep) - d[ep.name] = value + else: + d[ep.name] = value return d class entry_point_loading_dict(dict): http://git-wip-us.apache.org/repos/asf/allura/blob/407559b0/Allura/allura/lib/spam/__init__.py ---------------------------------------------------------------------- diff --git a/Allura/allura/lib/spam/__init__.py b/Allura/allura/lib/spam/__init__.py index a4405e3..aaa4412 100644 --- a/Allura/allura/lib/spam/__init__.py +++ b/Allura/allura/lib/spam/__init__.py @@ -16,6 +16,9 @@ # under the License. import logging +from copy import copy + +from paste.deploy.converters import aslist from allura.lib.helpers import exceptionless from allura.model.artifact import SpamCheckResult @@ -30,20 +33,23 @@ class SpamFilter(object): def __init__(self, config): pass + @property + def filter_name(self): + return self.__class__.__name__.replace('SpamFilter', '').lower() + def check(self, text, artifact=None, user=None, content_type='comment', **kw): """Return True if ``text`` is spam, else False.""" log.info("No spam checking enabled") return False def submit_spam(self, text, artifact=None, user=None, content_type='comment', **kw): - log.info("No spam checking enabled") + log.info("No submit_spam available for %s", self.filter_name) def submit_ham(self, text, artifact=None, user=None, content_type='comment', **kw): - log.info("No spam checking enabled") + log.info("No submit_ham available for %s", self.filter_name) def record_result(self, result, artifact, user): - filter_name = self.__class__.__name__.replace('SpamFilter', '').lower() - log.info("spam=%s (%s): %s" % (str(result), filter_name, artifact.url() if artifact else '')) + log.info("spam=%s (%s): %s" % (str(result), self.filter_name, artifact.url() if artifact else '')) r = SpamCheckResult( ref=artifact.ref if artifact else None, project_id=artifact.project_id if artifact else None, @@ -60,7 +66,36 @@ class SpamFilter(object): method = config.get('spam.method') if not method: return cls(config) - result = entry_points[method] - filter_obj = result(config) - filter_obj.check = exceptionless(False, log=log)(filter_obj.check) - return filter_obj + elif ' ' in method: + return ChainedSpamFilter(method, entry_points, config) + else: + result = entry_points[method] + filter_obj = result(config) + filter_obj.check = exceptionless(False, log=log)(filter_obj.check) + return filter_obj + + +class ChainedSpamFilter(SpamFilter): + + def __init__(self, methods_string, entry_points, config): + methods = aslist(methods_string) + self.filters = [] + for m in methods: + config = copy(config) + config['spam.method'] = m + spam_filter = SpamFilter.get(config=config, entry_points=entry_points) + self.filters.append(spam_filter) + + def check(self, *a, **kw): + for spam_filter in self.filters: + if spam_filter.check(*a, **kw): + return True + return False + + def submit_spam(self, *a, **kw): + for spam_filter in self.filters: + spam_filter.submit_spam(*a, **kw) + + def submit_ham(self, *a, **kw): + for spam_filter in self.filters: + spam_filter.submit_ham(*a, **kw) http://git-wip-us.apache.org/repos/asf/allura/blob/407559b0/Allura/allura/lib/spam/akismetfilter.py ---------------------------------------------------------------------- diff --git a/Allura/allura/lib/spam/akismetfilter.py b/Allura/allura/lib/spam/akismetfilter.py index 7a87bd9..26cccbd 100644 --- a/Allura/allura/lib/spam/akismetfilter.py +++ b/Allura/allura/lib/spam/akismetfilter.py @@ -38,13 +38,8 @@ class AkismetSpamFilter(SpamFilter): """Spam checking implementation via Akismet service. - To enable Akismet spam filtering in your Allura instance, first - enable the entry point in setup.py:: - - [allura.spam] - akismet = allura.lib.spam.akismetfilter:AkismetSpamFilter - - Then include the following parameters in your .ini file:: + To enable Akismet spam filtering in your Allura instance, first pip install akismet (see requirements-optional.txt) + and then include the following parameters in your .ini file:: spam.method = akismet spam.key = <your Akismet key here> http://git-wip-us.apache.org/repos/asf/allura/blob/407559b0/Allura/allura/lib/spam/mollomfilter.py ---------------------------------------------------------------------- diff --git a/Allura/allura/lib/spam/mollomfilter.py b/Allura/allura/lib/spam/mollomfilter.py index 533186b..f2d6b68 100644 --- a/Allura/allura/lib/spam/mollomfilter.py +++ b/Allura/allura/lib/spam/mollomfilter.py @@ -38,13 +38,8 @@ class MollomSpamFilter(SpamFilter): """Spam checking implementation via Mollom service. - To enable Mollom spam filtering in your Allura instance, first - enable the entry point in setup.py:: - - [allura.spam] - mollom = allura.lib.spam.mollomfilter:MollomSpamFilter - - Then include the following parameters in your .ini file:: + To enable Mollom spam filtering in your Allura instance, first pip install PyMollom (see requirements-optional.txt) + and then include the following parameters in your .ini file:: spam.method = mollom spam.public_key = <your Mollom public key here> http://git-wip-us.apache.org/repos/asf/allura/blob/407559b0/Allura/allura/lib/spam/stopforumspamfilter.py ---------------------------------------------------------------------- diff --git a/Allura/allura/lib/spam/stopforumspamfilter.py b/Allura/allura/lib/spam/stopforumspamfilter.py new file mode 100644 index 0000000..0e263be --- /dev/null +++ b/Allura/allura/lib/spam/stopforumspamfilter.py @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import logging +import csv +from sys import getsizeof + +import ipaddress +from pylons import request + +from allura.lib import utils +from allura.lib.spam import SpamFilter + +log = logging.getLogger(__name__) + + +class StopForumSpamSpamFilter(SpamFilter): + """Spam checking by IP address, using StopForumSpam data files + + To enable StopForumSpam spam filtering in your Allura instance, + include the following parameters in your .ini file:: + + spam.method = stopforumspam + spam.stopforumspam.ip_addr_file = /path/to/listed_ip_180_all.txt + + Of course you'll need to download the file from https://www.stopforumspam.com/downloads and do so on a regular + basis to keep it updated. + """ + + def __init__(self, config): + self.packed_ips = set() + with open(config['spam.stopforumspam.ip_addr_file']) as f: + csv_file = csv.reader(f) + for record in csv_file: + if int(record[1]) > int(config.get('spam.stopforumspam.threshold', 20)): + ip = record[0] + # int is the smallest memory representation of an IP addr + ip_int = int(ipaddress.ip_address(unicode(ip))) + self.packed_ips.add(ip_int) + # to get actual memory usage, use: from pympler.asizeof import asizeof + log.info('Read stopforumspam file; %s recs, probably %s bytes stored in memory', len(self.packed_ips), + len(self.packed_ips) * getsizeof(iter(self.packed_ips).next()) * 2) + + def check(self, text, artifact=None, user=None, content_type='comment', **kw): + ip = utils.ip_address(request) + ip_int = int(ipaddress.ip_address(unicode(ip))) + res = ip_int in self.packed_ips + self.record_result(res, artifact, user) + return res http://git-wip-us.apache.org/repos/asf/allura/blob/407559b0/Allura/allura/tests/unit/spam/test_spam_filter.py ---------------------------------------------------------------------- diff --git a/Allura/allura/tests/unit/spam/test_spam_filter.py b/Allura/allura/tests/unit/spam/test_spam_filter.py index 9d603f8..d05fa97 100644 --- a/Allura/allura/tests/unit/spam/test_spam_filter.py +++ b/Allura/allura/tests/unit/spam/test_spam_filter.py @@ -23,7 +23,7 @@ import unittest from ming.odm import ThreadLocalORMSession from nose.tools import assert_equal -from allura.lib.spam import SpamFilter +from allura.lib.spam import SpamFilter, ChainedSpamFilter from allura import model as M from allura.model.artifact import SpamCheckResult from alluratest.controller import setup_basic_test @@ -32,8 +32,19 @@ from forgewiki import model as WM class MockFilter(SpamFilter): + def __init__(self, config): + self.config = config + def check(*args, **kw): raise Exception("test exception") + + +class MockFilter2(SpamFilter): + + def __init__(self, config): + self.config = config + + def check(*args, **kw): return True @@ -82,3 +93,20 @@ class TestSpamFilterFunctional(object): assert_equal(len(results), 1) assert_equal(results[0].result, True) assert_equal(results[0].user.username, 'test-user') + + +class TestChainedSpamFilter(object): + + def test(self): + config = {'spam.method': 'mock1 mock2', 'spam.settingA': 'bcd'} + entry_points = {'mock1': MockFilter, 'mock2': MockFilter2} + checker = SpamFilter.get(config, entry_points) + assert isinstance(checker, ChainedSpamFilter) + assert len(checker.filters) == 2, checker.filters + assert_equal(checker.filters[0].config, {'spam.method': 'mock1', 'spam.settingA': 'bcd'}) + assert_equal(checker.filters[1].config, {'spam.method': 'mock2', 'spam.settingA': 'bcd'}) + + assert checker.check() # True because first filter errors out, and 2nd returns True + + checker.submit_spam('some text') + checker.submit_ham('some text') http://git-wip-us.apache.org/repos/asf/allura/blob/407559b0/Allura/allura/tests/unit/spam/test_stopforumspam.py ---------------------------------------------------------------------- diff --git a/Allura/allura/tests/unit/spam/test_stopforumspam.py b/Allura/allura/tests/unit/spam/test_stopforumspam.py new file mode 100644 index 0000000..64963cf --- /dev/null +++ b/Allura/allura/tests/unit/spam/test_stopforumspam.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import tempfile + +import mock + +from bson import ObjectId +from nose.tools import assert_equal + +from allura.lib.spam.stopforumspamfilter import StopForumSpamSpamFilter + + +class TestStopForumSpam(object): + + def setUp(self): + self.content = u'spåm text' + + self.artifact = mock.Mock() + self.artifact.project_id = ObjectId() + self.artifact.ref = None + + with tempfile.NamedTemporaryFile() as f: + f.write('''"1.1.1.1","2","2018-01-22 10:56:29" +"1.2.3.4","42","2017-09-24 18:33:00" +"4.3.2.1","1","2017-09-28 14:03:53"''') + f.flush() + self.sfs = StopForumSpamSpamFilter({'spam.stopforumspam.ip_addr_file': f.name}) + + @mock.patch('allura.lib.spam.stopforumspamfilter.request') + def test_check(self, request): + request.remote_addr = '1.2.3.4' + assert_equal(True, self.sfs.check(self.content, artifact=self.artifact)) + + request.remote_addr = '1.1.1.1' + assert_equal(False, self.sfs.check(self.content, artifact=self.artifact)) http://git-wip-us.apache.org/repos/asf/allura/blob/407559b0/Allura/development.ini ---------------------------------------------------------------------- diff --git a/Allura/development.ini b/Allura/development.ini index 4a7b9d9..d3f2700 100644 --- a/Allura/development.ini +++ b/Allura/development.ini @@ -237,13 +237,16 @@ user_prefs.maximum_claimed_emails = 20 site_admin_project = allura site_admin_project_nbhd = Projects -; Spam filtering service: mollom or akismet +; Spam filtering service: this can be one or more of: mollom akismet stopforumspam ;spam.method = akismet ; for akismet: ;spam.key = ; for mollom: ;spam.public_key = ;spam.private_key = +; for stopforumspam, should be a listed_ip_*_all.txt file +;spam.stopforumspam.ip_addr_file = +;spam.stopforumspam.threshold = 20 ; Phone verification service: Nexmo Verify ; phone.method = nexmo http://git-wip-us.apache.org/repos/asf/allura/blob/407559b0/Allura/setup.py ---------------------------------------------------------------------- diff --git a/Allura/setup.py b/Allura/setup.py index 3005d07..30bca7d 100644 --- a/Allura/setup.py +++ b/Allura/setup.py @@ -106,6 +106,7 @@ setup( [allura.spam] akismet = allura.lib.spam.akismetfilter:AkismetSpamFilter mollom = allura.lib.spam.mollomfilter:MollomSpamFilter + stopforumspam = allura.lib.spam.stopforumspamfilter:StopForumSpamSpamFilter [allura.phone] nexmo = allura.lib.phone.nexmo:NexmoPhoneService
