Dachary has uploaded a new change for review.

  https://gerrit.wikimedia.org/r/311302

Change subject: bot: rebase repository,qa on Bot
......................................................................

bot: rebase repository,qa on Bot

Change-Id: I43ab7051470ec420779784eb81d33e562a57d014
Signed-off-by: Loic Dachary <l...@dachary.org>
---
M FLOSSbot/main.py
M FLOSSbot/qa.py
M FLOSSbot/repository.py
M tests/test_main.py
M tests/test_repository.py
5 files changed, 132 insertions(+), 197 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/pywikibot/bots/FLOSSbot 
refs/changes/02/311302/1

diff --git a/FLOSSbot/main.py b/FLOSSbot/main.py
index d1227c8..8f3a970 100644
--- a/FLOSSbot/main.py
+++ b/FLOSSbot/main.py
@@ -48,11 +48,14 @@
             help='no side effect')
 
         self.parser.add_argument(
-            '--language-code',
-            default='wikidata',
-            choices=['test', 'wikidata'],
-            help='wikidata language code',
-        )
+            '--test',
+            action='store_true', default=None,
+            help='use test.wikidata.org instead of wikidata.org')
+
+        self.parser.add_argument(
+            '--user',
+            default=None,
+            help='wikidata user name')
 
         subparsers = self.parser.add_subparsers(
             title='subcommands',
diff --git a/FLOSSbot/qa.py b/FLOSSbot/qa.py
index 31fb29b..401630e 100644
--- a/FLOSSbot/qa.py
+++ b/FLOSSbot/qa.py
@@ -25,19 +25,12 @@
 from pywikibot import pagegenerators as pg
 
 from FLOSSbot import util
+from FLOSSbot import bot
 
 log = logging.getLogger(__name__)
 
-P_described_at_url = "P973"
-P_archive_url = "P1065"
-P_software_quality_assurance = "P2992"
-Q_continuous_integration = "Q965769"
 
-
-class QA(object):
-
-    def __init__(self, args):
-        self.args = args
+class QA(bot.Bot):
 
     @staticmethod
     def get_parser():
@@ -70,13 +63,12 @@
             FILTER NOT EXISTS { ?item p:P2992 ?qa }
         }
         """
-        site = pywikibot.Site(self.args.language_code, "wikidata")
         for item in pg.WikidataSPARQLPageGenerator(QUERY,
-                                                   site=site,
+                                                   site=self.site,
                                                    result_type=list):
-            self.fixup(site, item)
+            self.fixup(self.site, item)
 
-    def fixup(self, site, item):
+    def fixup(self, item):
         log.debug(str(item))
         item_dict = item.get()
         clm_dict = item_dict["claims"]
@@ -105,17 +97,18 @@
                 continue
             log.info("FOUND " + travis + " and " + travis_ci)
             continuous_integration = pywikibot.ItemPage(
-                site, Q_continuous_integration, 0)
+                self.site, self.Q_Continuous_integration, 0)
 
             software_quality_assurance = pywikibot.Claim(
-                site, P_software_quality_assurance, 0)
+                self.site, self.P_software_quality_assurance, 0)
             software_quality_assurance.setTarget(continuous_integration)
             item.addClaim(software_quality_assurance)
 
-            described_at_url = pywikibot.Claim(site, P_described_at_url, 0)
+            described_at_url = pywikibot.Claim(self.site,
+                                               self.P_described_at_URL, 0)
             described_at_url.setTarget(travis)
             software_quality_assurance.addQualifier(described_at_url, bot=True)
 
-            archive_url = pywikibot.Claim(site, P_archive_url, 0)
+            archive_url = pywikibot.Claim(self.site, self.P_archive_URL, 0)
             archive_url.setTarget(travis_ci)
             software_quality_assurance.addQualifier(archive_url, bot=True)
diff --git a/FLOSSbot/repository.py b/FLOSSbot/repository.py
index 6c3ce4d..45c32eb 100644
--- a/FLOSSbot/repository.py
+++ b/FLOSSbot/repository.py
@@ -24,24 +24,17 @@
 import requests
 from pywikibot import pagegenerators as pg
 
-from FLOSSbot import util
+from FLOSSbot import bot, util
 
 log = logging.getLogger(__name__)
-
-P_username = "P554"
-P_protocol = "P2700"
-P_source_code_repository = "P1324"
 
 FLOSS_doc = ("https://www.wikidata.org/wiki/Wikidata:";
              "WikiProject_Informatics/FLOSS#source_code_repository")
 
 
-class Repository(object):
+class Repository(bot.Bot):
 
     cache = None
-
-    def __init__(self, args):
-        self.args = args
 
     @staticmethod
     def get_parser():
@@ -161,20 +154,6 @@
     def factory(argv):
         return Repository(Repository.get_parser().parse_args(argv))
 
-    @staticmethod
-    def setup_cache(site):
-        if Repository.cache:
-            return
-        Repository.Q_git = pywikibot.ItemPage(site, "Q186055", 0)
-        Repository.Q_svn = pywikibot.ItemPage(site, "Q46794", 0)
-        Repository.Q_hg = pywikibot.ItemPage(site, "Q476543", 0)
-        Repository.Q_fossil = pywikibot.ItemPage(site, "Q1439431", 0)
-        Repository.Q_bzr = pywikibot.ItemPage(site, "Q812656", 0)
-        Repository.Q_cvs = pywikibot.ItemPage(site, "Q467252", 0)
-        Repository.Q_http = pywikibot.ItemPage(site, "Q8777", 0)
-        Repository.Q_ftp = pywikibot.ItemPage(site, "Q42283", 0)
-        Repository.cache = True
-
     def debug(self, item, message):
         self.log(log.debug, item, message)
 
@@ -188,18 +167,16 @@
         fun("http://wikidata.org/wiki/"; + item.getID() + " " + message)
 
     def run(self):
-        site = pywikibot.Site(self.args.language_code, "wikidata")
-        self.setup_cache(site)
         if len(self.args.item) > 0:
-            self.run_items(site)
+            self.run_items()
         else:
-            self.run_query(site)
+            self.run_query()
 
-    def run_items(self, site):
+    def run_items(self):
         for item in self.args.item:
-            self.fixup(site, pywikibot.ItemPage(site, item, 0))
+            self.fixup(pywikibot.ItemPage(self.site, item, 0))
 
-    def run_query(self, site):
+    def run_query(self):
         if self.args.filter == 'no-protocol':
             query = """
             SELECT DISTINCT ?item WHERE {
@@ -230,15 +207,15 @@
         query = query + " # " + str(time.time())
         log.debug(query)
         for item in pg.WikidataSPARQLPageGenerator(query,
-                                                   site=site,
+                                                   site=self.site,
                                                    result_type=list):
-            self.fixup(site, item)
+            self.fixup(item)
 
-    def fixup(self, site, item):
-        self.fixup_protocol(site, item)
-        self.fixup_rank(site, item)
+    def fixup(self, item):
+        self.fixup_protocol(item)
+        self.fixup_rank(item)
 
-    def fixup_rank(self, site, item):
+    def fixup_rank(self, item):
         item_dict = item.get()
         clm_dict = item_dict["claims"]
 
@@ -255,10 +232,10 @@
                 self.debug(item,
                            "SKIP because there already is a preferred URL")
                 return False
-            if P_protocol not in claim.qualifiers:
+            if self.P_protocol not in claim.qualifiers:
                 continue
-            for protocol in claim.qualifiers[P_protocol]:
-                if protocol.getTarget() == Repository.Q_http:
+            for protocol in claim.qualifiers[self.P_protocol]:
+                if protocol.getTarget() == self.Q_Hypertext_Transfer_Protocol:
                     http.append(claim)
         if len(http) != 1:
             self.debug(item, "SKIP because there are " + str(len(http)) +
@@ -269,8 +246,7 @@
         self.info(item, "PREFERRED set to " + http[0].getTarget())
         return True
 
-    def fixup_protocol(self, site, item):
-        self.setup_cache(site)
+    def fixup_protocol(self, item):
         item_dict = item.get()
         clm_dict = item_dict["claims"]
 
@@ -280,12 +256,14 @@
 
         for claim in clm_dict['P1324']:
             url = claim.getTarget()
-            extracted = Repository.extract_repository(url)
+            extracted = self.extract_repository(url)
             if extracted and extracted not in urls:
                 self.debug(item, "ADDING " + extracted +
                            " as a source repository discovered in " + url)
                 source_code_repository = pywikibot.Claim(
-                    site, P_source_code_repository, 0)
+                    self.site,
+                    self.P_source_code_repository,
+                    0)
                 source_code_repository.setTarget(extracted)
                 if not self.args.dry_run:
                     item.addClaim(source_code_repository)
@@ -296,32 +274,31 @@
                     self.info(item, "PREFERRED set to " + url)
 
         for claim in clm_dict['P1324']:
-            Repository.fixup_url(claim)
+            self.fixup_url(claim)
 
         for claim in clm_dict['P1324']:
-            if P_protocol in claim.qualifiers:
+            if self.P_protocol in claim.qualifiers:
                 self.debug(item, "IGNORE " + claim.getTarget() +
                            " because it already has a protocol")
                 continue
-            target_protocol = Repository.guess_protocol(claim)
+            target_protocol = self.guess_protocol(claim)
             if not target_protocol:
                 self.error(item,
                            claim.getTarget() + " misses a protocol qualifier")
                 continue
-            protocol = pywikibot.Claim(site, P_protocol, 0)
+            protocol = pywikibot.Claim(self.site, self.P_protocol, 0)
             protocol.setTarget(target_protocol)
             if not self.args.dry_run:
                 claim.addQualifier(protocol, bot=True)
             self.info(item, "SET protocol of " + claim.getTarget())
 
-    @staticmethod
-    def guess_protocol_from_url(url):
+    def guess_protocol_from_url(self, url):
         if 'github.com' in url:
-            return Repository.Q_git
+            return self.Q_git
         if 'code.launchpad.net' in url:
-            return Repository.Q_bzr
+            return self.Q_GNU_Bazaar
         if 'bitbucket.org' in url:
-            return Repository.Q_hg
+            return self.Q_Mercurial
         if url.lower().startswith('http'):
             known = (
                 'http://bxr.su/',
@@ -333,34 +310,31 @@
                 'http://svn.savannah.gnu.org/viewvc/?root=',
             )
             if url.lower().replace('https', 'http').startswith(known):
-                return Repository.Q_http
+                return self.Q_Hypertext_Transfer_Protocol
         if (re.match('https?://sourceforge.net/p/'
                      '.*/(svn|code|code-0)/HEAD/tree/', url) or
                 re.match('https?://sourceforge.net/p/'
                          '.*?/.*?/ci/(default|master)/tree/', url) or
                 re.match('https?://.*.codeplex.com/SourceControl', url)):
-            return Repository.Q_http
+            return self.Q_Hypertext_Transfer_Protocol
         if url.startswith('git://'):
-            return Repository.Q_git
+            return self.Q_git
         if url.startswith('svn://'):
-            return Repository.Q_svn
+            return self.Q_Apache_Subversion
         if url.startswith('ftp://'):
-            return Repository.Q_ftp
+            return self.Q_File_Transfer_Protocol
         return None
 
-    @staticmethod
-    def verify_git(url):
+    def verify_git(self, url):
         return util.sh_bool("timeout 30 git ls-remote " + url + " HEAD")
 
-    @staticmethod
-    def verify_hg(url):
+    def verify_hg(self, url):
         return util.sh_bool("""
         set -e
         timeout 30 hg identify {url}
         """.format(url=url))
 
-    @staticmethod
-    def verify_svn(url, credentials):
+    def verify_svn(self, url, credentials):
         if credentials:
             user = '--username=' + credentials[0]
         else:
@@ -374,8 +348,7 @@
         timeout 30 svn info {url} {user} {password}
         """.format(url=url, user=user, password=password))
 
-    @staticmethod
-    def verify_fossil(url):
+    def verify_fossil(self, url):
         return util.sh_bool("""
         set -e
         rm -fr /tmp/tmpclone
@@ -385,75 +358,68 @@
             grep -q -m 1 -e 'Round-trips'
         """.format(url=url))
 
-    @staticmethod
-    def verify_bzr(url):
+    def verify_bzr(self, url):
         return util.sh_bool("""
         set -e
         timeout 30 bzr version-info {url}
         """.format(url=url))
 
-    @staticmethod
-    def verify_ftp(url):
+    def verify_ftp(self, url):
         return util.sh_bool("""
         set -e
         timeout 30 lftp -e 'dir; quit' {url}
         """.format(url=url))
 
-    @staticmethod
-    def verify_http(url):
+    def verify_http(self, url):
         r = requests.head(url, allow_redirects=True)
         return r.status_code == requests.codes.ok
 
-    @staticmethod
-    def verify_protocol(url, protocol, credentials):
-        if protocol == Repository.Q_git:
-            return Repository.verify_git(url)
-        elif protocol == Repository.Q_hg:
-            return Repository.verify_hg(url)
-        elif protocol == Repository.Q_fossil:
-            return Repository.verify_fossil(url)
-        elif protocol == Repository.Q_bzr:
-            return Repository.verify_bzr(url)
-        elif protocol == Repository.Q_svn:
-            return Repository.verify_svn(url, credentials)
-        elif protocol == Repository.Q_http:
-            return Repository.verify_http(url)
-        elif protocol == Repository.Q_ftp:
-            return Repository.verify_ftp(url)
+    def verify_protocol(self, url, protocol, credentials):
+        if protocol == self.Q_git:
+            return self.verify_git(url)
+        elif protocol == self.Q_Mercurial:
+            return self.verify_hg(url)
+        elif protocol == self.Q_Fossil:
+            return self.verify_fossil(url)
+        elif protocol == self.Q_GNU_Bazaar:
+            return self.verify_bzr(url)
+        elif protocol == self.Q_Apache_Subversion:
+            return self.verify_svn(url, credentials)
+        elif protocol == self.Q_Hypertext_Transfer_Protocol:
+            return self.verify_http(url)
+        elif protocol == self.Q_File_Transfer_Protocol:
+            return self.verify_ftp(url)
         return None
 
-    @staticmethod
-    def try_protocol(url, credentials):
-        if Repository.verify_git(url):
-            return Repository.Q_git
-        elif Repository.verify_hg(url):
-            return Repository.Q_hg
-        elif Repository.verify_svn(url, credentials):
-            return Repository.Q_svn
-        elif Repository.verify_bzr(url):
-            return Repository.Q_bzr
-        elif Repository.verify_fossil(url):
-            return Repository.Q_fossil
+    def try_protocol(self, url, credentials):
+        if self.verify_git(url):
+            return self.Q_git
+        elif self.verify_hg(url):
+            return self.Q_Mercurial
+        elif self.verify_svn(url, credentials):
+            return self.Q_Apache_Subversion
+        elif self.verify_bzr(url):
+            return self.Q_GNU_Bazaar
+        elif self.verify_fossil(url):
+            return self.Q_Fossil
         return None
 
-    @staticmethod
-    def guess_protocol(repository):
+    def guess_protocol(self, repository):
         url = repository.getTarget()
-        if P_username in repository.qualifiers:
-            credentials = repository.qualifiers[P_username][0]
+        if self.P_username in repository.qualifiers:
+            credentials = repository.qualifiers[self.P_username][0]
             credentials = credentials.getTarget().split(':')
         else:
             credentials = None
-        protocol = Repository.guess_protocol_from_url(url)
+        protocol = self.guess_protocol_from_url(url)
         if protocol:
-            if not Repository.verify_protocol(url, protocol, credentials):
+            if not self.verify_protocol(url, protocol, credentials):
                 return None
             else:
                 return protocol
-        return Repository.try_protocol(url, credentials)
+        return self.try_protocol(url, credentials)
 
-    @staticmethod
-    def fixup_url(repository):
+    def fixup_url(self, repository):
         url = repository.getTarget()
         new_url = None
 
@@ -472,8 +438,7 @@
         else:
             return False
 
-    @staticmethod
-    def extract_repository(url):
+    def extract_repository(self, url):
         m = re.match('https://(.*).codeplex.com/SourceControl/latest', url)
         if m:
             return "https://git01.codeplex.com/"; + m.group(1)
diff --git a/tests/test_main.py b/tests/test_main.py
index 4aaa732..d12f27b 100644
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -17,8 +17,9 @@
 #
 import logging
 
-import FLOSSbot.qa
 import mock
+
+import FLOSSbot.qa
 from FLOSSbot import main
 
 
diff --git a/tests/test_repository.py b/tests/test_repository.py
index 7c2723f..95a5d79 100644
--- a/tests/test_repository.py
+++ b/tests/test_repository.py
@@ -15,65 +15,43 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
+import argparse
 
 from FLOSSbot.repository import Repository
-import mock
+from tests.wikidata import TestWikidata
 
 
 class TestRepository(object):
 
-    @mock.patch('pywikibot.ItemPage')
-    def test_setup_cache__first_populated(self, ItemPage):
-        assert(not Repository.cache)
-        Repository.setup_cache('enwiki')
-        ItemPage.assert_called()
-        assert(Repository.cache is True)
+    def setup_class(self):
+        TestWikidata().login()
 
-    @mock.patch('pywikibot.ItemPage')
-    def test_setup_cache__does_not_recache(self, ItemPage):
-        # Init Cache
-        Repository.setup_cache('enwiki')
-        ItemPage.reset_mock()
+    def setup(self):
+        self.r = Repository(argparse.Namespace(
+            test=True,
+            user='FLOSSbotCI',
+        ))
 
-        Repository.setup_cache('enwiki')
-        ItemPage.assert_not_called()
-
-    @mock.patch('pywikibot.ItemPage')
-    def test_setup_cache__can_be_forced_to_recache(self, ItemPage):
-        # Init Cache
-        Repository.setup_cache('enwiki')
-        ItemPage.reset_mock()
-
-        # Cache clearing
-        Repository.cache = False
-        Repository.setup_cache('enwiki')
-        ItemPage.assert_called()
-        assert(Repository.cache is True)
-
-    @mock.patch('pywikibot.ItemPage')
-    def test_guessproto__github_is_git(self, ItemPage):
+    def test_guessproto__github_is_git(self):
         assert(
-            Repository.guess_protocol_from_url('http://github.com/foo/bar')
-            == Repository.Q_git)
+            self.r.guess_protocol_from_url('http://github.com/foo/bar')
+            == self.r.Q_git)
 
-    @mock.patch('pywikibot.ItemPage')
-    def test_guessproto__launchpad_is_bazaar(self, ItemPage):
+    def test_guessproto__launchpad_is_bazaar(self):
         assert(
-            Repository.guess_protocol_from_url('https://code.launchpad.net')
-            == Repository.Q_bzr)
+            self.r.guess_protocol_from_url('https://code.launchpad.net')
+            == self.r.Q_GNU_Bazaar)
 
-    @mock.patch('pywikibot.ItemPage')
-    def test_guessproto__known_http_repo(self, ItemPage):
+    def test_guessproto__known_http_repo(self):
         assert(
-            Repository.guess_protocol_from_url('http://bxr.su/foo')
-            == Repository.Q_http)
+            self.r.guess_protocol_from_url('http://bxr.su/foo')
+            == self.r.Q_Hypertext_Transfer_Protocol)
         # https recognized as well
         assert(
-            Repository.guess_protocol_from_url('https://bxr.su/foo')
-            == Repository.Q_http)
+            self.r.guess_protocol_from_url('https://bxr.su/foo')
+            == self.r.Q_Hypertext_Transfer_Protocol)
 
-    @mock.patch('pywikibot.ItemPage')
-    def test_guessproto__sourceforge(self, ItemPage):
+    def test_guessproto__sourceforge(self):
         sf_cases = [
             'http://sourceforge.net/p/Foo/svn/HEAD/tree/',
             'http://sourceforge.net/p/Foo/code/HEAD/tree/',
@@ -83,36 +61,31 @@
             ]
         for case in sf_cases:
             assert(
-                Repository.guess_protocol_from_url(case)
-                == Repository.Q_http)
+                self.r.guess_protocol_from_url(case)
+                == self.r.Q_Hypertext_Transfer_Protocol)
 
-    @mock.patch('pywikibot.ItemPage')
-    def test_guessproto__codeplex_SourceControl_is_http(self, ItemPage):
+    def test_guessproto__codeplex_SourceControl_is_http(self):
         assert(
-            Repository.guess_protocol_from_url(
+            self.r.guess_protocol_from_url(
                 'http://foo.codeplex.com/SourceControl')
-            == Repository.Q_http)
+            == self.r.Q_Hypertext_Transfer_Protocol)
 
-    @mock.patch('pywikibot.ItemPage')
-    def test_guessproto__url_git_schema(self, ItemPage):
+    def test_guessproto__url_git_schema(self):
         assert(
-            Repository.guess_protocol_from_url('git://example.org')
-            == Repository.Q_git)
+            self.r.guess_protocol_from_url('git://example.org')
+            == self.r.Q_git)
 
-    @mock.patch('pywikibot.ItemPage')
-    def test_guessproto__url_svn_schema(self, ItemPage):
+    def test_guessproto__url_svn_schema(self):
         assert(
-            Repository.guess_protocol_from_url('svn://example.org')
-            == Repository.Q_svn)
+            self.r.guess_protocol_from_url('svn://example.org')
+            == self.r.Q_Apache_Subversion)
 
-    @mock.patch('pywikibot.ItemPage')
-    def test_guessproto__url_ftp_schema(self, ItemPage):
+    def test_guessproto__url_ftp_schema(self):
         assert(
-            Repository.guess_protocol_from_url('ftp://example.org')
-            == Repository.Q_ftp)
+            self.r.guess_protocol_from_url('ftp://example.org')
+            == self.r.Q_File_Transfer_Protocol)
 
-    @mock.patch('pywikibot.ItemPage')
-    def test_guessproto__uncovered_returns_none(self, ItemPage):
+    def test_guessproto__uncovered_returns_none(self):
         assert(
-            Repository.guess_protocol_from_url('example.org')
+            self.r.guess_protocol_from_url('example.org')
             is None)

-- 
To view, visit https://gerrit.wikimedia.org/r/311302
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I43ab7051470ec420779784eb81d33e562a57d014
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/bots/FLOSSbot
Gerrit-Branch: master
Gerrit-Owner: Dachary <l...@dachary.org>

_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

Reply via email to