jenkins-bot has submitted this change and it was merged.

Change subject: license: import licenses from wikipedia
......................................................................


license: import licenses from wikipedia

Change-Id: Ia9f52d1a1cf60de4a2d098cfecf6e056b3ca24ba
Signed-off-by: Loic Dachary <l...@dachary.org>
---
M FLOSSbot/bot.py
A FLOSSbot/license.py
A tests/test_license.py
3 files changed, 429 insertions(+), 1 deletion(-)

Approvals:
  Dachary: Looks good to me, approved
  jenkins-bot: Verified



diff --git a/FLOSSbot/bot.py b/FLOSSbot/bot.py
index 8642759..6962b54 100644
--- a/FLOSSbot/bot.py
+++ b/FLOSSbot/bot.py
@@ -22,7 +22,7 @@
 import pywikibot
 from pywikibot import pagegenerators as pg
 
-from FLOSSbot import fsd, qa, repository
+from FLOSSbot import fsd, license, qa, repository
 from FLOSSbot.plugin import Plugin
 
 logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s')
@@ -32,6 +32,7 @@
     repository.Repository,
     qa.QA,
     fsd.FSD,
+    license.License,
 ]
 
 name2plugin = dict([(p.__name__, p) for p in plugins])
diff --git a/FLOSSbot/license.py b/FLOSSbot/license.py
new file mode 100644
index 0000000..7f8fd09
--- /dev/null
+++ b/FLOSSbot/license.py
@@ -0,0 +1,291 @@
+#
+# Copyright (C) 2016 Loic Dachary <l...@dachary.org>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+import argparse
+import logging
+import re
+
+import pywikibot
+from pywikibot import pagegenerators as pg
+
+from FLOSSbot import plugin
+
+log = logging.getLogger(__name__)
+
+
+class License(plugin.Plugin):
+
+    def __init__(self, *args):
+        super(License, self).__init__(*args)
+        self.license2item = None
+        self.licenses = None
+
+    @staticmethod
+    def get_parser():
+        parser = argparse.ArgumentParser(add_help=False)
+        parser.add_argument(
+            '--license',
+            action='append',
+            default=[],
+            help='only consider this license (can be repeated)')
+        return parser
+
+    @staticmethod
+    def filter_names():
+        return ['license-verify', 'no-license']
+
+    def get_query(self, filter):
+        format_args = {
+            'license': self.P_license,
+            'subclass_of': self.P_subclass_of,
+            'instance_of': self.P_instance_of,
+            'open_source': self.Q_open_source_license.getID(),
+            'free_software': self.Q_free_software_license.getID(),
+            'retrieved': self.P_retrieved,
+            'delay': self.args.verification_delay,
+        }
+        if filter == 'license-verify':
+            query = """
+            SELECT DISTINCT ?item WHERE {{
+              {{
+                ?item p:{license} ?license .
+                ?license ps:{license}/wdt:{instance_of}?/wdt:{subclass_of}*
+                    wd:{open_source}.
+              }} Union {{
+                ?item p:{license} ?license .
+                ?license ps:{license}/wdt:{instance_of}?/wdt:{subclass_of}*
+                    wd:{free_software}.
+              }}
+              OPTIONAL {{
+                 ?license prov:wasDerivedFrom/
+                     <http://www.wikidata.org/prop/reference/{retrieved}>
+                     ?retrieved
+              }}
+              FILTER (!BOUND(?retrieved) ||
+                      ?retrieved < (now() - "P{delay}D"^^xsd:duration))
+            }} ORDER BY ?item
+            """.format(**format_args)
+        elif filter == 'no-license':
+            format_args.update({
+                'foss': self.Q_free_and_open_source_software.getID(),
+                'free_software': self.Q_free_software.getID(),
+                'open_source_software': self.Q_open_source_software.getID(),
+                'public_domain': self.Q_public_domain.getID(),
+                'software': self.Q_software.getID(),
+            })
+            query = """
+            SELECT DISTINCT ?item WHERE {{
+               {{
+                 ?item p:{instance_of}/ps:{instance_of}/wdt:{subclass_of}*
+                    wd:{foss}.
+               }} Union {{
+                 ?item p:{instance_of}/ps:{instance_of}/wdt:{subclass_of}*
+                    wd:{free_software}.
+               }} Union {{
+                 ?item p:{instance_of}/ps:{instance_of}/wdt:{subclass_of}*
+                    wd:{open_source_software}.
+               }} Union {{
+                 ?item p:{instance_of}/ps:{instance_of}/wdt:{subclass_of}*
+                    wd:{public_domain}.
+                 ?item p:{instance_of}/ps:{instance_of}/wdt:{subclass_of}*
+                    wd:{software}.
+               }}
+               FILTER NOT EXISTS {{ ?item p:{license} ?license }}
+            }} ORDER BY ?item
+            """.format(**format_args)
+        else:
+            query = None
+        return query
+
+    def run(self, item):
+        self.fixup(item)
+        self.verify(item)
+
+    def verify(self, item):
+        item.get()
+        self.debug(item, " verifying")
+
+    def get_names(self, lang):
+        if self.license2item is None:
+            self.set_license2item()
+            self.licenses = {
+                'en': {
+                    'names': dict([
+                        (l, l) for (l, i) in self.license2item.items()
+                    ]),
+                },
+            }
+            self.set_redirects('en')
+        if lang not in self.licenses:
+            self.set_names(lang)
+            self.set_redirects(lang)
+        licenses = self.licenses[lang]
+        return (list(licenses['names'].keys()) +
+                list(licenses['redirects'].keys()))
+
+    def set_redirects(self, lang):
+        redirects = {}
+        for name in self.licenses[lang]['names'].keys():
+            redirects[name] = name
+            for redirect in self.get_redirects(name, lang):
+                redirects[redirect] = name
+        self.licenses[lang]['redirects'] = redirects
+
+    def set_names(self, lang):
+        lang2en = {}
+        self.licenses[lang] = {'names': lang2en}
+        for english in self.licenses['en']['names'].keys():
+            title = self.translate_title(english, lang)
+            if title is not None:
+                log.debug("set_names " + lang + " " + english + " => " + title)
+                lang2en[title] = english
+
+    def get_item(self, license, lang):
+        licenses = self.licenses[lang]
+        canonical = licenses['redirects'][license]
+        english = licenses['names'][canonical]
+        return self.license2item[english]
+
+    def set_dbname2item(self):
+        query = """
+            SELECT DISTINCT ?item WHERE {{
+              ?item wdt:{dbname} ?dbname.
+        }}
+        """.format(dbname=self.P_Wikimedia_database_name)
+        log.debug("set_dbname2item " + query)
+        self.license2item = {}
+        enwiki = pywikibot.site.APISite.fromDBName('enwiki')
+        for item in pg.WikidataSPARQLPageGenerator(query,
+                                                   site=self.bot.site,
+                                                   result_type=list):
+            item.get()
+            log.debug("set_dbname2item " + item.title() +
+                      " " + str(item.labels.get('en')))
+            if 'enwiki' not in item.sitelinks:
+                log.debug('ignore ' + item.title() +
+                          " because it does not link to enwiki")
+                continue
+            p = pywikibot.Page(enwiki, item.sitelinks['enwiki'])
+            self.license2item[p.title()] = item
+
+    def set_license2item(self):
+        format_args = {
+            'subclass_of': self.P_subclass_of,
+            'instance_of': self.P_instance_of,
+            'open_source': self.Q_open_source_license.getID(),
+            'free_software': self.Q_free_software_license.getID(),
+            'licenses': '',
+        }
+        if self.args.license:
+            licenses = []
+            for license in self.args.license:
+                licenses.append("STR(?label) = '" + license + "'")
+            licenses = ('?item rdfs:label ?label FILTER(' +
+                        " || ".join(licenses) + ")")
+            format_args['licenses'] = licenses
+        query = """
+            SELECT DISTINCT ?item WHERE {{
+              {{
+                ?item wdt:{instance_of}?/wdt:{subclass_of}* wd:{open_source}.
+              }} Union {{
+                ?item wdt:{instance_of}?/wdt:{subclass_of}* wd:{free_software}.
+              }}
+              {licenses}
+        }}
+        """.format(**format_args)
+        log.debug("set_license2item " + query)
+        self.license2item = {}
+        enwiki = pywikibot.site.APISite.fromDBName('enwiki')
+        for item in pg.WikidataSPARQLPageGenerator(query,
+                                                   site=self.bot.site,
+                                                   result_type=list):
+            item.get()
+            log.debug("set_license2item " + item.title() +
+                      " " + str(item.labels.get('en')))
+            if 'enwiki' not in item.sitelinks:
+                log.debug('set_license2item ignore ' + item.title() +
+                          " because it does not link to enwiki")
+                continue
+            p = pywikibot.Page(enwiki, item.sitelinks['enwiki'])
+            self.license2item[p.title()] = item
+
+    def template_parse_license(self, license, lang):
+        free_software_licenses = self.get_names(lang)
+        results = set()
+        for name in (re.findall('\[\[([^|\]]+?)\]\]', license) +
+                     re.findall('\[\[([^|\]]+?)\|[^\]]*\]\]', license)):
+            log.debug("template_parse_license: " + name)
+            if name in free_software_licenses:
+                results.add(self.get_item(name, lang))
+        return list(results)
+
+    def fixup(self, item):
+        item.get()
+
+        if self.P_license in item.claims:
+            return ['exists']
+
+        lang2field = {
+            'ca': 'Llicència',
+            'en': 'License',
+            'ja': 'license',
+            'ml': 'license',
+            'ru': 'license',
+            'zh': 'license',
+        }
+        lang2template = {
+            'ca': 'Caixa Programari',
+            'es': 'Ficha de software',
+            'en': 'Infobox',
+            'it': 'Software',
+            'pt': 'Info/Software',
+            'ru': 'Карточка программы',
+            '*': 'Infobox',
+        }
+        lang2value = {}
+        for (lang, license) in self.get_template_field(
+                item, lang2field, lang2template).items():
+            lang2value[lang] = self.template_parse_license(license, lang)
+        if len(lang2value) == 0:
+            return ['nothing']
+        self.debug(item, "fixup " + str(lang2value))
+        values = list(lang2value.values())
+        # if one wikipedia disagrees with the others, do nothing
+        if values.count(values[0]) != len(values):
+            self.error(item,
+                       "inconsistent license information between wikipedia" +
+                       str(lang2value))
+            return ['inconsistent']
+        status = []
+        for license in lang2value[list(lang2value.keys())[0]]:
+            license.get()
+            langs = list(lang2value.keys())
+            self.info(item, "ADD license " + license.labels['en'] +
+                      " from " + str(langs))
+            status.append(license.labels['en'])
+            claim = pywikibot.Claim(self.bot.site, self.P_license, 0)
+            claim.setTarget(license)
+            if not self.args.dry_run:
+                item.addClaim(claim)
+            for lang in langs:
+                imported = pywikibot.Claim(self.bot.site,
+                                           self.P_imported_from,
+                                           isReference=True)
+                imported.setTarget(self.get_sitelink_item(lang + "wiki"))
+                if not self.args.dry_run:
+                    claim.addSource(imported)
+            self.set_retrieved(item, claim)
+        return status
diff --git a/tests/test_license.py b/tests/test_license.py
new file mode 100644
index 0000000..62de15a
--- /dev/null
+++ b/tests/test_license.py
@@ -0,0 +1,136 @@
+# -*- mode: python; coding: utf-8 -*-
+#
+# Copyright (C) 2016 Loic Dachary <l...@dachary.org>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+import logging
+
+import mock
+import pywikibot
+
+from FLOSSbot.bot import Bot
+from FLOSSbot.license import License
+from tests.wikidata import WikidataHelper
+
+log = logging.getLogger('FLOSSbot')
+
+
+class TestLicense(object):
+
+    def setup_class(self):
+        WikidataHelper().login()
+
+    def setup(self):
+        self.gpl = 'GNU General Public License'
+        self.mit = 'MIT license'
+        self.args = [
+            '--license', self.gpl,
+            '--license', self.mit,
+        ]
+
+    def test_get_item(self):
+        bot = Bot.factory(['--verbose'] + self.args)
+        license = License(bot, bot.args)
+        redirect = 'GPL'
+        license.get_names('en')
+        canonical_item = license.get_item(self.gpl, 'en')
+        assert canonical_item == license.get_item(redirect, 'en')
+        gpl_fr = 'Licence publique générale GNU'
+        names_fr = license.get_names('fr')
+        assert gpl_fr in names_fr
+        assert canonical_item == license.get_item(gpl_fr, 'fr')
+
+    def test_get_names(self):
+        bot = Bot.factory(['--verbose'] + self.args)
+        license = License(bot, bot.args)
+        redirect = 'GPL'
+        names = license.get_names('en')
+        assert self.gpl in names
+        assert redirect in names
+
+        canonical_fr = 'Licence publique générale GNU'
+        names = license.get_names('fr')
+        assert canonical_fr in names
+        assert self.gpl in names
+
+    def test_template_parse_license(self):
+        bot = Bot.factory(['--verbose'] + self.args)
+        license = License(bot, bot.args)
+        found = license.template_parse_license('[[GPL]] [[MIT|]]', 'en')
+        for item in found:
+            item.get()
+            license.debug(item, "FOUND")
+            assert item.labels['en'] in (self.gpl, self.mit)
+
+    @mock.patch('FLOSSbot.license.License.set_license2item')
+    @mock.patch('FLOSSbot.plugin.Plugin.get_sitelink_item')
+    def test_fixup(self, m_get_sitelink_item, m_set_license2item):
+        bot = Bot.factory([
+            '--verbose',
+            '--test',
+            '--user=FLOSSbotCI',
+        ])
+        l = License(bot, bot.args)
+
+        gpl = l.Q_GNU_General_Public_License
+        gpl.get()
+        found = False
+        if gpl.claims:
+            for claim in gpl.claims.get(l.P_subclass_of, []):
+                if claim.type != 'wikibase-item':
+                    continue
+                if (claim.getTarget().getID() ==
+                        l.Q_free_software_license.getID()):
+                    found = True
+                    break
+        if not found:
+            subclass_of = pywikibot.Claim(l.bot.site, l.P_subclass_of, 0)
+            subclass_of.setTarget(l.Q_free_software_license)
+            gpl.addClaim(subclass_of)
+        gpl.setSitelink({'site': 'enwiki', 'title': self.gpl})
+        gpl.get(force=True)
+
+        emacs = l.Q_GNU_Emacs
+        emacs.get()
+        if emacs.claims:
+            licenses = emacs.claims.get(l.P_license, [])
+            if licenses:
+                emacs.removeClaims(licenses)
+                emacs.get(force=True)
+
+        def set_license2item():
+            l.license2item = {self.gpl: l.Q_GNU_General_Public_License}
+        m_set_license2item.side_effect = set_license2item
+
+        def get_sitelink_item(dbname):
+            if dbname == 'enwiki':
+                return l.Q_English_Wikipedia
+            elif dbname == 'frwiki':
+                return l.Q_French_Wikipedia
+            else:
+                assert 0, "unexpected " + dbname
+        m_get_sitelink_item.side_effect = get_sitelink_item
+        emacs.removeSitelinks(['enwiki'])
+        emacs.removeSitelinks(['frwiki'])
+        emacs.get(force=True)
+        assert ['nothing'] == l.fixup(emacs)
+        emacs.setSitelink({'site': 'enwiki', 'title': 'GNU Emacs'})
+        emacs.setSitelink({'site': 'frwiki', 'title': 'GNU Emacs'})
+        emacs.get(force=True)
+        assert [self.gpl] == l.fixup(emacs)
+
+# Local Variables:
+# compile-command: "cd .. ; tox -e py3 tests/test_license.py"
+# End:

-- 
To view, visit https://gerrit.wikimedia.org/r/316059
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: merged
Gerrit-Change-Id: Ia9f52d1a1cf60de4a2d098cfecf6e056b3ca24ba
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/bots/FLOSSbot
Gerrit-Branch: master
Gerrit-Owner: Dachary <l...@dachary.org>
Gerrit-Reviewer: Dachary <l...@dachary.org>
Gerrit-Reviewer: jenkins-bot <>

_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

Reply via email to