Dachary has uploaded a new change for review.

  https://gerrit.wikimedia.org/r/311669

Change subject: repository: verify the URL obeys the protocol
......................................................................

repository: verify the URL obeys the protocol

If enough time has passed since the URL was last retrieved (30 days by
default), try to access it using a method that depends on the
protocol. The retrieved qualifier is updated if it succeeds, otherwise
an error is displayed.

Change-Id: Ib01897068ba4a492a7ea2d3cec89570240ea0a1a
Signed-off-by: Loic Dachary <l...@dachary.org>
---
M FLOSSbot/bot.py
M FLOSSbot/repository.py
M tests/test_repository.py
3 files changed, 78 insertions(+), 9 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/pywikibot/bots/FLOSSbot 
refs/changes/69/311669/1

diff --git a/FLOSSbot/bot.py b/FLOSSbot/bot.py
index 16fa42a..1625f61 100644
--- a/FLOSSbot/bot.py
+++ b/FLOSSbot/bot.py
@@ -122,7 +122,7 @@
             },
         }
         if type == 'property':
-            assert found, type + " " + name + " must exist wikidata"
+            assert found, type + " " + name + " must exist in wikidata"
             id = found['id']
             found = self.wikidata_site.loadcontent({'ids': id}, 'datatype')
             assert found, "datatype of " + id + " " + name + " is not found"
@@ -168,10 +168,11 @@
             previous = datetime(year=previous.year,
                                 month=previous.month,
                                 day=previous.day)
-            if now - previous > timedelta(
-                    days=self.args.verification_delay):
-                return True
-        return False
+            print("now - previous " + str(now - previous))
+            return (now - previous >=
+                    timedelta(days=self.args.verification_delay))
+        else:
+            return True
 
     def set_retrieved(self, item, claim, now=datetime.utcnow()):
         when = pywikibot.WbTime(now.year, now.month, now.day)
diff --git a/FLOSSbot/repository.py b/FLOSSbot/repository.py
index 4026cdb..873c0f3 100644
--- a/FLOSSbot/repository.py
+++ b/FLOSSbot/repository.py
@@ -202,6 +202,31 @@
                                                    site=self.site,
                                                    result_type=list):
             self.fixup(item)
+            self.verify(item)
+
+    def verify(self, item):
+        item_dict = item.get()
+        clm_dict = item_dict["claims"]
+
+        status = {}
+        for claim in clm_dict[self.P_source_code_repository]:
+            url = claim.getTarget()
+            if not self.need_verification(claim):
+                status[url] = 'no need'
+                continue
+            if self.P_protocol not in claim.qualifiers:
+                status[url] = 'no protocol'
+                continue
+            protocol = claim.qualifiers[self.P_protocol][0].getTarget()
+            credentials = self.get_credentials(claim)
+            if self.verify_protocol(url, protocol, credentials):
+                self.info(item, "VERIFIED " + url)
+                status[url] = 'verified'
+                self.set_retrieved(item, claim)
+            else:
+                self.error(item, "VERIFY FAIL " + url)
+                status[url] = 'fail'
+        return status
 
     def fixup(self, item):
         self.fixup_protocol(item)
@@ -398,13 +423,17 @@
             return self.Q_Fossil
         return None
 
-    def guess_protocol(self, repository):
-        url = repository.getTarget()
-        if self.P_username in repository.qualifiers:
-            credentials = repository.qualifiers[self.P_username][0]
+    def get_credentials(self, repository):
+        if self.P_website_username in repository.qualifiers:
+            credentials = repository.qualifiers[self.P_website_username][0]
             credentials = credentials.getTarget().split(':')
         else:
             credentials = None
+        return credentials
+
+    def guess_protocol(self, repository):
+        url = repository.getTarget()
+        credentials = self.get_credentials(repository)
         protocol = self.guess_protocol_from_url(url)
         if protocol:
             if not self.verify_protocol(url, protocol, credentials):
diff --git a/tests/test_repository.py b/tests/test_repository.py
index 81123af..47a05e9 100644
--- a/tests/test_repository.py
+++ b/tests/test_repository.py
@@ -15,6 +15,10 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
+import argparse
+
+import pywikibot
+
 from FLOSSbot.repository import Repository
 from tests.wikidata import TestWikidata
 
@@ -87,3 +91,38 @@
         assert(
             self.r.guess_protocol_from_url('example.org')
             is None)
+
+    def test_verify(self):
+        r = Repository(argparse.Namespace(
+            test=True,
+            user='FLOSSbotCI',
+            dry_run=False,
+            verification_delay=30,
+        ))
+        item = r.__getattribute__('Q_' + TestWikidata.random_name())
+        claim = pywikibot.Claim(r.site,
+                                r.P_source_code_repository,
+                                0)
+        url = "http://github.com/ceph/ceph";
+        claim.setTarget(url)
+        item.addClaim(claim)
+
+        to_verify = pywikibot.ItemPage(r.site, item.getID(), 0)
+        assert {url: 'no protocol'} == r.verify(to_verify)
+
+        protocol = pywikibot.Claim(r.site, r.P_protocol, 0)
+        protocol.setTarget(r.Q_git)
+        claim.addQualifier(protocol, bot=True)
+
+        to_verify = pywikibot.ItemPage(r.site, item.getID(), 0)
+        assert {url: 'verified'} == r.verify(to_verify)
+
+        to_verify = pywikibot.ItemPage(r.site, item.getID(), 0)
+        assert {url: 'no need'} == r.verify(to_verify)
+
+        claim.changeTarget("http://example.org";)
+
+        to_verify = pywikibot.ItemPage(r.site, item.getID(), 0)
+        assert {"http://example.org": 'fail'} == r.verify(to_verify)
+
+        r.clear_entity_label(item.getID())

-- 
To view, visit https://gerrit.wikimedia.org/r/311669
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib01897068ba4a492a7ea2d3cec89570240ea0a1a
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/bots/FLOSSbot
Gerrit-Branch: master
Gerrit-Owner: Dachary <l...@dachary.org>

_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

Reply via email to