jenkins-bot has submitted this change and it was merged.

Change subject: repository: verify the URL obeys the protocol
......................................................................


repository: verify the URL obeys the protocol

If enough time has passed since the URL was last retrieved (30 days by
default), try to access it using a method that depends on the
protocol. The retrieved qualifier is updated if it succeeds, otherwise
an error is displayed.

Change-Id: Ib01897068ba4a492a7ea2d3cec89570240ea0a1a
Signed-off-by: Loic Dachary <l...@dachary.org>
---
M FLOSSbot/bot.py
M FLOSSbot/repository.py
M tests/test_repository.py
3 files changed, 85 insertions(+), 12 deletions(-)

Approvals:
  Dachary: Looks good to me, approved
  jenkins-bot: Verified



diff --git a/FLOSSbot/bot.py b/FLOSSbot/bot.py
index 16fa42a..6381027 100644
--- a/FLOSSbot/bot.py
+++ b/FLOSSbot/bot.py
@@ -122,7 +122,7 @@
             },
         }
         if type == 'property':
-            assert found, type + " " + name + " must exist wikidata"
+            assert found, type + " " + name + " must exist in wikidata"
             id = found['id']
             found = self.wikidata_site.loadcontent({'ids': id}, 'datatype')
             assert found, "datatype of " + id + " " + name + " is not found"
@@ -168,10 +168,10 @@
             previous = datetime(year=previous.year,
                                 month=previous.month,
                                 day=previous.day)
-            if now - previous > timedelta(
-                    days=self.args.verification_delay):
-                return True
-        return False
+            return (now - previous >=
+                    timedelta(days=self.args.verification_delay))
+        else:
+            return True
 
     def set_retrieved(self, item, claim, now=datetime.utcnow()):
         when = pywikibot.WbTime(now.year, now.month, now.day)
diff --git a/FLOSSbot/repository.py b/FLOSSbot/repository.py
index 4026cdb..634304d 100644
--- a/FLOSSbot/repository.py
+++ b/FLOSSbot/repository.py
@@ -163,7 +163,9 @@
 
     def run_items(self):
         for item in self.args.item:
-            self.fixup(pywikibot.ItemPage(self.site, item, 0))
+            item = pywikibot.ItemPage(self.site, item, 0)
+            self.fixup(item)
+            self.verify(item)
 
     def run_query(self):
         if self.args.filter == 'no-protocol':
@@ -202,6 +204,33 @@
                                                    site=self.site,
                                                    result_type=list):
             self.fixup(item)
+            self.verify(item)
+
+    def verify(self, item):
+        item_dict = item.get()
+        clm_dict = item_dict["claims"]
+
+        status = {}
+        for claim in clm_dict[self.P_source_code_repository]:
+            url = claim.getTarget()
+            if not self.need_verification(claim):
+                status[url] = 'no need'
+                continue
+            if self.P_protocol not in claim.qualifiers:
+                status[url] = 'no protocol'
+                continue
+            protocol = claim.qualifiers[self.P_protocol][0].getTarget()
+            self.debug(item, url + " protocol " + protocol.getID() + " " +
+                       protocol.get()['labels']['en'])
+            credentials = self.get_credentials(claim)
+            if self.verify_protocol(url, protocol, credentials):
+                self.info(item, "VERIFIED " + url)
+                status[url] = 'verified'
+                self.set_retrieved(item, claim)
+            else:
+                self.error(item, "VERIFY FAIL " + url)
+                status[url] = 'fail'
+        return status
 
     def fixup(self, item):
         self.fixup_protocol(item)
@@ -379,7 +408,8 @@
             return self.verify_bzr(url)
         elif protocol == self.Q_Apache_Subversion:
             return self.verify_svn(url, credentials)
-        elif protocol == self.Q_Hypertext_Transfer_Protocol:
+        elif (protocol == self.Q_Hypertext_Transfer_Protocol or
+              protocol == self.Q_HTTPS):
             return self.verify_http(url)
         elif protocol == self.Q_File_Transfer_Protocol:
             return self.verify_ftp(url)
@@ -398,13 +428,17 @@
             return self.Q_Fossil
         return None
 
-    def guess_protocol(self, repository):
-        url = repository.getTarget()
-        if self.P_username in repository.qualifiers:
-            credentials = repository.qualifiers[self.P_username][0]
+    def get_credentials(self, repository):
+        if self.P_website_username in repository.qualifiers:
+            credentials = repository.qualifiers[self.P_website_username][0]
             credentials = credentials.getTarget().split(':')
         else:
             credentials = None
+        return credentials
+
+    def guess_protocol(self, repository):
+        url = repository.getTarget()
+        credentials = self.get_credentials(repository)
         protocol = self.guess_protocol_from_url(url)
         if protocol:
             if not self.verify_protocol(url, protocol, credentials):
@@ -426,7 +460,7 @@
             new_url = "https://code.launchpad.net/"; + m.group(1)
 
         if new_url:
-            print("REPLACE " + url + " with " + new_url)
+            self.info(repository, "REPLACE " + url + " with " + new_url)
             repository.changeTarget(new_url)
             return True
         else:
diff --git a/tests/test_repository.py b/tests/test_repository.py
index 81123af..47a05e9 100644
--- a/tests/test_repository.py
+++ b/tests/test_repository.py
@@ -15,6 +15,10 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
+import argparse
+
+import pywikibot
+
 from FLOSSbot.repository import Repository
 from tests.wikidata import TestWikidata
 
@@ -87,3 +91,38 @@
         assert(
             self.r.guess_protocol_from_url('example.org')
             is None)
+
+    def test_verify(self):
+        r = Repository(argparse.Namespace(
+            test=True,
+            user='FLOSSbotCI',
+            dry_run=False,
+            verification_delay=30,
+        ))
+        item = r.__getattribute__('Q_' + TestWikidata.random_name())
+        claim = pywikibot.Claim(r.site,
+                                r.P_source_code_repository,
+                                0)
+        url = "http://github.com/ceph/ceph";
+        claim.setTarget(url)
+        item.addClaim(claim)
+
+        to_verify = pywikibot.ItemPage(r.site, item.getID(), 0)
+        assert {url: 'no protocol'} == r.verify(to_verify)
+
+        protocol = pywikibot.Claim(r.site, r.P_protocol, 0)
+        protocol.setTarget(r.Q_git)
+        claim.addQualifier(protocol, bot=True)
+
+        to_verify = pywikibot.ItemPage(r.site, item.getID(), 0)
+        assert {url: 'verified'} == r.verify(to_verify)
+
+        to_verify = pywikibot.ItemPage(r.site, item.getID(), 0)
+        assert {url: 'no need'} == r.verify(to_verify)
+
+        claim.changeTarget("http://example.org";)
+
+        to_verify = pywikibot.ItemPage(r.site, item.getID(), 0)
+        assert {"http://example.org": 'fail'} == r.verify(to_verify)
+
+        r.clear_entity_label(item.getID())

-- 
To view, visit https://gerrit.wikimedia.org/r/311669
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: merged
Gerrit-Change-Id: Ib01897068ba4a492a7ea2d3cec89570240ea0a1a
Gerrit-PatchSet: 4
Gerrit-Project: pywikibot/bots/FLOSSbot
Gerrit-Branch: master
Gerrit-Owner: Dachary <l...@dachary.org>
Gerrit-Reviewer: Dachary <l...@dachary.org>
Gerrit-Reviewer: Hashar <has...@free.fr>
Gerrit-Reviewer: Xqt <i...@gno.de>
Gerrit-Reviewer: jenkins-bot <>

_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

Reply via email to