Xqt has submitted this change and it was merged.
Change subject: Do not force preloading patch.exe for windows user. Ask for it
if needed only and prohibit this message every time a script was started.
Otherwise automatic batch scripts could not run without operator inputs. Some
additional PEP8 changes and small improv
......................................................................
Do not force preloading patch.exe for windows user.
Ask for it if needed only and prohibit this message every time a script was
started. Otherwise automatic batch scripts could not run without operator
inputs.
Some additional PEP8 changes and small improvements.
Change-Id: Icc887384f2d8dea1a03c17213a212c7f5bbb3533
---
M externals/__init__.py
1 file changed, 106 insertions(+), 98 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/externals/__init__.py b/externals/__init__.py
index db6baf5..314c55d 100644
--- a/externals/__init__.py
+++ b/externals/__init__.py
@@ -31,26 +31,26 @@
# mercurial (hg)
# patch (unix/linux & gnuwin32 version/flavour)
modules_needed = {
- 'patch.exe': ({}, # for win32 only, unix/linux is already equipped
with a patch tool
+ 'patch.exe': ({}, # for win32 only, unix/linux is already equipped
with a patch tool
{ 'url':
'http://downloads.sourceforge.net/project/gnuwin32/patch/2.5.9-7/patch-2.5.9-7-bin.zip',
'path': 'bin/patch.exe'},
- {}), # OK
+ {}), # OK
'crontab': ({},
- #{ 'url':
'https://github.com/josiahcarlson/parse-crontab/archive/master.zip',
- # 'path': 'parse-crontab-master/crontab',}), #
OK
+## { 'url':
'https://github.com/josiahcarlson/parse-crontab/archive/master.zip',
+## 'path': 'parse-crontab-master/crontab',}), # OK
{ 'url':
'https://github.com/josiahcarlson/parse-crontab/archive/1ec538ff67df6a207993a6c5b6988f4f628c5776.zip',
'path':
'parse-crontab-1ec538ff67df6a207993a6c5b6988f4f628c5776/crontab',},
- {}), # OK
+ {}), # OK
'odf': ({},
- #{ 'url':
'https://pypi.python.org/packages/source/o/odfpy/odfpy-0.9.6.tar.gz',
- # 'path': 'odfpy-0.9.6/odf',}), #
OK
+## { 'url':
'https://pypi.python.org/packages/source/o/odfpy/odfpy-0.9.6.tar.gz',
+## 'path': 'odfpy-0.9.6/odf',}), # OK
{ 'url':
'https://pypi.python.org/packages/source/o/odfpy/odfpy-0.9.4.tar.gz',
'path': 'odfpy-0.9.4/odf'},
- {}), # OK
+ {}), # OK
'openpyxl': ({},
{ 'url':
'https://bitbucket.org/ericgazoni/openpyxl/get/1.5.6.tar.gz',
'path': 'ericgazoni-openpyxl-e5934500ffac/openpyxl'},
- {}), # OK
+ {}), # OK
# 'spelling': $ svn propedit svn:externals externals/.
# spelling
http://svn.wikimedia.org/svnroot/pywikipedia/trunk/spelling/
# $ git submodule add
https://gerrit.wikimedia.org/r/p/pywikibot/spelling.git externals/spelling
@@ -58,46 +58,46 @@
'linux-ubuntu': ['python-beautifulsoup']},
{ 'url':
'https://pypi.python.org/packages/source/B/BeautifulSoup/BeautifulSoup-3.2.0.tar.gz',
'path': 'BeautifulSoup-3.2.0/BeautifulSoup.py'},
- {}), # OK
+ {}), # OK
'irclib': ({'linux-fedora': ['python-irclib'],
'linux-ubuntu': ['python-irclib']},
- {}, # http://python-irclib.sourceforge.net/
- {}), # OK
+ {}, # http://python-irclib.sourceforge.net/
+ {}), # OK
'mwparserfromhell': ({},
{ 'url':
'https://github.com/earwig/mwparserfromhell/archive/v0.2.zip',
- #{ 'url':
'https://github.com/earwig/mwparserfromhell/archive/master.zip',
+## { 'url':
'https://github.com/earwig/mwparserfromhell/archive/master.zip',
'path': 'mwparserfromhell-0.2/mwparserfromhell'},
- {}), # OK
+ {}), # OK
'colormath': ({'linux-fedora': [],
'linux-ubuntu': ['python-colormath'],},
{ 'url':
'https://github.com/gtaylor/python-colormath/archive/master.zip',
'path': 'python-colormath-master/colormath',},
- {}), # OK
+ {}), # OK
'jseg': ({},
{ 'url':
'http://vision.ece.ucsb.edu/segmentation/jseg/software/jseg.zip',
'path': 'jseg',
#$ diff -Nau --exclude="*.o" --exclude="*.pyc"
--exclude="segdist_cpp*" TEST_jseg/ jseg/ > patch-jseg
'patch': 'patch-jseg'},
- {}), # OK
+ {}), # OK
'jseg/jpeg-6b': ({},
{ 'url':
'http://vision.ece.ucsb.edu/segmentation/jseg/software/jpeg-6b.zip',
'path': 'jpeg-6b',},
- {}), # OK
+ {}), # OK
'_mlpy': ({},
{ 'url':
'http://downloads.sourceforge.net/project/mlpy/mlpy%203.5.0/mlpy-3.5.0.tar.gz',
'path': 'mlpy-3.5.0/mlpy'},
- {}), # OK
+ {}), # OK
'_music21': ({},
{ 'url':
'http://music21.googlecode.com/files/music21-1.4.0.tar.gz',
'path': 'music21-1.4.0',
#$ diff -Naur --exclude="*.pyc" TEST__music21/
_music21/ > patch-music21
'patch': 'patch-music21'},
- {}), # OK
+ {}), # OK
# TODO: vvv (future; enable for and use in 'catimages.py', patch needed)
'_ocropus': ({},
{},
{ 'url': 'https://code.google.com/p/ocropus',
- 'rev': 'ocropus-0.6'}), # OK
+ 'rev': 'ocropus-0.6'}), # OK
# TODO: vvv (further clean-up and unlink - check with 'svn list')
# 'opencv': $ svn propedit svn:externals externals/.
# opencv
https://svn.toolserver.org/svnroot/drtrigon/externals/opencv
@@ -111,7 +111,7 @@
'opencv/haarcascades': ({},
{ 'url':
'https://svn.toolserver.org/svnroot/drtrigon/externals/haarcascades-full.tar.gz',
'path': 'haarcascades'},
- {}), # OK
+ {}), # OK
# 'pdfminer' is not used anymore/at the moment...
# 'pycolorname': $ svn propset svn:externals 'pycolorname
https://svn.toolserver.org/svnroot/drtrigon/externals/pycolorname' externals/.
'pydmtx': ({'linux-fedora': ['python-libdmtx'],
@@ -120,11 +120,11 @@
'path': 'dmtx-wrappers-master/python',
#$ diff -Nau --exclude="*.pyc" TEST_pydmtx/ pydmtx/ >
patch-pydmtx
'patch': 'patch-pydmtx'},
- {}), # OK
+ {}), # OK
'py_w3c': ({},
{ 'url':
'https://bitbucket.org/nmb10/py_w3c/downloads/py_w3c-v0.1.0.tar.gz',
'path': 'py_w3c-0.1.0/py_w3c'},
- {}), # OK
+ {}), # OK
# TODO: vvv (include)
# 'TEST_slic': ({},
# { 'url':
'http://ivrg.epfl.ch/files/content/sites/ivrg/files/supplementary_material/RK_SLICsuperpixels/SLICSuperpixelsAndSupervoxelsCode.zip',
@@ -139,7 +139,7 @@
'path': 'zbar-0.10',
#$ diff -Nau --exclude="*.pyc" TEST__zbar/ _zbar/ >
patch-zbar
'patch': 'patch-zbar'},
- {}), # OK
+ {}), # OK
# TODO: vvv (include)
# 'TEST__bob': ({},
# { 'url':
'https://www.idiap.ch/software/bob/packages/bob-1.1.2.zip',
@@ -168,14 +168,14 @@
import os
import sys
import inspect
-import wikipedia as pywikibot # sets externals path
+import wikipedia as pywikibot # sets externals path
#from pywikibot.comms import http
# allow imports from externals
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
-### BEGIN of VisTrails inspired and copied code ### ### ### ### ### ### ### ###
+### BEGIN of VisTrails inspired and copied code ###
def has_logger():
#return hasattr(sys.modules['wikipedia'], 'logger')
@@ -190,7 +190,7 @@
if has_logger():
pywikibot.warning(text)
else:
- print "WARNING:", text
+ print "\nWARNING:", text
def guess_system():
@@ -201,19 +201,20 @@
def show_question(module):
lowlevel_warning("Required package missing: %s\n"
"This package is not installed, but required by the file"
- " '%s'." % (module, inspect.stack()[2][1]))
+ "\n'%s'." % (module, inspect.stack()[2][1]))
lowlevel_warning("For more and additional information, please confer:\n"
"http://www.mediawiki.org/wiki/Manual:Pywikipediabot/";
"Installation#Dependencies")
options = [(i+1) for i, item in enumerate(modules_needed[module]) if item]
options += [0, 's', '']
options.sort()
- options_msg = ("There are multiple ways to solve this:\n"
- "RECOMMENDED for admins: always option [0] or the next available"
- " (e.g. [1])\n"
- "RECOMMENDED for non-admins: always option [2] (if available)\n"
- "0: automatically determine the best of the following methods (may need\n"
- " administrator privileges)\n")
+ options_msg = ("""There are multiple ways to solve this:
+RECOMMENDED for admins: always option [0] or the next available (e.g. [1])
+RECOMMENDED for non-admins: always option [2] (if available)
+0: automatically determine the best of the following methods (may need
+ administrator privileges)
+
+""")
if 1 in options:
options_msg += ("1: install the package using the OS package"
" management system like yum\n"
@@ -230,8 +231,8 @@
lowlevel_warning(options_msg)
v = None
while (v not in options):
- lowlevel_warning("Please choose [%s, s - default]: " %
- (", ".join(map(str, options[:-2]))))
+ lowlevel_warning("Please choose [%s, s - default]: "
+ % (", ".join(map(str, options[:-2]))))
v = raw_input().lower()
try:
v = int(v)
@@ -239,11 +240,13 @@
pass
return v
+
def show_patch_question():
global _patch_permission
if _patch_permission is None:
- lowlevel_warning("Give externals permission to execute the patch
command?"
- " [y(es), n(o) - default]: ")
+ lowlevel_warning(
+ "Give externals permission to execute the patch command?"
+ " [y(es), n(o) - default]: ")
v = raw_input().upper()
_patch_permission = (v == 'Y') or (v == 'YES')
return _patch_permission
@@ -251,7 +254,9 @@
def python_module_exists(module_name):
"""python_module_exists(module_name): Boolean.
-Returns if python module of given name can be safely imported."""
+ Returns if python module of given name can be safely imported.
+
+ """
module_name = module_name.replace(u'.py', u'')
module_name = module_name[1:] if module_name[0] == u'_' else module_name
@@ -301,9 +306,11 @@
def linux_install(dependency_dictionary):
- """Tries to import a python module. If unsuccessful, tries to install
-the appropriate bundle and then reimport. py_import tries to be smart
-about which system it runs on."""
+ """ Tries to import a python module. If unsuccessful, tries to install
+ the appropriate bundle and then reimport. py_import tries to be smart
+ about which system it runs on.
+
+ """
# Ugly fix to avoid circular import
distro = guess_system()
@@ -330,54 +337,60 @@
def windows_install(dependency_dictionary):
- lowlevel_warning(u'Not available in windows OS, use download mode (2)
instead.')
+ lowlevel_warning(
+ u'Not available in windows OS, use download mode (2) instead.')
return False # skip this in order to trigger 'download_install' next
### END of VisTrails inspired and copied code ### ### ### ### ### ### ### ###
def download_install(package, module, path):
- if package:
- lowlevel_warning(u'Download package "%s" from %s'
- % (module, package['url']))
- import mimetypes
- import urllib2
- for i in range(3):
- response = urllib2.urlopen(package['url'])
- #response = http.request(pywikibot.getSite(), package['url'],
- # no_hostname = True, back_response =
True)[0]
- if 'Content-Length' in response.headers:
- break
- lowlevel_warning(u'Could not retrieve data, re-trying ...')
- lowlevel_warning(u'Size of download: %s byte(s)'
- % response.headers['Content-Length'])
- #mime = response.headers['Content-Type'].lower().split('/')
- mime = mimetypes.guess_type(package['url'],
- strict=True)[0].lower().split('/')
- lowlevel_warning(u'MIME type: %s' % mime)
+ if not package:
+ return
+ lowlevel_warning(u'Download package "%s" from %s'
+ % (module, package['url']))
+ import mimetypes
+ import urllib2
+ for i in range(3):
+ response = urllib2.urlopen(package['url'])
+ #response = http.request(pywikibot.getSite(), package['url'],
+ # no_hostname = True, back_response = True)[0]
+ if 'Content-Length' in response.headers:
+ break
+ lowlevel_warning(u'Could not retrieve data, re-trying ...')
+ lowlevel_warning(u'Size of download: %s byte(s)'
+ % response.headers['Content-Length'])
+ #mime = response.headers['Content-Type'].lower().split('/')
+ mime = mimetypes.guess_type(package['url'],
+ strict=True)[0].lower().split('/')
+ lowlevel_warning(u'MIME type: %s' % mime)
- lowlevel_warning(u'Extract package "%s" to %s.'
- % (module, os.path.join(path, module)))
- if len(mime) > 1:
- import StringIO
- if mime[1] in ['zip', 'x-zip-compressed']:
- import zipfile
- arch = zipfile.ZipFile(StringIO.StringIO(response.read()))
- elif mime[1] == 'x-tar':
- import tarfile
- arch = tarfile.open(fileobj=StringIO.StringIO(response.read()))
- else:
- raise NotImplementedError(u'Not implemented mime type %s'
- % mime[1])
- arch.extractall(os.path.join(path, '__setup_tmp/'))
- arch.close()
- import shutil
- shutil.move(os.path.join(path, '__setup_tmp/', package['path']),
- os.path.join(path, module))
- shutil.rmtree(os.path.join(path, '__setup_tmp/'))
+ lowlevel_warning(u'Extract package "%s" to %s.'
+ % (module, os.path.join(path, module)))
+ if len(mime) > 1:
+ import StringIO
+ if mime[1] in ['zip', 'x-zip-compressed']:
+ import zipfile
+ arch = zipfile.ZipFile(StringIO.StringIO(response.read()))
+ elif mime[1] == 'x-tar':
+ import tarfile
+ arch = tarfile.open(fileobj=StringIO.StringIO(response.read()))
+ else:
+ raise NotImplementedError(u'Not implemented mime type %s'
+ % mime[1])
+ arch.extractall(os.path.join(path, '__setup_tmp/'))
+ arch.close()
+ import shutil
+ shutil.move(os.path.join(path, '__setup_tmp/', package['path']),
+ os.path.join(path, module))
+ shutil.rmtree(os.path.join(path, '__setup_tmp/'))
- result = 0
- if ('patch' in package) and show_patch_question():
+ result = 0
+ if 'patch' in package:
+ if sys.platform == 'win32':
+ if not check_setup('patch.exe'):
+ result = -1
+ if result == 0 and show_patch_question():
lowlevel_warning(u'Applying patch to %s in order to finish'
u' the installation of package "%s".'
% (os.path.join(path, module), module))
@@ -391,9 +404,9 @@
os.path.join(path, package['patch']))
result = os.system(cmd)
- lowlevel_warning(u'Package "%s" installed to %s.'
- % (module, os.path.join(path, module)))
- return (result == 0)
+ lowlevel_warning(u'Package "%s" installed to %s.'
+ % (module, os.path.join(path, module)))
+ return (result == 0)
def mercurial_repo_install(package, module, path):
@@ -423,19 +436,21 @@
dist = guess_system()
func = dist.split(u'-')[0] + '_install'
if sel in [0, 1]:
- lowlevel_warning(u'(1) Trying to install by use of "%s" package
management system:' % dist)
+ lowlevel_warning(
+ u'(1) Trying to install by use of "%s" package management system:'
+ % dist)
if (func in globals()) and globals()[func](modules_needed[m][0]):
- return
+ return sel
if sel in [0, 2]:
lowlevel_warning(u'(2) Trying to install by download from source URL:')
if download_install(modules_needed[m][1], m, path):
- return
+ return sel
if sel in [0, 3]:
lowlevel_warning(u'(3) Trying to install by use of mercurial:')
- if (len(modules_needed[m]) > 2) and\
+ if (len(modules_needed[m]) > 2) and \
mercurial_repo_install(modules_needed[m][2], m, path):
- return
- if sel in [0, 1, 2, 3]:
+ return sel
+ if sel in range(4):
lowlevel_warning(u'No suitable package could be found nor installed!')
lowlevel_warning(u'Several scripts might fail, if the modules are not'
@@ -448,10 +463,3 @@
def check_setup_all():
for m in modules_order:
check_setup(m)
-
-
-# check and install modules NEEDED
-if sys.platform == 'win32':
- check_setup('patch.exe')
-#check_setup('BeautifulSoup.py')
-#check_setup('mwparserfromhell')
--
To view, visit https://gerrit.wikimedia.org/r/83114
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Icc887384f2d8dea1a03c17213a212c7f5bbb3533
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/compat
Gerrit-Branch: master
Gerrit-Owner: Xqt <[email protected]>
Gerrit-Reviewer: Ladsgroup <[email protected]>
Gerrit-Reviewer: Legoktm <[email protected]>
Gerrit-Reviewer: Xqt <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
Pywikibot-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/pywikibot-commits
