Changeset: 6ed9a3b729dc for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6ed9a3b729dc
Modified Files:
testing/Mfilter.py
testing/Mtest.py.in
Branch: Apr2019
Log Message:
Port to Python 3 on Windows.
Always use UTF-8 encoding when reading/writing files.
diffs (truncated from 600 to 300 lines):
diff --git a/testing/Mfilter.py b/testing/Mfilter.py
--- a/testing/Mfilter.py
+++ b/testing/Mfilter.py
@@ -32,6 +32,13 @@ def wlen(str) :
return len(' '.join(str.split()))
### wlen(str) #
+def openutf8(file, mode='r'):
+ try:
+ f = open(file, mode, encoding='utf-8') # Python 3
+ except TypeError:
+ f = open(file, mode) # Python 2
+ return f
+
test = (
# potential differences, which we want to ignore
re.compile('(?:'+')|(?:'.join([
@@ -140,7 +147,7 @@ def mFilter (FILE, IGNORE) :
i += 1
IGNORE = ''.join(ign)
- fin = open(FILE, "rU")
+ fin = openutf8(FILE, "rU")
LINE = fin.readline()
while len(LINE) and ( len(LINE) < 15 or LINE[:15] not in ("stdout of
test ", "stderr of test ") ):
LINE = fin.readline()
@@ -154,7 +161,7 @@ def mFilter (FILE, IGNORE) :
ftmp = []
ig = n = 0
il = iw = ic = el = ew = ec = al = aw = ac = 0
- for iline in open(FILE, 'rU'):
+ for iline in openutf8(FILE, 'rU'):
iline = iline.replace('\033[?1034h','')
if iline.startswith('# builtin opt') or \
iline.startswith('# cmdline opt') or \
@@ -240,7 +247,7 @@ def mFilter (FILE, IGNORE) :
ic = ic + c
ftmp.append(ln)
- fout = open(FILE + ".FILTERED", "w")
+ fout = openutf8(FILE + ".FILTERED", "w")
if (al > 0) or (aw > 0) or (ac >0):
fout.write("Statistics about std"+WHAT+" of test '"+TST+"` in
directory '"+TSTDIR+"`:\n")
fout.write(" %9d lines, %9d words, %9d chars in lines not matching
'^$|%s|^=`\n" % (il,iw,ic,IGNORE))
@@ -289,7 +296,7 @@ def mFilter (FILE, IGNORE) :
warn(THISFILE, "Removing input file '%s'." % FILE)
try:
os.remove(FILE)
- fin = open(FILE,"w")
+ fin = openutf8(FILE,"w")
fin.write("%s: Removed '%s' to create space for
'%s'.\n" % (THISFILE, FILE, fout.name))
fin.close()
except:
diff --git a/testing/Mtest.py.in b/testing/Mtest.py.in
--- a/testing/Mtest.py.in
+++ b/testing/Mtest.py.in
@@ -260,11 +260,12 @@ if isatty and os.isatty(sys.stdin.fileno
# ridiculously narrow tty, ignore value
ttywidth = 0
-if os.name == 'nt':
- p = os.path.join(sys.prefix, 'Lib')
- if 'PYTHONPATH' in os.environ:
- p = os.environ['PYTHONPATH'] + os.pathsep + p
- os.environ['PYTHONPATH'] = p
+# if os.name == 'nt':
+# p = os.path.join(sys.prefix, 'Lib')
+# if 'PYTHONPATH' in os.environ:
+# p = os.environ['PYTHONPATH'] + os.pathsep + p
+# os.environ['PYTHONPATH'] = p
+# print('setting PYTHONPATH to %s' % p)
import string # for whitespace
def splitcommand(cmd):
@@ -696,9 +697,16 @@ elif sys.version_info[0] == 3:
url = url[2:]
return url
+def openutf8(file, mode='r'):
+ try:
+ f = open(file, mode, encoding='utf-8') # Python 3
+ except TypeError:
+ f = open(file, mode) # Python 2
+ return f
+
def try_open(path, mode) :
try:
- f = open(path, mode)
+ f = openutf8(path, mode)
except IOError as err:
Warn("Opening file '%s' in mode '%s' failed with #%d: '%s'." % (path,
mode, err.errno, err.strerror))
f = None
@@ -760,7 +768,7 @@ def CreateHtmlIndex (env, *body) :
Element('title', {}, Text(HTMLTITLE)),
stylesheet),
hbody)
- f = open("%s.head.html" % INDEX,"w")
+ f = openutf8("%s.head.html" % INDEX,"w")
html.write(f, True)
f.close()
@@ -797,7 +805,7 @@ def CreateHtmlIndex (env, *body) :
'bordercolor': white,
'marginwidth': '0',
'marginheight': '0'})))
- f = open("%s.html" % INDEX, "w")
+ f = openutf8("%s.html" % INDEX, "w")
html.write(f, True)
f.close()
env['_%s_BODY_' % TSTDIR] = ["", 0]
@@ -868,10 +876,10 @@ def CreateTstWhatXhtml (env, TST, stable
'bordercolor': white,
'marginwidth': '0',
'marginheight': '0'})))
- f = open(".%s%s.html" % (TST, WHAT), "w")
+ f = openutf8(".%s%s.html" % (TST, WHAT), "w")
html.write(f, True)
f.close()
- f = open(".%s%s.head.html" % (TST, WHAT),"w")
+ f = openutf8(".%s%s.head.html" % (TST, WHAT),"w")
target = '%s_%s_%s_%s_body' % (DISTVER, TSTDIR, TST, WHAT[1:])
if REV: # implies URLPREFIX is not None
urlpref = '%s%s/%s' % (URLPREFIX, url(TSTDIR), TSTSUFF)
@@ -895,7 +903,7 @@ def CreateTstWhatXhtml (env, TST, stable
d = urlpref
if os.path.isfile(TST + stableWHAT + '.src'):
# there's only one file like this...
- fl = open(TST + stableWHAT + '.src').readline().strip()
+ fl = openutf8(TST + stableWHAT + '.src').readline().strip()
if fl.startswith('$RELSRCDIR/'):
fl = fl[11:]
while fl.startswith('../'):
@@ -920,7 +928,7 @@ def CreateTstWhatXhtml (env, TST, stable
if REV:
d = urlpref
if os.path.isfile(TST + EXT + '.src'):
- fl = open(TST + EXT + '.src').readline().strip()
+ fl = openutf8(TST + EXT + '.src').readline().strip()
if fl.startswith('$RELSRCDIR/'):
fl = fl[11:]
while fl.startswith('../'):
@@ -984,7 +992,7 @@ def CreateSrcIndex (env, TST, EXT) :
if URLPREFIX:
framesrc = '%s%s/%s/%s%s' % (URLPREFIX, url(TSTDIR), TSTSUFF, TST, EXT)
else:
- f = open(".%s.nosrc.index.html" % TST, "w")
+ f = openutf8(".%s.nosrc.index.html" % TST, "w")
html = Element('html', {},
Element('head', {},
Element('title', {},
@@ -1025,7 +1033,7 @@ def CreateSrcIndex (env, TST, EXT) :
'bordercolor': white,
'marginwidth': '0',
'marginheight': '0'})))
- f = open(".%s.src.index.html" % TST,"w")
+ f = openutf8(".%s.src.index.html" % TST,"w")
html.write(f, True)
f.close()
@@ -1060,7 +1068,7 @@ def CreateSrcIndex (env, TST, EXT) :
'cellspacing': '0',
'cellpadding': '3'},
tr))))
- f = open(".%s.src.index.head.html" % TST, "w")
+ f = openutf8(".%s.src.index.head.html" % TST, "w")
html.write(f, True)
f.close()
### CreateSrcIndex (env, TST, EXT) #
@@ -1182,7 +1190,7 @@ def SkipTest(env, TST, EXT, REASON, leng
if testweb:
return None
- f = open(".%s.SKIPPED" % TST, "w")
+ f = openutf8(".%s.SKIPPED" % TST, "w")
f.write("\n%s Warning: %s\n\n" % (THISFILE, TEXT))
f.close()
target = '%s_%s_body' % (DISTVER, TSTDIR)
@@ -1237,7 +1245,7 @@ def PerformDir(env, testdir, testlist, B
alltests = []
try:
- allf = open(os.path.join(TSTSRCDIR, "All"))
+ allf = openutf8(os.path.join(TSTSRCDIR, "All"))
except IOError:
pass
else:
@@ -1458,7 +1466,7 @@ def ApproveOutput (env, TST) :
if os.path.isfile(stableOUT):
shutil.copy(stableOUT,stableOUTPUT)
else:
- open(stableOUTPUT,"w").close()
+ openutf8(stableOUTPUT,"w").close()
for d in ('TMPDIR', 'TMP', 'TEMP'):
if d in os.environ:
@@ -1467,7 +1475,7 @@ def ApproveOutput (env, TST) :
else:
patch = os.path.join(os.sep, 'tmp')
patch = os.path.join(patch, "%s.patch-%s" %
(os.path.basename(stableOUTPUT), str(os.getpid())))
- f = open(patch + '.0', 'w')
+ f = openutf8(patch + '.0', 'w')
proc = process.Popen(['diff', '-Bb', '-I^[#=]', '-I^MAPI *=',
'-U0',
stableOUTPUT, testOUTPUT],
stdout = f, universal_newlines = True)
@@ -1485,8 +1493,8 @@ def ApproveOutput (env, TST) :
oc = ''
print("Approving %s -> stable.%s%s%s" %
(os.path.join(TSTDIR, "%s.test.%s" % (TST, WHAT)), WHAT, SYSTEM, oc))
- f = open(patch + ".1", "wb")
- for l in open(patch + ".0"):
+ f = openutf8(patch + ".1", "w")
+ for l in openutf8(patch + ".0"):
if TO.search(l):
f.write(l[:1] + '\n')
Warn('Rejecting (error) message: "%s"' %
l[1:].replace(os.linesep, ''))
@@ -1507,16 +1515,16 @@ def ApproveOutput (env, TST) :
sem = 1
f.flush()
f.close()
- o = open(stableOUTPUT).read()
- open(stableOUTPUT + ".ORG", 'wb').write(o)
- open(stableOUTPUT, 'wb').write(o)
+ o = openutf8(stableOUTPUT).read()
+ openutf8(stableOUTPUT + ".ORG", 'w').write(o)
+ openutf8(stableOUTPUT, 'w').write(o)
patchcmd = ['patch']
if not verbose:
patchcmd.append('--quiet')
proc = process.Popen(patchcmd + [stableOUTPUT, patch + '.1'],
universal_newlines = True)
proc.wait()
- f = open(patch, 'w')
+ f = openutf8(patch, 'w')
proc = process.Popen(['diff', '-u', stableOUTPUT + '.ORG',
stableOUTPUT],
stdout = f, universal_newlines = True)
@@ -1524,8 +1532,8 @@ def ApproveOutput (env, TST) :
f.close()
remove(stableOUTPUT + ".ORG")
remove(patch + ".1")
- o = open(stableOUTPUT).read()
- open(stableOUTPUT, 'w').write(o)
+ o = openutf8(stableOUTPUT).read()
+ openutf8(stableOUTPUT, 'w').write(o)
o = None
thefile = os.path.split(stableOUTPUT)[1]
@@ -1539,7 +1547,7 @@ def ApproveOutput (env, TST) :
elif f != thefile and test.match(f):
remove(os.path.join(dir or os.curdir, f + '.rej'))
remove(os.path.join(dir or os.curdir, f + '.orig'))
- proc = process.Popen(patchcmd + ['--forward',
os.path.join(dir or os.curdir, f)], stdin = open(patch), universal_newlines =
True)
+ proc = process.Popen(patchcmd + ['--forward',
os.path.join(dir or os.curdir, f)], stdin = openutf8(patch), universal_newlines
= True)
proc.wait()
if os.path.exists(os.path.join(dir or os.curdir, f
+ '.rej')):
list.append(f)
@@ -1797,7 +1805,7 @@ def GetBitsAndModsAndThreads(env) :
def CheckMods(env, TST, SERVER, CALL) :
missing = []
if os.path.isfile(TST + ".modules"):
- for m in open(TST + ".modules"):
+ for m in openutf8(TST + ".modules"):
m = m.strip()
if m and m[0] != "#" and m not in env['TST_MODS']:
missing.append(m)
@@ -1818,7 +1826,7 @@ def CheckTests(env, TST, oktests):
# we didn't clean up from a previous run, assume tests were done
return missing
- for test in open(TST + '.reqtests'):
+ for test in openutf8(TST + '.reqtests'):
test = test.strip()
if not test or test.startswith('#'):
continue
@@ -1861,7 +1869,7 @@ def StableOutErr(env,par,TST,SYST,RELEAS
def CategorizeResult(TST, SockTime, outmissing, errmissing):
l = '<!--MajorDiffs-->' # assign something in case file is empty
- for l in open("%s.out.diff.html" % TST):
+ for l in openutf8("%s.out.diff.html" % TST):
pass
if l.startswith('<!--NoDiffs-->'):
o = F_OK
@@ -1874,12 +1882,12 @@ def CategorizeResult(TST, SockTime, outm
o = F_ERROR
else:
Warn("Unexpected last line in %s.out.diff.html:\n%s" % (TST, l))
- ff = open("%s.out.diff.html" % TST, "a")
+ ff = openutf8("%s.out.diff.html" % TST, "a")
ff.write("\n<!--MajorDiffs-->\n")
ff.close()
o = F_ERROR
l = '<!--MajorDiffs-->' # assign something in case file is empty
- for l in open("%s.err.diff.html" % TST):
+ for l in openutf8("%s.err.diff.html" % TST):
pass
if l.startswith('<!--NoDiffs-->'):
e = F_OK
@@ -1892,7 +1900,7 @@ def CategorizeResult(TST, SockTime, outm
e = F_ERROR
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
