On Wed, 23 Oct 2019 02:33:20 +0000 [email protected] wrote:
> Source: clickhouse
> Version: 18.16.1+ds-5
> Severity: normal
> Tags: sid bullseye
> User: [email protected]
> Usertags: py2removal
i started working on this (see attached patch): it looks like python2
is only used when running tests; i got to the point where tests are
running but now there are 35 errors which i'm not really equipped to
debug.
I'll upload the package now to fix the FTBFS and i'll revisit this at
a later time.
Cheers,
Sandro
--- a/dbms/tests/clickhouse-test
+++ b/dbms/tests/clickhouse-test
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
import sys
import os
import os.path
@@ -19,7 +19,6 @@ from time import sleep
from errno import ESRCH
from termcolor import colored
from random import random
-import commands
OP_SQUARE_BRACKET = colored("[", attrs=['bold'])
@@ -39,9 +38,9 @@ def remove_control_characters(s):
if int(s, base) < 0x10000:
return unichr(int(s, base))
return default
- s = re.sub(ur"&#(\d+);?", lambda c: str_to_int(c.group(1), c.group(0)), s)
- s = re.sub(ur"&#[xX]([0-9a-fA-F]+);?", lambda c: str_to_int(c.group(1), c.group(0), base=16), s)
- s = re.sub(ur"[\x00-\x08\x0b\x0e-\x1f\x7f]", "", s)
+ s = re.sub(r"&#(\d+);?", lambda c: str_to_int(c.group(1), c.group(0)), s)
+ s = re.sub(r"&#[xX]([0-9a-fA-F]+);?", lambda c: str_to_int(c.group(1), c.group(0), base=16), s)
+ s = re.sub(r"[\x00-\x08\x0b\x0e-\x1f\x7f]", "", s)
return s
def main(args):
@@ -49,7 +48,7 @@ def main(args):
SERVER_DIED = False
def is_data_present():
- clickhouse_proc = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE)
+ clickhouse_proc = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True)
(stdout, stderr) = clickhouse_proc.communicate("EXISTS TABLE test.hits")
if clickhouse_proc.returncode != 0:
raise CalledProcessError(clickhouse_proc.returncode, args.client, stderr)
@@ -87,7 +86,7 @@ def main(args):
os.environ.setdefault("CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL", server_logs_level)
if args.zookeeper is None:
- code, out = commands.getstatusoutput(args.extract_from_config +" --try --config " + args.configserver + ' --key zookeeper | grep . | wc -l')
+ code, out = subprocess.getstatusoutput(args.extract_from_config +" --try --config " + args.configserver + ' --key zookeeper | grep . | wc -l')
try:
if int(out) > 0:
args.zookeeper = True
@@ -97,7 +96,7 @@ def main(args):
args.zookeeper = False
if args.shard is None:
- code, out = commands.getstatusoutput(args.extract_from_config + " --try --config " + args.configserver + ' --key listen_host | grep -E "127.0.0.2|::"')
+ code, out = subprocess.getstatusoutput(args.extract_from_config + " --try --config " + args.configserver + ' --key listen_host | grep -E "127.0.0.2|::"')
if out:
args.shard = True
else:
@@ -107,7 +106,7 @@ def main(args):
skipped_total = 0
failures_total = 0
- clickhouse_proc_create = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE)
+ clickhouse_proc_create = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True)
clickhouse_proc_create.communicate("CREATE DATABASE IF NOT EXISTS test")
for suite in sorted(os.listdir(base_dir)):
@@ -151,7 +150,13 @@ def main(args):
except ValueError:
return 99997
- for case in sorted(filter(lambda case: re.search(args.test, case) if args.test else True, os.listdir(suite_dir)), key=key_func):
+ # taken from upstream at https://github.com/ClickHouse/ClickHouse/blob/9c9ce0aa37d835fb5f2bf60a05ca77a848d603fb/dbms/tests/clickhouse-test#L425-L429
+ all_tests = os.listdir(suite_dir)
+ if args.test:
+ all_tests = [t for t in all_tests if any([re.search(r, t) for r in args.test])]
+ # commented for now, it fails with "TypeError: '<' not supported between instances of 'tuple' and 'int'" not sure why
+ # all_tests.sort(key=key_func)
+ for case in all_tests:
if SERVER_DIED:
break
@@ -162,8 +167,7 @@ def main(args):
report_testcase = et.Element("testcase", attrib = {"name": name})
try:
- print "{0:72}".format(name + ": "),
- sys.stdout.flush()
+ print("{0:72}".format(name + ": "), end='', flush=True)
if args.skip and any(s in name for s in args.skip):
report_testcase.append(et.Element("skipped", attrib = {"message": "skip"}))
@@ -191,7 +195,7 @@ def main(args):
else:
if args.testname:
- clickhouse_proc = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE)
+ clickhouse_proc = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True)
clickhouse_proc.communicate("SELECT 'Running test {suite}/{case} from pid={pid}';".format(pid = os.getpid(), case = case, suite = suite))
reference_file = os.path.join(suite_dir, name) + '.reference'
@@ -222,9 +226,7 @@ def main(args):
print("{0} - Timeout!".format(MSG_FAIL))
else:
stdout = open(stdout_file, 'r').read() if os.path.exists(stdout_file) else ''
- stdout = unicode(stdout, errors='replace', encoding='utf-8')
stderr = open(stderr_file, 'r').read() if os.path.exists(stderr_file) else ''
- stderr = unicode(stderr, errors='replace', encoding='utf-8')
if proc.returncode != 0:
failure = et.Element("failure", attrib = {"message": "return code {}".format(proc.returncode)})
@@ -277,8 +279,7 @@ def main(args):
result_is_different = subprocess.call(['cmp', '-s', reference_file, stdout_file], stdout = PIPE)
if result_is_different:
- (diff, _) = Popen(['diff', '--unified', reference_file, stdout_file], stdout = PIPE).communicate()
- diff = unicode(diff, errors='replace', encoding='utf-8')
+ (diff, _) = Popen(['diff', '--unified', reference_file, stdout_file], stdout = PIPE, text=True).communicate()
failure = et.Element("failure", attrib = {"message": "result differs with reference"})
report_testcase.append(failure)
