On Wed, 23 Oct 2019 02:33:20 +0000 mo...@debian.org wrote: > Source: clickhouse > Version: 18.16.1+ds-5 > Severity: normal > Tags: sid bullseye > User: debian-pyt...@lists.debian.org > Usertags: py2removal
i started working on this (see attached patch): it looks like python2 is only used when running tests; i got to the point where tests are running but now there are 35 errors which i'm not really equipped to debug. I'll upload the package now to fix the FTBFS and i'll revisit this at a later time. Cheers, Sandro
--- a/dbms/tests/clickhouse-test +++ b/dbms/tests/clickhouse-test @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import sys import os import os.path @@ -19,7 +19,6 @@ from time import sleep from errno import ESRCH from termcolor import colored from random import random -import commands OP_SQUARE_BRACKET = colored("[", attrs=['bold']) @@ -39,9 +38,9 @@ def remove_control_characters(s): if int(s, base) < 0x10000: return unichr(int(s, base)) return default - s = re.sub(ur"&#(\d+);?", lambda c: str_to_int(c.group(1), c.group(0)), s) - s = re.sub(ur"&#[xX]([0-9a-fA-F]+);?", lambda c: str_to_int(c.group(1), c.group(0), base=16), s) - s = re.sub(ur"[\x00-\x08\x0b\x0e-\x1f\x7f]", "", s) + s = re.sub(r"&#(\d+);?", lambda c: str_to_int(c.group(1), c.group(0)), s) + s = re.sub(r"&#[xX]([0-9a-fA-F]+);?", lambda c: str_to_int(c.group(1), c.group(0), base=16), s) + s = re.sub(r"[\x00-\x08\x0b\x0e-\x1f\x7f]", "", s) return s def main(args): @@ -49,7 +48,7 @@ def main(args): SERVER_DIED = False def is_data_present(): - clickhouse_proc = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) + clickhouse_proc = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True) (stdout, stderr) = clickhouse_proc.communicate("EXISTS TABLE test.hits") if clickhouse_proc.returncode != 0: raise CalledProcessError(clickhouse_proc.returncode, args.client, stderr) @@ -87,7 +86,7 @@ def main(args): os.environ.setdefault("CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL", server_logs_level) if args.zookeeper is None: - code, out = commands.getstatusoutput(args.extract_from_config +" --try --config " + args.configserver + ' --key zookeeper | grep . | wc -l') + code, out = subprocess.getstatusoutput(args.extract_from_config +" --try --config " + args.configserver + ' --key zookeeper | grep . | wc -l') try: if int(out) > 0: args.zookeeper = True @@ -97,7 +96,7 @@ def main(args): args.zookeeper = False if args.shard is None: - code, out = commands.getstatusoutput(args.extract_from_config + " --try --config " + args.configserver + ' --key listen_host | grep -E "127.0.0.2|::"') + code, out = subprocess.getstatusoutput(args.extract_from_config + " --try --config " + args.configserver + ' --key listen_host | grep -E "127.0.0.2|::"') if out: args.shard = True else: @@ -107,7 +106,7 @@ def main(args): skipped_total = 0 failures_total = 0 - clickhouse_proc_create = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) + clickhouse_proc_create = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True) clickhouse_proc_create.communicate("CREATE DATABASE IF NOT EXISTS test") for suite in sorted(os.listdir(base_dir)): @@ -151,7 +150,13 @@ def main(args): except ValueError: return 99997 - for case in sorted(filter(lambda case: re.search(args.test, case) if args.test else True, os.listdir(suite_dir)), key=key_func): + # taken from upstream at https://github.com/ClickHouse/ClickHouse/blob/9c9ce0aa37d835fb5f2bf60a05ca77a848d603fb/dbms/tests/clickhouse-test#L425-L429 + all_tests = os.listdir(suite_dir) + if args.test: + all_tests = [t for t in all_tests if any([re.search(r, t) for r in args.test])] + # commented for now, it fails with "TypeError: '<' not supported between instances of 'tuple' and 'int'" not sure why + # all_tests.sort(key=key_func) + for case in all_tests: if SERVER_DIED: break @@ -162,8 +167,7 @@ def main(args): report_testcase = et.Element("testcase", attrib = {"name": name}) try: - print "{0:72}".format(name + ": "), - sys.stdout.flush() + print("{0:72}".format(name + ": "), end='', flush=True) if args.skip and any(s in name for s in args.skip): report_testcase.append(et.Element("skipped", attrib = {"message": "skip"})) @@ -191,7 +195,7 @@ def main(args): else: if args.testname: - clickhouse_proc = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) + clickhouse_proc = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True) clickhouse_proc.communicate("SELECT 'Running test {suite}/{case} from pid={pid}';".format(pid = os.getpid(), case = case, suite = suite)) reference_file = os.path.join(suite_dir, name) + '.reference' @@ -222,9 +226,7 @@ def main(args): print("{0} - Timeout!".format(MSG_FAIL)) else: stdout = open(stdout_file, 'r').read() if os.path.exists(stdout_file) else '' - stdout = unicode(stdout, errors='replace', encoding='utf-8') stderr = open(stderr_file, 'r').read() if os.path.exists(stderr_file) else '' - stderr = unicode(stderr, errors='replace', encoding='utf-8') if proc.returncode != 0: failure = et.Element("failure", attrib = {"message": "return code {}".format(proc.returncode)}) @@ -277,8 +279,7 @@ def main(args): result_is_different = subprocess.call(['cmp', '-s', reference_file, stdout_file], stdout = PIPE) if result_is_different: - (diff, _) = Popen(['diff', '--unified', reference_file, stdout_file], stdout = PIPE).communicate() - diff = unicode(diff, errors='replace', encoding='utf-8') + (diff, _) = Popen(['diff', '--unified', reference_file, stdout_file], stdout = PIPE, text=True).communicate() failure = et.Element("failure", attrib = {"message": "result differs with reference"}) report_testcase.append(failure)