commit 22ca9cb4886dac3daceaab5ebac185b639e0e874 Author: Marcin Krol <h...@tld-linux.org> Date: Mon May 3 11:49:50 2021 +0200
- more python 3.x fixes, can't test on PLD but TLD version works fine PLD_Builder/bqueue.py | 2 +- PLD_Builder/chroot.py | 54 ++++++++++++++---------------------------- PLD_Builder/file_sender.py | 15 ++++++------ PLD_Builder/install.py | 12 +++++----- PLD_Builder/load_balancer.py | 2 +- PLD_Builder/mailer.py | 6 ++--- PLD_Builder/maintainer.py | 2 +- PLD_Builder/request.py | 2 +- PLD_Builder/request_fetcher.py | 18 +++++++------- PLD_Builder/request_handler.py | 3 --- PLD_Builder/rpm_builder.py | 31 +++++++++++++----------- PLD_Builder/srpm_builder.py | 7 ++---- PLD_Builder/util.py | 12 +++------- 13 files changed, 69 insertions(+), 97 deletions(-) --- diff --git a/PLD_Builder/bqueue.py b/PLD_Builder/bqueue.py index 4699fd7..e33483e 100644 --- a/PLD_Builder/bqueue.py +++ b/PLD_Builder/bqueue.py @@ -133,7 +133,7 @@ class B_Queue: sio.seek(0) (fdno, tmpname) = tempfile.mkstemp(dir=os.path.dirname(name)) f = os.fdopen(fdno, "w") - util.sendfile(sio, f) + shutil.copyfileobj(sio, f) f.flush() os.fsync(f.fileno()) f.close() diff --git a/PLD_Builder/chroot.py b/PLD_Builder/chroot.py index 8acd58d..6e7976a 100644 --- a/PLD_Builder/chroot.py +++ b/PLD_Builder/chroot.py @@ -4,11 +4,8 @@ import os import re import random import util - -try: - from hashlib import md5 as md5 -except ImportError: - from md5 import md5 +import shutil +import subprocess from config import config @@ -27,8 +24,13 @@ def command_sh(cmd): return "%s sudo chroot %s /bin/sh -c \"export LC_ALL=C; exec < /dev/null; %s\"" \ % (config.sudo_chroot_wrapper, config.chroot, quote(cmd)) -def popen(cmd, user = "builder", mode = "r"): - f = os.popen(command(cmd, user), mode) +def popen(cmd, user = "builder", mode = "r", encoding = None): + if mode == "r": + p = subprocess.Popen(command(cmd, user), shell=True, stdout=subprocess.PIPE, close_fds=True, encoding=encoding) + f = p.stdout + else: + p = subprocess.Popen(command(cmd, user), shell=True, stdin=subprocess.PIPE, close_fds=True, encoding=encoding) + f = p.stdin return f def run(cmd, user = "builder", logfile = None, logstdout = None): @@ -49,36 +51,16 @@ def run(cmd, user = "builder", logfile = None, logstdout = None): return r def cp(file, outfile, user="builder", rm=False): - m = md5() - m.update(util.to_bytes(str(random.sample(range(100000), 500)))) - digest = m.hexdigest() - - marker_start = "--- FILE BEGIN DIGEST %s ---" % digest - marker_end = "--- FILE END DIGEST %s ---" % digest - - f = open(outfile, 'wb') - cmd = "echo \"%s\"; cat %s; echo \"%s\"" % (marker_start, file, marker_end) + f_out = open(outfile, 'wb') + cmd = "cat %s" % file if rm: cmd += "; rm %s" % file - c = command(cmd, user) - p = os.popen(c) - # get file contents - marker = False - for l in p: - if not marker and l.strip() == marker_start: - marker = True - continue - me = l.find(marker_end) - if me != -1: - l = l[:me] - f.write(util.to_bytes(l)) - marker = False - break - if marker: - f.write(util.to_bytes(l)) - rp = p.close() - rf = f.close() - if rp == None: + p = subprocess.Popen(command(cmd, user), shell=True, stdout=subprocess.PIPE, close_fds=True) + f_in = p.stdout + shutil.copyfileobj(f_in, f_out) + f_out.close() + r = f_in.close() + if r == None: return 0 else: - return rp + return r diff --git a/PLD_Builder/file_sender.py b/PLD_Builder/file_sender.py index c6cc459..3ebf240 100644 --- a/PLD_Builder/file_sender.py +++ b/PLD_Builder/file_sender.py @@ -8,8 +8,7 @@ import time import shutil import sys import traceback -import urllib2 - +import urllib.request from config import config, init_conf import mailer import path @@ -94,11 +93,11 @@ def post_file(src, url): global problems try: f = open(src, 'r') - data = f.read() + data = f.read().encode('utf-8') f.close() - req = urllib2.Request(url, data) - req.add_header('X-Filename', os.path.basename(src)) - f = urllib2.urlopen(req) + headers = { 'X-Filename' : os.path.basename(src) } + req = urllib.request.Request(url, data=data, headers=headers) + f = urllib.request.urlopen(req) f.close() except Exception as e: problems[src] = e @@ -164,9 +163,9 @@ def flush_queue(dir): d = read_name_val(f) if d != None: q.append(d) def mycmp(x, y): - rc = cmp(x['Time'], y['Time']) + rc = util.cmp(x['Time'], y['Time']) if rc == 0 and 'Type' in x and 'Type' in y: - return cmp(x['Type'], y['Type']) + return util.cmp(x['Type'], y['Type']) else: return rc q.sort(key=util.cmp_to_key(mycmp)) diff --git a/PLD_Builder/install.py b/PLD_Builder/install.py index 3e11817..91a115e 100644 --- a/PLD_Builder/install.py +++ b/PLD_Builder/install.py @@ -28,7 +28,7 @@ def close_killset(killset): del killset[p] errors += "cannot remove %s because it's crucial\n" % p else: - f = chroot.popen("poldek --noask --test --test --erase %s" % p, user = "root") + f = chroot.popen("poldek --noask --test --test --erase %s" % p, user = "root", encoding = "utf-8") crucial = 0 e = [] for l in f: @@ -49,7 +49,7 @@ def close_killset(killset): return errors def upgrade_from_batch(r, b): - f = chroot.popen("rpm --test -F %s 2>&1" % ' '.join(b.files), user = "root") + f = chroot.popen("rpm --test -F %s 2>&1" % ' '.join(b.files), user = "root", encoding = "utf-8") killset = {} rx = re.compile(r' \(installed\) (?P<name>[^\s]+)-[^-]+-[^-]+$') for l in f: @@ -107,7 +107,7 @@ def uninstall_self_conflict(b): 'rpmdefs' : b.rpmbuild_opts(), 'topdir' : b.get_topdir(), 'spec': b.spec, - }) + }, encoding = "utf-8") # java-sun >= 1.5 conflicts with soprano-2.1.67-1.src # java-sun conflicts with soprano-2.1.67-1.src rx = re.compile(r"\s+(?P<name>[\w-]+)\s+.*conflicts with [^\s]+-[^-]+-[^-]+\.src($| .*)") @@ -125,7 +125,7 @@ def uninstall_self_conflict(b): def install_br(r, b): def is_rpmorg(): - f = chroot.popen("rpm --version 2>&1") + f = chroot.popen("rpm --version 2>&1", encoding = "utf-8") v = re.compile(r'(RPM version|rpm \(RPM\)) (?P<major>\d)\.(?P<minor>\d+)(\.\d+)?') for l in f: m = v.search(l) @@ -154,7 +154,7 @@ def install_br(r, b): 'rpmdefs' : b.rpmbuild_opts(), 'spec': b.spec, } - f = chroot.popen(cmd) + f = chroot.popen(cmd, encoding = "utf-8") rx = re.compile(r"^\s*(?P<name>[^\s]+) .*is needed by") needed = {} b.log_line("checking BR") @@ -180,7 +180,7 @@ def install_br(r, b): chroot.run("poldek --up --upa", user = "root", logfile = b.logfile) # check conflicts in BRed packages b.log_line("checking conflicting packages in BRed packages") - f = chroot.popen("poldek --test --test --noask --caplookup -Q -v %s --upgrade %s" % (b.ignores(), br), user = "root") + f = chroot.popen("poldek --test --test --noask --caplookup -Q -v %s --upgrade %s" % (b.ignores(), br), user = "root", encoding = "utf-8") # phonon-devel-4.3.1-1.i686 conflicts with qt4-phonon-devel-4.5.0-6.i686 # jdbc-stdext >= 2.0 is required by installed java-struts-1.3.10-1.noarch # jmx is needed by (installed) java-commons-modeler-2.0-1.noarch diff --git a/PLD_Builder/load_balancer.py b/PLD_Builder/load_balancer.py index 43d85d5..459034b 100644 --- a/PLD_Builder/load_balancer.py +++ b/PLD_Builder/load_balancer.py @@ -39,7 +39,7 @@ def builders_order(): log.alert("found strange lock in got-lock: %s" % b) def mycmp(b1, b2): - return cmp(bs[b1], bs[b2]) + return util.cmp(bs[b1], bs[b2]) bl.sort(key=util.cmp_to_key(mycmp)) diff --git a/PLD_Builder/mailer.py b/PLD_Builder/mailer.py index 2cf324d..7d90091 100644 --- a/PLD_Builder/mailer.py +++ b/PLD_Builder/mailer.py @@ -5,7 +5,7 @@ import os import sys from io import StringIO from config import config -import util +import shutil import log def recode(s): @@ -58,7 +58,7 @@ class Message: self.body.write("\n\n[...]\n\n") line += 1 else: - util.sendfile(open(log), self.body) + shutil.copyfileobj(open(log), self.body) def set_std_headers(self): self.headers["Date"] = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()) @@ -73,7 +73,7 @@ class Message: f.write("%s: %s\n" % (k, v)) f.write("\n") self.body.seek(0) - util.sendfile(self.body, f) + shutil.copyfileobj(self.body, f) def send(self): if not os.path.exists("/usr/lib/sendmail"): diff --git a/PLD_Builder/maintainer.py b/PLD_Builder/maintainer.py index 6b1b45a..4d909df 100644 --- a/PLD_Builder/maintainer.py +++ b/PLD_Builder/maintainer.py @@ -39,7 +39,7 @@ def handle_src(): def handle_bin(): send_rpmqa() - f=chroot.popen("""ls -l --time-style +%s /spools/ready""", 'root') + f=chroot.popen("""ls -l --time-style +%s /spools/ready""", "root", encoding = "utf-8") rmpkgs=[] curtime=time.time() for i in f: diff --git a/PLD_Builder/request.py b/PLD_Builder/request.py index 71a422c..0b71d5f 100644 --- a/PLD_Builder/request.py +++ b/PLD_Builder/request.py @@ -7,7 +7,7 @@ import time import xml.sax.saxutils import fnmatch import os -import urllib +import urllib.parse import html import pytz import tempfile diff --git a/PLD_Builder/request_fetcher.py b/PLD_Builder/request_fetcher.py index 25b130f..83229ae 100644 --- a/PLD_Builder/request_fetcher.py +++ b/PLD_Builder/request_fetcher.py @@ -3,16 +3,16 @@ import string import signal import os -import urllib -import urllib2 +import urllib.request import sys -from io import StringIO +from io import StringIO, BytesIO import gzip import path import log import status import lock import util +import shutil import gpg import request import loop @@ -42,8 +42,8 @@ def has_new(control_url): signal.alarm(300) try: headers = { 'Cache-Control': 'no-cache', 'Pragma': 'no-cache' } - req = urllib2.Request(url=control_url + "/max_req_no", headers=headers) - f = urllib2.urlopen(req) + req = urllib.request.Request(url=control_url + "/max_req_no", headers=headers) + f = urllib.request.urlopen(req) count = int(f.readline().strip()) signal.alarm(0) except Exception as e: @@ -62,15 +62,15 @@ def fetch_queue(control_url): signal.alarm(300) try: headers = { 'Cache-Control': 'no-cache', 'Pragma': 'no-cache' } - req = urllib2.Request(url=control_url + "/queue.gz", headers=headers) - f = urllib2.urlopen(req) + req = urllib.request.Request(url=control_url + "/queue.gz", headers=headers) + f = urllib.request.urlopen(req) signal.alarm(0) except Exception as e: signal.alarm(0) log.error("can't fetch %s: %s" % (control_url + "/queue.gz", e)) sys.exit(1) - sio = StringIO() - util.sendfile(f, sio) + sio = BytesIO() + shutil.copyfileobj(f, sio) f.close() sio.seek(0) f = gzip.GzipFile(fileobj = sio) diff --git a/PLD_Builder/request_handler.py b/PLD_Builder/request_handler.py index 70a1866..74c7f1c 100644 --- a/PLD_Builder/request_handler.py +++ b/PLD_Builder/request_handler.py @@ -19,7 +19,6 @@ from lock import lock from bqueue import B_Queue from config import config, init_conf from mailer import Message -#import messagebus def check_double_id(id): id_nl = id + "\n" @@ -196,10 +195,8 @@ def handle_request(req, filename = None): status.push("request from %s" % user.login) r = request.parse_request(body) if r.kind == 'group': -# messagebus.notify(topic="request.group", user=user.login, **r.dump_json()) handle_group(r, user) elif r.kind == 'notification': -# messagebus.notify(topic="request.notify", user=user.login, **r.dump_json()) handle_notification(r, user) else: msg = "%s: don't know how to handle requests of this kind '%s'" \ diff --git a/PLD_Builder/rpm_builder.py b/PLD_Builder/rpm_builder.py index 1f1ed2c..f5d4e95 100644 --- a/PLD_Builder/rpm_builder.py +++ b/PLD_Builder/rpm_builder.py @@ -8,13 +8,15 @@ import atexit import time import datetime import string -import urllib -import urllib2 +import urllib.request +import urllib.parse +import urllib.error from config import config, init_conf from bqueue import B_Queue import lock import util +import shutil import loop import path import status @@ -46,9 +48,9 @@ def pick_request(q): def mycmp(r1, r2): if r1.kind != 'group' or r2.kind != 'group': raise Exception("non-group requests") - pri_diff = cmp(r1.priority, r2.priority) + pri_diff = util.cmp(r1.priority, r2.priority) if pri_diff == 0: - return cmp(r1.time, r2.time) + return util.cmp(r1.time, r2.time) else: return pri_diff q.requests.sort(key=util.cmp_to_key(mycmp)) @@ -62,12 +64,12 @@ def check_skip_build(r, b): while not good: try: headers = { 'Cache-Control': 'no-cache', 'Pragma': 'no-cache' } - req = urllib2.Request(url=src_url, headers=headers) - f = urllib2.urlopen(req) + req = urllib.request.Request(url=src_url, headers=headers) + f = urllib.request.urlopen(req) good = True - except urllib2.HTTPError as error: + except urllib.error.HTTPError as error: return False - except urllib2.URLError as error: + except urllib.error.URLError as error: # see errno.h try: errno = error.errno @@ -92,10 +94,10 @@ def fetch_src(r, b): while not good: try: headers = { 'Cache-Control': 'no-cache', 'Pragma': 'no-cache' } - req = urllib2.Request(url=src_url, headers=headers) - f = urllib2.urlopen(req) + req = urllib.request.Request(url=src_url, headers=headers) + f = urllib.request.urlopen(req) good = True - except urllib2.HTTPError as error: + except urllib.error.HTTPError as error: # fail in a way where cron job will retry msg = "unable to fetch url %s, http code: %d" % (src_url, error.code) b.log_line(msg) @@ -107,7 +109,7 @@ def fetch_src(r, b): msg = "in queue for more than 6 hours, download failing" b.log_line(msg) return False - except urllib2.URLError as error: + except urllib.error.URLError as error: errno = 0 if isinstance(error.args[0], IOError): errno = error.args[0].errno @@ -126,14 +128,15 @@ def fetch_src(r, b): print("error.reason exception %s" % e) raise - o = chroot.popen("cat > %s" % b.src_rpm, mode = "wb") + o = chroot.popen("cat > %s" % b.src_rpm, mode = "w") try: - bytes = util.sendfile(f, o) + shutil.copyfileobj(f, o) except IOError as e: b.log_line("error: unable to write to `%s': %s" % (b.src_rpm, e)) raise + bytes = float(f.headers['content-length']) f.close() o.close() t = time.time() - start diff --git a/PLD_Builder/srpm_builder.py b/PLD_Builder/srpm_builder.py index 562b0c5..55d164e 100644 --- a/PLD_Builder/srpm_builder.py +++ b/PLD_Builder/srpm_builder.py @@ -23,7 +23,6 @@ import notify import status import build import report -#import messagebus from lock import lock from bqueue import B_Queue @@ -33,9 +32,9 @@ def pick_request(q): def mycmp(r1, r2): if r1.kind != 'group' or r2.kind != 'group': raise Exception("non-group requests") - pri_diff = cmp(r1.priority, r2.priority) + pri_diff = util.cmp(r1.priority, r2.priority) if pri_diff == 0: - return cmp(r1.time, r2.time) + return util.cmp(r1.time, r2.time) else: return pri_diff q.requests.sort(key=util.cmp_to_key(mycmp)) @@ -96,7 +95,6 @@ def build_srpm(r, b): return "FAIL" status.push("building %s" % b.spec) -# messagebus.notify(topic="build_srpm.start", spec=b.spec, flags=r.flags, batch=b, request=r) b.src_rpm = "" builder_opts = "-nu -nm --nodeps --http --define \'_pld_builder 1\'" @@ -142,7 +140,6 @@ def build_srpm(r, b): if res: res = "FAIL" -# messagebus.notify(topic="build_srpm.finish", spec=b.spec) return res def handle_request(r): diff --git a/PLD_Builder/util.py b/PLD_Builder/util.py index 1d485a1..af6b8f1 100644 --- a/PLD_Builder/util.py +++ b/PLD_Builder/util.py @@ -32,15 +32,6 @@ def pkg_name(nvr): def msg(m): sys.stderr.write(m) -def sendfile(src, dst): - cnt = 0 - while 1: - s = src.read(10000) - if s == "": break - cnt += len(s) - dst.write(s) - return cnt - def append_to(log, msg): f = open(log, "a") f.write("%s\n" % msg) @@ -112,3 +103,6 @@ def to_str(s): return s else: raise TypeError("Expected bytes or string, but got %s." % type(s)) + +def cmp(a, b): + return (a > b) - (a < b) ================================================================ ---- gitweb: http://git.pld-linux.org/gitweb.cgi/projects/pld-builder.new.git/commitdiff/22ca9cb4886dac3daceaab5ebac185b639e0e874 _______________________________________________ pld-cvs-commit mailing list pld-cvs-commit@lists.pld-linux.org http://lists.pld-linux.org/mailman/listinfo/pld-cvs-commit