[projects/pld-builder.new] - more python 3.x fixes, can't test on PLD but TLD version works fine
hawk
hawk at pld-linux.org
Mon May 3 11:51:57 CEST 2021
commit 22ca9cb4886dac3daceaab5ebac185b639e0e874
Author: Marcin Krol <hawk at tld-linux.org>
Date: Mon May 3 11:49:50 2021 +0200
- more python 3.x fixes, can't test on PLD but TLD version works fine
PLD_Builder/bqueue.py | 2 +-
PLD_Builder/chroot.py | 54 ++++++++++++++----------------------------
PLD_Builder/file_sender.py | 15 ++++++------
PLD_Builder/install.py | 12 +++++-----
PLD_Builder/load_balancer.py | 2 +-
PLD_Builder/mailer.py | 6 ++---
PLD_Builder/maintainer.py | 2 +-
PLD_Builder/request.py | 2 +-
PLD_Builder/request_fetcher.py | 18 +++++++-------
PLD_Builder/request_handler.py | 3 ---
PLD_Builder/rpm_builder.py | 31 +++++++++++++-----------
PLD_Builder/srpm_builder.py | 7 ++----
PLD_Builder/util.py | 12 +++-------
13 files changed, 69 insertions(+), 97 deletions(-)
---
diff --git a/PLD_Builder/bqueue.py b/PLD_Builder/bqueue.py
index 4699fd7..e33483e 100644
--- a/PLD_Builder/bqueue.py
+++ b/PLD_Builder/bqueue.py
@@ -133,7 +133,7 @@ class B_Queue:
sio.seek(0)
(fdno, tmpname) = tempfile.mkstemp(dir=os.path.dirname(name))
f = os.fdopen(fdno, "w")
- util.sendfile(sio, f)
+ shutil.copyfileobj(sio, f)
f.flush()
os.fsync(f.fileno())
f.close()
diff --git a/PLD_Builder/chroot.py b/PLD_Builder/chroot.py
index 8acd58d..6e7976a 100644
--- a/PLD_Builder/chroot.py
+++ b/PLD_Builder/chroot.py
@@ -4,11 +4,8 @@ import os
import re
import random
import util
-
-try:
- from hashlib import md5 as md5
-except ImportError:
- from md5 import md5
+import shutil
+import subprocess
from config import config
@@ -27,8 +24,13 @@ def command_sh(cmd):
return "%s sudo chroot %s /bin/sh -c \"export LC_ALL=C; exec < /dev/null; %s\"" \
% (config.sudo_chroot_wrapper, config.chroot, quote(cmd))
-def popen(cmd, user = "builder", mode = "r"):
- f = os.popen(command(cmd, user), mode)
+def popen(cmd, user = "builder", mode = "r", encoding = None):
+ if mode == "r":
+ p = subprocess.Popen(command(cmd, user), shell=True, stdout=subprocess.PIPE, close_fds=True, encoding=encoding)
+ f = p.stdout
+ else:
+ p = subprocess.Popen(command(cmd, user), shell=True, stdin=subprocess.PIPE, close_fds=True, encoding=encoding)
+ f = p.stdin
return f
def run(cmd, user = "builder", logfile = None, logstdout = None):
@@ -49,36 +51,16 @@ def run(cmd, user = "builder", logfile = None, logstdout = None):
return r
def cp(file, outfile, user="builder", rm=False):
- m = md5()
- m.update(util.to_bytes(str(random.sample(range(100000), 500))))
- digest = m.hexdigest()
-
- marker_start = "--- FILE BEGIN DIGEST %s ---" % digest
- marker_end = "--- FILE END DIGEST %s ---" % digest
-
- f = open(outfile, 'wb')
- cmd = "echo \"%s\"; cat %s; echo \"%s\"" % (marker_start, file, marker_end)
+ f_out = open(outfile, 'wb')
+ cmd = "cat %s" % file
if rm:
cmd += "; rm %s" % file
- c = command(cmd, user)
- p = os.popen(c)
- # get file contents
- marker = False
- for l in p:
- if not marker and l.strip() == marker_start:
- marker = True
- continue
- me = l.find(marker_end)
- if me != -1:
- l = l[:me]
- f.write(util.to_bytes(l))
- marker = False
- break
- if marker:
- f.write(util.to_bytes(l))
- rp = p.close()
- rf = f.close()
- if rp == None:
+ p = subprocess.Popen(command(cmd, user), shell=True, stdout=subprocess.PIPE, close_fds=True)
+ f_in = p.stdout
+ shutil.copyfileobj(f_in, f_out)
+ f_out.close()
+ r = f_in.close()
+ if r == None:
return 0
else:
- return rp
+ return r
diff --git a/PLD_Builder/file_sender.py b/PLD_Builder/file_sender.py
index c6cc459..3ebf240 100644
--- a/PLD_Builder/file_sender.py
+++ b/PLD_Builder/file_sender.py
@@ -8,8 +8,7 @@ import time
import shutil
import sys
import traceback
-import urllib2
-
+import urllib.request
from config import config, init_conf
import mailer
import path
@@ -94,11 +93,11 @@ def post_file(src, url):
global problems
try:
f = open(src, 'r')
- data = f.read()
+ data = f.read().encode('utf-8')
f.close()
- req = urllib2.Request(url, data)
- req.add_header('X-Filename', os.path.basename(src))
- f = urllib2.urlopen(req)
+ headers = { 'X-Filename' : os.path.basename(src) }
+ req = urllib.request.Request(url, data=data, headers=headers)
+ f = urllib.request.urlopen(req)
f.close()
except Exception as e:
problems[src] = e
@@ -164,9 +163,9 @@ def flush_queue(dir):
d = read_name_val(f)
if d != None: q.append(d)
def mycmp(x, y):
- rc = cmp(x['Time'], y['Time'])
+ rc = util.cmp(x['Time'], y['Time'])
if rc == 0 and 'Type' in x and 'Type' in y:
- return cmp(x['Type'], y['Type'])
+ return util.cmp(x['Type'], y['Type'])
else:
return rc
q.sort(key=util.cmp_to_key(mycmp))
diff --git a/PLD_Builder/install.py b/PLD_Builder/install.py
index 3e11817..91a115e 100644
--- a/PLD_Builder/install.py
+++ b/PLD_Builder/install.py
@@ -28,7 +28,7 @@ def close_killset(killset):
del killset[p]
errors += "cannot remove %s because it's crucial\n" % p
else:
- f = chroot.popen("poldek --noask --test --test --erase %s" % p, user = "root")
+ f = chroot.popen("poldek --noask --test --test --erase %s" % p, user = "root", encoding = "utf-8")
crucial = 0
e = []
for l in f:
@@ -49,7 +49,7 @@ def close_killset(killset):
return errors
def upgrade_from_batch(r, b):
- f = chroot.popen("rpm --test -F %s 2>&1" % ' '.join(b.files), user = "root")
+ f = chroot.popen("rpm --test -F %s 2>&1" % ' '.join(b.files), user = "root", encoding = "utf-8")
killset = {}
rx = re.compile(r' \(installed\) (?P<name>[^\s]+)-[^-]+-[^-]+$')
for l in f:
@@ -107,7 +107,7 @@ def uninstall_self_conflict(b):
'rpmdefs' : b.rpmbuild_opts(),
'topdir' : b.get_topdir(),
'spec': b.spec,
- })
+ }, encoding = "utf-8")
# java-sun >= 1.5 conflicts with soprano-2.1.67-1.src
# java-sun conflicts with soprano-2.1.67-1.src
rx = re.compile(r"\s+(?P<name>[\w-]+)\s+.*conflicts with [^\s]+-[^-]+-[^-]+\.src($| .*)")
@@ -125,7 +125,7 @@ def uninstall_self_conflict(b):
def install_br(r, b):
def is_rpmorg():
- f = chroot.popen("rpm --version 2>&1")
+ f = chroot.popen("rpm --version 2>&1", encoding = "utf-8")
v = re.compile(r'(RPM version|rpm \(RPM\)) (?P<major>\d)\.(?P<minor>\d+)(\.\d+)?')
for l in f:
m = v.search(l)
@@ -154,7 +154,7 @@ def install_br(r, b):
'rpmdefs' : b.rpmbuild_opts(),
'spec': b.spec,
}
- f = chroot.popen(cmd)
+ f = chroot.popen(cmd, encoding = "utf-8")
rx = re.compile(r"^\s*(?P<name>[^\s]+) .*is needed by")
needed = {}
b.log_line("checking BR")
@@ -180,7 +180,7 @@ def install_br(r, b):
chroot.run("poldek --up --upa", user = "root", logfile = b.logfile)
# check conflicts in BRed packages
b.log_line("checking conflicting packages in BRed packages")
- f = chroot.popen("poldek --test --test --noask --caplookup -Q -v %s --upgrade %s" % (b.ignores(), br), user = "root")
+ f = chroot.popen("poldek --test --test --noask --caplookup -Q -v %s --upgrade %s" % (b.ignores(), br), user = "root", encoding = "utf-8")
# phonon-devel-4.3.1-1.i686 conflicts with qt4-phonon-devel-4.5.0-6.i686
# jdbc-stdext >= 2.0 is required by installed java-struts-1.3.10-1.noarch
# jmx is needed by (installed) java-commons-modeler-2.0-1.noarch
diff --git a/PLD_Builder/load_balancer.py b/PLD_Builder/load_balancer.py
index 43d85d5..459034b 100644
--- a/PLD_Builder/load_balancer.py
+++ b/PLD_Builder/load_balancer.py
@@ -39,7 +39,7 @@ def builders_order():
log.alert("found strange lock in got-lock: %s" % b)
def mycmp(b1, b2):
- return cmp(bs[b1], bs[b2])
+ return util.cmp(bs[b1], bs[b2])
bl.sort(key=util.cmp_to_key(mycmp))
diff --git a/PLD_Builder/mailer.py b/PLD_Builder/mailer.py
index 2cf324d..7d90091 100644
--- a/PLD_Builder/mailer.py
+++ b/PLD_Builder/mailer.py
@@ -5,7 +5,7 @@ import os
import sys
from io import StringIO
from config import config
-import util
+import shutil
import log
def recode(s):
@@ -58,7 +58,7 @@ class Message:
self.body.write("\n\n[...]\n\n")
line += 1
else:
- util.sendfile(open(log), self.body)
+ shutil.copyfileobj(open(log), self.body)
def set_std_headers(self):
self.headers["Date"] = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())
@@ -73,7 +73,7 @@ class Message:
f.write("%s: %s\n" % (k, v))
f.write("\n")
self.body.seek(0)
- util.sendfile(self.body, f)
+ shutil.copyfileobj(self.body, f)
def send(self):
if not os.path.exists("/usr/lib/sendmail"):
diff --git a/PLD_Builder/maintainer.py b/PLD_Builder/maintainer.py
index 6b1b45a..4d909df 100644
--- a/PLD_Builder/maintainer.py
+++ b/PLD_Builder/maintainer.py
@@ -39,7 +39,7 @@ def handle_src():
def handle_bin():
send_rpmqa()
- f=chroot.popen("""ls -l --time-style +%s /spools/ready""", 'root')
+ f=chroot.popen("""ls -l --time-style +%s /spools/ready""", "root", encoding = "utf-8")
rmpkgs=[]
curtime=time.time()
for i in f:
diff --git a/PLD_Builder/request.py b/PLD_Builder/request.py
index 71a422c..0b71d5f 100644
--- a/PLD_Builder/request.py
+++ b/PLD_Builder/request.py
@@ -7,7 +7,7 @@ import time
import xml.sax.saxutils
import fnmatch
import os
-import urllib
+import urllib.parse
import html
import pytz
import tempfile
diff --git a/PLD_Builder/request_fetcher.py b/PLD_Builder/request_fetcher.py
index 25b130f..83229ae 100644
--- a/PLD_Builder/request_fetcher.py
+++ b/PLD_Builder/request_fetcher.py
@@ -3,16 +3,16 @@
import string
import signal
import os
-import urllib
-import urllib2
+import urllib.request
import sys
-from io import StringIO
+from io import StringIO, BytesIO
import gzip
import path
import log
import status
import lock
import util
+import shutil
import gpg
import request
import loop
@@ -42,8 +42,8 @@ def has_new(control_url):
signal.alarm(300)
try:
headers = { 'Cache-Control': 'no-cache', 'Pragma': 'no-cache' }
- req = urllib2.Request(url=control_url + "/max_req_no", headers=headers)
- f = urllib2.urlopen(req)
+ req = urllib.request.Request(url=control_url + "/max_req_no", headers=headers)
+ f = urllib.request.urlopen(req)
count = int(f.readline().strip())
signal.alarm(0)
except Exception as e:
@@ -62,15 +62,15 @@ def fetch_queue(control_url):
signal.alarm(300)
try:
headers = { 'Cache-Control': 'no-cache', 'Pragma': 'no-cache' }
- req = urllib2.Request(url=control_url + "/queue.gz", headers=headers)
- f = urllib2.urlopen(req)
+ req = urllib.request.Request(url=control_url + "/queue.gz", headers=headers)
+ f = urllib.request.urlopen(req)
signal.alarm(0)
except Exception as e:
signal.alarm(0)
log.error("can't fetch %s: %s" % (control_url + "/queue.gz", e))
sys.exit(1)
- sio = StringIO()
- util.sendfile(f, sio)
+ sio = BytesIO()
+ shutil.copyfileobj(f, sio)
f.close()
sio.seek(0)
f = gzip.GzipFile(fileobj = sio)
diff --git a/PLD_Builder/request_handler.py b/PLD_Builder/request_handler.py
index 70a1866..74c7f1c 100644
--- a/PLD_Builder/request_handler.py
+++ b/PLD_Builder/request_handler.py
@@ -19,7 +19,6 @@ from lock import lock
from bqueue import B_Queue
from config import config, init_conf
from mailer import Message
-#import messagebus
def check_double_id(id):
id_nl = id + "\n"
@@ -196,10 +195,8 @@ def handle_request(req, filename = None):
status.push("request from %s" % user.login)
r = request.parse_request(body)
if r.kind == 'group':
-# messagebus.notify(topic="request.group", user=user.login, **r.dump_json())
handle_group(r, user)
elif r.kind == 'notification':
-# messagebus.notify(topic="request.notify", user=user.login, **r.dump_json())
handle_notification(r, user)
else:
msg = "%s: don't know how to handle requests of this kind '%s'" \
diff --git a/PLD_Builder/rpm_builder.py b/PLD_Builder/rpm_builder.py
index 1f1ed2c..f5d4e95 100644
--- a/PLD_Builder/rpm_builder.py
+++ b/PLD_Builder/rpm_builder.py
@@ -8,13 +8,15 @@ import atexit
import time
import datetime
import string
-import urllib
-import urllib2
+import urllib.request
+import urllib.parse
+import urllib.error
from config import config, init_conf
from bqueue import B_Queue
import lock
import util
+import shutil
import loop
import path
import status
@@ -46,9 +48,9 @@ def pick_request(q):
def mycmp(r1, r2):
if r1.kind != 'group' or r2.kind != 'group':
raise Exception("non-group requests")
- pri_diff = cmp(r1.priority, r2.priority)
+ pri_diff = util.cmp(r1.priority, r2.priority)
if pri_diff == 0:
- return cmp(r1.time, r2.time)
+ return util.cmp(r1.time, r2.time)
else:
return pri_diff
q.requests.sort(key=util.cmp_to_key(mycmp))
@@ -62,12 +64,12 @@ def check_skip_build(r, b):
while not good:
try:
headers = { 'Cache-Control': 'no-cache', 'Pragma': 'no-cache' }
- req = urllib2.Request(url=src_url, headers=headers)
- f = urllib2.urlopen(req)
+ req = urllib.request.Request(url=src_url, headers=headers)
+ f = urllib.request.urlopen(req)
good = True
- except urllib2.HTTPError as error:
+ except urllib.error.HTTPError as error:
return False
- except urllib2.URLError as error:
+ except urllib.error.URLError as error:
# see errno.h
try:
errno = error.errno
@@ -92,10 +94,10 @@ def fetch_src(r, b):
while not good:
try:
headers = { 'Cache-Control': 'no-cache', 'Pragma': 'no-cache' }
- req = urllib2.Request(url=src_url, headers=headers)
- f = urllib2.urlopen(req)
+ req = urllib.request.Request(url=src_url, headers=headers)
+ f = urllib.request.urlopen(req)
good = True
- except urllib2.HTTPError as error:
+ except urllib.error.HTTPError as error:
# fail in a way where cron job will retry
msg = "unable to fetch url %s, http code: %d" % (src_url, error.code)
b.log_line(msg)
@@ -107,7 +109,7 @@ def fetch_src(r, b):
msg = "in queue for more than 6 hours, download failing"
b.log_line(msg)
return False
- except urllib2.URLError as error:
+ except urllib.error.URLError as error:
errno = 0
if isinstance(error.args[0], IOError):
errno = error.args[0].errno
@@ -126,14 +128,15 @@ def fetch_src(r, b):
print("error.reason exception %s" % e)
raise
- o = chroot.popen("cat > %s" % b.src_rpm, mode = "wb")
+ o = chroot.popen("cat > %s" % b.src_rpm, mode = "w")
try:
- bytes = util.sendfile(f, o)
+ shutil.copyfileobj(f, o)
except IOError as e:
b.log_line("error: unable to write to `%s': %s" % (b.src_rpm, e))
raise
+ bytes = float(f.headers['content-length'])
f.close()
o.close()
t = time.time() - start
diff --git a/PLD_Builder/srpm_builder.py b/PLD_Builder/srpm_builder.py
index 562b0c5..55d164e 100644
--- a/PLD_Builder/srpm_builder.py
+++ b/PLD_Builder/srpm_builder.py
@@ -23,7 +23,6 @@ import notify
import status
import build
import report
-#import messagebus
from lock import lock
from bqueue import B_Queue
@@ -33,9 +32,9 @@ def pick_request(q):
def mycmp(r1, r2):
if r1.kind != 'group' or r2.kind != 'group':
raise Exception("non-group requests")
- pri_diff = cmp(r1.priority, r2.priority)
+ pri_diff = util.cmp(r1.priority, r2.priority)
if pri_diff == 0:
- return cmp(r1.time, r2.time)
+ return util.cmp(r1.time, r2.time)
else:
return pri_diff
q.requests.sort(key=util.cmp_to_key(mycmp))
@@ -96,7 +95,6 @@ def build_srpm(r, b):
return "FAIL"
status.push("building %s" % b.spec)
-# messagebus.notify(topic="build_srpm.start", spec=b.spec, flags=r.flags, batch=b, request=r)
b.src_rpm = ""
builder_opts = "-nu -nm --nodeps --http --define \'_pld_builder 1\'"
@@ -142,7 +140,6 @@ def build_srpm(r, b):
if res:
res = "FAIL"
-# messagebus.notify(topic="build_srpm.finish", spec=b.spec)
return res
def handle_request(r):
diff --git a/PLD_Builder/util.py b/PLD_Builder/util.py
index 1d485a1..af6b8f1 100644
--- a/PLD_Builder/util.py
+++ b/PLD_Builder/util.py
@@ -32,15 +32,6 @@ def pkg_name(nvr):
def msg(m):
sys.stderr.write(m)
-def sendfile(src, dst):
- cnt = 0
- while 1:
- s = src.read(10000)
- if s == "": break
- cnt += len(s)
- dst.write(s)
- return cnt
-
def append_to(log, msg):
f = open(log, "a")
f.write("%s\n" % msg)
@@ -112,3 +103,6 @@ def to_str(s):
return s
else:
raise TypeError("Expected bytes or string, but got %s." % type(s))
+
+def cmp(a, b):
+ return (a > b) - (a < b)
================================================================
---- gitweb:
http://git.pld-linux.org/gitweb.cgi/projects/pld-builder.new.git/commitdiff/22ca9cb4886dac3daceaab5ebac185b639e0e874
More information about the pld-cvs-commit
mailing list