[projects/pld-builder.new] - more python 3.x fixes, can't test on PLD but TLD version works fine

hawk hawk at pld-linux.org
Mon May 3 11:51:57 CEST 2021


commit 22ca9cb4886dac3daceaab5ebac185b639e0e874
Author: Marcin Krol <hawk at tld-linux.org>
Date:   Mon May 3 11:49:50 2021 +0200

    - more python 3.x fixes, can't test on PLD but TLD version works fine

 PLD_Builder/bqueue.py          |  2 +-
 PLD_Builder/chroot.py          | 54 ++++++++++++++----------------------------
 PLD_Builder/file_sender.py     | 15 ++++++------
 PLD_Builder/install.py         | 12 +++++-----
 PLD_Builder/load_balancer.py   |  2 +-
 PLD_Builder/mailer.py          |  6 ++---
 PLD_Builder/maintainer.py      |  2 +-
 PLD_Builder/request.py         |  2 +-
 PLD_Builder/request_fetcher.py | 18 +++++++-------
 PLD_Builder/request_handler.py |  3 ---
 PLD_Builder/rpm_builder.py     | 31 +++++++++++++-----------
 PLD_Builder/srpm_builder.py    |  7 ++----
 PLD_Builder/util.py            | 12 +++-------
 13 files changed, 69 insertions(+), 97 deletions(-)
---
diff --git a/PLD_Builder/bqueue.py b/PLD_Builder/bqueue.py
index 4699fd7..e33483e 100644
--- a/PLD_Builder/bqueue.py
+++ b/PLD_Builder/bqueue.py
@@ -133,7 +133,7 @@ class B_Queue:
         sio.seek(0)
         (fdno, tmpname) = tempfile.mkstemp(dir=os.path.dirname(name))
         f = os.fdopen(fdno, "w")
-        util.sendfile(sio, f)
+        shutil.copyfileobj(sio, f)
         f.flush()
         os.fsync(f.fileno())
         f.close()
diff --git a/PLD_Builder/chroot.py b/PLD_Builder/chroot.py
index 8acd58d..6e7976a 100644
--- a/PLD_Builder/chroot.py
+++ b/PLD_Builder/chroot.py
@@ -4,11 +4,8 @@ import os
 import re
 import random
 import util
-
-try:
-    from hashlib import md5 as md5
-except ImportError:
-    from md5 import md5
+import shutil
+import subprocess
 
 from config import config
 
@@ -27,8 +24,13 @@ def command_sh(cmd):
     return "%s sudo chroot %s /bin/sh -c \"export LC_ALL=C; exec < /dev/null; %s\"" \
             % (config.sudo_chroot_wrapper, config.chroot, quote(cmd))
 
-def popen(cmd, user = "builder", mode = "r"):
-    f = os.popen(command(cmd, user), mode)
+def popen(cmd, user = "builder", mode = "r", encoding = None):
+    if mode == "r":
+        p = subprocess.Popen(command(cmd, user), shell=True, stdout=subprocess.PIPE, close_fds=True, encoding=encoding)
+        f = p.stdout
+    else:
+        p = subprocess.Popen(command(cmd, user), shell=True, stdin=subprocess.PIPE, close_fds=True, encoding=encoding)
+        f = p.stdin
     return f
 
 def run(cmd, user = "builder", logfile = None, logstdout = None):
@@ -49,36 +51,16 @@ def run(cmd, user = "builder", logfile = None, logstdout = None):
         return r
 
 def cp(file, outfile, user="builder", rm=False):
-    m = md5()
-    m.update(util.to_bytes(str(random.sample(range(100000), 500))))
-    digest = m.hexdigest()
-
-    marker_start = "--- FILE BEGIN DIGEST %s ---" % digest
-    marker_end = "--- FILE END DIGEST %s ---" % digest
-
-    f = open(outfile, 'wb')
-    cmd = "echo \"%s\"; cat %s; echo \"%s\"" % (marker_start, file, marker_end)
+    f_out = open(outfile, 'wb')
+    cmd = "cat %s" % file
     if rm:
         cmd += "; rm %s" % file
-    c = command(cmd, user)
-    p = os.popen(c)
-    # get file contents
-    marker = False
-    for l in p:
-        if not marker and l.strip() == marker_start:
-            marker = True
-            continue
-        me = l.find(marker_end)
-        if me != -1:
-            l = l[:me]
-            f.write(util.to_bytes(l))
-            marker = False
-            break
-        if marker:
-            f.write(util.to_bytes(l))
-    rp = p.close()
-    rf = f.close()
-    if rp == None:
+    p = subprocess.Popen(command(cmd, user), shell=True, stdout=subprocess.PIPE, close_fds=True)
+    f_in = p.stdout
+    shutil.copyfileobj(f_in, f_out)
+    f_out.close()
+    r = f_in.close()
+    if r == None:
         return 0
     else:
-        return rp
+        return r
diff --git a/PLD_Builder/file_sender.py b/PLD_Builder/file_sender.py
index c6cc459..3ebf240 100644
--- a/PLD_Builder/file_sender.py
+++ b/PLD_Builder/file_sender.py
@@ -8,8 +8,7 @@ import time
 import shutil
 import sys
 import traceback
-import urllib2
-
+import urllib.request
 from config import config, init_conf
 import mailer
 import path
@@ -94,11 +93,11 @@ def post_file(src, url):
     global problems
     try:
         f = open(src, 'r')
-        data = f.read()
+        data = f.read().encode('utf-8')
         f.close()
-        req = urllib2.Request(url, data)
-        req.add_header('X-Filename', os.path.basename(src))
-        f = urllib2.urlopen(req)
+        headers = { 'X-Filename' : os.path.basename(src) }
+        req = urllib.request.Request(url, data=data, headers=headers)
+        f = urllib.request.urlopen(req)
         f.close()
     except Exception as e:
         problems[src] = e
@@ -164,9 +163,9 @@ def flush_queue(dir):
         d = read_name_val(f)
         if d != None: q.append(d)
     def mycmp(x, y):
-        rc = cmp(x['Time'], y['Time'])
+        rc = util.cmp(x['Time'], y['Time'])
         if rc == 0 and 'Type' in x and 'Type' in y:
-            return cmp(x['Type'], y['Type'])
+            return util.cmp(x['Type'], y['Type'])
         else:
             return rc
     q.sort(key=util.cmp_to_key(mycmp))
diff --git a/PLD_Builder/install.py b/PLD_Builder/install.py
index 3e11817..91a115e 100644
--- a/PLD_Builder/install.py
+++ b/PLD_Builder/install.py
@@ -28,7 +28,7 @@ def close_killset(killset):
             del killset[p]
             errors += "cannot remove %s because it's crucial\n" % p
         else:
-            f = chroot.popen("poldek --noask --test --test --erase %s" % p, user = "root")
+            f = chroot.popen("poldek --noask --test --test --erase %s" % p, user = "root", encoding = "utf-8")
             crucial = 0
             e = []
             for l in f:
@@ -49,7 +49,7 @@ def close_killset(killset):
     return errors
 
 def upgrade_from_batch(r, b):
-    f = chroot.popen("rpm --test -F %s 2>&1" % ' '.join(b.files), user = "root")
+    f = chroot.popen("rpm --test -F %s 2>&1" % ' '.join(b.files), user = "root", encoding = "utf-8")
     killset = {}
     rx = re.compile(r' \(installed\) (?P<name>[^\s]+)-[^-]+-[^-]+$')
     for l in f:
@@ -107,7 +107,7 @@ def uninstall_self_conflict(b):
         'rpmdefs' : b.rpmbuild_opts(),
         'topdir' : b.get_topdir(),
         'spec': b.spec,
-    })
+    }, encoding = "utf-8")
     # java-sun >= 1.5 conflicts with soprano-2.1.67-1.src
     # java-sun conflicts with soprano-2.1.67-1.src
     rx = re.compile(r"\s+(?P<name>[\w-]+)\s+.*conflicts with [^\s]+-[^-]+-[^-]+\.src($| .*)")
@@ -125,7 +125,7 @@ def uninstall_self_conflict(b):
 
 def install_br(r, b):
     def is_rpmorg():
-        f = chroot.popen("rpm --version 2>&1")
+        f = chroot.popen("rpm --version 2>&1", encoding = "utf-8")
         v = re.compile(r'(RPM version|rpm \(RPM\)) (?P<major>\d)\.(?P<minor>\d+)(\.\d+)?')
         for l in f:
             m = v.search(l)
@@ -154,7 +154,7 @@ def install_br(r, b):
             'rpmdefs' : b.rpmbuild_opts(),
             'spec': b.spec,
         }
-        f = chroot.popen(cmd)
+        f = chroot.popen(cmd, encoding = "utf-8")
         rx = re.compile(r"^\s*(?P<name>[^\s]+) .*is needed by")
         needed = {}
         b.log_line("checking BR")
@@ -180,7 +180,7 @@ def install_br(r, b):
     chroot.run("poldek --up --upa", user = "root", logfile = b.logfile)
     # check conflicts in BRed packages
     b.log_line("checking conflicting packages in BRed packages")
-    f = chroot.popen("poldek --test --test --noask --caplookup -Q -v %s --upgrade %s" % (b.ignores(), br), user = "root")
+    f = chroot.popen("poldek --test --test --noask --caplookup -Q -v %s --upgrade %s" % (b.ignores(), br), user = "root", encoding = "utf-8")
     # phonon-devel-4.3.1-1.i686 conflicts with qt4-phonon-devel-4.5.0-6.i686
     # jdbc-stdext >= 2.0 is required by installed java-struts-1.3.10-1.noarch
     # jmx is needed by (installed) java-commons-modeler-2.0-1.noarch
diff --git a/PLD_Builder/load_balancer.py b/PLD_Builder/load_balancer.py
index 43d85d5..459034b 100644
--- a/PLD_Builder/load_balancer.py
+++ b/PLD_Builder/load_balancer.py
@@ -39,7 +39,7 @@ def builders_order():
             log.alert("found strange lock in got-lock: %s" % b)
 
     def mycmp(b1, b2):
-        return cmp(bs[b1], bs[b2])
+        return util.cmp(bs[b1], bs[b2])
 
     bl.sort(key=util.cmp_to_key(mycmp))
 
diff --git a/PLD_Builder/mailer.py b/PLD_Builder/mailer.py
index 2cf324d..7d90091 100644
--- a/PLD_Builder/mailer.py
+++ b/PLD_Builder/mailer.py
@@ -5,7 +5,7 @@ import os
 import sys
 from io import StringIO
 from config import config
-import util
+import shutil
 import log
 
 def recode(s):
@@ -58,7 +58,7 @@ class Message:
                     self.body.write("\n\n[...]\n\n")
                 line += 1
         else:
-            util.sendfile(open(log), self.body)
+            shutil.copyfileobj(open(log), self.body)
 
     def set_std_headers(self):
         self.headers["Date"] = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())
@@ -73,7 +73,7 @@ class Message:
             f.write("%s: %s\n" % (k, v))
         f.write("\n")
         self.body.seek(0)
-        util.sendfile(self.body, f)
+        shutil.copyfileobj(self.body, f)
 
     def send(self):
         if not os.path.exists("/usr/lib/sendmail"):
diff --git a/PLD_Builder/maintainer.py b/PLD_Builder/maintainer.py
index 6b1b45a..4d909df 100644
--- a/PLD_Builder/maintainer.py
+++ b/PLD_Builder/maintainer.py
@@ -39,7 +39,7 @@ def handle_src():
 
 def handle_bin():
     send_rpmqa()
-    f=chroot.popen("""ls -l --time-style +%s /spools/ready""", 'root')
+    f=chroot.popen("""ls -l --time-style +%s /spools/ready""", "root", encoding = "utf-8")
     rmpkgs=[]
     curtime=time.time()
     for i in f:
diff --git a/PLD_Builder/request.py b/PLD_Builder/request.py
index 71a422c..0b71d5f 100644
--- a/PLD_Builder/request.py
+++ b/PLD_Builder/request.py
@@ -7,7 +7,7 @@ import time
 import xml.sax.saxutils
 import fnmatch
 import os
-import urllib
+import urllib.parse
 import html
 import pytz
 import tempfile
diff --git a/PLD_Builder/request_fetcher.py b/PLD_Builder/request_fetcher.py
index 25b130f..83229ae 100644
--- a/PLD_Builder/request_fetcher.py
+++ b/PLD_Builder/request_fetcher.py
@@ -3,16 +3,16 @@
 import string
 import signal
 import os
-import urllib
-import urllib2
+import urllib.request
 import sys
-from io import StringIO
+from io import StringIO, BytesIO
 import gzip
 import path
 import log
 import status
 import lock
 import util
+import shutil
 import gpg
 import request
 import loop
@@ -42,8 +42,8 @@ def has_new(control_url):
     signal.alarm(300)
     try:
         headers = { 'Cache-Control': 'no-cache', 'Pragma': 'no-cache' }
-        req = urllib2.Request(url=control_url + "/max_req_no", headers=headers)
-        f = urllib2.urlopen(req)
+        req = urllib.request.Request(url=control_url + "/max_req_no", headers=headers)
+        f = urllib.request.urlopen(req)
         count = int(f.readline().strip())
         signal.alarm(0)
     except Exception as e:
@@ -62,15 +62,15 @@ def fetch_queue(control_url):
     signal.alarm(300)
     try:
         headers = { 'Cache-Control': 'no-cache', 'Pragma': 'no-cache' }
-        req = urllib2.Request(url=control_url + "/queue.gz", headers=headers)
-        f = urllib2.urlopen(req)
+        req = urllib.request.Request(url=control_url + "/queue.gz", headers=headers)
+        f = urllib.request.urlopen(req)
         signal.alarm(0)
     except Exception as e:
         signal.alarm(0)
         log.error("can't fetch %s: %s" % (control_url + "/queue.gz", e))
         sys.exit(1)
-    sio = StringIO()
-    util.sendfile(f, sio)
+    sio = BytesIO()
+    shutil.copyfileobj(f, sio)
     f.close()
     sio.seek(0)
     f = gzip.GzipFile(fileobj = sio)
diff --git a/PLD_Builder/request_handler.py b/PLD_Builder/request_handler.py
index 70a1866..74c7f1c 100644
--- a/PLD_Builder/request_handler.py
+++ b/PLD_Builder/request_handler.py
@@ -19,7 +19,6 @@ from lock import lock
 from bqueue import B_Queue
 from config import config, init_conf
 from mailer import Message
-#import messagebus
 
 def check_double_id(id):
     id_nl = id + "\n"
@@ -196,10 +195,8 @@ def handle_request(req, filename = None):
     status.push("request from %s" % user.login)
     r = request.parse_request(body)
     if r.kind == 'group':
-#        messagebus.notify(topic="request.group", user=user.login, **r.dump_json())
         handle_group(r, user)
     elif r.kind == 'notification':
-#        messagebus.notify(topic="request.notify", user=user.login, **r.dump_json())
         handle_notification(r, user)
     else:
         msg = "%s: don't know how to handle requests of this kind '%s'" \
diff --git a/PLD_Builder/rpm_builder.py b/PLD_Builder/rpm_builder.py
index 1f1ed2c..f5d4e95 100644
--- a/PLD_Builder/rpm_builder.py
+++ b/PLD_Builder/rpm_builder.py
@@ -8,13 +8,15 @@ import atexit
 import time
 import datetime
 import string
-import urllib
-import urllib2
+import urllib.request
+import urllib.parse
+import urllib.error
 
 from config import config, init_conf
 from bqueue import B_Queue
 import lock
 import util
+import shutil
 import loop
 import path
 import status
@@ -46,9 +48,9 @@ def pick_request(q):
     def mycmp(r1, r2):
         if r1.kind != 'group' or r2.kind != 'group':
             raise Exception("non-group requests")
-        pri_diff = cmp(r1.priority, r2.priority)
+        pri_diff = util.cmp(r1.priority, r2.priority)
         if pri_diff == 0:
-            return cmp(r1.time, r2.time)
+            return util.cmp(r1.time, r2.time)
         else:
             return pri_diff
     q.requests.sort(key=util.cmp_to_key(mycmp))
@@ -62,12 +64,12 @@ def check_skip_build(r, b):
     while not good:
         try:
             headers = { 'Cache-Control': 'no-cache', 'Pragma': 'no-cache' }
-            req = urllib2.Request(url=src_url, headers=headers)
-            f = urllib2.urlopen(req)
+            req = urllib.request.Request(url=src_url, headers=headers)
+            f = urllib.request.urlopen(req)
             good = True
-        except urllib2.HTTPError as error:
+        except urllib.error.HTTPError as error:
             return False
-        except urllib2.URLError as error:
+        except urllib.error.URLError as error:
             # see errno.h
             try:
                 errno = error.errno
@@ -92,10 +94,10 @@ def fetch_src(r, b):
     while not good:
         try:
             headers = { 'Cache-Control': 'no-cache', 'Pragma': 'no-cache' }
-            req = urllib2.Request(url=src_url, headers=headers)
-            f = urllib2.urlopen(req)
+            req = urllib.request.Request(url=src_url, headers=headers)
+            f = urllib.request.urlopen(req)
             good = True
-        except urllib2.HTTPError as error:
+        except urllib.error.HTTPError as error:
             # fail in a way where cron job will retry
             msg = "unable to fetch url %s, http code: %d" % (src_url, error.code)
             b.log_line(msg)
@@ -107,7 +109,7 @@ def fetch_src(r, b):
                 msg = "in queue for more than 6 hours, download failing"
                 b.log_line(msg)
                 return False
-        except urllib2.URLError as error:
+        except urllib.error.URLError as error:
             errno = 0
             if isinstance(error.args[0], IOError):
                 errno = error.args[0].errno
@@ -126,14 +128,15 @@ def fetch_src(r, b):
                     print("error.reason exception %s" % e)
                 raise
 
-    o = chroot.popen("cat > %s" % b.src_rpm, mode = "wb")
+    o = chroot.popen("cat > %s" % b.src_rpm, mode = "w")
 
     try:
-        bytes = util.sendfile(f, o)
+        shutil.copyfileobj(f, o)
     except IOError as e:
         b.log_line("error: unable to write to `%s': %s" % (b.src_rpm, e))
         raise
 
+    bytes = float(f.headers['content-length'])
     f.close()
     o.close()
     t = time.time() - start
diff --git a/PLD_Builder/srpm_builder.py b/PLD_Builder/srpm_builder.py
index 562b0c5..55d164e 100644
--- a/PLD_Builder/srpm_builder.py
+++ b/PLD_Builder/srpm_builder.py
@@ -23,7 +23,6 @@ import notify
 import status
 import build
 import report
-#import messagebus
 
 from lock import lock
 from bqueue import B_Queue
@@ -33,9 +32,9 @@ def pick_request(q):
     def mycmp(r1, r2):
         if r1.kind != 'group' or r2.kind != 'group':
             raise Exception("non-group requests")
-        pri_diff = cmp(r1.priority, r2.priority)
+        pri_diff = util.cmp(r1.priority, r2.priority)
         if pri_diff == 0:
-            return cmp(r1.time, r2.time)
+            return util.cmp(r1.time, r2.time)
         else:
             return pri_diff
     q.requests.sort(key=util.cmp_to_key(mycmp))
@@ -96,7 +95,6 @@ def build_srpm(r, b):
         return "FAIL"
 
     status.push("building %s" % b.spec)
-#    messagebus.notify(topic="build_srpm.start", spec=b.spec, flags=r.flags, batch=b, request=r)
 
     b.src_rpm = ""
     builder_opts = "-nu -nm --nodeps --http --define \'_pld_builder 1\'"
@@ -142,7 +140,6 @@ def build_srpm(r, b):
     if res:
         res = "FAIL"
 
-#    messagebus.notify(topic="build_srpm.finish", spec=b.spec)
     return res
 
 def handle_request(r):
diff --git a/PLD_Builder/util.py b/PLD_Builder/util.py
index 1d485a1..af6b8f1 100644
--- a/PLD_Builder/util.py
+++ b/PLD_Builder/util.py
@@ -32,15 +32,6 @@ def pkg_name(nvr):
 def msg(m):
     sys.stderr.write(m)
 
-def sendfile(src, dst):
-    cnt = 0
-    while 1:
-        s = src.read(10000)
-        if s == "": break
-        cnt += len(s)
-        dst.write(s)
-    return cnt
-
 def append_to(log, msg):
     f = open(log, "a")
     f.write("%s\n" % msg)
@@ -112,3 +103,6 @@ def to_str(s):
         return s
     else:
         raise TypeError("Expected bytes or string, but got %s." % type(s))
+
+def cmp(a, b):
+    return (a > b) - (a < b)
================================================================

---- gitweb:

http://git.pld-linux.org/gitweb.cgi/projects/pld-builder.new.git/commitdiff/22ca9cb4886dac3daceaab5ebac185b639e0e874



More information about the pld-cvs-commit mailing list