pld-builder.new: PLD_Builder/config.py, PLD_Builder/request.py, PLD_Builder...

[arekm]

Author: arekm                        Date: Tue Mar  3 22:56:14 2009 GMT
Module: pld-builder.new               Tag: HEAD
---- Log message:
- add parallel builds support

---- Files affected:
pld-builder.new/PLD_Builder:
   config.py (1.37 -> 1.38) , request.py (1.57 -> 1.58) , rpm_builder.py (1.69 -> 1.70) 
pld-builder.new/config:
   builder.conf (1.38 -> 1.39) 

---- Diffs:

================================================================
Index: pld-builder.new/PLD_Builder/config.py
diff -u pld-builder.new/PLD_Builder/config.py:1.37 pld-builder.new/PLD_Builder/config.py:1.38
--- pld-builder.new/PLD_Builder/config.py:1.37	Mon Aug 25 15:36:03 2008
+++ pld-builder.new/PLD_Builder/config.py	Tue Mar  3 23:56:09 2009
@@ -95,6 +95,7 @@
         self.rpmqa_filename = get("rpmqa_filename")
         self.job_slots = int(get("job_slots"))
         self.max_load = float(get("max_load"))
+        self.max_jobs = int(get("max_jobs", "1"))
         self.rpm_cache_dir = get("rpm_cache_dir", "/spools/ready")
         self.builder_user = get("builder_user", "builder")
         self.sudo_chroot_wrapper = get("sudo_chroot_wrapper", "")

================================================================
Index: pld-builder.new/PLD_Builder/request.py
diff -u pld-builder.new/PLD_Builder/request.py:1.57 pld-builder.new/PLD_Builder/request.py:1.58
--- pld-builder.new/PLD_Builder/request.py:1.57	Fri Feb 27 23:04:43 2009
+++ pld-builder.new/PLD_Builder/request.py	Tue Mar  3 23:56:09 2009
@@ -9,6 +9,7 @@
 import util
 import log
 from acl import acl
+from config import config
 
 __all__ = ['parse_request', 'parse_requests']
     
@@ -43,6 +44,7 @@
         self.priority = 2
         self.time = time.time()
         self.requester = ""
+        self.max_jobs = min(os.sysconf('SC_NPROCESSORS_ONLN'), config.max_jobs)
         self.requester_email = ""
         self.flags = string.split(attr(e, "flags", ""))
         for c in e.childNodes:
@@ -58,6 +60,8 @@
                 self.priority = int(text(c))
             elif c.nodeName == "time":
                 self.time = int(text(c))
+            elif c.nodeName == "maxjobs":
+                self.max_jobs = min(int(text(c)), config.max_jobs)
             else:
                 log.panic("xml: evil group child (%s)" % c.nodeName)
         # note that we also check that group is sorted WRT deps
@@ -86,11 +90,11 @@
         f.write("\n")
 
     def dump_html(self, f):
-        f.write("<p><b>%d</b>. %s from %s <small>%s, %d, %s</small><br/>\n" % \
+        f.write("<p><b>%d</b>. %s from %s <small>%s, prio=%d, jobs=%d, %s</small><br/>\n" % \
                 (self.no,
                  escape(time.strftime("%Y.%m.%d %H:%M:%S", time.localtime(self.time))),
                  escape(self.requester),
-                 self.id, self.priority, string.join(self.flags)))
+                 self.id, self.priority, self.max_jobs, string.join(self.flags)))
         f.write("<ul>\n")
         for b in self.batches:
             b.dump_html(f, self.id)
@@ -102,9 +106,10 @@
        <group id="%s" no="%d" flags="%s">
          <requester email='%s'>%s</requester>
          <time>%d</time>
-         <priority>%d</priority>\n""" % (self.id, self.no, string.join(self.flags),
+         <priority>%d</priority>
+         <maxjobs>%d</maxjobs>\n""" % (self.id, self.no, string.join(self.flags),
                     escape(self.requester_email), escape(self.requester), 
-                    self.time, self.priority))
+                    self.time, self.priority, self.max_jobs))
         for b in self.batches:
             b.write_to(f)
         f.write("       </group>\n\n")

================================================================
Index: pld-builder.new/PLD_Builder/rpm_builder.py
diff -u pld-builder.new/PLD_Builder/rpm_builder.py:1.69 pld-builder.new/PLD_Builder/rpm_builder.py:1.70
--- pld-builder.new/PLD_Builder/rpm_builder.py:1.69	Fri Feb 27 23:04:43 2009
+++ pld-builder.new/PLD_Builder/rpm_builder.py	Tue Mar  3 23:56:09 2009
@@ -124,8 +124,8 @@
             if ("no-install-br" not in r.flags) and not install.install_br(r, b):
                 res = "FAIL_DEPS_INSTALL"
             if not res:
-                cmd = "cd rpm/SPECS; TMPDIR=%s nice -n %s rpmbuild -bb %s %s" % \
-                            (tmpdir, config.nice, rpmbuild_opt, b.spec)
+                cmd = "cd rpm/SPECS; TMPDIR=%s nice -n %s rpmbuild -bb --define '_smp_mflags -j%d' %s %s" % \
+                            (tmpdir, config.nice, r.max_jobs, rpmbuild_opt, b.spec)
                 b.log_line("building RPM using: %s" % cmd)
                 res = chroot.run(cmd, logfile = b.logfile)
                 if res:

================================================================
Index: pld-builder.new/config/builder.conf
diff -u pld-builder.new/config/builder.conf:1.38 pld-builder.new/config/builder.conf:1.39
--- pld-builder.new/config/builder.conf:1.38	Tue Aug 26 05:35:15 2008
+++ pld-builder.new/config/builder.conf	Tue Mar  3 23:56:09 2009
@@ -44,6 +44,9 @@
 # Maximal load, above which builds won't get started.
 max_load = 10.0
 
+# make -jX for single build
+max_jobs = 1
+
 # Nice control.
 #nice = 0
 
================================================================

---- CVS-web:
    http://cvs.pld-linux.org/cgi-bin/cvsweb.cgi/pld-builder.new/PLD_Builder/config.py?r1=1.37&r2=1.38&f=u
    http://cvs.pld-linux.org/cgi-bin/cvsweb.cgi/pld-builder.new/PLD_Builder/request.py?r1=1.57&r2=1.58&f=u
    http://cvs.pld-linux.org/cgi-bin/cvsweb.cgi/pld-builder.new/PLD_Builder/rpm_builder.py?r1=1.69&r2=1.70&f=u
    http://cvs.pld-linux.org/cgi-bin/cvsweb.cgi/pld-builder.new/config/builder.conf?r1=1.38&r2=1.39&f=u