pld-builder.new: PLD_Builder/bqueue.py, PLD_Builder/request_handler.py, PLD...

arekm arekm at pld-linux.org
Sat Nov 13 22:38:12 CET 2010


Author: arekm                        Date: Sat Nov 13 21:38:12 2010 GMT
Module: pld-builder.new               Tag: HEAD
---- Log message:
dump to temporary file, try to fsync and then rename (all that to avoid queue file corruption on server hangs )

---- Files affected:
pld-builder.new/PLD_Builder:
   bqueue.py (1.17 -> 1.18) , request_handler.py (1.52 -> 1.53) , srpm_builder.py (1.72 -> 1.73) 

---- Diffs:

================================================================
Index: pld-builder.new/PLD_Builder/bqueue.py
diff -u pld-builder.new/PLD_Builder/bqueue.py:1.17 pld-builder.new/PLD_Builder/bqueue.py:1.18
--- pld-builder.new/PLD_Builder/bqueue.py:1.17	Thu Sep  9 22:54:29 2010
+++ pld-builder.new/PLD_Builder/bqueue.py	Sat Nov 13 22:38:07 2010
@@ -7,6 +7,7 @@
 import os
 import fcntl
 import string
+import tempfile
 
 # PLD_Builder:
 import gpg
@@ -20,13 +21,31 @@
         self.requests = []
         self.fd = None
 
-    def dump(self, f):
+    def dump(self, fname):
+        (f, tmpfname) = tempfile.mkstemp(dir=os.path.dirname(fname))
+        self.dump_fobj(f)
+        f.flush()
+        os.fsync(f.fileno())
+        f.close()
+        os.chmod(tmpfname, 0644)
+        os.rename(tmpfname, fname)
+
+    def dump_fobj(self, f):
         self.requests.reverse()
         for r in self.requests:
-            r.dump(f)
+            r.dump_fobj(f)
         self.requests.reverse()
 
-    def dump_html(self, f):
+    def dump_html(self, fname):
+        (f, tmpfname) = tempfile.mkstemp(dir=os.path.dirname(fname))
+        self.dump_html_fobj(f)
+        f.flush()
+        os.fsync(f.fileno())
+        f.close()
+        os.chmod(tmpfname, 0644)
+        os.rename(tmpfname, fname)
+
+    def dump_html_fobj(self, f):
         f.write("""
 <html>
     <head>
@@ -39,7 +58,7 @@
         )
         self.requests.reverse()
         for r in self.requests:
-            r.dump_html(f)
+            r.dump_html_fobj(f)
         self.requests.reverse()
         f.write("</body></html>\n")
 
@@ -106,14 +125,19 @@
         self._write_to(sio)
         sio.seek(0)
         sio.write(gpg.sign(sio.read()))
-        if os.access(name, os.F_OK): os.unlink(name)
+        sio.seek(0)
+        (f, tmpname) = tempfile.mkstemp(dir=os.path.dirname(name))
         if re.search(r"\.gz$", name):
-            f = gzip.open(name, "w", 6)
+            fgz = gzip.GzipFile(filename=name, mode="w", compresslevel=6, fileobj=f)
+            util.sendfile(sio, fgz)
+            fgz.close()
         else:
-            f = open(name, "w")
-        sio.seek(0)
-        util.sendfile(sio, f)
+            util.sendfile(sio, f)
+        f.flush()
+        os.fsync(f.fileno())
         f.close()
+        os.chmod(tmpname, 0644)
+        os.rename(tmpfname, name)
 
     def add(self, req):
         self.requests.append(req)

================================================================
Index: pld-builder.new/PLD_Builder/request_handler.py
diff -u pld-builder.new/PLD_Builder/request_handler.py:1.52 pld-builder.new/PLD_Builder/request_handler.py:1.53
--- pld-builder.new/PLD_Builder/request_handler.py:1.52	Wed Jun  2 20:03:23 2010
+++ pld-builder.new/PLD_Builder/request_handler.py	Sat Nov 13 22:38:07 2010
@@ -149,12 +149,9 @@
         return True
     q.requests = filter(leave_it, q.requests)
     q.write()
-    q.dump(open(path.queue_stats_file, "w"))
-    q.dump_html(open(path.queue_html_stats_file, "w"))
-    os.chmod(path.queue_html_stats_file, 0644)
-    os.chmod(path.queue_stats_file, 0644)
+    q.dump(path.queue_stats_file)
+    q.dump_html(path.queue_html_stats_file)
     q.write_signed(path.req_queue_signed_file)
-    os.chmod(path.req_queue_signed_file, 0644)
     q.unlock()
 
 def handle_request(req, filename = None):

================================================================
Index: pld-builder.new/PLD_Builder/srpm_builder.py
diff -u pld-builder.new/PLD_Builder/srpm_builder.py:1.72 pld-builder.new/PLD_Builder/srpm_builder.py:1.73
--- pld-builder.new/PLD_Builder/srpm_builder.py:1.72	Sat Feb 13 22:45:31 2010
+++ pld-builder.new/PLD_Builder/srpm_builder.py	Sat Nov 13 22:38:07 2010
@@ -58,12 +58,9 @@
     q.read()
     q.add(r)
     q.write()
-    q.dump(open(path.queue_stats_file, "w"))
-    q.dump_html(open(path.queue_html_stats_file, "w"))
-    os.chmod(path.queue_stats_file, 0644)
-    os.chmod(path.queue_html_stats_file, 0644)
+    q.dump(path.queue_stats_file)
+    q.dump_html(path.queue_html_stats_file)
     q.write_signed(path.req_queue_signed_file)
-    os.chmod(path.req_queue_signed_file, 0644)
     q.unlock()
     cnt_f.seek(0)
     cnt_f.write("%d\n" % num)
================================================================

---- CVS-web:
    http://cvs.pld-linux.org/cgi-bin/cvsweb.cgi/pld-builder.new/PLD_Builder/bqueue.py?r1=1.17&r2=1.18&f=u
    http://cvs.pld-linux.org/cgi-bin/cvsweb.cgi/pld-builder.new/PLD_Builder/request_handler.py?r1=1.52&r2=1.53&f=u
    http://cvs.pld-linux.org/cgi-bin/cvsweb.cgi/pld-builder.new/PLD_Builder/srpm_builder.py?r1=1.72&r2=1.73&f=u



More information about the pld-cvs-commit mailing list