[projects/pld-ftp-admin] Process all trees. Some speed improvements.

[arekm]

commit d1dd5b813ea377d903f5de59993d55472b1c8bd6
Author: Arkadiusz Miśkiewicz <arekm at maven.pl>
Date:   Tue Mar 10 23:42:40 2026 +0100

    Process all trees. Some speed improvements.

 wwwbin/list-no-digest-packages.sh |  12 ++---
 wwwbin/no-digest-packages.py      | 105 +++++++++++++++++++++++++++++++-------
 2 files changed, 89 insertions(+), 28 deletions(-)
---
diff --git a/wwwbin/list-no-digest-packages.sh b/wwwbin/list-no-digest-packages.sh
index 3c15fd4..0ed663d 100755
--- a/wwwbin/list-no-digest-packages.sh
+++ b/wwwbin/list-no-digest-packages.sh
@@ -2,12 +2,6 @@
 
 export LC_ALL=C
 
-date > $HOME/www/no-digest.txt.new
-
-/usr/bin/time -f '\nElapsed time: %E' -o $HOME/www/no-digest.txt.time $HOME/bin/no-digest-packages.py >> $HOME/www/no-digest.txt.new 2>/dev/null
-
-cat $HOME/www/no-digest.txt.time >>$HOME/www/no-digest.txt.new
-rm -f $HOME/www/no-digest.txt.time
-
-mv $HOME/www/no-digest.txt.new $HOME/www/no-digest.txt
-chmod 644 $HOME/www/no-digest.txt
+$HOME/bin/no-digest-packages.py -o $HOME/www/no-digest.txt
+$HOME/bin/no-digest-packages.py -t PLD,ready -o $HOME/www/no-digest-ready.txt
+$HOME/bin/no-digest-packages.py -t PLD,ready,test -o $HOME/www/no-digest-ready-test.txt
diff --git a/wwwbin/no-digest-packages.py b/wwwbin/no-digest-packages.py
index fa926b2..67dce2c 100755
--- a/wwwbin/no-digest-packages.py
+++ b/wwwbin/no-digest-packages.py
@@ -1,42 +1,109 @@
 #!/usr/bin/python3
 
+import argparse
 from datetime import date
+import io
 import os
-import subprocess
+import rpm
 import sys
+import tempfile
 import time
 
 sys.path.insert(0, os.environ['HOME']+'/pld-ftp-admin/modules')
 import config
 import ftptree
 
+parser = argparse.ArgumentParser(description='Find packages without RPM digests')
+parser.add_argument('--tree', '-t', default='PLD,ready,test',
+    help='comma-separated list of trees to scan in order (default: PLD,ready,test)')
+parser.add_argument('--output', '-o', default=None,
+    help='output file (default: stdout)')
+args = parser.parse_args()
+tree_names = [t.strip() for t in args.tree.split(',')]
+out = io.StringIO() if args.output else sys.stdout
+
 # The day we switched from rpm5 to rpm.org...
 MAX_DATE = date.fromisoformat('2021-03-15')
 
+start_time = time.monotonic()
+print(f'Started: {time.strftime("%a %b %d %H:%M:%S %Z %Y")}. Trees: {", ".join(tree_names)}', file=out)
+print('---', file=out)
+
 pkgs = {}
 bad = {}
 
-try:
-    tree = ftptree.FtpTree('PLD')
-except ftptree.SomeError as e:
-    print(e)
+trees = {}
+for treename in tree_names:
+    try:
+        trees[treename] = ftptree.FtpTree(treename)
+    except ftptree.SomeError:
+        pass
+
+if not trees:
+    print("No trees available")
     sys.exit(1)
 
-with os.scandir(config.value['ftp_dir']+'/PLD/SRPMS/.metadata') as it:
-    for entry in it:
-        if not entry.name.endswith('.src.rpm.info'):
-            continue
-        stat = entry.stat()
-        if date.fromtimestamp(stat.st_mtime) >= MAX_DATE:
-            continue
-        pkgs[entry.name[:-13]] = ftptree.Pkg(entry.name[:-13], tree)
+# Scan trees in order of freshness: PLD < ready < test
+# Later trees override earlier ones, so we keep only the freshest version
+for treename in tree_names:
+    if treename not in trees:
+        continue
+    metadata_dir = config.value['ftp_dir'] + '/' + treename + '/SRPMS/.metadata'
+    if not os.path.isdir(metadata_dir):
+        continue
+    with os.scandir(metadata_dir) as it:
+        for entry in it:
+            if not entry.name.endswith('.src.rpm.info'):
+                continue
+            nvr = entry.name[:-13]
+            pkg = ftptree.Pkg(nvr, trees[treename])
+            if treename == 'PLD':
+                # Only old packages (pre rpm.org switch) may lack digests
+                stat = entry.stat()
+                if date.fromtimestamp(stat.st_mtime) >= MAX_DATE:
+                    continue
+                pkgs[pkg.name] = pkg
+            else:
+                # ready/test have no date filter - they are rebuilds;
+                # only override existing PLD entries (skip new-only packages)
+                if pkg.name in pkgs:
+                    pkgs[pkg.name] = pkg
+
+ts = rpm.TransactionSet()
+ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES | rpm._RPMVSF_NODIGESTS)
+
+# Replaces: rpm --install --test --nodeps --justdb <file>
+# and checking stderr for "does not verify: no digest"
+def has_no_digest(rpm_file):
+    fd = rpm.fd.open(rpm_file)
+    try:
+        h = ts.hdrFromFdno(fd)
+    except rpm.error:
+        return False
+    finally:
+        fd.close()
+    return (h[rpm.RPMTAG_SHA256HEADER] is None
+            and h[rpm.RPMTAG_SHA3_256HEADER] is None)
 
 for name in sorted(pkgs.keys()):
-    for rpm in pkgs[name].rpmfiles(debugfiles=False, sourcefiles=False):
-        proc = subprocess.run(['/bin/rpm', '--install', '--test', '--nodeps', '--justdb', rpm], capture_output=True, text=True)
-        if 'does not verify: no digest' in proc.stderr:
-            bad[pkgs[name].name] = name
+    pkg = pkgs[name]
+    nvr = pkg.nvr
+    for rpmfile in pkg.rpmfiles(debugfiles=False, sourcefiles=False):
+        if has_no_digest(rpmfile):
+            bad[name] = (nvr, pkg.tree.treename)
             break
 
-for package in bad:
-    print(f'{package}.spec: {bad[package]}')
+for name in bad:
+    nvr, treename = bad[name]
+    print(f'{name}.spec: {nvr}: {treename}', file=out)
+
+elapsed = time.monotonic() - start_time
+print('---', file=out)
+print(f'{len(bad)} packages without digest out of {len(pkgs)} checked in {elapsed:.1f}s', file=out)
+
+if args.output:
+    output_dir = os.path.dirname(args.output) or '.'
+    with tempfile.NamedTemporaryFile(mode='w', dir=output_dir, delete=False) as tmp:
+        tmp.write(out.getvalue())
+    os.chmod(tmp.name, 0o644)
+    os.rename(tmp.name, args.output)
================================================================

---- gitweb:

http://git.pld-linux.org/gitweb.cgi/projects/pld-ftp-admin.git/commitdiff/d1dd5b813ea377d903f5de59993d55472b1c8bd6