[projects/git-slug: 7/9] gitrepo.check_remote() is expensive, so make it run in parallel.

[glen]

commit 1eab7322f498de90fb09096f0541e88bb8cc8caf
Author: Arkadiusz Miśkiewicz <arekm at maven.pl>
Date:   Fri Nov 28 12:47:40 2014 +0100

    gitrepo.check_remote() is expensive, so make it run in parallel.
    
    gitrepo.check_remote() now runs in parallel (as part of
    fetch_package()). Unfortunately lambdas cannot be pickled,
    so we have to get rid of it.

 git_slug/refsdata.py |  8 +++++++-
 slug.py              | 19 +++++++++++--------
 2 files changed, 18 insertions(+), 9 deletions(-)
---
diff --git a/git_slug/refsdata.py b/git_slug/refsdata.py
index 4354ac4..67592f8 100644
--- a/git_slug/refsdata.py
+++ b/git_slug/refsdata.py
@@ -16,7 +16,7 @@ class NoMatchedRepos(Exception):
 
 class RemoteRefsData:
     def __init__(self, stream, pattern, dirpattern=('*',)):
-        self.heads = collections.defaultdict(lambda: collections.defaultdict(lambda: EMPTYSHA1))
+        self.heads = collections.defaultdict(self.__dict_var__)
         pats = re.compile('|'.join(fnmatch.translate(os.path.join('refs/heads', p)) for p in pattern))
         dirpat = re.compile('|'.join(fnmatch.translate(p) for p in dirpattern))
         for line in stream.readlines():
@@ -28,6 +28,12 @@ class RemoteRefsData:
         if not self.heads:
             raise NoMatchedRepos
 
+    def __dict_init__(self):
+        return EMPTYSHA1
+
+    def __dict_var__(self):
+        return collections.defaultdict(self.__dict_init__)
+
     def put(self, repo, data):
         for line in data:
             (sha1_old, sha1, ref) = line.split()
diff --git a/slug.py b/slug.py
index b576df8..b4ac0fd 100755
--- a/slug.py
+++ b/slug.py
@@ -96,7 +96,16 @@ def getrefs(*args):
         sys.exit(2)
     return refs
 
-def fetch_package(gitrepo, ref2fetch, options):
+def fetch_package(gitrepo, refs_heads, options):
+    ref2fetch = []
+    for ref in refs_heads:
+        if gitrepo.check_remote(ref) != refs_heads[ref]:
+            ref2fetch.append('+{}:{}/{}'.format(ref, REMOTEREFS, ref[len('refs/heads/'):]))
+    if ref2fetch:
+        ref2fetch.append('refs/notes/*:refs/notes/*')
+    else:
+        return
+
     try:
         (stdout, stderr) = gitrepo.fetch(ref2fetch, options.depth)
         if stderr != b'':
@@ -130,13 +139,7 @@ def fetch_packages(options, return_all=False):
             continue
         else:
             gitrepo = GitRepo(os.path.join(options.packagesdir, pkgdir))
-        ref2fetch = []
-        for ref in refs.heads[pkgdir]:
-            if gitrepo.check_remote(ref) != refs.heads[pkgdir][ref]:
-                ref2fetch.append('+{}:{}/{}'.format(ref, REMOTEREFS, ref[len('refs/heads/'):]))
-        if ref2fetch:
-            ref2fetch.append('refs/notes/*:refs/notes/*')
-            args.append((gitrepo, ref2fetch, options))
+            args.append((gitrepo, refs.heads[pkgdir], options))
 
     updated_repos = []
     pool = WorkerPool(options.jobs, pool_worker_init)
================================================================

---- gitweb:

http://git.pld-linux.org/gitweb.cgi/projects/git-slug.git/commitdiff/d9fdf7d60e5aea39f9ac7065a09a645261fc712b