[packages/git-core-slug] - as simple diff
arekm
arekm at pld-linux.org
Fri Nov 28 13:24:21 CET 2014
commit 8794fb23c91d328ace89beb41ea97055eae9378a
Author: Arkadiusz Miśkiewicz <arekm at maven.pl>
Date: Fri Nov 28 13:24:15 2014 +0100
- as simple diff
git-core-slug-git.patch | 385 +++++++++++++++++++-----------------------------
1 file changed, 148 insertions(+), 237 deletions(-)
---
diff --git a/git-core-slug-git.patch b/git-core-slug-git.patch
index 571f246..bc0406e 100644
--- a/git-core-slug-git.patch
+++ b/git-core-slug-git.patch
@@ -1,13 +1,30 @@
-commit 697db3c82877c46c4567e744a8bc15f1748b94f7
-Author: Arkadiusz Miśkiewicz <arekm at maven.pl>
-Date: Fri Nov 28 12:47:40 2014 +0100
-
- gitrepo.check_remote() is expensive, so make it run in parallel.
-
- gitrepo.check_remote() now runs in parallel (as part of
- fetch_package()). Unfortunately lambdas cannot be pickled,
- so we have to get rid of it.
-
+diff --git a/git_slug/gitrepo.py b/git_slug/gitrepo.py
+index 5234deb..d9f88ee 100644
+--- a/git_slug/gitrepo.py
++++ b/git_slug/gitrepo.py
+@@ -82,12 +82,21 @@ class GitRepo:
+ 'refs/notes/*:refs/notes/*'])
+
+ def check_remote(self, ref, remote=REMOTE_NAME):
++ localref = EMPTYSHA1
+ ref = ref.replace(REFFILE, os.path.join('remotes', remote))
+ try:
+ with open(os.path.join(self.gdir, ref), 'r') as f:
+ localref = f.readline().strip()
+ except IOError:
+- localref = EMPTYSHA1
++ try:
++ with open(os.path.join(self.gdir, 'packed-refs')) as f:
++ for line in f:
++ line_data = line.split()
++ if len(line_data) == 2 and line_data[1] == ref:
++ localref = line_data[0].strip()
++ break
++ except IOError:
++ pass
+ return localref
+
+ def showfile(self, filename, ref="/".join([REMOTE_NAME, "master"])):
diff --git a/git_slug/refsdata.py b/git_slug/refsdata.py
index 4354ac4..67592f8 100644
--- a/git_slug/refsdata.py
@@ -35,173 +52,7 @@ index 4354ac4..67592f8 100644
for line in data:
(sha1_old, sha1, ref) = line.split()
diff --git a/slug.py b/slug.py
-index b576df8..68f68cd 100755
---- a/slug.py
-+++ b/slug.py
-@@ -96,7 +96,14 @@ def getrefs(*args):
- sys.exit(2)
- return refs
-
--def fetch_package(gitrepo, ref2fetch, options):
-+def fetch_package(gitrepo, refs_heads, options):
-+ ref2fetch = []
-+ for ref in refs_heads:
-+ if gitrepo.check_remote(ref) != refs_heads[ref]:
-+ ref2fetch.append('+{}:{}/{}'.format(ref, REMOTEREFS, ref[len('refs/heads/'):]))
-+ if ref2fetch:
-+ ref2fetch.append('refs/notes/*:refs/notes/*')
-+
- try:
- (stdout, stderr) = gitrepo.fetch(ref2fetch, options.depth)
- if stderr != b'':
-@@ -130,13 +137,7 @@ def fetch_packages(options, return_all=False):
- continue
- else:
- gitrepo = GitRepo(os.path.join(options.packagesdir, pkgdir))
-- ref2fetch = []
-- for ref in refs.heads[pkgdir]:
-- if gitrepo.check_remote(ref) != refs.heads[pkgdir][ref]:
-- ref2fetch.append('+{}:{}/{}'.format(ref, REMOTEREFS, ref[len('refs/heads/'):]))
-- if ref2fetch:
-- ref2fetch.append('refs/notes/*:refs/notes/*')
-- args.append((gitrepo, ref2fetch, options))
-+ args.append((gitrepo, refs.heads[pkgdir], options))
-
- updated_repos = []
- pool = WorkerPool(options.jobs, pool_worker_init)
-
-commit 3aa5fead45cce8c63eef64b98ce5dd215cd7dc24
-Author: Arkadiusz Miśkiewicz <arekm at maven.pl>
-Date: Fri Nov 28 12:15:20 2014 +0100
-
- Initialize updated_repos to empty list.
-
-diff --git a/slug.py b/slug.py
-index 914e894..b576df8 100755
---- a/slug.py
-+++ b/slug.py
-@@ -138,6 +138,7 @@ def fetch_packages(options, return_all=False):
- ref2fetch.append('refs/notes/*:refs/notes/*')
- args.append((gitrepo, ref2fetch, options))
-
-+ updated_repos = []
- pool = WorkerPool(options.jobs, pool_worker_init)
- try:
- updated_repos = pool.starmap(fetch_package, args)
-
-commit 3482f3141eb1ecc9cc44d7b6d5af359960a49e73
-Author: Arkadiusz Miśkiewicz <arekm at maven.pl>
-Date: Fri Nov 28 12:13:27 2014 +0100
-
- Parallelize initpackage operation.
-
-diff --git a/slug.py b/slug.py
-index fa8fd89..914e894 100755
---- a/slug.py
-+++ b/slug.py
-@@ -108,15 +108,25 @@ def fetch_package(gitrepo, ref2fetch, options):
- def fetch_packages(options, return_all=False):
- refs = getrefs(options.branch, options.repopattern)
- print('Read remotes data')
-+ pkgs_new = []
-+ if options.newpkgs:
-+ for pkgdir in sorted(refs.heads):
-+ gitdir = os.path.join(options.packagesdir, pkgdir, '.git')
-+ if not os.path.isdir(gitdir):
-+ pkgs_new.append(pkgdir)
-+
-+ pool = WorkerPool(options.jobs, pool_worker_init)
-+ try:
-+ pool.starmap(initpackage, zip(pkgs_new, [options] * len(pkgs_new)))
-+ except KeyboardInterrupt:
-+ pool.terminate()
-+ else:
-+ pool.close()
-+ pool.join()
-+
- args = []
- for pkgdir in sorted(refs.heads):
-- gitdir = os.path.join(options.packagesdir, pkgdir, '.git')
-- if not os.path.isdir(gitdir):
-- if options.newpkgs:
-- gitrepo = initpackage(pkgdir, options)
-- else:
-- continue
-- elif options.omitexisting:
-+ if options.omitexisting and pkgdir not in pkgs_new:
- continue
- else:
- gitrepo = GitRepo(os.path.join(options.packagesdir, pkgdir))
-
-commit 29ab16f193cf3ebccb0c044b98f2ba9be98c3090
-Author: Arkadiusz Miśkiewicz <arekm at maven.pl>
-Date: Sun Nov 23 00:32:46 2014 +0100
-
- Make updateall off by default.
-
- Turn updateall to off by default so 'git pld pull' will only checkout
- packages that were fetched at this session. If you want old behaviour
- use 'git pld pull --all'.
-
-diff --git a/slug.py b/slug.py
-index d083cf4..fa8fd89 100755
---- a/slug.py
-+++ b/slug.py
-@@ -274,7 +274,7 @@ default_options['fetch'] = {'branch': '[*]', 'prune': False, 'newpkgs': False, '
-
- pull = subparsers.add_parser('pull', help='git-pull in all existing repositories', parents=[common_fetchoptions],
- formatter_class=argparse.RawDescriptionHelpFormatter)
--pull.add_argument('--all', help='update local branches in all repositories', dest='updateall', action='store_true', default=True)
-+pull.add_argument('--all', help='update local branches in all repositories', dest='updateall', action='store_true', default=False)
- pull.add_argument('--noall', help='update local branches only when something has been fetched', dest='updateall', action='store_false', default=True)
- newpkgsopt = pull.add_mutually_exclusive_group()
- newpkgsopt.add_argument('-n', '--newpkgs', help='download packages that do not exist on local side',
-
-commit da9abb0e6d7ef1a1440e7f5ac4ad4dbf5538dc99
-Author: Arkadiusz Miśkiewicz <arekm at maven.pl>
-Date: Sun Nov 23 00:28:22 2014 +0100
-
- Add --newpkgs/--nonewpkgs to pull command.
-
- Allow 'git pld pull --newpkgs' to also fetch and pull new packages. Off
- by default.
-
-diff --git a/slug.py b/slug.py
-index da9c050..d083cf4 100755
---- a/slug.py
-+++ b/slug.py
-@@ -276,8 +276,12 @@ pull = subparsers.add_parser('pull', help='git-pull in all existing repositories
- formatter_class=argparse.RawDescriptionHelpFormatter)
- pull.add_argument('--all', help='update local branches in all repositories', dest='updateall', action='store_true', default=True)
- pull.add_argument('--noall', help='update local branches only when something has been fetched', dest='updateall', action='store_false', default=True)
-+newpkgsopt = pull.add_mutually_exclusive_group()
-+newpkgsopt.add_argument('-n', '--newpkgs', help='download packages that do not exist on local side',
-+ action='store_true')
-+newpkgsopt.add_argument('-nn', '--nonewpkgs', help='do not download new packages', dest='newpkgs', action='store_false')
- pull.set_defaults(func=pull_packages, branch='[*]', prune=False, newpkgs=False, omitexisting=False)
--default_options['pull'] = {'branch': ['*'], 'prune': False, 'newpkgs': False, 'omitexisting': False}
-+default_options['pull'] = {'branch': ['*'], 'prune': False, 'omitexisting': False}
-
- checkout =subparsers.add_parser('checkout', help='checkout repositories', parents=[common_fetchoptions],
- formatter_class=argparse.RawDescriptionHelpFormatter)
-
-commit b1096c634ea9b262bd791863d68e2aed3847078d
-Author: Arkadiusz Miśkiewicz <arekm at maven.pl>
-Date: Sun Nov 23 00:18:38 2014 +0100
-
- Parallelize fetching, checking out, cloning.
-
- Parallelize fetching, checking out, cloning using multiprocessing
- module.
-
- By default use number of parallel processes equal to number of
- system CPUs (use old value, 4, as fallback).
-
- Also replace thread based ThreadFetch() with the same multiprocessing
- mechanism as above for consistency.
-
-diff --git a/slug.py b/slug.py
-index 69bd3b9..da9c050 100755
+index 69bd3b9..68f68cd 100755
--- a/slug.py
+++ b/slug.py
@@ -7,26 +7,18 @@ import os
@@ -269,20 +120,11 @@ index 69bd3b9..da9c050 100755
def readconfig(path):
config = UnquoteConfig(delimiters='=', interpolation=None, strict=False)
-@@ -114,18 +96,19 @@ def getrefs(*args):
+@@ -114,38 +96,60 @@ def getrefs(*args):
sys.exit(2)
return refs
-+def fetch_package(gitrepo, ref2fetch, options):
-+ try:
-+ (stdout, stderr) = gitrepo.fetch(ref2fetch, options.depth)
-+ if stderr != b'':
-+ print('------', gitrepo.gdir[:-len('.git')], '------\n' + stderr.decode('utf-8'))
-+ return gitrepo
-+ except GitRepoError as e:
-+ print('------', gitrepo.gdir[:-len('.git')], '------\n', e)
-+
- def fetch_packages(options, return_all=False):
+-def fetch_packages(options, return_all=False):
- fetch_queue = queue.Queue()
- updated_repos = Store()
- for i in range(options.jobs):
@@ -291,21 +133,65 @@ index 69bd3b9..da9c050 100755
- t.start()
-
- signal.signal(signal.SIGINT, signal.SIG_DFL)
--
++def fetch_package(gitrepo, refs_heads, options):
++ ref2fetch = []
++ for ref in refs_heads:
++ if gitrepo.check_remote(ref) != refs_heads[ref]:
++ ref2fetch.append('+{}:{}/{}'.format(ref, REMOTEREFS, ref[len('refs/heads/'):]))
++ if ref2fetch:
++ ref2fetch.append('refs/notes/*:refs/notes/*')
+
++ try:
++ (stdout, stderr) = gitrepo.fetch(ref2fetch, options.depth)
++ if stderr != b'':
++ print('------', gitrepo.gdir[:-len('.git')], '------\n' + stderr.decode('utf-8'))
++ return gitrepo
++ except GitRepoError as e:
++ print('------', gitrepo.gdir[:-len('.git')], '------\n', e)
++
++def fetch_packages(options, return_all=False):
refs = getrefs(options.branch, options.repopattern)
print('Read remotes data')
++ pkgs_new = []
++ if options.newpkgs:
++ for pkgdir in sorted(refs.heads):
++ gitdir = os.path.join(options.packagesdir, pkgdir, '.git')
++ if not os.path.isdir(gitdir):
++ pkgs_new.append(pkgdir)
++
++ pool = WorkerPool(options.jobs, pool_worker_init)
++ try:
++ pool.starmap(initpackage, zip(pkgs_new, [options] * len(pkgs_new)))
++ except KeyboardInterrupt:
++ pool.terminate()
++ else:
++ pool.close()
++ pool.join()
++
+ args = []
for pkgdir in sorted(refs.heads):
- gitdir = os.path.join(options.packagesdir, pkgdir, '.git')
- if not os.path.isdir(gitdir):
-@@ -143,9 +126,18 @@ def fetch_packages(options, return_all=False):
- ref2fetch.append('+{}:{}/{}'.format(ref, REMOTEREFS, ref[len('refs/heads/'):]))
- if ref2fetch:
- ref2fetch.append('refs/notes/*:refs/notes/*')
+- gitdir = os.path.join(options.packagesdir, pkgdir, '.git')
+- if not os.path.isdir(gitdir):
+- if options.newpkgs:
+- gitrepo = initpackage(pkgdir, options)
+- else:
+- continue
+- elif options.omitexisting:
++ if options.omitexisting and pkgdir not in pkgs_new:
+ continue
+ else:
+ gitrepo = GitRepo(os.path.join(options.packagesdir, pkgdir))
+- ref2fetch = []
+- for ref in refs.heads[pkgdir]:
+- if gitrepo.check_remote(ref) != refs.heads[pkgdir][ref]:
+- ref2fetch.append('+{}:{}/{}'.format(ref, REMOTEREFS, ref[len('refs/heads/'):]))
+- if ref2fetch:
+- ref2fetch.append('refs/notes/*:refs/notes/*')
- fetch_queue.put((gitrepo, ref2fetch))
-+ args.append((gitrepo, ref2fetch, options))
++ args.append((gitrepo, refs.heads[pkgdir], options))
- fetch_queue.join()
++ updated_repos = []
+ pool = WorkerPool(options.jobs, pool_worker_init)
+ try:
+ updated_repos = pool.starmap(fetch_package, args)
@@ -319,7 +205,7 @@ index 69bd3b9..da9c050 100755
if options.prune:
refs = getrefs('*')
-@@ -158,26 +150,60 @@ def fetch_packages(options, return_all=False):
+@@ -158,26 +162,60 @@ def fetch_packages(options, return_all=False):
if return_all:
return refs.heads
else:
@@ -391,7 +277,7 @@ index 69bd3b9..da9c050 100755
def pull_packages(options):
repolist = []
-@@ -189,19 +215,14 @@ def pull_packages(options):
+@@ -189,19 +227,14 @@ def pull_packages(options):
else:
repolist = fetch_packages(options, False)
print('--------Pulling------------')
@@ -419,7 +305,7 @@ index 69bd3b9..da9c050 100755
def list_packages(options):
refs = getrefs(options.branch, options.repopattern)
-@@ -213,7 +234,7 @@ common_options.add_argument('-d', '--packagesdir', help='local directory with gi
+@@ -213,7 +246,7 @@ common_options.add_argument('-d', '--packagesdir', help='local directory with gi
default=os.path.expanduser('~/rpm/packages'))
common_fetchoptions = argparse.ArgumentParser(add_help=False, parents=[common_options])
@@ -428,43 +314,68 @@ index 69bd3b9..da9c050 100755
common_fetchoptions.add_argument('repopattern', nargs='*', default = ['*'])
common_fetchoptions.add_argument('--depth', help='depth of fetch', default=0)
-
-commit fac30722a98a4d6300822fd3f790ce1fa48e7d83
-Author: Arkadiusz Miśkiewicz <arekm at maven.pl>
-Date: Sun Nov 23 00:15:10 2014 +0100
-
- check_remote(): Add support to packed refs database.
-
- check_remote() did not handle git packed refs database. That made
- fetch_packages() to always fetch packages even if we already had
- them fetched.
-
- Supporting packed refs database fixes this problem.
-
-diff --git a/git_slug/gitrepo.py b/git_slug/gitrepo.py
-index 5234deb..d9f88ee 100644
---- a/git_slug/gitrepo.py
-+++ b/git_slug/gitrepo.py
-@@ -82,12 +82,21 @@ class GitRepo:
- 'refs/notes/*:refs/notes/*'])
+@@ -253,10 +286,14 @@ default_options['fetch'] = {'branch': '[*]', 'prune': False, 'newpkgs': False, '
- def check_remote(self, ref, remote=REMOTE_NAME):
-+ localref = EMPTYSHA1
- ref = ref.replace(REFFILE, os.path.join('remotes', remote))
- try:
- with open(os.path.join(self.gdir, ref), 'r') as f:
- localref = f.readline().strip()
- except IOError:
-- localref = EMPTYSHA1
-+ try:
-+ with open(os.path.join(self.gdir, 'packed-refs')) as f:
-+ for line in f:
-+ line_data = line.split()
-+ if len(line_data) == 2 and line_data[1] == ref:
-+ localref = line_data[0].strip()
-+ break
-+ except IOError:
-+ pass
- return localref
+ pull = subparsers.add_parser('pull', help='git-pull in all existing repositories', parents=[common_fetchoptions],
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+-pull.add_argument('--all', help='update local branches in all repositories', dest='updateall', action='store_true', default=True)
++pull.add_argument('--all', help='update local branches in all repositories', dest='updateall', action='store_true', default=False)
+ pull.add_argument('--noall', help='update local branches only when something has been fetched', dest='updateall', action='store_false', default=True)
++newpkgsopt = pull.add_mutually_exclusive_group()
++newpkgsopt.add_argument('-n', '--newpkgs', help='download packages that do not exist on local side',
++ action='store_true')
++newpkgsopt.add_argument('-nn', '--nonewpkgs', help='do not download new packages', dest='newpkgs', action='store_false')
+ pull.set_defaults(func=pull_packages, branch='[*]', prune=False, newpkgs=False, omitexisting=False)
+-default_options['pull'] = {'branch': ['*'], 'prune': False, 'newpkgs': False, 'omitexisting': False}
++default_options['pull'] = {'branch': ['*'], 'prune': False, 'omitexisting': False}
- def showfile(self, filename, ref="/".join([REMOTE_NAME, "master"])):
+ checkout =subparsers.add_parser('checkout', help='checkout repositories', parents=[common_fetchoptions],
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+diff --git a/slug_watch b/slug_watch
+index 7b64460..b077de9 100755
+--- a/slug_watch
++++ b/slug_watch
+@@ -23,6 +23,7 @@ LOCKFILE = 'slug_watch.lock'
+ PROJECTS_LIST = 'projects.list'
+ PROJECTS_LIST_NEW = PROJECTS_LIST + '.new'
+ PROJECTS_LIST_HEAD = PROJECTS_LIST + '.head'
++PROJECTS_LIST_GITWEB = PROJECTS_LIST + ".gitweb"
+ REFFILE_NEW = REFFILE + '.new'
+ REFREPO_WDIR = 'Refs'
+
+@@ -63,10 +64,11 @@ def process_file(pathname):
+ print('{} is not an ordinary file'.format(pathname))
+ return
+
+- try:
+- shutil.copyfile(PROJECTS_LIST_HEAD, PROJECTS_LIST_NEW)
+- except (OSError, shutil.Error):
+- logger.error('Cannot write {}'.format(PROJECTS_LIST_NEW))
++ if os.path.isfile(PROJECTS_LIST_HEAD):
++ try:
++ shutil.copyfile(PROJECTS_LIST_HEAD, PROJECTS_LIST_NEW)
++ except (OSError, shutil.Error):
++ logger.error('Cannot write {}'.format(PROJECTS_LIST_NEW))
+
+ with open(os.path.join(REFREPO_WDIR, REFFILE),'w') as headfile_new, open(pathname, 'r') as newfile, \
+ open(PROJECTS_LIST_NEW,'a') as projects:
+@@ -82,7 +84,7 @@ def process_file(pathname):
+ if sha1 != EMPTYSHA1:
+ print(sha1, ref, repo, file=headfile_new)
+ if repo != oldtuple[0]:
+- print(quote_plus('packages/'+repo+'.git', safe='/'), file=projects)
++ print('packages/'+repo+'.git', file=projects)
+ oldtuple = (repo, ref)
+ except ValueError:
+ logger.error("Problem with file: {}".format(pathname))
+@@ -90,6 +92,10 @@ def process_file(pathname):
+ process.wait()
+
+ os.rename(PROJECTS_LIST_NEW, PROJECTS_LIST)
++ with open(PROJECTS_LIST, 'r') as projects, open(PROJECTS_LIST_GITWEB, 'w') as output:
++ for line in projects:
++ print(quote_plus(line, safe='/\n'), end='', file=output)
++
+ headrepo = GitRepo(REFREPO_WDIR, REFREPO_GDIR)
+ headrepo.commitfile(REFFILE, 'Changes by {}'.format(committer))
+ os.remove(pathname)
================================================================
---- gitweb:
http://git.pld-linux.org/gitweb.cgi/packages/git-core-slug.git/commitdiff/8794fb23c91d328ace89beb41ea97055eae9378a
More information about the pld-cvs-commit
mailing list