[projects/git-slug: 1/170] Init commit (there is an earlier private history)
glen
glen at pld-linux.org
Mon Sep 21 21:38:53 CEST 2015
commit 35d474408f50e2fb10e9eff2cd02dbb4fcd77bcf
Author: Kacper Kornet <draenog at pld-linux.org>
Date: Fri Sep 9 06:26:13 2011 +0100
Init commit (there is an earlier private history)
gitconst.py | 13 ++++++++
gitrepo.py | 63 +++++++++++++++++++++++++++++++++++++
post-receive | 31 ++++++++++++++++++
refsdata.py | 41 ++++++++++++++++++++++++
slug.py | 100 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
5 files changed, 248 insertions(+)
---
diff --git a/gitconst.py b/gitconst.py
new file mode 100644
index 0000000..b068909
--- /dev/null
+++ b/gitconst.py
@@ -0,0 +1,13 @@
+from os.path import join
+
+EMPTYSHA1='0000000000000000000000000000000000000000'
+REMOTE_NAME = 'origin'
+REMOTEREFS=join('refs/remotes/',REMOTE_NAME)
+
+GITSERVER = 'carme.pld-linux.org'
+_packages_dir = 'packages'
+_packages_remote = join(GITSERVER, _packages_dir)
+GIT_REPO = 'git://' + _packages_remote
+GIT_REPO_PUSH = '@' + _packages_remote
+REFREPO = 'Refs'
+REFFILE = 'heads'
diff --git a/gitrepo.py b/gitrepo.py
new file mode 100644
index 0000000..6b3e616
--- /dev/null
+++ b/gitrepo.py
@@ -0,0 +1,63 @@
+from gitconst import *
+
+import os
+from subprocess import PIPE
+import subprocess
+
+class GitRepoError(Exception):
+ pass
+
+class GitRepo:
+ def __init__(self, working_tree = None, git_dir = None):
+ self.wtree = working_tree
+ self.command_prefix = ['git']
+ if git_dir is None and working_tree is not None:
+ self.gdir = os.path.join(working_tree, '.git')
+ else:
+ self.gdir = git_dir
+ if self.gdir is not None:
+ self.command_prefix.append('--git-dir='+self.gdir)
+ if self.wtree is not None:
+ self.command_prefix.append('--work-tree='+self.wtree)
+
+ def command(self, clist):
+ return subprocess.Popen(self.command_prefix + clist, stdout=PIPE, stderr=PIPE)
+
+ def commandio(self, clist):
+ return self.command(clist).communicate()
+
+ def commitfile(self, path, message):
+ clist = ['commit', '-m', message, path]
+ self.commandio(clist)
+
+ def fetch(self, fetchlist=[], depth = 0, remotename=REMOTE_NAME):
+ clist = ['fetch']
+ if depth:
+ clist.append('--depth={}'.format(depth))
+ clist += [ remotename ] + fetchlist
+ return self.commandio(clist)
+
+ def init(self, remotepull, remotepush = None, remotename=REMOTE_NAME):
+ clist = ['git', 'init']
+ if self.wtree is not None:
+ clist.append(self.wtree)
+ else:
+ clist.extend(['--bare', self.gdir])
+ if subprocess.call(clist):
+ raise GitRepoError(self.gdir)
+ self.commandio(['remote', 'add', remotename, remotepull])
+ if remotepush is not None:
+ self.commandio(['remote', 'set-url', '--push', remotename, remotepush])
+
+ def check_remote(self, ref, remote=REMOTE_NAME):
+ ref = ref.replace(REFFILE, os.path.join('remotes', remote))
+ try:
+ with open(os.path.join(self.gdir, ref), 'r') as f:
+ localref = f.readline().strip()
+ except IOError:
+ localref = EMPTYSHA1
+ return localref
+
+ def showfile(self, filename, ref='origin/master'):
+ clist = ['show', ref + ':' + filename]
+ return self.command(clist)
diff --git a/post-receive b/post-receive
new file mode 100755
index 0000000..2d1d2df
--- /dev/null
+++ b/post-receive
@@ -0,0 +1,31 @@
+#!/usr/bin/python -u
+
+import fcntl
+import os
+import sys
+
+from refsdata import RemoteRefsData
+from gitconst import REFREPO, REFFILE
+from gitrepo import GitRepo
+
+
+data = sys.stdin.readlines()
+
+gitrepo = os.environ.get('GL_REPO')
+if gitrepo.startswith('packages/'):
+ gitrepo = gitrepo[len('packages/'):]
+else:
+ sys.exit()
+
+with open(os.path.join(os.getenv('HOME'), REFREPO, REFFILE), 'r+') as headfile:
+ fcntl.lockf(headfile, fcntl.LOCK_EX)
+ print 'Lock obtained'
+ refs = RemoteRefsData(headfile, '*')
+ refs.put(gitrepo, data)
+ headfile.seek(0)
+ headfile.truncate()
+ refs.dump(headfile)
+ headfile.flush()
+ os.fsync(headfile.fileno())
+ headrepo = GitRepo(os.path.join(os.getenv('HOME'), REFREPO), os.path.join(os.getenv('GL_REPO_BASE_ABS'), REFREPO+'.git'))
+ headrepo.commitfile(REFFILE, 'Changes by {}'.format(os.getenv('GL_USER')))
diff --git a/refsdata.py b/refsdata.py
new file mode 100644
index 0000000..e27b30d
--- /dev/null
+++ b/refsdata.py
@@ -0,0 +1,41 @@
+
+import collections
+import fnmatch
+import os
+from gitconst import EMPTYSHA1, REFFILE, REFREPO, GITSERVER
+from gitrepo import GitRepo
+
+
+class RemoteRefsError(Exception):
+ pass
+
+class RemoteRefsData:
+ def __init__(self, stream, pattern, dirpattern='*'):
+ self.heads = collections.defaultdict(lambda: collections.defaultdict(lambda: EMPTYSHA1))
+ pattern = os.path.join('refs/heads', pattern)
+ for line in stream.readlines():
+ (sha1, ref, repo) = line.split()
+ if fnmatch.fnmatchcase(ref, pattern) and fnmatch.fnmatchcase(repo, dirpattern):
+ self.heads[repo][ref] = sha1
+
+ def put(self, repo, data):
+ for line in data:
+ (sha1_old, sha1, ref) = line.split()
+ self.heads[repo][ref] = sha1
+
+ def dump(self, stream):
+ for repo in sorted(self.heads):
+ for ref in sorted(self.heads[repo]):
+ if self.heads[repo][ref] != EMPTYSHA1:
+ stream.write('{} {} {}\n'.format(self.heads[repo][ref], ref, repo))
+
+class GitRemoteRefsData(RemoteRefsData):
+ def __init__(self, path, pattern, dirpattern='*'):
+ refsrepo = GitRepo(git_dir=path)
+ if not os.path.isdir(refsrepo.gdir):
+ refsrepo.init('git://' + os.path.join(GITSERVER, REFREPO))
+ refsrepo.fetch(depth=1)
+ showfile = refsrepo.showfile(REFFILE)
+ RemoteRefsData.__init__(self, showfile.stdout, pattern, dirpattern)
+ if showfile.wait():
+ raise RemoteRefsError(REFFILE, path)
diff --git a/slug.py b/slug.py
new file mode 100755
index 0000000..a4d27a3
--- /dev/null
+++ b/slug.py
@@ -0,0 +1,100 @@
+#!/usr/bin/python
+
+import glob
+import sys
+import os
+import shutil
+import Queue
+import threading
+
+import argparse
+
+import signal
+
+from gitconst import *
+from gitrepo import GitRepo, GitRepoError
+from refsdata import *
+
+fetch_queue = Queue.Queue()
+
+class ThreadFetch(threading.Thread):
+ def __init__(self, queue, dir, depth=0):
+ threading.Thread.__init__(self)
+ self.queue = queue
+ self.packagesdir = dir
+ self.depth = depth
+
+ def run(self):
+ while True:
+ (package, ref2fetch) = self.queue.get()
+ gitrepo = GitRepo(os.path.join(self.packagesdir, package))
+ (stdout, stderr) = gitrepo.fetch(ref2fetch, self.depth)
+ print '------', package, '------\n' + stderr
+ self.queue.task_done()
+
+
+
+parser = argparse.ArgumentParser(description='PLD tool for interaction with git repos',
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument('-b', '--branch', help='branch to fetch', default = 'master')
+parser.add_argument('-P', '--prune', help='prune git repositories that do no exist upstream', action='store_true')
+parser.add_argument('-j', help='number of threads to use', default=4, type=int)
+parser.add_argument('-d', '--packagesdir', help='local directory with git repositories',
+ default=os.path.join(os.getenv('HOME'),'PLD_clone/packages'))
+parser.add_argument('--depth', help='depth of fetch', default=0)
+parser.add_argument('-n', '--newpkgs', help='download packages that do not exist on local side', action='store_true')
+parser.add_argument('-r', '--remoterefs', help='repository with list of all refs',
+ default=os.path.join(os.getenv('HOME'),'PLD_clone/Refs.git'))
+parser.add_argument('-u', '--user', help='the user name to register for pushes for new repositories')
+parser.add_argument('dirpattern', nargs='?', default = '*')
+options = parser.parse_args()
+
+for i in range(options.j):
+ t = ThreadFetch(fetch_queue, options.packagesdir, options.depth)
+ t.setDaemon(True)
+ t.start()
+
+signal.signal(signal.SIGINT, signal.SIG_DFL)
+
+try:
+ refs = GitRemoteRefsData(options.remoterefs, options.branch, options.dirpattern)
+except GitRepoError as e:
+ print >> sys.stderr, 'Cannot create repository {}'.format(e)
+ sys.exit()
+except RemoteRefsError as e:
+ print >> sys.stderr, 'Problem with file {} in repository {}'.format(*e)
+ sys.exit()
+
+print 'Read remotes data'
+for dir in sorted(refs.heads):
+ gitdir = os.path.join(options.packagesdir, dir, '.git')
+ gitrepo = GitRepo(os.path.join(options.packagesdir, dir))
+ if not os.path.isdir(gitdir):
+ if options.newpkgs:
+ if options.user:
+ remotepush = 'ssh://' + os.path.join(options.user+GIT_REPO_PUSH ,dir)
+ else:
+ remotepush = None
+ gitrepo.init(os.path.join(GIT_REPO,dir), remotepush)
+ else:
+ continue
+ ref2fetch = []
+ for ref in refs.heads[dir]:
+ if gitrepo.check_remote(ref) != refs.heads[dir][ref]:
+ ref2fetch.append('+{}:{}/{}'.format(ref, REMOTEREFS, ref[len('refs/heads/'):]))
+ if ref2fetch:
+ fetch_queue.put((dir, ref2fetch))
+
+fetch_queue.join()
+
+if options.prune:
+ try:
+ refs = GitRemoteRefsData(options.remoterefs, '*')
+ except RemoteRefsError as e:
+ print >> sys.stderr, 'Problem with file {} in repository {}'.format(*e)
+ sys.exit()
+ for fulldir in glob.iglob(os.path.join(options.packagesdir,options.dirpattern)):
+ dir = os.path.basename(fulldir)
+ if len(refs.heads[dir]) == 0 and os.path.isdir(os.path.join(fulldir, '.git')):
+ print 'Removing', fulldir
+ shutil.rmtree(fulldir)
================================================================
---- gitweb:
http://git.pld-linux.org/gitweb.cgi/projects/git-slug.git/commitdiff/4ed64f73960519a2f4fd04c42950b2c96ae795c5
More information about the pld-cvs-commit
mailing list