SVN: packages-ng: mergerepos output packages/managers.py packages/models.py packages/templates/base....
patrys
patrys at pld-linux.org
Fri Apr 3 16:33:57 CEST 2009
Author: patrys
Date: Fri Apr 3 16:33:57 2009
New Revision: 10307
Added:
packages-ng/output/
Removed:
packages-ng/packages/managers.py
Modified:
packages-ng/mergerepos
packages-ng/packages/models.py
packages-ng/packages/templates/base.html
packages-ng/packages/templates/details.html
packages-ng/packages/templates/group.html
packages-ng/packages/templates/search.html
packages-ng/packages/views.py
packages-ng/settings.py
Log:
- repo merging script works
- use single db
Modified: packages-ng/mergerepos
==============================================================================
--- packages-ng/mergerepos (original)
+++ packages-ng/mergerepos Fri Apr 3 16:33:57 2009
@@ -4,15 +4,19 @@
'file:///home/users/patrys/Desktop/web/packages-ng/repo1/',
'file:///home/users/patrys/Desktop/web/packages-ng/repo2/',
]
-OUTPUT = './repo3'
+OUTPUT = './output'
-from createrepo import merge
from createrepo import MetaDataGenerator
+from createrepo import merge
+from createrepo.utils import _gzipOpen, bzipFile, checkAndMakeDir, GzipFile, checksum_and_rename
+import libxml2
import os
import shutil
+import sqlitecachec
import tempfile
import yum
import yum.Errors
+import yum.misc
class FixedMetaDataGenerator(MetaDataGenerator):
'''
@@ -64,9 +68,9 @@
complete_path = os.path.join(repopath, file)
zfo = _gzipOpen(complete_path)
- uncsum = misc.checksum(sumtype, zfo)
+ uncsum = yum.misc.checksum(sumtype, zfo)
zfo.close()
- csum = misc.checksum(sumtype, complete_path)
+ csum = yum.misc.checksum(sumtype, complete_path)
timestamp = os.stat(complete_path)[8]
db_csums = {}
@@ -94,24 +98,20 @@
os.rename(tmp_result_path, resultpath)
compressed_name = '%s.bz2' % good_name
result_compressed = os.path.join(repopath, compressed_name)
- db_csums[ftype] = misc.checksum(sumtype, resultpath)
+ db_csums[ftype] = yum.misc.checksum(sumtype, resultpath)
- # compress the files
- bzipFile(resultpath, result_compressed)
# csum the compressed file
- db_compressed_sums[ftype] = misc.checksum(sumtype, result_compressed)
- # remove the uncompressed file
- os.unlink(resultpath)
+ db_compressed_sums[ftype] = yum.misc.checksum(sumtype, resultpath)
if self.conf.unique_md_filenames:
- csum_compressed_name = '%s-%s.bz2' % (db_compressed_sums[ftype], good_name)
+ csum_compressed_name = '%s-%s' % (db_compressed_sums[ftype], good_name)
csum_result_compressed = os.path.join(repopath, csum_compressed_name)
- os.rename(result_compressed, csum_result_compressed)
- result_compressed = csum_result_compressed
+ os.rename(resultpath, csum_result_compressed)
+ resultpath = csum_result_compressed
compressed_name = csum_compressed_name
# timestamp the compressed file
- db_timestamp = os.stat(result_compressed)[8]
+ db_timestamp = os.stat(resultpath)[8]
# add this data as a section to the repomdxml
db_data_type = '%s_db' % ftype
@@ -162,11 +162,6 @@
attribs = {'database_version':dbversion}
self.addArbitraryMetadata(fn, ftype, reporoot, compress=True,
compress_type='bzip2', attribs=attribs)
- try:
- os.unlink(fn)
- except (IOError, OSError), e:
- pass
-
if self.conf.groupfile is not None:
self.addArbitraryMetadata(self.conf.groupfile, 'group_gz', reporoot)
@@ -198,26 +193,6 @@
'''
def write_metadata(self, outputdir=None):
mytempdir = tempfile.mkdtemp()
- if self.groups:
- comps_fn = mytempdir + '/groups.xml'
- compsfile = open(comps_fn, 'w')
- compsfile.write(self.yumbase.comps.xml())
- compsfile.close()
- self.mdconf.groupfile=comps_fn
-
- if self.updateinfo:
- ui_fn = mytempdir + '/updateinfo.xml'
- uifile = open(ui_fn, 'w')
- umd = yum.update_md.UpdateMetadata()
- for repo in self.yumbase.repos.listEnabled():
- try: # attempt to grab the updateinfo.xml.gz from the repodata
- umd.add(repo)
- except yum.Errors.RepoMDError:
- continue
- umd.xml(fileobj=uifile)
- uifile.close()
- self.mdconf.additional_metadata['updateinfo'] = ui_fn
-
self.mdconf.pkglist = self.yumbase.pkgSack
self.mdconf.directory = self.outputdir
@@ -237,8 +212,7 @@
rmbase = RepoMerge(REPOS)
rmbase.mdconf.database = True
+#rmbase.mdconf.database_only = True
rmbase.outputdir = OUTPUT
-rmbase.database = True
-rmbase.database_only = True
rmbase.merge_repos()
rmbase.write_metadata()
Modified: packages-ng/packages/models.py
==============================================================================
--- packages-ng/packages/models.py (original)
+++ packages-ng/packages/models.py Fri Apr 3 16:33:57 2009
@@ -1,5 +1,4 @@
from django.db import models
-from managers import custom_db_manager
def flags_format(self):
if self.flags == 'EQ':
@@ -29,7 +28,7 @@
res = self.name + u' ' + self.version_format()
return res
-class ArchPackage(models.Model):
+class Package(models.Model):
pkgkey = models.IntegerField(primary_key = True)
pkgid = models.TextField(blank = True)
name = models.TextField(blank = True)
@@ -56,7 +55,6 @@
location_href = models.TextField(blank = True)
location_base = models.TextField(blank = True)
checksum_type = models.TextField(blank = True)
- objects = custom_db_manager('arch')
class Meta:
db_table = u'packages'
@@ -64,14 +62,13 @@
def __unicode__(self):
return u'%s-%s:%s-%s.%s' % (self.name, self.epoch, self.version, self.release, self.arch)
-class ArchConflict(models.Model):
+class Conflict(models.Model):
name = models.CharField(blank = True, max_length = 255, primary_key = True)
flags = models.TextField(blank = True)
epoch = models.TextField(blank = True)
version = models.TextField(blank = True)
release = models.TextField(blank = True)
- package = models.ForeignKey(ArchPackage, db_column = 'pkgkey', related_name = 'conflicts')
- _default_manager = custom_db_manager('arch')
+ package = models.ForeignKey(Package, db_column = 'pkgkey', related_name = 'conflicts')
class Meta:
db_table = u'conflicts'
@@ -82,11 +79,10 @@
version_format = version_format
__unicode__ = header_format
-class ArchFile(models.Model):
+class File(models.Model):
name = models.TextField(blank = True)
type = models.TextField(blank = True)
- package = models.ForeignKey(ArchPackage, db_column = 'pkgkey', related_name = 'files')
- _default_manager = custom_db_manager('arch')
+ package = models.ForeignKey(Package, db_column = 'pkgkey', related_name = 'files')
class Meta:
db_table = u'files'
@@ -94,14 +90,13 @@
def __unicode__(self):
return u'%s %s' % (self.name, self.type)
-class ArchObsolete(models.Model):
+class Obsolete(models.Model):
name = models.CharField(blank = True, max_length = 255, primary_key = True)
flags = models.TextField(blank = True)
epoch = models.TextField(blank = True)
version = models.TextField(blank = True)
release = models.TextField(blank = True)
- package = models.ForeignKey(ArchPackage, db_column = 'pkgkey', related_name = 'obsoletes')
- _default_manager = custom_db_manager('arch')
+ package = models.ForeignKey(Package, db_column = 'pkgkey', related_name = 'obsoletes')
class Meta:
db_table = u'obsoletes'
@@ -112,14 +107,13 @@
version_format = version_format
__unicode__ = header_format
-class ArchProvide(models.Model):
+class Provide(models.Model):
name = models.CharField(blank = True, max_length = 255, primary_key = True)
flags = models.CharField(blank = True, max_length = 32)
epoch = models.CharField(blank = True, max_length = 255)
version = models.CharField(blank = True, max_length = 255)
release = models.CharField(blank = True, max_length = 255)
- package = models.ForeignKey(ArchPackage, db_column = 'pkgkey', related_name = 'provides')
- _default_manager = custom_db_manager('arch')
+ package = models.ForeignKey(Package, db_column = 'pkgkey', related_name = 'provides')
class Meta:
db_table = u'provides'
@@ -130,137 +124,14 @@
version_format = version_format
__unicode__ = header_format
-class ArchRequire(models.Model):
+class Require(models.Model):
name = models.CharField(blank = True, max_length = 255, primary_key = True)
flags = models.TextField(blank = True)
epoch = models.TextField(blank = True)
version = models.TextField(blank = True)
release = models.TextField(blank = True)
- package = models.ForeignKey(ArchPackage, db_column = 'pkgkey', related_name = 'requires')
+ package = models.ForeignKey(Package, db_column = 'pkgkey', related_name = 'requires')
pre = models.BooleanField(null = True, blank = True)
- _default_manager = custom_db_manager('arch')
-
- class Meta:
- db_table = u'requires'
- ordering = ['name', ]
- unique_together = ['name', 'flags', 'epoch', 'version', 'release', 'package']
-
- flags_format = flags_format
- version_format = version_format
- __unicode__ = header_format
-
-# ugly hack to use two DBs
-
-class NoarchPackage(models.Model):
- pkgkey = models.IntegerField(primary_key = True)
- pkgid = models.TextField(blank = True)
- name = models.TextField(blank = True)
- arch = models.TextField(blank = True)
- version = models.TextField(blank = True)
- epoch = models.TextField(blank = True)
- release = models.TextField(blank = True)
- summary = models.TextField(blank = True)
- description = models.TextField(blank = True)
- url = models.TextField(blank = True)
- time_file = models.IntegerField(null = True, blank = True)
- time_build = models.IntegerField(null = True, blank = True)
- rpm_license = models.TextField(blank = True)
- rpm_vendor = models.TextField(blank = True)
- rpm_group = models.TextField(blank = True)
- rpm_buildhost = models.TextField(blank = True)
- rpm_sourcerpm = models.TextField(blank = True)
- rpm_header_start = models.IntegerField(null = True, blank = True)
- rpm_header_end = models.IntegerField(null = True, blank = True)
- rpm_packager = models.TextField(blank = True)
- size_package = models.IntegerField(null = True, blank = True)
- size_installed = models.IntegerField(null = True, blank = True)
- size_archive = models.IntegerField(null = True, blank = True)
- location_href = models.TextField(blank = True)
- location_base = models.TextField(blank = True)
- checksum_type = models.TextField(blank = True)
- objects = custom_db_manager('noarch')
-
- class Meta:
- db_table = u'packages'
- ordering = ['name', '-epoch', '-version', '-release', 'arch']
- def __unicode__(self):
- return u'%s-%s:%s-%s.%s' % (self.name, self.epoch, self.version, self.release, self.arch)
-
-class NoarchConflict(models.Model):
- name = models.CharField(blank = True, max_length = 255, primary_key = True)
- flags = models.TextField(blank = True)
- epoch = models.TextField(blank = True)
- version = models.TextField(blank = True)
- release = models.TextField(blank = True)
- package = models.ForeignKey(NoarchPackage, db_column = 'pkgkey', related_name = 'conflicts')
- _default_manager = custom_db_manager('noarch')
-
- class Meta:
- db_table = u'conflicts'
- ordering = ['name', ]
- unique_together = ['name', 'flags', 'epoch', 'version', 'release', 'package']
-
- flags_format = flags_format
- version_format = version_format
- __unicode__ = header_format
-
-class NoarchFile(models.Model):
- name = models.TextField(blank = True)
- type = models.TextField(blank = True)
- package = models.ForeignKey(NoarchPackage, db_column = 'pkgkey', related_name = 'files')
- _default_manager = custom_db_manager('noarch')
-
- class Meta:
- db_table = u'files'
-
- def __unicode__(self):
- return u'%s %s' % (self.name, self.type)
-
-class NoarchObsolete(models.Model):
- name = models.CharField(blank = True, max_length = 255, primary_key = True)
- flags = models.TextField(blank = True)
- epoch = models.TextField(blank = True)
- version = models.TextField(blank = True)
- release = models.TextField(blank = True)
- package = models.ForeignKey(NoarchPackage, db_column = 'pkgkey', related_name = 'obsoletes')
- _default_manager = custom_db_manager('noarch')
-
- class Meta:
- db_table = u'obsoletes'
- ordering = ['name', ]
- unique_together = ['name', 'flags', 'epoch', 'version', 'release', 'package']
-
- flags_format = flags_format
- version_format = version_format
- __unicode__ = header_format
-
-class NoarchProvide(models.Model):
- name = models.CharField(blank = True, max_length = 255, primary_key = True)
- flags = models.CharField(blank = True, max_length = 32)
- epoch = models.CharField(blank = True, max_length = 255)
- version = models.CharField(blank = True, max_length = 255)
- release = models.CharField(blank = True, max_length = 255)
- package = models.ForeignKey(NoarchPackage, db_column = 'pkgkey', related_name = 'provides')
- objects = custom_db_manager('noarch')
-
- class Meta:
- db_table = u'provides'
- ordering = ['name', ]
- unique_together = ['name', 'flags', 'epoch', 'version', 'release', 'package']
-
- flags_format = flags_format
- version_format = version_format
- __unicode__ = header_format
-
-class NoarchRequire(models.Model):
- name = models.CharField(blank = True, max_length = 255, primary_key = True)
- flags = models.TextField(blank = True)
- epoch = models.TextField(blank = True)
- version = models.TextField(blank = True)
- release = models.TextField(blank = True)
- package = models.ForeignKey(NoarchPackage, db_column = 'pkgkey', related_name = 'requires')
- pre = models.BooleanField(null = True, blank = True)
- _default_manager = custom_db_manager('noarch')
class Meta:
db_table = u'requires'
Modified: packages-ng/packages/templates/base.html
==============================================================================
--- packages-ng/packages/templates/base.html (original)
+++ packages-ng/packages/templates/base.html Fri Apr 3 16:33:57 2009
@@ -3,6 +3,7 @@
<title>{% block title %}PLD Linux Packages{% endblock %}</title>
</head>
<html>
+ <h1><a href="{% url packages.views.index %}">PLD Linux Packages</a></h1>
{% block content %}
{% endblock %}
</html>
Modified: packages-ng/packages/templates/details.html
==============================================================================
--- packages-ng/packages/templates/details.html (original)
+++ packages-ng/packages/templates/details.html Fri Apr 3 16:33:57 2009
@@ -3,49 +3,58 @@
{% load markup %}
{% block content %}
-{% for package in packages %}
-<h2>Package: {{ package.name }} ({{ package.epoch }}:{{ package.version }}-{{ package.release }}.{{ package.arch }})</h2>
+<h2>{{ packages.0.name }} — {{ packages.0.summary }}</h2>
-<h3>{{ package.summary }}</h3>
+{{ packages.0.description|markdown }}
-{{ package.description|markdown }}
+<p>Group: <a href="{% url packages.views.browse_group packages.0.rpm_group %}">{{ packages.0.rpm_group }}</a></p>
-<p>Source package: <a href="{% url packages.views.browse_source package.rpm_sourcerpm %}">{{ package.rpm_sourcerpm }}</a></p>
+<p><a href="{% url packages.views.install packages.0.name %}">Install this package</a></p>
-<p>Group: <a href="{% url packages.views.browse_group package.rpm_group %}">{{ package.rpm_group }}</a></p>
-
-<p><a href="{% url packages.views.install package.name %}">Install this package</a></p>
-
-<h3>Provides:</h3>
+{% regroup packages|dictsort:"arch" by arch as packages_grouped %}
+{% for arch in packages_grouped %}
+<h3 id="arch-{{ arch.grouper }}">{{ arch.grouper }}</h3>
<ul>
-{% for provide in package.provides.all %}
- <li><a href="{% url packages.views.browse_requires provide.name %}">{{ provide.name }}</a>{% if provide.version_format %} ({{ provide.version_format }}){% endif %}</li>
-{% endfor %}
-</ul>
-
-<h3>Dependencies:</h3>
-
-<ul>
-{% for require in package.requires.all %}
- <li><a href="{% url packages.views.browse_provides require.name %}">{{ require.name }}</a>{% if require.version_format %} ({{ require.version_format }}){% endif %}</li>
-{% endfor %}
-</ul>
+ {% for package in arch.list %}
+ <li>
+ <h4>{{ package.epoch }}:{{ package.version }}-{{ package.release }}</h4>
+ <p>Source package: <a href="{% url packages.views.browse_source package.rpm_sourcerpm %}">{{ package.rpm_sourcerpm }}</a></p>
+
+ <h5>Provides:</h5>
+
+ <ul>
+ {% for provide in package.provides.all %}
+ <li><a href="{% url packages.views.browse_requires provide.name %}">{{ provide.name }}</a>{% if provide.version_format %} ({{ provide.version_format }}){% endif %}</li>
+ {% endfor %}
+ </ul>
+
+ <h5>Dependencies:</h5>
+
+ <ul>
+ {% for require in package.requires.all %}
+ <li><a href="{% url packages.views.browse_provides require.name %}">{{ require.name }}</a>{% if require.version_format %} ({{ require.version_format }}){% endif %}</li>
+ {% endfor %}
+ </ul>
+
+ <h5>Conflicts:</h5>
+
+ <ul>
+ {% for conflict in package.conflicts.all %}
+ <li>{{ conflict.name }}{% if conflict.version_format %} ({{ conflict.version_format }}){% endif %}</li>
+ {% endfor %}
+ </ul>
+
+ <h5>Obsoletes:</h5>
+
+ <ul>
+ {% for obsolete in package.obsoletes.all %}
+ <li>{{ obsolete.name }}{% if obsolete.version_format %} ({{ obsolete.version_format }}){% endif %}</li>
+ {% endfor %}
+ </ul>
-<h3>Conflicts:</h3>
+ {% endfor %}
+</li>
-<ul>
-{% for conflict in package.conflicts.all %}
- <li>{{ conflict.name }}{% if conflict.version_format %} ({{ conflict.version_format }}){% endif %}</li>
-{% endfor %}
-</ul>
-
-<h3>Obsoletes:</h3>
-
-<ul>
-{% for obsolete in package.obsoletes.all %}
- <li>{{ obsolete.name }}{% if obsolete.version_format %} ({{ obsolete.version_format }}){% endif %}</li>
-{% endfor %}
-</ul>
{% endfor %}
{% endblock %}
Modified: packages-ng/packages/templates/group.html
==============================================================================
--- packages-ng/packages/templates/group.html (original)
+++ packages-ng/packages/templates/group.html Fri Apr 3 16:33:57 2009
@@ -11,6 +11,12 @@
<dt><a href="{% url packages.views.details name=package.grouper %}">{{ package.grouper }}</a> — {{ package.list.0.summary }}</dt>
<dd>
{{ package.list.0.description|markdown }}
+ {% regroup package.list|dictsort:"arch" by arch as arches %}
+ <p>
+ {% for arch in arches %}
+ [<a href="{% url packages.views.details name=package.grouper %}#arch-{{ arch.grouper }}">{{ arch.grouper }}</a>]
+ {% endfor %}
+ </p>
</dd>
{% endfor %}
</dl>
Modified: packages-ng/packages/templates/search.html
==============================================================================
--- packages-ng/packages/templates/search.html (original)
+++ packages-ng/packages/templates/search.html Fri Apr 3 16:33:57 2009
@@ -17,6 +17,12 @@
<dt><a href="{% url packages.views.details name=package.grouper %}">{{ package.grouper }}</a> — {{ package.list.0.summary }}</dt>
<dd>
{{ package.list.0.description|markdown }}
+ {% regroup package.list|dictsort:"arch" by arch as arches %}
+ <p>
+ {% for arch in arches %}
+ [<a href="{% url packages.views.details name=package.grouper %}#arch-{{ arch.grouper }}">{{ arch.grouper }}</a>]
+ {% endfor %}
+ </p>
</dd>
{% endfor %}
</dl>
Modified: packages-ng/packages/views.py
==============================================================================
--- packages-ng/packages/views.py (original)
+++ packages-ng/packages/views.py Fri Apr 3 16:33:57 2009
@@ -2,52 +2,45 @@
from django.shortcuts import get_list_or_404, get_object_or_404
from django.views.generic.simple import direct_to_template
from django import forms
-from models import ArchPackage, NoarchPackage
+from models import Package
from operator import attrgetter
class PkSearchForm(forms.Form):
query = forms.CharField(u'Query')
-def filter_packages(**kwargs):
- packages1 = ArchPackage.objects.filter(**kwargs)
- packages2 = NoarchPackage.objects.filter(**kwargs)
- packages = list(set(list(packages1) + list(packages2)))
- packages.sort(key = attrgetter('name'))
- return packages
-
def index(request):
form = PkSearchForm()
- groups = list(set([x['rpm_group'] for x in ArchPackage.objects.order_by('rpm_group').values('rpm_group').distinct()] + [x['rpm_group'] for x in NoarchPackage.objects.order_by('rpm_group').values('rpm_group').distinct()]))
+ groups = [x['rpm_group'] for x in Package.objects.order_by('rpm_group').values('rpm_group').distinct()]
groups.sort()
return direct_to_template(request, 'index.html', {'form': form, 'groups': groups})
def search(request):
form = PkSearchForm(request.GET)
if form.is_valid():
- packages = filter_packages(name__icontains = form.cleaned_data['query'])
+ packages = Package.objects.filter(name__icontains = form.cleaned_data['query'])
return direct_to_template(request, 'search.html', {'form': form, 'packages': packages})
else:
return direct_to_template(request, 'search.html', {'form': form})
def browse_group(request, group):
print group
- packages = filter_packages(rpm_group = group)
+ packages = Package.objects.filter(rpm_group = group)
return direct_to_template(request, 'group.html', {'packages': packages, 'group': group})
def browse_provides(request, provide):
- packages = filter_packages(provides__name = provide)
+ packages = Package.objects.filter(provides__name = provide)
return direct_to_template(request, 'provides.html', {'packages': packages, 'provide': provide})
def browse_requires(request, require):
- packages = filter_packages(requires__name = require)
+ packages = Package.objects.filter(requires__name = require)
return direct_to_template(request, 'requires.html', {'packages': packages, 'require': require})
def browse_source(request, source):
- packages = filter_packages(rpm_sourcerpm = source)
+ packages = Package.objects.filter(rpm_sourcerpm = source)
return direct_to_template(request, 'source.html', {'packages': packages, 'source': source})
def details(request, name):
- packages = filter_packages(name = name)
+ packages = Package.objects.filter(name = name)
return direct_to_template(request, 'details.html', {'packages': packages})
def install(request, name):
Modified: packages-ng/settings.py
==============================================================================
--- packages-ng/settings.py (original)
+++ packages-ng/settings.py Fri Apr 3 16:33:57 2009
@@ -10,18 +10,7 @@
MANAGERS = ADMINS
DATABASE_ENGINE = 'sqlite3'
-DATABASE_NAME = 'arch.sqlite'
-
-DATABASES = {
- 'arch': dict(
- DATABASE_ENGINE = 'sqlite3',
- DATABASE_NAME = 'arch.sqlite'
- ),
- 'noarch': dict (
- DATABASE_ENGINE = 'sqlite3',
- DATABASE_NAME = 'noarch.sqlite'
- )
-}
+DATABASE_NAME = 'output/repodata/primary.sqlite'
# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
More information about the pld-cvs-commit
mailing list