poldek: poldek/pkgdir/pkgdir.c, poldek/pkgdir/pkgdir_dirindex.c, p...
mis
mis at pld-linux.org
Wed Jun 20 19:11:48 CEST 2007
Author: mis Date: Wed Jun 20 17:11:48 2007 GMT
Module: poldek Tag: HEAD
---- Log message:
- dirindex is updateable now -> no more scanning rpmdb after its every change
---- Files affected:
poldek/poldek/pkgdir:
pkgdir.c (1.46 -> 1.47) , pkgdir_dirindex.c (1.8 -> 1.9) , pkgdir_dirindex.h (1.2 -> 1.3) , pkgdir_intern.h (1.9 -> 1.10)
poldek/poldek/pkgdir/pdir:
pdir.c (1.29 -> 1.30) , pdir.h (1.8 -> 1.9)
---- Diffs:
================================================================
Index: poldek/poldek/pkgdir/pkgdir.c
diff -u poldek/poldek/pkgdir/pkgdir.c:1.46 poldek/poldek/pkgdir/pkgdir.c:1.47
--- poldek/poldek/pkgdir/pkgdir.c:1.46 Wed Aug 23 21:52:42 2006
+++ poldek/poldek/pkgdir/pkgdir.c Wed Jun 20 19:11:43 2007
@@ -1,5 +1,5 @@
/*
- Copyright (C) 2000 - 2005 Pawel A. Gajda <mis at k2.net.pl>
+ Copyright (C) 2000 - 2007 Pawel A. Gajda <mis at pld-linux.org>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License, version 2 as
@@ -577,13 +577,6 @@
return;
pkgdir->dirindex = pkgdir_dirindex_open(pkgdir);
-
- /* broken or outdated dirindex, shouldn't happen, but... */
- if (pkgdir->dirindex == NULL) {
- msgn(1, "Rebuilding %s's directory index...", pkgdir_idstr(pkgdir));
- pkgdir_dirindex_create(pkgdir); /* rebuild, open removes index on fail*/
- pkgdir->dirindex = pkgdir_dirindex_open(pkgdir);
- }
}
@@ -633,9 +626,6 @@
rc = 1;
pkgdir->flags |= PKGDIR_LOADED;
- if (ldflags & PKGDIR_LD_DIRINDEX)
- pkgdir_dirindex_create(pkgdir);
-
n_array_sort(pkgdir->pkgs);
for (i=0; i < n_array_size(pkgdir->pkgs); i++) {
struct pkg *pkg = n_array_nth(pkgdir->pkgs, i);
@@ -806,7 +796,7 @@
}
}
-const char *pkgdir_localidxpath(struct pkgdir *pkgdir)
+const char *pkgdir_localidxpath(const struct pkgdir *pkgdir)
{
if (pkgdir->mod->localidxpath)
return pkgdir->mod->localidxpath(pkgdir);
@@ -1002,7 +992,7 @@
pkgdir->avlangs_h = avlangs_h_tmp;
n_hash_free(avlangs_h);
}
-
+
return nerr == 0;
}
@@ -1011,6 +1001,8 @@
{
if (n_array_bsearch(pkgdir->pkgs, pkg))
return 0;
+
+ pkg->recno = 0; /* local to pkgdir */
n_array_push(pkgdir->pkgs, pkg_link(pkg));
pkgdir->flags |= PKGDIR_CHANGED;
return 1;
@@ -1019,10 +1011,10 @@
int pkgdir_remove_package(struct pkgdir *pkgdir, struct pkg *pkg)
{
int n;
-
+
if ((n = n_array_bsearch_idx(pkgdir->pkgs, pkg)) < 0)
return 0;
-
+
n_array_remove_nth(pkgdir->pkgs, n);
pkgdir->flags |= PKGDIR_CHANGED;
return 1;
================================================================
Index: poldek/poldek/pkgdir/pkgdir_dirindex.c
diff -u poldek/poldek/pkgdir/pkgdir_dirindex.c:1.8 poldek/poldek/pkgdir/pkgdir_dirindex.c:1.9
--- poldek/poldek/pkgdir/pkgdir_dirindex.c:1.8 Fri Jun 15 00:49:12 2007
+++ poldek/poldek/pkgdir/pkgdir_dirindex.c Wed Jun 20 19:11:43 2007
@@ -1,5 +1,5 @@
/*
- Copyright (C) 2000 - 2006 Pawel A. Gajda <mis at k2.net.pl>
+ Copyright (C) 2000 - 2007 Pawel A. Gajda <mis at pld-linux.org>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License, version 2 as
@@ -52,166 +52,201 @@
#include "pndir/pndir.h" /* for pndir_make_pkgkey() */
#include "pkgdir_dirindex.h"
-#define KEY_REQDIR '.'
-#define KEY_PKGID 'i'
+/* prefixes for non-path hash entries */
+#define PREFIX_PKGKEY_REQDIR '\\'
+#define PREFIX_PKGKEY_OWNDIR '~'
+#define PREFIX_PKGKEY_NO '`'
+
+#define PREFIXLEN 2
struct pkgdir_dirindex {
struct tndb *db;
tn_alloc *na;
- tn_hash *idmap;
+ tn_hash *idmap; /* { package_no => pkg } pairs */
+ tn_hash *keymap; /* { package_key => package_no } */
};
-/* key prefixed by KEY_REQDIR */
-static int req_pkgkey(char *key, int size, const struct pkg *pkg)
+static
+const char **get_package_directories(struct tndb *db, const char *key, int klen,
+ char *val, int vsize, int *ndirs);
+
+
+/* package_no as db key */
+static int package_no_key(char *buf, int size, uint32_t package_no, int prefixed)
+{
+ if (prefixed)
+ return n_snprintf(buf, size, "_%c%u", PREFIX_PKGKEY_NO, package_no);
+ return n_snprintf(buf, size, "%u", package_no);
+}
+
+/* package as db key */
+static int package_key(char *key, int size, const struct pkg *pkg, int prefix)
{
n_assert(size >= UINT8_MAX);
key[0] = '_';
- key[1] = KEY_REQDIR;
- return pndir_make_pkgkey(&key[2], size - 2, pkg) + 2;
+ key[1] = prefix;
+ return pndir_make_pkgkey(&key[PREFIXLEN], size - PREFIXLEN, pkg) + PREFIXLEN;
}
static tn_buf *dirarray_join(tn_buf *nbuf, tn_array *arr, char *sep)
{
- int i;
- for (i=0; i < n_array_size(arr); i++) {
+ int i, size = n_array_size(arr);
+ for (i=0; i < size; i++) {
n_buf_printf(nbuf, "%s%s", (char*)n_array_nth(arr, i),
- i < n_array_size(arr) - 1 ? sep : "");
+ i < size - 1 ? sep : "");
}
return nbuf;
}
+/* store { PREFIX[2] . package_no => pkg } pair */
static
-int nth2str(char *buf, int size, uint32_t nth)
-{
- return n_snprintf(buf, size, "_%c%u", KEY_PKGID, nth);
-}
-
-/* store nth => pkg pair */
-static
-void index_nth(uint32_t nth, struct pkg *pkg, struct tndb *db)
+void store_package_no(uint32_t package_no, struct tndb *db, const struct pkg *pkg)
{
char key[512], val[512];
int klen, vlen;
+
+ klen = package_no_key(key, sizeof(key), package_no, 1);
+ vlen = package_key(val, sizeof(val), pkg, PREFIX_PKGKEY_REQDIR);
- vlen = pndir_make_pkgkey(val, sizeof(val), pkg);
- klen = nth2str(key, sizeof(key), nth);
- tndb_put(db, key, klen, val, vlen);
+ tndb_put(db, key, klen, val + PREFIXLEN, vlen - PREFIXLEN); /* without prefix */
}
-/* build hash of path => ids[] */
+/* build hash of path => package_no[] */
static
-void add_to_hash(tn_hash *ht, const char *path, uint32_t nth)
+void add_to_path_index(tn_hash *path_index, const char *path, uint32_t package_no)
{
char val[512];
int vlen;
tn_array *keys;
- if ((keys = n_hash_get(ht, path)) == NULL) {
+ if ((keys = n_hash_get(path_index, path)) == NULL) {
keys = n_array_new(16, free, (tn_fn_cmp)strcmp);
- n_hash_insert(ht, path, keys);
+ n_hash_insert(path_index, path, keys);
}
- vlen = nth2str(val, sizeof(val), nth);
- //printf("add %s: %s\n", path, val);
+ vlen = package_no_key(val, sizeof(val), package_no, 0);
+ //printf("add %s: %s\n", path, val);
n_array_push(keys, n_strdupl(val, vlen));
}
-#if 0 /* XXX not used */
-static
-void index_package_allfiles(uint32_t nth, struct pkg *pkg, struct tndb *db,
- tn_hash *index)
+static int store_from_previous(uint32_t package_no, struct pkg *pkg, struct tndb *db,
+ tn_hash *path_index, struct pkgdir_dirindex *dirindex)
{
- struct pkgflist *flist;
- int i, j;
+ const char **tl, **tl_save;
+ char key[512], val[16 * 1024];
+ int klen, vlen, found = 0, ndirs;
+
+ n_assert(dirindex);
- index_nth(nth, pkg, db);
+ klen = package_key(key, sizeof(key), pkg, PREFIX_PKGKEY_REQDIR);
+
+ if (n_hash_exists(dirindex->keymap, &key[2])) { /* got it */
+ DBGF("HIT %s\n", pkg_id(pkg));
+ found = 1;
+ }
- if ((flist = pkg_get_flist(pkg)) == NULL)
- return;
+ if ((vlen = tndb_get(dirindex->db, key, klen, val, sizeof(val))) > 0)
+ tndb_put(db, key, klen, val, vlen);
+
+ key[1] = PREFIX_PKGKEY_OWNDIR;
+
- for (i=0; i < n_tuple_size(flist->fl); i++) {
- struct pkgfl_ent *flent = n_tuple_nth(flist->fl, i);
+ tl = tl_save = get_package_directories(dirindex->db, key, klen, val, sizeof(val),
+ &ndirs);
+
+ if (tl == NULL)
+ return found;
+
+ tndb_put(db, key, klen, val, vlen);
- for (j=0; j < flent->items; j++) {
- struct flfile *f = flent->files[j];
- char buf[1024], *slash = "";
- int n;
-
-
- if (S_ISDIR(f->mode)) {
- if (*flent->dirname != '/')
- slash = "/";
- }
-
- n = n_snprintf(buf, sizeof(buf), "%s%s%s%s%s",
- *flent->dirname == '/' ? "":"/",
- flent->dirname,
- *flent->dirname == '/' ? "":"/",
- f->basename, slash);
- n_assert(n < UINT8_MAX);
- add_to_hash(index, buf, nth);
- }
+ while (*tl) {
+ const char *dir = *tl;
+ add_to_path_index(path_index, dir, package_no);
+ tl++;
}
- pkgflist_free(flist);
+
+ n_str_tokl_free(tl_save);
+ return found;
}
-#endif
+/* process package files and add them to dirindex */
static
-void index_package(uint32_t nth, struct pkg *pkg, struct tndb *db,
- tn_hash *index, tn_buf *nbuf)
+void store_package(uint32_t package_no, struct pkg *pkg, struct tndb *db,
+ tn_hash *path_index, tn_buf *nbuf, struct pkgdir_dirindex *prev)
{
tn_array *required = NULL, *owned = NULL;
struct pkgflist *flist;
- int i;
- if ((flist = pkg_get_flist(pkg)) == NULL)
+ if (prev && store_from_previous(package_no, pkg, db, path_index, prev))
return;
- //if (strcmp(pkg_id(pkg), "arachne-common-1.66b-2") != 0)
- // return;
+ if ((flist = pkg_get_flist(pkg)) == NULL)
+ return;
if (pkgfl_owned_and_required_dirs(flist->fl, &owned, &required) == 0) {
DBGF("%s: NULL\n", pkg_id(pkg));
pkgflist_free(flist);
return;
}
+
+ msgn_i(3, 4, " new package %s\n", pkg_id(pkg));
+
DBGF("%s: %d %d\n", pkg_id(pkg), owned ? n_array_size(owned): -1,
required ? n_array_size(required): -1);
if (owned) {
+ int i;
+
for (i=0; i < n_array_size(owned); i++) {
const char *dir = n_array_nth(owned, i);
- add_to_hash(index, dir, nth);
+ add_to_path_index(path_index, dir, package_no);
}
- n_array_free(owned);
}
- if (required) {
- if (n_array_size(required)) {
- char key[512];
- int klen;
-
- klen = req_pkgkey(key, sizeof(key), pkg);
+ if (required || owned) { /* write { package_key => package.{r,o}dirs.join(':') } */
+ char key[512];
+ int klen;
+
+ klen = package_key(key, sizeof(key), pkg, PREFIX_PKGKEY_REQDIR);
+
+ if (owned)
+ n_assert(n_array_size(owned) > 0);
+
+ if (required)
+ n_assert(n_array_size(required) > 0);
+
+ if (required) {
n_buf_clean(nbuf);
- n_buf_printf(nbuf, "/");
+ n_buf_printf(nbuf, "/"); /* prefix all by '/' */
nbuf = dirarray_join(nbuf, required, ":/");
tndb_put(db, key, klen, n_buf_ptr(nbuf), n_buf_size(nbuf));
}
- n_array_free(required);
+
+ if (owned) {
+ n_buf_clean(nbuf);
+ n_buf_printf(nbuf, "/"); /* prefix all by '/' */
+ nbuf = dirarray_join(nbuf, owned, ":/");
+
+ /* ugly, but what for another package_key() call */
+ key[1] = PREFIX_PKGKEY_OWNDIR;
+ tndb_put(db, key, klen, n_buf_ptr(nbuf), n_buf_size(nbuf));
+ }
}
+ n_array_cfree(&owned);
+ n_array_cfree(&required);
pkgflist_free(flist);
}
-static
-int do_pkgdir_dirindex_create(struct pkgdir *pkgdir, const char *path)
+static int dirindex_create(const struct pkgdir *pkgdir, const char *path,
+ struct pkgdir_dirindex *prev_dirindex)
{
struct tndb *db;
tn_buf *nbuf;
- tn_hash *index;
+ tn_hash *path_index;
tn_array *paths;
tn_alloc *na;
struct vflock *lock;
@@ -223,7 +258,8 @@
MEMINF("START");
- msgn(2, "Creating directory index of %s...", pkgdir_idstr(pkgdir));
+ msgn_i(2, 2, "%s directory index of %s...",
+ prev_dirindex ? "Updating" : "Creating", pkgdir_idstr(pkgdir));
n_strdupap(path, &tmp);
dir = n_dirname(tmp);
@@ -242,34 +278,35 @@
}
na = n_alloc_new(4, TN_ALLOC_OBSTACK);
- index = n_hash_new_na(na, n_array_size(pkgdir->pkgs) * 16, (tn_fn_free)n_array_free);
+ path_index = n_hash_new_na(na, n_array_size(pkgdir->pkgs) * 16, (tn_fn_free)n_array_free);
nbuf = n_buf_new(1024 * 16);
for (i=0; i < n_array_size(pkgdir->pkgs); i++) {
struct pkg *pkg = n_array_nth(pkgdir->pkgs, i);
- index_nth(i, pkg, db);
+ store_package_no(i, db, pkg);
}
for (i=0; i < n_array_size(pkgdir->pkgs); i++) {
struct pkg *pkg = n_array_nth(pkgdir->pkgs, i);
- index_package(i, pkg, db, index, nbuf);
+ store_package(i, pkg, db, path_index, nbuf, prev_dirindex);
if (i % 1000 == 0) {
MEMINF("%d packages", i);
}
}
- /* store index to db */
- paths = n_hash_keys(index);
- msgn(3, " saving %d paths\n", n_array_size(paths));
+ /* store { path => packages_no[] } pairs */
+ paths = n_hash_keys(path_index);
+ msgn_i(3, 3, "Saving %d paths\n", n_array_size(paths));
for (i=0; i < n_array_size(paths); i++) {
const char *path = n_array_nth(paths, i);
- tn_array *ids = n_hash_get(index, path);
+ tn_array *ids = n_hash_get(path_index, path);
n_buf_clean(nbuf);
nbuf = dirarray_join(nbuf, ids, ":");
+
DBGF("%s %s\n", path, n_buf_ptr(nbuf));
tndb_put(db, path, strlen(path), n_buf_ptr(nbuf), n_buf_size(nbuf));
@@ -277,7 +314,7 @@
n_array_free(paths);
n_buf_free(nbuf);
- n_hash_free(index);
+ n_hash_free(path_index);
n_alloc_free(na);
tndb_close(db);
vf_lock_release(lock);
@@ -286,7 +323,8 @@
return 1;
}
-static int dirindex_path(char *path, int size, struct pkgdir *pkgdir)
+/* build dirindex path based on pkgdir one */
+static int dirindex_path(char *path, int size, const struct pkgdir *pkgdir)
{
char tmp[PATH_MAX], tmp2[PATH_MAX];
char *ofpath;
@@ -304,52 +342,22 @@
ofpath = dn;
}
- n_snprintf(tmp2, sizeof(tmp2), "%s/dirindex-of-%s.tndb", ofpath, pkgdir->type);
+ n_snprintf(tmp2, sizeof(tmp2), "%s/dirindex.%s.tndb", ofpath, pkgdir->type);
n_snprintf(tmp2, sizeof(tmp2), "%s", ofpath);
DBGF("path = %s\n", ofpath);
n = vf_cachepath(path, size, ofpath);
DBGF("cache path = %s\n", path);
n_assert(n > 0);
- n += n_snprintf(&path[n], size - n, "/dirindex-of-%s.tndb", pkgdir->type);
+ n += n_snprintf(&path[n], size - n, "/dirindex.%s.tndb", pkgdir->type);
DBGF("result = %s\n", path);
n_assert(n > 0);
return n;
}
-
-int pkgdir_dirindex_create(struct pkgdir *pkgdir)
-{
- char path[1024];
- time_t mtime;
-
- dirindex_path(path, sizeof(path), pkgdir);
-
- mtime = poldek_util_mtime(path);
- n_assert(pkgdir->ts);
-
- if (mtime == 0 || mtime < pkgdir->ts) {
- if (do_pkgdir_dirindex_create(pkgdir, path)) {
- struct utimbuf ut;
- ut.actime = ut.modtime = pkgdir->ts;
- utime(path, &ut);
- }
- }
-
- return 1;
-}
-
-void pkgdir_dirindex_close(struct pkgdir_dirindex *dirindex)
-{
- tndb_close(dirindex->db);
- n_hash_free(dirindex->idmap);
- n_alloc_free(dirindex->na);
- /* no free(dirindex), it is allocated by na */
-}
-
-
-static tn_hash *load_ids(struct tndb *db, int npackages)
+/* load { packages_key => package_no } into hash */
+static tn_hash *load_keymap(struct tndb *db, int npackages)
{
struct tndb_it it;
char key[TNDB_KEY_MAX + 1], *val = NULL;
@@ -373,13 +381,17 @@
goto l_end;
}
- while (*key == '_' && *(key + 1) == KEY_PKGID) {
+ while (*key == '_' && *(key + 1) == PREFIX_PKGKEY_NO) {
char *id;
val[vlen] = '\0';
-
- id = na->na_malloc(na, klen);
- memcpy(id, key + 2, klen); /* key + 2 => skipping _KEY_PKGID */
+ DBGF("key = %s\n", key);
+
+ id = na->na_malloc(na, klen - PREFIXLEN + 1);
+ memcpy(id, key + PREFIXLEN, klen - PREFIXLEN); /* skipping prefix */
+ id[klen - PREFIXLEN] = '\0';
+
+ DBGF("%s => %s\n", val, id);
n_hash_replace(keymap, val, id);
@@ -408,110 +420,180 @@
}
static
-const char **get_req_directories(struct tndb *db, char *key, int klen,
- char *val, int vsize, int *n);
+const char **get_package_directories(struct tndb *db, const char *key, int klen,
+ char *val, int vsize, int *ndirs)
+{
+ const char **tl;
+ int vlen;
-struct pkgdir_dirindex *pkgdir_dirindex_open(struct pkgdir *pkgdir)
+ if ((vlen = tndb_get(db, key, klen, val, vsize)) == 0)
+ return NULL;
+
+ n_assert(vlen < vsize);
+ val[vlen] = '\0';
+
+ *ndirs = 0;
+ tl = n_str_tokl_n(val, ":", ndirs);
+
+ if (tl && *ndirs == 0) {
+ n_str_tokl_free(tl);
+ tl = NULL;
+ }
+ return tl;
+}
+
+
+static
+struct tndb *open_index_database(const struct pkgdir *pkgdir, const char *path,
+ tn_hash **keymap)
{
struct tndb *db;
- char path[PATH_MAX];
- int i, rc = 0;
- tn_alloc *na = NULL;
- tn_hash *idmap = NULL, *keymap = NULL;
- struct pkgdir_dirindex *dirindex = NULL;
+ tn_hash *kmap = NULL;
+ int rc = 0;
n_assert(n_array_size(pkgdir->pkgs)); /* XXX: tndb w/o */
-
- dirindex_path(path, sizeof(path), pkgdir);
-
- msgn(2, "Opening directory index of %s...", pkgdir_idstr(pkgdir));
- MEMINF("start");
+ MEMINF("start");
+
rc = 0;
if ((db = tndb_open(path)) == NULL) {
logn(LOGERR, "%s: open failed", path);
- goto l_end;
+ goto l_error_end;
}
if (!tndb_verify(db)) {
logn(LOGERR, _("%s: broken directory index"), path);
- goto l_end;
+ goto l_error_end;
}
MEMINF("opened");
+
+ if ((kmap = load_keymap(db, n_array_size(pkgdir->pkgs))) == NULL)
+ goto l_error_end;
+
+ MEMINF("keymap");
+
+ *keymap = kmap;
+ return db;
+
+l_error_end:
+ if (kmap)
+ n_hash_free(kmap);
<<Diff was trimmed, longer than 597 lines>>
---- CVS-web:
http://cvs.pld-linux.org/poldek/poldek/pkgdir/pkgdir.c?r1=1.46&r2=1.47&f=u
http://cvs.pld-linux.org/poldek/poldek/pkgdir/pkgdir_dirindex.c?r1=1.8&r2=1.9&f=u
http://cvs.pld-linux.org/poldek/poldek/pkgdir/pkgdir_dirindex.h?r1=1.2&r2=1.3&f=u
http://cvs.pld-linux.org/poldek/poldek/pkgdir/pkgdir_intern.h?r1=1.9&r2=1.10&f=u
http://cvs.pld-linux.org/poldek/poldek/pkgdir/pdir/pdir.c?r1=1.29&r2=1.30&f=u
http://cvs.pld-linux.org/poldek/poldek/pkgdir/pdir/pdir.h?r1=1.8&r2=1.9&f=u
More information about the pld-cvs-commit
mailing list