[Frugalware-devel] [RFC] pacman-g2 fix and speed up -Sy / -Su

Miklos Vajna vmiklos at frugalware.org
Mon Jun 20 02:27:00 CEST 2011


Hi,

So Marius came up with the "but having to sometimes use -Syy on -current
is annoying", and in the past we were generally blamed for having a slow
-Sy / -Su operation.

One possible solution is to avoid unpacking the fdb at all, and use
libarchive to read from the fdb when needed. This obviously makes random
access within the fdb hard, but as long as reading the whole fdb at once
is about as fast as reading a single entry from the old db, I would not
worry.

So here is what I have:

1) -Si is a bit slower now:

$ time pacman -Si glibc
real    0m2.957s

$ time ../src/pacman-g2/pacman-g2 --config root/etc/pacman-g2.conf -r root -Si glibc
real    0m3.262s

2) -Sy / -Su is tad faster:

I did not benchmark -Sy, but it basically depends on your bandwidth,
since it's just about downloading frugalware-current.fdb to
/var/lib/pacman-g2.

A noop -Su is way faster:

$ time sudo pacman -Su
:: Starting local database upgrade...
real    1m20.296s

$ time sudo ../src/pacman-g2/pacman-g2 --config root/etc/pacman-g2.conf -r root -Su
:: Starting local database upgrade...
real    0m3.866s

And given that we always read from the fdb, data is always up to date,
no need to use -Syy in case a pkg was uploaded with the same pkgrel
twice.

Needless to say, the testsuite passes.

Why I did not push these patches directly to git:

1) A proof-read of the third patch from others (Michel? :) ) would be
nice. The first two is quite trivial at the moment.

2) I ran out of time, updating the rest of _pacman_db_* functions is not
strictly necessary, but it's ugly to call closedir() on a libarchive
handle.

3) I think further optimisation of _pacman_archive_fgets is possible, if
we do not each character separately (so -Si would not be a slowdown,
either).

4) Other issues: islocal() uses strcmp() at the moment, while it could
just compare to the handle->local_db pointer, seeking to the start of
the fdb is currently done by closing and reopening it, which is ugly.

Thanks.
-------------- next part --------------
From 6790b9d402b997dc3137453c9f4b382afaa50172 Mon Sep 17 00:00:00 2001
From: Miklos Vajna <vmiklos at frugalware.org>
Date: Sun, 19 Jun 2011 14:51:41 +0200
Subject: [PATCH 1/3] pacman_db_update: do not unpack the fdb

---
 lib/libpacman/db.c     |    2 +-
 lib/libpacman/pacman.c |   11 +++--------
 2 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/lib/libpacman/db.c b/lib/libpacman/db.c
index 1e6b47b..d628143 100644
--- a/lib/libpacman/db.c
+++ b/lib/libpacman/db.c
@@ -200,7 +200,7 @@ pmdb_t *_pacman_db_register(const char *treename, pacman_cb_db_register callback
 
 	/* make sure the database directory exists */
 	snprintf(path, PATH_MAX, "%s%s/%s", handle->root, handle->dbpath, treename);
-	if(stat(path, &buf) != 0 || !S_ISDIR(buf.st_mode)) {
+	if(!strcmp(treename, "local") && (stat(path, &buf) != 0 || !S_ISDIR(buf.st_mode))) {
 		_pacman_log(PM_LOG_FLOW1, _("database directory '%s' does not exist -- try creating it"), path);
 		if(_pacman_makepath(path) != 0) {
 			RET_ERR(PM_ERR_SYSTEM, NULL);
diff --git a/lib/libpacman/pacman.c b/lib/libpacman/pacman.c
index 49a8935..e6b0f58 100644
--- a/lib/libpacman/pacman.c
+++ b/lib/libpacman/pacman.c
@@ -316,7 +316,7 @@ int pacman_db_setserver(pmdb_t *db, char *url)
 int pacman_db_update(int force, PM_DB *db)
 {
 	pmlist_t *lp;
-	char path[PATH_MAX], lckpath[PATH_MAX];
+	char path[PATH_MAX], dirpath[PATH_MAX], lckpath[PATH_MAX];
 	pmlist_t *files = NULL;
 	char newmtime[16] = "";
 	char lastupdate[16] = "";
@@ -367,10 +367,10 @@ int pacman_db_update(int force, PM_DB *db)
 			_pacman_log(PM_LOG_DEBUG, _("sync: new mtime for %s: %s\n"), db->treename, newmtime);
 			updated = 1;
 		}
+		snprintf(dirpath, PATH_MAX, "%s%s/%s", handle->root, handle->dbpath, db->treename);
 		snprintf(path, PATH_MAX, "%s%s/%s" PM_EXT_DB, handle->root, handle->dbpath, db->treename);
 
 		/* remove the old dir */
-		if (force) {
 		_pacman_log(PM_LOG_FLOW2, _("flushing database %s/%s"), handle->dbpath, db->treename);
 		for(lp = _pacman_db_get_pkgcache(db); lp; lp = lp->next) {
 			if(_pacman_db_remove(db, lp->data) == -1) {
@@ -381,16 +381,11 @@ int pacman_db_update(int force, PM_DB *db)
 				RET_ERR(PM_ERR_DB_REMOVE, -1);
 			}
 		}
-		}
+		rmdir(dirpath);
 
 		/* Cache needs to be rebuild */
 		_pacman_db_free_pkgcache(db);
 
-		/* uncompress the sync database */
-		if(_pacman_db_install(db, path) == -1) {
-			status = -1;
-			goto rmlck;
-		}
 		if(updated) {
 			_pacman_db_setlastupdate(db, newmtime);
 		}
-- 
1.7.5.4

-------------- next part --------------
From dccf5c94f7bfd7de95f3281eb5926f11a5e49e50 Mon Sep 17 00:00:00 2001
From: Miklos Vajna <vmiklos at frugalware.org>
Date: Mon, 20 Jun 2011 00:12:42 +0200
Subject: [PATCH 2/3] Introduce _pacman_archive_fgets

- this is a port of commit f8c737d3b6b28373a69d5979a4ba2e31b77baa62of pacman.git
- original author: Dan McGee <dan at archlinux.org>
---
 lib/libpacman/util.c |   29 +++++++++++++++++++++++++++++
 lib/libpacman/util.h |    1 +
 2 files changed, 30 insertions(+), 0 deletions(-)

diff --git a/lib/libpacman/util.c b/lib/libpacman/util.c
index 0df66e0..1b0ec8e 100644
--- a/lib/libpacman/util.c
+++ b/lib/libpacman/util.c
@@ -805,6 +805,35 @@ int _pacman_reg_match(char *string, char *pattern)
 	return(!(result));
 }
 
+char *_pacman_archive_fgets(char *line, size_t size, struct archive *a)
+{
+	/* for now, just read one char at a time until we get to a
+	 * '\n' char. we can optimize this later with an internal
+	 * buffer. */
+	/* leave room for zero terminator */
+	char *last = line + size - 1;
+	char *i;
+
+	for(i = line; i < last; i++) {
+		int ret = archive_read_data(a, i, 1);
+		/* special check for first read- if null, return null,
+		 * this indicates EOF */
+		if(i == line && (ret <= 0 || *i == '\0')) {
+			return(NULL);
+		}
+		/* check if read value was null or newline */
+		if(ret <= 0 || *i == '\0' || *i == '\n') {
+			last = i + 1;
+			break;
+		}
+	}
+
+	/* always null terminate the buffer */
+	*last = '\0';
+
+	return(line);
+}
+
 #endif
 
 /* vim: set ts=2 sw=2 noet: */
diff --git a/lib/libpacman/util.h b/lib/libpacman/util.h
index 7171da3..7ff4a02 100644
--- a/lib/libpacman/util.h
+++ b/lib/libpacman/util.h
@@ -76,6 +76,7 @@ int _pacman_reg_match(char *string, char *pattern);
 char* strsep(char** str, const char* delims);
 char* mkdtemp(char *template);
 #endif
+char *_pacman_archive_fgets(char *line, size_t size, struct archive *a);
 
 #endif /* _PACMAN_UTIL_H */
 
-- 
1.7.5.4

-------------- next part --------------
From 0bc2b804ea3e3e4642efcd85cbdeebb8ba455190 Mon Sep 17 00:00:00 2001
From: Miklos Vajna <vmiklos at frugalware.org>
Date: Mon, 20 Jun 2011 00:42:05 +0200
Subject: [PATCH 3/3] Introduce _pacman_db_read_fgets

---
 lib/libpacman/be_files.c |  295 +++++++++++++++++++++++++++++++++-------------
 lib/libpacman/cache.c    |    7 +-
 2 files changed, 220 insertions(+), 82 deletions(-)

diff --git a/lib/libpacman/be_files.c b/lib/libpacman/be_files.c
index f9ccfbb..8357a5d 100644
--- a/lib/libpacman/be_files.c
+++ b/lib/libpacman/be_files.c
@@ -44,6 +44,10 @@
 #include "error.h"
 #include "handle.h"
 
+static inline int islocal(pmdb_t *db)
+{
+	return strcmp(db->treename, "local") == 0;
+}
 
 /* This function is used to convert the downloaded db file to the proper backend
  * format
@@ -103,9 +107,29 @@ int _pacman_db_open(pmdb_t *db)
 		RET_ERR(PM_ERR_DB_NULL, -1);
 	}
 
-	db->handle = opendir(db->path);
-	if(db->handle == NULL) {
-		RET_ERR(PM_ERR_DB_OPEN, -1);
+	if (islocal(db)) {
+		db->handle = opendir(db->path);
+		if(db->handle == NULL) {
+			RET_ERR(PM_ERR_DB_OPEN, -1);
+		}
+	} else {
+		char dbpath[PATH_MAX];
+		snprintf(dbpath, PATH_MAX, "%s" PM_EXT_DB, db->path);
+		struct stat buf;
+		if(stat(dbpath, &buf) != 0) {
+			// db is not there, we'll open it later
+			db->handle = NULL;
+			return 0;
+		}
+		if((db->handle = archive_read_new()) == NULL) {
+			RET_ERR(PM_ERR_DB_OPEN, -1);
+		}
+		archive_read_support_compression_all(db->handle);
+		archive_read_support_format_all(db->handle);
+		if(archive_read_open_filename(db->handle, dbpath, ARCHIVE_DEFAULT_BYTES_PER_BLOCK) != ARCHIVE_OK) {
+			archive_read_finish(db->handle);
+			RET_ERR(PM_ERR_DB_OPEN, -1);
+		}
 	}
 	if(_pacman_db_getlastupdate(db, db->lastupdate) == -1) {
 		db->lastupdate[0] = '\0';
@@ -132,7 +156,17 @@ void _pacman_db_rewind(pmdb_t *db)
 		return;
 	}
 
-	rewinddir(db->handle);
+	if (islocal(db)) {
+		rewinddir(db->handle);
+	} else {
+		char dbpath[PATH_MAX];
+		snprintf(dbpath, PATH_MAX, "%s" PM_EXT_DB, db->path);
+		archive_read_finish(db->handle);
+		db->handle = archive_read_new();
+		archive_read_support_compression_all(db->handle);
+		archive_read_support_format_all(db->handle);
+		archive_read_open_filename(db->handle, dbpath, ARCHIVE_DEFAULT_BYTES_PER_BLOCK);
+	}
 }
 
 pmpkg_t *_pacman_db_scan(pmdb_t *db, const char *target, unsigned int inforeq)
@@ -145,33 +179,66 @@ pmpkg_t *_pacman_db_scan(pmdb_t *db, const char *target, unsigned int inforeq)
 	int found = 0;
 	pmpkg_t *pkg;
 
+	char dbpath[PATH_MAX];
+	snprintf(dbpath, PATH_MAX, "%s" PM_EXT_DB, db->path);
+	struct archive_entry *entry = NULL;
+
 	if(db == NULL) {
 		RET_ERR(PM_ERR_DB_NULL, NULL);
 	}
 
 	if(target != NULL) {
 		/* search for a specific package (by name only) */
-		rewinddir(db->handle);
-		while(!found && (ent = readdir(db->handle)) != NULL) {
-			if(!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) {
-				continue;
-			}
-			/* stat the entry, make sure it's a directory */
-			snprintf(path, PATH_MAX, "%s/%s", db->path, ent->d_name);
-			if(stat(path, &sbuf) || !S_ISDIR(sbuf.st_mode)) {
-				continue;
-			}
-			STRNCPY(name, ent->d_name, PKG_FULLNAME_LEN);
-			/* truncate the string at the second-to-last hyphen, */
-			/* which will give us the package name */
-			if((ptr = rindex(name, '-'))) {
-				*ptr = '\0';
-			}
-			if((ptr = rindex(name, '-'))) {
-				*ptr = '\0';
+		if (islocal(db)) {
+			rewinddir(db->handle);
+			while(!found && (ent = readdir(db->handle)) != NULL) {
+				if(!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) {
+					continue;
+				}
+				/* stat the entry, make sure it's a directory */
+				snprintf(path, PATH_MAX, "%s/%s", db->path, ent->d_name);
+				if(stat(path, &sbuf) || !S_ISDIR(sbuf.st_mode)) {
+					continue;
+				}
+				STRNCPY(name, ent->d_name, PKG_FULLNAME_LEN);
+				/* truncate the string at the second-to-last hyphen, */
+				/* which will give us the package name */
+				if((ptr = rindex(name, '-'))) {
+					*ptr = '\0';
+				}
+				if((ptr = rindex(name, '-'))) {
+					*ptr = '\0';
+				}
+				if(!strcmp(name, target)) {
+					found = 1;
+				}
 			}
-			if(!strcmp(name, target)) {
-				found = 1;
+		} else {
+			// seek to start
+			if (db->handle)
+				archive_read_finish(db->handle);
+			db->handle = archive_read_new();
+			archive_read_support_compression_all(db->handle);
+			archive_read_support_format_all(db->handle);
+			archive_read_open_filename(db->handle, dbpath, ARCHIVE_DEFAULT_BYTES_PER_BLOCK);
+
+			while (!found && archive_read_next_header(db->handle, &entry) == ARCHIVE_OK) {
+				// make sure it's a directory
+				const char *pathname = archive_entry_pathname(entry);
+				if (pathname[strlen(pathname)-1] != '/')
+					continue;
+				STRNCPY(name, pathname, PKG_FULLNAME_LEN);
+				// truncate the string at the second-to-last hyphen,
+				// which will give us the package name
+				if((ptr = rindex(name, '-'))) {
+					*ptr = '\0';
+				}
+				if((ptr = rindex(name, '-'))) {
+					*ptr = '\0';
+				}
+				if(!strcmp(name, target)) {
+					found = 1;
+				}
 			}
 		}
 		if(!found) {
@@ -181,18 +248,34 @@ pmpkg_t *_pacman_db_scan(pmdb_t *db, const char *target, unsigned int inforeq)
 		/* normal iteration */
 		int isdir = 0;
 		while(!isdir) {
-			ent = readdir(db->handle);
-			if(ent == NULL) {
-				return(NULL);
-			}
-			if(!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) {
-				isdir = 0;
-				continue;
-			}
-			/* stat the entry, make sure it's a directory */
-			snprintf(path, PATH_MAX, "%s/%s", db->path, ent->d_name);
-			if(!stat(path, &sbuf) && S_ISDIR(sbuf.st_mode)) {
-				isdir = 1;
+			if (islocal(db)) {
+				ent = readdir(db->handle);
+				if(ent == NULL) {
+					return(NULL);
+				}
+				if(!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) {
+					isdir = 0;
+					continue;
+				}
+				/* stat the entry, make sure it's a directory */
+				snprintf(path, PATH_MAX, "%s/%s", db->path, ent->d_name);
+				if(!stat(path, &sbuf) && S_ISDIR(sbuf.st_mode)) {
+					isdir = 1;
+				}
+			} else {
+				if (!db->handle) {
+					db->handle = archive_read_new();
+					archive_read_support_compression_all(db->handle);
+					archive_read_support_format_all(db->handle);
+					archive_read_open_filename(db->handle, dbpath, ARCHIVE_DEFAULT_BYTES_PER_BLOCK);
+				}
+				if (archive_read_next_header(db->handle, &entry) != ARCHIVE_OK) {
+					return NULL;
+				}
+				// make sure it's a directory
+				const char *pathname = archive_entry_pathname(entry);
+				if (pathname[strlen(pathname)-1] == '/')
+					isdir = 1;
 			}
 		}
 	}
@@ -201,8 +284,15 @@ pmpkg_t *_pacman_db_scan(pmdb_t *db, const char *target, unsigned int inforeq)
 	if(pkg == NULL) {
 		return(NULL);
 	}
-	if(_pacman_pkg_splitname(ent->d_name, pkg->name, pkg->version, 0) == -1) {
-		_pacman_log(PM_LOG_ERROR, _("invalid name for dabatase entry '%s'"), ent->d_name);
+	char *dname;
+	if (islocal(db)) {
+		dname = ent->d_name;
+	} else {
+		dname = strdup(archive_entry_pathname(entry));
+		dname[strlen(dname)-1] = '\0'; // drop trailing slash
+	}
+	if(_pacman_pkg_splitname(dname, pkg->name, pkg->version, 0) == -1) {
+		_pacman_log(PM_LOG_ERROR, _("invalid name for dabatase entry '%s'"), dname);
 		return(NULL);
 	}
 	if(_pacman_db_read(db, inforeq, pkg) == -1) {
@@ -212,6 +302,14 @@ pmpkg_t *_pacman_db_scan(pmdb_t *db, const char *target, unsigned int inforeq)
 	return(pkg);
 }
 
+static char *_pacman_db_read_fgets(pmdb_t *db, char *line, size_t size, FILE *fp)
+{
+	if (islocal(db))
+		return fgets(line, size, fp);
+	else
+		return _pacman_archive_fgets(line, size, db->handle);
+}
+
 static int _pacman_db_read_desc(pmdb_t *db, unsigned int inforeq, pmpkg_t *info)
 {
 	FILE *fp = NULL;
@@ -221,19 +319,21 @@ static int _pacman_db_read_desc(pmdb_t *db, unsigned int inforeq, pmpkg_t *info)
 	pmlist_t *i;
 
 	if(inforeq & INFRQ_DESC) {
-		snprintf(path, PATH_MAX, "%s/%s-%s/desc", db->path, info->name, info->version);
-		fp = fopen(path, "r");
-		if(fp == NULL) {
-			_pacman_log(PM_LOG_DEBUG, "%s (%s)", path, strerror(errno));
-			goto error;
+		if (islocal(db)) {
+			snprintf(path, PATH_MAX, "%s/%s-%s/desc", db->path, info->name, info->version);
+			fp = fopen(path, "r");
+			if(fp == NULL) {
+				_pacman_log(PM_LOG_DEBUG, "%s (%s)", path, strerror(errno));
+				goto error;
+			}
 		}
-		while(!feof(fp)) {
-			if(fgets(line, 256, fp) == NULL) {
+		while(!islocal(db) || !feof(fp)) {
+			if(_pacman_db_read_fgets(db, line, 256, fp) == NULL) {
 				break;
 			}
 			_pacman_strtrim(line);
 			if(!strcmp(line, "%DESC%")) {
-				while(fgets(line, sline, fp) && strlen(_pacman_strtrim(line))) {
+				while(_pacman_db_read_fgets(db, line, sline, fp) && strlen(_pacman_strtrim(line))) {
 					info->desc_localized = _pacman_list_add(info->desc_localized, strdup(line));
 				}
 				STRNCPY(info->desc, (char*)info->desc_localized->data, sizeof(info->desc));
@@ -245,46 +345,46 @@ static int _pacman_db_read_desc(pmdb_t *db, unsigned int inforeq, pmpkg_t *info)
 				}
 				_pacman_strtrim(info->desc);
 			} else if(!strcmp(line, "%GROUPS%")) {
-				while(fgets(line, sline, fp) && strlen(_pacman_strtrim(line))) {
+				while(_pacman_db_read_fgets(db, line, sline, fp) && strlen(_pacman_strtrim(line))) {
 					info->groups = _pacman_list_add(info->groups, strdup(line));
 				}
 			} else if(!strcmp(line, "%URL%")) {
-				if(fgets(info->url, sizeof(info->url), fp) == NULL) {
+				if(_pacman_db_read_fgets(db, info->url, sizeof(info->url), fp) == NULL) {
 					goto error;
 				}
 				_pacman_strtrim(info->url);
 			} else if(!strcmp(line, "%LICENSE%")) {
-				while(fgets(line, sline, fp) && strlen(_pacman_strtrim(line))) {
+				while(_pacman_db_read_fgets(db, line, sline, fp) && strlen(_pacman_strtrim(line))) {
 					info->license = _pacman_list_add(info->license, strdup(line));
 				}
 			} else if(!strcmp(line, "%ARCH%")) {
-				if(fgets(info->arch, sizeof(info->arch), fp) == NULL) {
+				if(_pacman_db_read_fgets(db, info->arch, sizeof(info->arch), fp) == NULL) {
 					goto error;
 				}
 				_pacman_strtrim(info->arch);
 			} else if(!strcmp(line, "%BUILDDATE%")) {
-				if(fgets(info->builddate, sizeof(info->builddate), fp) == NULL) {
+				if(_pacman_db_read_fgets(db, info->builddate, sizeof(info->builddate), fp) == NULL) {
 					goto error;
 				}
 				_pacman_strtrim(info->builddate);
 			} else if(!strcmp(line, "%BUILDTYPE%")) {
-				if(fgets(info->buildtype, sizeof(info->buildtype), fp) == NULL) {
+				if(_pacman_db_read_fgets(db, info->buildtype, sizeof(info->buildtype), fp) == NULL) {
 					goto error;
 				}
 				_pacman_strtrim(info->buildtype);
 			} else if(!strcmp(line, "%INSTALLDATE%")) {
-				if(fgets(info->installdate, sizeof(info->installdate), fp) == NULL) {
+				if(_pacman_db_read_fgets(db, info->installdate, sizeof(info->installdate), fp) == NULL) {
 					goto error;
 				}
 				_pacman_strtrim(info->installdate);
 			} else if(!strcmp(line, "%PACKAGER%")) {
-				if(fgets(info->packager, sizeof(info->packager), fp) == NULL) {
+				if(_pacman_db_read_fgets(db, info->packager, sizeof(info->packager), fp) == NULL) {
 					goto error;
 				}
 				_pacman_strtrim(info->packager);
 			} else if(!strcmp(line, "%REASON%")) {
 				char tmp[32];
-				if(fgets(tmp, sizeof(tmp), fp) == NULL) {
+				if(_pacman_db_read_fgets(db, tmp, sizeof(tmp), fp) == NULL) {
 					goto error;
 				}
 				_pacman_strtrim(tmp);
@@ -296,7 +396,7 @@ static int _pacman_db_read_desc(pmdb_t *db, unsigned int inforeq, pmpkg_t *info)
 				 *       only used in local databases.
 				 */
 				char tmp[32];
-				if(fgets(tmp, sizeof(tmp), fp) == NULL) {
+				if(_pacman_db_read_fgets(db, tmp, sizeof(tmp), fp) == NULL) {
 					goto error;
 				}
 				_pacman_strtrim(tmp);
@@ -305,7 +405,7 @@ static int _pacman_db_read_desc(pmdb_t *db, unsigned int inforeq, pmpkg_t *info)
 				/* USIZE (uncompressed size) tag only appears in sync repositories,
 				 * not the local one. */
 				char tmp[32];
-				if(fgets(tmp, sizeof(tmp), fp) == NULL) {
+				if(_pacman_db_read_fgets(db, tmp, sizeof(tmp), fp) == NULL) {
 					goto error;
 				}
 				_pacman_strtrim(tmp);
@@ -313,13 +413,13 @@ static int _pacman_db_read_desc(pmdb_t *db, unsigned int inforeq, pmpkg_t *info)
 			} else if(!strcmp(line, "%SHA1SUM%")) {
 				/* SHA1SUM tag only appears in sync repositories,
 				 * not the local one. */
-				if(fgets(info->sha1sum, sizeof(info->sha1sum), fp) == NULL) {
+				if(_pacman_db_read_fgets(db, info->sha1sum, sizeof(info->sha1sum), fp) == NULL) {
 					goto error;
 				}
 			} else if(!strcmp(line, "%MD5SUM%")) {
 				/* MD5SUM tag only appears in sync repositories,
 				 * not the local one. */
-				if(fgets(info->md5sum, sizeof(info->md5sum), fp) == NULL) {
+				if(_pacman_db_read_fgets(db, info->md5sum, sizeof(info->md5sum), fp) == NULL) {
 					goto error;
 				}
 			/* XXX: these are only here as backwards-compatibility for pacman
@@ -329,7 +429,7 @@ static int _pacman_db_read_desc(pmdb_t *db, unsigned int inforeq, pmpkg_t *info)
 			} else if(!strcmp(line, "%REPLACES%")) {
 				/* the REPLACES tag is special -- it only appears in sync repositories,
 				 * not the local one. */
-				while(fgets(line, sline, fp) && strlen(_pacman_strtrim(line))) {
+				while(_pacman_db_read_fgets(db, line, sline, fp) && strlen(_pacman_strtrim(line))) {
 					info->replaces = _pacman_list_add(info->replaces, strdup(line));
 				}
 			} else if(!strcmp(line, "%FORCE%")) {
@@ -342,7 +442,8 @@ static int _pacman_db_read_desc(pmdb_t *db, unsigned int inforeq, pmpkg_t *info)
 				info->stick = 1;
 			}
 		}
-		fclose(fp);
+		if (fp)
+			fclose(fp);
 		fp = NULL;
 	}
 
@@ -363,35 +464,39 @@ static int _pacman_db_read_depends(pmdb_t *db, unsigned int inforeq, pmpkg_t *in
 	int sline = sizeof(line)-1;
 
 	if(inforeq & INFRQ_DEPENDS) {
-		snprintf(path, PATH_MAX, "%s/%s-%s/depends", db->path, info->name, info->version);
-		fp = fopen(path, "r");
-		if(fp == NULL) {
-			_pacman_log(PM_LOG_WARNING, "%s (%s)", path, strerror(errno));
-			goto error;
+		if (islocal(db)) {
+			snprintf(path, PATH_MAX, "%s/%s-%s/depends", db->path, info->name, info->version);
+			fp = fopen(path, "r");
+			if(fp == NULL) {
+				_pacman_log(PM_LOG_WARNING, "%s (%s)", path, strerror(errno));
+				goto error;
+			}
 		}
-		while(!feof(fp)) {
-			fgets(line, 255, fp);
+		while(!islocal(db) || !feof(fp)) {
+			if(_pacman_db_read_fgets(db, line, 256, fp) == NULL) {
+				break;
+			}
 			_pacman_strtrim(line);
 			if(!strcmp(line, "%DEPENDS%")) {
-				while(fgets(line, sline, fp) && strlen(_pacman_strtrim(line))) {
+				while(_pacman_db_read_fgets(db, line, sline, fp) && strlen(_pacman_strtrim(line))) {
 					info->depends = _pacman_list_add(info->depends, strdup(line));
 				}
 			} else if(!strcmp(line, "%REQUIREDBY%")) {
-				while(fgets(line, sline, fp) && strlen(_pacman_strtrim(line))) {
+				while(_pacman_db_read_fgets(db, line, sline, fp) && strlen(_pacman_strtrim(line))) {
 					info->requiredby = _pacman_list_add(info->requiredby, strdup(line));
 				}
 			} else if(!strcmp(line, "%CONFLICTS%")) {
-				while(fgets(line, sline, fp) && strlen(_pacman_strtrim(line))) {
+				while(_pacman_db_read_fgets(db, line, sline, fp) && strlen(_pacman_strtrim(line))) {
 					info->conflicts = _pacman_list_add(info->conflicts, strdup(line));
 				}
 			} else if(!strcmp(line, "%PROVIDES%")) {
-				while(fgets(line, sline, fp) && strlen(_pacman_strtrim(line))) {
+				while(_pacman_db_read_fgets(db, line, sline, fp) && strlen(_pacman_strtrim(line))) {
 					info->provides = _pacman_list_add(info->provides, strdup(line));
 				}
 			} else if(!strcmp(line, "%REPLACES%")) {
 				/* the REPLACES tag is special -- it only appears in sync repositories,
 				 * not the local one. */
-				while(fgets(line, sline, fp) && strlen(_pacman_strtrim(line))) {
+				while(_pacman_db_read_fgets(db, line, sline, fp) && strlen(_pacman_strtrim(line))) {
 					info->replaces = _pacman_list_add(info->replaces, strdup(line));
 				}
 			} else if(!strcmp(line, "%FORCE%")) {
@@ -404,7 +509,8 @@ static int _pacman_db_read_depends(pmdb_t *db, unsigned int inforeq, pmpkg_t *in
 				info->stick = 1;
 			}
 		}
-		fclose(fp);
+		if (fp)
+			fclose(fp);
 		fp = NULL;
 	}
 
@@ -417,6 +523,15 @@ error:
 	return(-1);
 }
 
+static int suffixcmp(const char *str, const char *suffix)
+{
+	int len = strlen(str), suflen = strlen(suffix);
+	if (len < suflen)
+		return -1;
+	else
+		return strcmp(str + len - suflen, suffix);
+}
+
 int _pacman_db_read(pmdb_t *db, unsigned int inforeq, pmpkg_t *info)
 {
 	FILE *fp = NULL;
@@ -436,16 +551,36 @@ int _pacman_db_read(pmdb_t *db, unsigned int inforeq, pmpkg_t *info)
 	}
 
 	snprintf(path, PATH_MAX, "%s/%s-%s", db->path, info->name, info->version);
-	if(stat(path, &buf)) {
+	if(islocal(db) && stat(path, &buf)) {
 		/* directory doesn't exist or can't be opened */
 		return(-1);
 	}
 
-	if (_pacman_db_read_desc(db, inforeq, info) == -1)
-		return -1;
+	if (islocal(db)) {
+		if (_pacman_db_read_desc(db, inforeq, info) == -1)
+			return -1;
 
-	if (_pacman_db_read_depends(db, inforeq, info) == -1)
-		return -1;
+		if (_pacman_db_read_depends(db, inforeq, info) == -1)
+			return -1;
+	} else {
+		int descdone = 0, depsdone = 0;
+		while (!descdone || !depsdone) {
+			struct archive_entry *entry = NULL;
+			if (archive_read_next_header(db->handle, &entry) != ARCHIVE_OK)
+				return -1;
+			const char *pathname = archive_entry_pathname(entry);
+			if (!suffixcmp(pathname, "/desc")) {
+				if (_pacman_db_read_desc(db, inforeq, info) == -1)
+					return -1;
+				descdone = 1;
+			}
+			if (!suffixcmp(pathname, "/depends")) {
+				if (_pacman_db_read_depends(db, inforeq, info) == -1)
+					return -1;
+				depsdone = 1;
+			}
+		}
+	}
 
 	/* FILES */
 	if(inforeq & INFRQ_FILES) {
@@ -514,7 +649,7 @@ int _pacman_db_write(pmdb_t *db, pmpkg_t *info, unsigned int inforeq)
 	/* make sure we have a sane umask */
 	umask(0022);
 
-	if(strcmp(db->treename, "local") == 0) {
+	if(islocal(db)) {
 		local = 1;
 	}
 
diff --git a/lib/libpacman/cache.c b/lib/libpacman/cache.c
index 29b1afe..6fbbd1b 100644
--- a/lib/libpacman/cache.c
+++ b/lib/libpacman/cache.c
@@ -54,11 +54,14 @@ int _pacman_db_load_pkgcache(pmdb_t *db)
 
 	_pacman_db_free_pkgcache(db);
 
+	unsigned int inforeq = 0;
+	if (strcmp(db->treename, "local") != 0)
+		inforeq = INFRQ_DESC | INFRQ_DEPENDS;
 	_pacman_log(PM_LOG_DEBUG, _("loading package cache (infolevel=%#x) for repository '%s'"),
-	                        0, db->treename);
+	                        inforeq, db->treename);
 
 	_pacman_db_rewind(db);
-	while((info = _pacman_db_scan(db, NULL, 0)) != NULL) {
+	while((info = _pacman_db_scan(db, NULL, inforeq)) != NULL) {
 		info->origin = PKG_FROM_CACHE;
 		info->data = db;
 		/* add to the collective */
-- 
1.7.5.4

-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 198 bytes
Desc: not available
URL: </pipermail/attachments/20110620/c28dd163/attachment-0001.asc>


More information about the Frugalware-devel mailing list