Bug#340312: mkimage not stat every file in -f (MD5) list

[Ian Jackson]

Package: jigit
Version: 1.15-2
Severity: wishlist
Tags: patch

If you feed mkimage an MD5 list then it stats every file in that list,
even if you say -q.

In my situation I have a magicmirror repository, which contains all
200k files in Debian, Ubuntu and a few other sites, available via NFS
and indexed by md5sum (in one giant directory).  I have a script to
produce an MD5 list in the format expected by mkimage.

But, when I run mkimage, while it is reading the md5 list, it wants to
stat every file in the list.  This is very slow over NFS and of course
quite unnecessary because most of those files aren't going to be used.

The patch below changes this behaviour.  It defers statting files
until the size is needed.

Regards,
Ian.

diff -u jigit-1.15/debian/changelog jigit-1.15/debian/changelog
--- jigit-1.15/debian/changelog
+++ jigit-1.15/debian/changelog
@@ -1,3 +1,10 @@
+jigit (1.15-2.0iwj1) unstable; urgency=low
+
+  * mkimage: Do not stat all files in MD5 file - instead, just
+    stat them when we need them.
+
+ -- Ian Jackson <[EMAIL PROTECTED]>  Tue, 22 Nov 2005 16:29:09 +0000
+
 jigit (1.15-2) unstable; urgency=low
 
   * Fixes for GCC4 compilation. Thanks to Andreas Jochens for the
diff -u jigit-1.15/mkimage.c jigit-1.15/mkimage.c
--- jigit-1.15/mkimage.c
+++ jigit-1.15/mkimage.c
@@ -38,6 +38,7 @@
 
 #define BUF_SIZE 65536
 #define MISSING -1
+#define UNKNOWN -2
 
 #ifndef MIN
 #define MIN(x,y)        ( ((x) < (y)) ? (x) : (y))
@@ -209,14 +210,20 @@
     return 0;
 }
 
-static md5_list_t *find_file_in_md5_list(unsigned char *base64_md5)
+static md5_list_t *find_file_in_md5_list(unsigned char *base64_md5,
+                                        int need_size)
 {
     md5_list_t *md5_list_entry = md5_list_head;
     
     while (md5_list_entry)
     {        
-        if (!memcmp(md5_list_entry->md5, base64_md5, 16))
+        if (!memcmp(md5_list_entry->md5, base64_md5, 16)) {
+            if (need_size &&
+               md5_list_entry->file_size == UNKNOWN)
+               md5_list_entry->file_size =
+                   get_file_size(md5_list_entry->full_path);
             return md5_list_entry;
+        }
         /* else */
         md5_list_entry = md5_list_entry->next;
     }
@@ -278,7 +285,6 @@
     int error = 0;
     char *file_name = NULL;
     char *md5 = NULL;
-    INT64 file_size = 0;
 
     md5_entry[22] = 0;
     md5_entry[23] = 0;
@@ -289,9 +295,7 @@
     if ('\n' == file_name[strlen(file_name) -1])
         file_name[strlen(file_name) - 1] = 0;
     
-    file_size = get_file_size(file_name);
-
-    error = add_md5_entry(file_size, md5, file_name);
+    error = add_md5_entry(UNKNOWN, md5, file_name);
     return 0;
 }
 
@@ -355,7 +359,7 @@
             ptr++;
     }
 
-    if (find_file_in_md5_list((unsigned char *)base64_md5))
+    if (find_file_in_md5_list((unsigned char *)base64_md5, 0))
         return 0; /* We already have an entry for this file; don't
                    * waste any more time on it */
 
@@ -696,7 +700,7 @@
     if (!quick)
         mk_MD5Init(&file_context);
 
-    md5_list_entry = find_file_in_md5_list((unsigned char *)base64_md5);
+    md5_list_entry = find_file_in_md5_list((unsigned char *)base64_md5, 1);
     if (md5_list_entry && file_size == md5_list_entry->file_size)
     {
         input_file = fopen(md5_list_entry->full_path, "rb");


-- 
To UNSUBSCRIBE, email to [EMAIL PROTECTED]
with a subject of "unsubscribe". Trouble? Contact [EMAIL PROTECTED]