Re: [PATCH 08/23] midx: read packfiles from pack directory

2018-06-07 Thread Duy Nguyen
On Thu, Jun 7, 2018 at 4:03 PM, Derrick Stolee  wrote:
> @@ -114,14 +119,56 @@ int write_midx_file(const char *object_dir)
>   midx_name);
> }
>
> +   strbuf_addf(_dir, "%s/pack", object_dir);
> +   dir = opendir(pack_dir.buf);
> +
> +   if (!dir) {
> +   error_errno("unable to open pack directory: %s",
> +   pack_dir.buf);

_()

> +   strbuf_release(_dir);
> +   return 1;
> +   }
> +
> +   strbuf_addch(_dir, '/');
> +   pack_dir_len = pack_dir.len;
> +   ALLOC_ARRAY(packs, alloc_packs);
> +   while ((de = readdir(dir)) != NULL) {
> +   if (is_dot_or_dotdot(de->d_name))
> +   continue;
> +
> +   if (ends_with(de->d_name, ".idx")) {
> +   ALLOC_GROW(packs, nr_packs + 1, alloc_packs);
> +
> +   strbuf_setlen(_dir, pack_dir_len);
> +   strbuf_addstr(_dir, de->d_name);
> +
> +   packs[nr_packs] = add_packed_git(pack_dir.buf,
> +pack_dir.len,
> +0);
> +   if (!packs[nr_packs])
> +   warning("failed to add packfile '%s'",
> +   pack_dir.buf);
> +   else
> +   nr_packs++;
> +   }
> +   }
> +   closedir(dir);
> +   strbuf_release(_dir);

Can we refactor and share this scanning-for-packs code with
packfile.c? I'm pretty sure it does something similar in there.

> -   write_midx_header(f, num_chunks, num_packs);
> +   write_midx_header(f, num_chunks, nr_packs);

Hmm.. could have stuck to one name from the beginning...
-- 
Duy


[PATCH 08/23] midx: read packfiles from pack directory

2018-06-07 Thread Derrick Stolee
When constructing a multi-pack-index file for a given object directory,
read the files within the enclosed pack directory and find matches that
end with ".idx" and find the correct paired packfile using
add_packed_git().

Signed-off-by: Derrick Stolee 
---
 midx.c  | 51 +++--
 t/t5319-midx.sh | 15 ---
 2 files changed, 57 insertions(+), 9 deletions(-)

diff --git a/midx.c b/midx.c
index fa18770f1d..9fb89c80a2 100644
--- a/midx.c
+++ b/midx.c
@@ -102,10 +102,15 @@ static size_t write_midx_header(struct hashfile *f,
 int write_midx_file(const char *object_dir)
 {
unsigned char num_chunks = 0;
-   uint32_t num_packs = 0;
char *midx_name;
struct hashfile *f;
struct lock_file lk;
+   struct packed_git **packs = NULL;
+   uint32_t i, nr_packs = 0, alloc_packs = 0;
+   DIR *dir;
+   struct dirent *de;
+   struct strbuf pack_dir = STRBUF_INIT;
+   size_t pack_dir_len;
 
midx_name = get_midx_filename(object_dir);
if (safe_create_leading_directories(midx_name)) {
@@ -114,14 +119,56 @@ int write_midx_file(const char *object_dir)
  midx_name);
}
 
+   strbuf_addf(_dir, "%s/pack", object_dir);
+   dir = opendir(pack_dir.buf);
+
+   if (!dir) {
+   error_errno("unable to open pack directory: %s",
+   pack_dir.buf);
+   strbuf_release(_dir);
+   return 1;
+   }
+
+   strbuf_addch(_dir, '/');
+   pack_dir_len = pack_dir.len;
+   ALLOC_ARRAY(packs, alloc_packs);
+   while ((de = readdir(dir)) != NULL) {
+   if (is_dot_or_dotdot(de->d_name))
+   continue;
+
+   if (ends_with(de->d_name, ".idx")) {
+   ALLOC_GROW(packs, nr_packs + 1, alloc_packs);
+
+   strbuf_setlen(_dir, pack_dir_len);
+   strbuf_addstr(_dir, de->d_name);
+
+   packs[nr_packs] = add_packed_git(pack_dir.buf,
+pack_dir.len,
+0);
+   if (!packs[nr_packs])
+   warning("failed to add packfile '%s'",
+   pack_dir.buf);
+   else
+   nr_packs++;
+   }
+   }
+   closedir(dir);
+   strbuf_release(_dir);
+
hold_lock_file_for_update(, midx_name, LOCK_DIE_ON_ERROR);
f = hashfd(lk.tempfile->fd, lk.tempfile->filename.buf);
FREE_AND_NULL(midx_name);
 
-   write_midx_header(f, num_chunks, num_packs);
+   write_midx_header(f, num_chunks, nr_packs);
 
finalize_hashfile(f, NULL, CSUM_FSYNC | CSUM_HASH_IN_STREAM);
commit_lock_file();
 
+   for (i = 0; i < nr_packs; i++) {
+   close_pack(packs[i]);
+   FREE_AND_NULL(packs[i]);
+   }
+
+   FREE_AND_NULL(packs);
return 0;
 }
diff --git a/t/t5319-midx.sh b/t/t5319-midx.sh
index 2c25a69744..abe545c7c4 100755
--- a/t/t5319-midx.sh
+++ b/t/t5319-midx.sh
@@ -4,8 +4,9 @@ test_description='multi-pack-indexes'
 . ./test-lib.sh
 
 midx_read_expect() {
+   NUM_PACKS=$1
cat >expect <<- EOF
-   header: 4d494458 1 1 0 0
+   header: 4d494458 1 1 0 $NUM_PACKS
object_dir: .
EOF
git midx read --object-dir=. >actual &&
@@ -16,7 +17,7 @@ test_expect_success 'write midx with no packs' '
git midx --object-dir=. write &&
test_when_finished rm pack/multi-pack-index &&
test_path_is_file pack/multi-pack-index &&
-   midx_read_expect
+   midx_read_expect 0
 '
 
 test_expect_success 'create objects' '
@@ -47,14 +48,14 @@ test_expect_success 'write midx with one v1 pack' '
pack=$(git pack-objects --index-version=1 pack/test obj-list &&
git update-ref HEAD $commit &&
-   git pack-objects --index-version=2 test-pack