Re: [PATCH v4 16/23] config: create core.multiPackIndex setting
On 7/12/2018 5:05 PM, Junio C Hamano wrote:
Derrick Stolee writes:
The core.multiPackIndex config setting controls the multi-pack-
index (MIDX) feature. If false, the setting will disable all reads
from the multi-pack-index file.
Read this config setting in the new prepare_multi_pack_index_one()
which is called during prepare_packed_git(). This check is run once
per repository.
Add comparison commands in t5319-multi-pack-index.sh to check
typical Git behavior remains the same as the config setting is turned
on and off. This currently includes 'git rev-list' and 'git log'
commands to trigger several object database reads. Currently, these
would only catch an error in the prepare_multi_pack_index_one(), but
with later commits will catch errors in object lookups, abbreviations,
and approximate object counts.
Signed-off-by: Derrick Stolee
midx: prepare midxed_git struct
Signed-off-by: Derrick Stolee
What is going on around here?
Sorry. I squashed the commits, and intended to drop this second commit
message.
diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh
index 4a4fa26f7a..601e28a2f0 100755
--- a/t/t5319-multi-pack-index.sh
+++ b/t/t5319-multi-pack-index.sh
@@ -3,6 +3,8 @@
test_description='multi-pack-indexes'
. ./test-lib.sh
+objdir=.git/objects
+
midx_read_expect () {
NUM_PACKS=$1
NUM_OBJECTS=$2
@@ -76,18 +78,35 @@ test_expect_success 'create objects' '
'
test_expect_success 'write midx with one v1 pack' '
- pack=$(git pack-objects --index-version=1 pack/test
Hmph, so we used to run tests as if $cwd were GIT_OBJECT_DIRECTORY
but now we are running them from the top-level of the working tree,
just like all the other tests? Interesting.
This is the first time we _need_ them in the .git/object directory.
'
+midx_git_two_modes() {
+ git -c core.multiPackIndex=false $1 >expect &&
+ git -c core.multiPackIndex=true $1 >actual &&
+ test_cmp expect actual
+}
+
+compare_results_with_midx() {
Style: "compare_results_with_midx () {", just like mdx_read_expect
near the top of the file, but unlike midx_git_two_modes we see
nearby. Please keep "git grep 'funcname () {'" a usable way to
locate where a shell function is defined without forcing people to
type an asterisk.
Sorry that I missed these two.
Thanks,
-Stolee
Re: [PATCH v4 16/23] config: create core.multiPackIndex setting
Derrick Stolee writes:
> The core.multiPackIndex config setting controls the multi-pack-
> index (MIDX) feature. If false, the setting will disable all reads
> from the multi-pack-index file.
>
> Read this config setting in the new prepare_multi_pack_index_one()
> which is called during prepare_packed_git(). This check is run once
> per repository.
>
> Add comparison commands in t5319-multi-pack-index.sh to check
> typical Git behavior remains the same as the config setting is turned
> on and off. This currently includes 'git rev-list' and 'git log'
> commands to trigger several object database reads. Currently, these
> would only catch an error in the prepare_multi_pack_index_one(), but
> with later commits will catch errors in object lookups, abbreviations,
> and approximate object counts.
>
> Signed-off-by: Derrick Stolee
>
> midx: prepare midxed_git struct
>
> Signed-off-by: Derrick Stolee
What is going on around here?
> diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh
> index 4a4fa26f7a..601e28a2f0 100755
> --- a/t/t5319-multi-pack-index.sh
> +++ b/t/t5319-multi-pack-index.sh
> @@ -3,6 +3,8 @@
> test_description='multi-pack-indexes'
> . ./test-lib.sh
>
> +objdir=.git/objects
> +
> midx_read_expect () {
> NUM_PACKS=$1
> NUM_OBJECTS=$2
> @@ -76,18 +78,35 @@ test_expect_success 'create objects' '
> '
>
> test_expect_success 'write midx with one v1 pack' '
> - pack=$(git pack-objects --index-version=1 pack/test - test_when_finished rm pack/test-$pack.pack pack/test-$pack.idx
> pack/multi-pack-index &&
> - git multi-pack-index --object-dir=. write &&
> - midx_read_expect 1 18 4 .
> + pack=$(git pack-objects --index-version=1 $objdir/pack/test &&
> + test_when_finished rm $objdir/pack/test-$pack.pack \
> + $objdir/pack/test-$pack.idx $objdir/pack/multi-pack-index &&
> + git multi-pack-index --object-dir=$objdir write &&
> + midx_read_expect 1 18 4 $objdir
Hmph, so we used to run tests as if $cwd were GIT_OBJECT_DIRECTORY
but now we are running them from the top-level of the working tree,
just like all the other tests? Interesting.
> '
>
> +midx_git_two_modes() {
> + git -c core.multiPackIndex=false $1 >expect &&
> + git -c core.multiPackIndex=true $1 >actual &&
> + test_cmp expect actual
> +}
> +
> +compare_results_with_midx() {
Style: "compare_results_with_midx () {", just like mdx_read_expect
near the top of the file, but unlike midx_git_two_modes we see
nearby. Please keep "git grep 'funcname () {'" a usable way to
locate where a shell function is defined without forcing people to
type an asterisk.
[PATCH v4 16/23] config: create core.multiPackIndex setting
The core.multiPackIndex config setting controls the multi-pack-
index (MIDX) feature. If false, the setting will disable all reads
from the multi-pack-index file.
Read this config setting in the new prepare_multi_pack_index_one()
which is called during prepare_packed_git(). This check is run once
per repository.
Add comparison commands in t5319-multi-pack-index.sh to check
typical Git behavior remains the same as the config setting is turned
on and off. This currently includes 'git rev-list' and 'git log'
commands to trigger several object database reads. Currently, these
would only catch an error in the prepare_multi_pack_index_one(), but
with later commits will catch errors in object lookups, abbreviations,
and approximate object counts.
Signed-off-by: Derrick Stolee
midx: prepare midxed_git struct
Signed-off-by: Derrick Stolee
---
Documentation/config.txt| 5
midx.c | 25 ++
midx.h | 5
object-store.h | 7 +
packfile.c | 6 -
t/t5319-multi-pack-index.sh | 51 +++--
6 files changed, 85 insertions(+), 14 deletions(-)
diff --git a/Documentation/config.txt b/Documentation/config.txt
index ab641bf5a9..25f817ca42 100644
--- a/Documentation/config.txt
+++ b/Documentation/config.txt
@@ -908,6 +908,11 @@ core.commitGraph::
Enable git commit graph feature. Allows reading from the
commit-graph file.
+core.multiPackIndex::
+ Use the multi-pack-index file to track multiple packfiles using a
+ single index. See link:technical/multi-pack-index.html[the
+ multi-pack-index design document].
+
core.sparseCheckout::
Enable "sparse checkout" feature. See section "Sparse checkout" in
linkgit:git-read-tree[1] for more information.
diff --git a/midx.c b/midx.c
index e83110ae92..4090cf4ca4 100644
--- a/midx.c
+++ b/midx.c
@@ -1,4 +1,5 @@
#include "cache.h"
+#include "config.h"
#include "csum-file.h"
#include "dir.h"
#include "lockfile.h"
@@ -177,6 +178,30 @@ struct multi_pack_index *load_multi_pack_index(const char
*object_dir)
return NULL;
}
+int prepare_multi_pack_index_one(struct repository *r, const char *object_dir)
+{
+ struct multi_pack_index *m = r->objects->multi_pack_index;
+ struct multi_pack_index *m_search;
+ int config_value;
+
+ if (repo_config_get_bool(r, "core.multipackindex", _value) ||
+ !config_value)
+ return 0;
+
+ for (m_search = m; m_search; m_search = m_search->next)
+ if (!strcmp(object_dir, m_search->object_dir))
+ return 1;
+
+ r->objects->multi_pack_index = load_multi_pack_index(object_dir);
+
+ if (r->objects->multi_pack_index) {
+ r->objects->multi_pack_index->next = m;
+ return 1;
+ }
+
+ return 0;
+}
+
static size_t write_midx_header(struct hashfile *f,
unsigned char num_chunks,
uint32_t num_packs)
diff --git a/midx.h b/midx.h
index e15966272f..9bcfc82d2e 100644
--- a/midx.h
+++ b/midx.h
@@ -1,7 +1,11 @@
#ifndef __MIDX_H__
#define __MIDX_H__
+#include "repository.h"
+
struct multi_pack_index {
+ struct multi_pack_index *next;
+
int fd;
const unsigned char *data;
@@ -25,6 +29,7 @@ struct multi_pack_index {
};
struct multi_pack_index *load_multi_pack_index(const char *object_dir);
+int prepare_multi_pack_index_one(struct repository *r, const char *object_dir);
int write_midx_file(const char *object_dir);
diff --git a/object-store.h b/object-store.h
index 13a766aea8..c2b162489a 100644
--- a/object-store.h
+++ b/object-store.h
@@ -105,6 +105,13 @@ struct raw_object_store {
*/
struct oidmap *replace_map;
+ /*
+* private data
+*
+* should only be accessed directly by packfile.c and midx.c
+*/
+ struct multi_pack_index *multi_pack_index;
+
/*
* private data
*
diff --git a/packfile.c b/packfile.c
index 3d652212c6..5d4493dbf4 100644
--- a/packfile.c
+++ b/packfile.c
@@ -15,6 +15,7 @@
#include "tree-walk.h"
#include "tree.h"
#include "object-store.h"
+#include "midx.h"
char *odb_pack_name(struct strbuf *buf,
const unsigned char *sha1,
@@ -935,10 +936,13 @@ static void prepare_packed_git(struct repository *r)
if (r->objects->packed_git_initialized)
return;
+ prepare_multi_pack_index_one(r, r->objects->objectdir);
prepare_packed_git_one(r, r->objects->objectdir, 1);
prepare_alt_odb(r);
- for (alt = r->objects->alt_odb_list; alt; alt = alt->next)
+ for (alt = r->objects->alt_odb_list; alt; alt = alt->next) {
+ prepare_multi_pack_index_one(r, alt->path);
prepare_packed_git_one(r, alt->path, 0);
+ }
rearrange_packed_git(r);
