[PATCH 06/24] untracked cache: record/validate dir mtime and reuse cached output
The main readdir loop in read_directory_recursive() is replaced with a
new one that checks if cached results of a directory is still valid.
If a file is added or removed from the index, the containing directory
is invalidated (but not its subdirs). If directory's mtime is changed,
the same happens. If a .gitignore is updated, the containing directory
and all subdirs are invalidated recursively. If dir_struct#flags or
other conditions change, the cache is ignored.
If a directory is invalidated, we opendir/readdir/closedir and run the
exclude machinery on that directory listing as usual. If untracked
cache is also enabled, we'll update the cache along the way. If a
directory is validated, we simply pull the untracked listing out from
the cache. The cache also records the list of direct subdirs that we
have to recurse in. Fully excluded directories are seen as untracked
files.
In the best case when no dirs are invalidated, read_directory()
becomes a series of
stat(dir), open(.gitignore), fstat(), read(), close() and optionally
hash_sha1_file()
For comparison, standard read_directory() is a sequence of
opendir(), readdir(), open(.gitignore), fstat(), read(), close(), the
expensive last_exclude_matching() and closedir().
We already try not to open(.gitignore) if we know it does not exist,
so open/fstat/read/close sequence does not apply to every
directory. The sequence could be reduced further, as noted in
prep_exclude() in another patch. So in theory, the entire best-case
read_directory sequence could be reduced to a series of stat() and
nothing else.
This is not a silver bullet approach. When you compile a C file, for
example, the old .o file is removed and a new one with the same name
created, effectively invalidating the containing directory's cache
(but not its subdirectories). If your build process touches every
directory, this cache adds extra overhead for nothing, so it's a good
idea to separate generated files from tracked files.. Editors may use
the same strategy for saving files. And of course you're out of luck
running your repo on an unsupported filesystem and/or operating system.
Helped-by: Eric Sunshine sunsh...@sunshineco.com
Signed-off-by: Nguyễn Thái Ngọc Duy pclo...@gmail.com
Signed-off-by: Junio C Hamano gits...@pobox.com
---
dir.c | 121 --
dir.h | 2 ++
2 files changed, 121 insertions(+), 2 deletions(-)
diff --git a/dir.c b/dir.c
index 3ea920c..9bbe122 100644
--- a/dir.c
+++ b/dir.c
@@ -37,7 +37,12 @@ enum path_treatment {
struct cached_dir {
DIR *fdir;
struct untracked_cache_dir *untracked;
+ int nr_files;
+ int nr_dirs;
+
struct dirent *de;
+ const char *file;
+ struct untracked_cache_dir *ucd;
};
static enum path_treatment read_directory_recursive(struct dir_struct *dir,
@@ -607,6 +612,14 @@ static void invalidate_gitignore(struct untracked_cache
*uc,
do_invalidate_gitignore(dir);
}
+static void invalidate_directory(struct untracked_cache *uc,
+struct untracked_cache_dir *dir)
+{
+ uc-dir_invalidated++;
+ dir-valid = 0;
+ dir-untracked_nr = 0;
+}
+
/*
* Given a file with name fname, read it (either from disk, or from
* the index if check_index is non-zero), parse it and store the
@@ -1425,6 +1438,39 @@ static enum path_treatment treat_one_path(struct
dir_struct *dir,
}
}
+static enum path_treatment treat_path_fast(struct dir_struct *dir,
+ struct untracked_cache_dir
*untracked,
+ struct cached_dir *cdir,
+ struct strbuf *path,
+ int baselen,
+ const struct path_simplify *simplify)
+{
+ strbuf_setlen(path, baselen);
+ if (!cdir-ucd) {
+ strbuf_addstr(path, cdir-file);
+ return path_untracked;
+ }
+ strbuf_addstr(path, cdir-ucd-name);
+ /* treat_one_path() does this before it calls treat_directory() */
+ if (path-buf[path-len - 1] != '/')
+ strbuf_addch(path, '/');
+ if (cdir-ucd-check_only)
+ /*
+* check_only is set as a result of treat_directory() getting
+* to its bottom. Verify again the same set of directories
+* with check_only set.
+*/
+ return read_directory_recursive(dir, path-buf, path-len,
+ cdir-ucd, 1, simplify);
+ /*
+* We get path_recurse in the first run when
+* directory_exists_in_index() returns index_nonexistent. We
+* are sure that new changes in the index does not impact the
+* outcome. Return now.
+*/
+ return path_recurse;
+}
+
static enum path_treatment treat_path(struct
[PATCH 06/24] untracked cache: record/validate dir mtime and reuse cached output
The main readdir loop in read_directory_recursive() is replaced with a
new one that checks if cached results of a directory is still valid.
If a file is added or removed from the index, the containing directory
is invalidated (but not its subdirs). If directory's mtime is changed,
the same happens. If a .gitignore is updated, the containing directory
and all subdirs are invalidated recursively. If dir_struct#flags or
other conditions change, the cache is ignored.
If a directory is invalidated, we opendir/readdir/closedir and run the
exclude machinery on that directory listing as usual. If untracked
cache is also enabled, we'll update the cache along the way. If a
directory is validated, we simply pull the untracked listing out from
the cache. The cache also records the list of direct subdirs that we
have to recurse in. Fully excluded directories are seen as untracked
files.
In the best case when no dirs are invalidated, read_directory()
becomes a series of
stat(dir), open(.gitignore), fstat(), read(), close() and optionally
hash_sha1_file()
For comparison, standard read_directory() is a sequence of
opendir(), readdir(), open(.gitignore), fstat(), read(), close(), the
expensive last_exclude_matching() and closedir().
We already try not to open(.gitignore) if we know it does not exist,
so open/fstat/read/close sequence does not apply to every
directory. The sequence could be reduced further, as noted in
prep_exclude() in another patch. So in theory, the entire best-case
read_directory sequence could be reduced to a series of stat() and
nothing else.
This is not a silver bullet approach. When you compile a C file, for
example, the old .o file is removed and a new one with the same name
created, effectively invalidating the containing directory's cache
(but not its subdirectories). If your build process touches every
directory, this cache adds extra overhead for nothing, so it's a good
idea to separate generated files from tracked files.. Editors may use
the same strategy for saving files. And of course you're out of luck
running your repo on an unsupported filesystem and/or operating system.
Helped-by: Eric Sunshine sunsh...@sunshineco.com
Signed-off-by: Nguyễn Thái Ngọc Duy pclo...@gmail.com
Signed-off-by: Junio C Hamano gits...@pobox.com
---
dir.c | 121 --
dir.h | 2 ++
2 files changed, 121 insertions(+), 2 deletions(-)
diff --git a/dir.c b/dir.c
index fb6ed86..8c989e3 100644
--- a/dir.c
+++ b/dir.c
@@ -37,7 +37,12 @@ enum path_treatment {
struct cached_dir {
DIR *fdir;
struct untracked_cache_dir *untracked;
+ int nr_files;
+ int nr_dirs;
+
struct dirent *de;
+ const char *file;
+ struct untracked_cache_dir *ucd;
};
static enum path_treatment read_directory_recursive(struct dir_struct *dir,
@@ -607,6 +612,14 @@ static void invalidate_gitignore(struct untracked_cache
*uc,
do_invalidate_gitignore(dir);
}
+static void invalidate_directory(struct untracked_cache *uc,
+struct untracked_cache_dir *dir)
+{
+ uc-dir_invalidated++;
+ dir-valid = 0;
+ dir-untracked_nr = 0;
+}
+
/*
* Given a file with name fname, read it (either from disk, or from
* the index if check_index is non-zero), parse it and store the
@@ -1424,6 +1437,39 @@ static enum path_treatment treat_one_path(struct
dir_struct *dir,
}
}
+static enum path_treatment treat_path_fast(struct dir_struct *dir,
+ struct untracked_cache_dir
*untracked,
+ struct cached_dir *cdir,
+ struct strbuf *path,
+ int baselen,
+ const struct path_simplify *simplify)
+{
+ strbuf_setlen(path, baselen);
+ if (!cdir-ucd) {
+ strbuf_addstr(path, cdir-file);
+ return path_untracked;
+ }
+ strbuf_addstr(path, cdir-ucd-name);
+ /* treat_one_path() does this before it calls treat_directory() */
+ if (path-buf[path-len - 1] != '/')
+ strbuf_addch(path, '/');
+ if (cdir-ucd-check_only)
+ /*
+* check_only is set as a result of treat_directory() getting
+* to its bottom. Verify again the same set of directories
+* with check_only set.
+*/
+ return read_directory_recursive(dir, path-buf, path-len,
+ cdir-ucd, 1, simplify);
+ /*
+* We get path_recurse in the first run when
+* directory_exists_in_index() returns index_nonexistent. We
+* are sure that new changes in the index does not impact the
+* outcome. Return now.
+*/
+ return path_recurse;
+}
+
static enum path_treatment treat_path(struct
[PATCH 06/24] untracked cache: record/validate dir mtime and reuse cached output
The main readdir loop in read_directory_recursive() is replaced with a
new one that checks if cached results of a directory is still valid.
If a file is added or removed from the index, the containing directory
is invalidated (but not its subdirs). If directory's mtime is changed,
the same happens. If a .gitignore is updated, the containing directory
and all subdirs are invalidated recursively. If dir_struct#flags or
other conditions change, the cache is ignored.
If a directory is invalidated, we opendir/readdir/closedir and run the
exclude machinery on that directory listing as usual. If untracked
cache is also enabled, we'll update the cache along the way. If a
directory is validated, we simply pull the untracked listing out from
the cache. The cache also records the list of direct subdirs that we
have to recurse in. Fully excluded directories are seen as untracked
files.
In the best case when no dirs are invalidated, read_directory()
becomes a series of
stat(dir), open(.gitignore), fstat(), read(), close() and optionally
hash_sha1_file()
For comparison, standard read_directory() is a sequence of
opendir(), readdir(), open(.gitignore), fstat(), read(), close(), the
expensive last_exclude_matching() and closedir().
We already try not to open(.gitignore) if we know it does not exist,
so open/fstat/read/close sequence does not apply to every
directory. The sequence could be reduced further, as noted in
prep_exclude() in another patch. So in theory, the entire best-case
read_directory sequence could be reduced to a series of stat() and
nothing else.
This is not a silver bullet approach. When you compile a C file, for
example, the old .o file is removed and a new one with the same name
created, effectively invalidating the containing directory's cache
(but not its subdirectories). If your build process touches every
directory, this cache adds extra overhead for nothing, so it's a good
idea to separate generated files from tracked files.. Editors may use
the same strategy for saving files. And of course you're out of luck
running your repo on an unsupported filesystem and/or operating system.
Helped-by: Eric Sunshine sunsh...@sunshineco.com
Signed-off-by: Nguyễn Thái Ngọc Duy pclo...@gmail.com
Signed-off-by: Junio C Hamano gits...@pobox.com
---
dir.c | 121 --
dir.h | 2 ++
2 files changed, 121 insertions(+), 2 deletions(-)
diff --git a/dir.c b/dir.c
index fb6ed86..8c989e3 100644
--- a/dir.c
+++ b/dir.c
@@ -37,7 +37,12 @@ enum path_treatment {
struct cached_dir {
DIR *fdir;
struct untracked_cache_dir *untracked;
+ int nr_files;
+ int nr_dirs;
+
struct dirent *de;
+ const char *file;
+ struct untracked_cache_dir *ucd;
};
static enum path_treatment read_directory_recursive(struct dir_struct *dir,
@@ -607,6 +612,14 @@ static void invalidate_gitignore(struct untracked_cache
*uc,
do_invalidate_gitignore(dir);
}
+static void invalidate_directory(struct untracked_cache *uc,
+struct untracked_cache_dir *dir)
+{
+ uc-dir_invalidated++;
+ dir-valid = 0;
+ dir-untracked_nr = 0;
+}
+
/*
* Given a file with name fname, read it (either from disk, or from
* the index if check_index is non-zero), parse it and store the
@@ -1424,6 +1437,39 @@ static enum path_treatment treat_one_path(struct
dir_struct *dir,
}
}
+static enum path_treatment treat_path_fast(struct dir_struct *dir,
+ struct untracked_cache_dir
*untracked,
+ struct cached_dir *cdir,
+ struct strbuf *path,
+ int baselen,
+ const struct path_simplify *simplify)
+{
+ strbuf_setlen(path, baselen);
+ if (!cdir-ucd) {
+ strbuf_addstr(path, cdir-file);
+ return path_untracked;
+ }
+ strbuf_addstr(path, cdir-ucd-name);
+ /* treat_one_path() does this before it calls treat_directory() */
+ if (path-buf[path-len - 1] != '/')
+ strbuf_addch(path, '/');
+ if (cdir-ucd-check_only)
+ /*
+* check_only is set as a result of treat_directory() getting
+* to its bottom. Verify again the same set of directories
+* with check_only set.
+*/
+ return read_directory_recursive(dir, path-buf, path-len,
+ cdir-ucd, 1, simplify);
+ /*
+* We get path_recurse in the first run when
+* directory_exists_in_index() returns index_nonexistent. We
+* are sure that new changes in the index does not impact the
+* outcome. Return now.
+*/
+ return path_recurse;
+}
+
static enum path_treatment treat_path(struct
