Re: [PATCH v5 5/8] kbuild: add fine grained build dependencies for exported symbols

[Nicolas Pitre]

On Sat, 5 Mar 2016, Nicolas Pitre wrote:

> On Sat, 5 Mar 2016, Michal Marek wrote:
> 
> > I reproduced the SIGBUS after a few iterations, and it crashes in
> > parse_dep_file(). I'm now testing this
> > 
> > diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include
> > index 0be7e09ba381..4fdd8348acf6 100644
> > --- a/scripts/Kbuild.include
> > +++ b/scripts/Kbuild.include
> > @@ -270,10 +270,12 @@ else
> >  
> >  # Filter out exported kernel symbol names from the preprocessor output.
> >  # See also __KSYM_DEPS__ in include/linux/export.h.
> > +# We disable the depfile generation here, so as not to overwrite the 
> > existing
> > +# depfile while fixdep is parsing it
> >  ksym_dep_filter =  
> >   \
> > case "$(1)" in   \
> > -   cc_*_c) $(CPP) $(c_flags) -D__KSYM_DEPS__ $< ;;  \
> > -   as_*_S) $(CPP) $(a_flags) -D__KSYM_DEPS__ $< ;;  \
> > +   cc_*_c) $(CPP) $(filter-out -Wp$(comma)-M%, $(c_flags)) -D__KSYM_DEPS__ 
> > $< ;; \
> > +   as_*_S) $(CPP) $(filter-out -Wp$(comma)-M%, $(a_flags)) -D__KSYM_DEPS__ 
> > $< ;; \
> > cpp_lds_S) : ;;  \
> > *) echo "Don't know how to preprocess $(1)"; false ;;\
> > esac | sed -rn 's/^.*=== __KSYM_(.*) ===.*$$/KSYM_\1/p'
> 
> This makes perfect sense even if I can't reproduce on my side.

I folded the following patch into my tree and pushed it out.

diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include
index 0be7e09ba3..2c14a27e39 100644
--- a/scripts/Kbuild.include
+++ b/scripts/Kbuild.include
@@ -270,14 +270,17 @@ else
 
 # Filter out exported kernel symbol names from the preprocessor output.
 # See also __KSYM_DEPS__ in include/linux/export.h.
+# We disable the depfile generation here, so as not to overwrite the existing
+# depfile while fixdep is parsing it
+flags_nodeps = $(filter-out -Wp$(comma)-M%, $($(1)))
 ksym_dep_filter =\
case "$(1)" in   \
-   cc_*_c) $(CPP) $(c_flags) -D__KSYM_DEPS__ $< ;;  \
-   as_*_S) $(CPP) $(a_flags) -D__KSYM_DEPS__ $< ;;  \
+   cc_*_c) $(CPP) $(call flags_nodeps,c_flags) -D__KSYM_DEPS__ $< ;;\
+   as_*_S) $(CPP) $(call flags_nodeps,a_flags) -D__KSYM_DEPS__ $< ;;\
cpp_lds_S) : ;;  \
-   *) echo "Don't know how to preprocess $(1)"; false ;;\
+   *) echo "Don't know how to preprocess $(1)" >&2; false ;;\
esac | sed -rn 's/^.*=== __KSYM_(.*) ===.*$$/KSYM_\1/p'
-   
+
 cmd_and_fixdep = \
$(echo-cmd) $(cmd_$(1)); \
$(ksym_dep_filter) | \

Nicolas

Re: [PATCH v5 5/8] kbuild: add fine grained build dependencies for exported symbols

[Nicolas Pitre]

On Sat, 5 Mar 2016, Nicolas Pitre wrote:

> On Sat, 5 Mar 2016, Michal Marek wrote:
> 
> > I reproduced the SIGBUS after a few iterations, and it crashes in
> > parse_dep_file(). I'm now testing this
> > 
> > diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include
> > index 0be7e09ba381..4fdd8348acf6 100644
> > --- a/scripts/Kbuild.include
> > +++ b/scripts/Kbuild.include
> > @@ -270,10 +270,12 @@ else
> >  
> >  # Filter out exported kernel symbol names from the preprocessor output.
> >  # See also __KSYM_DEPS__ in include/linux/export.h.
> > +# We disable the depfile generation here, so as not to overwrite the 
> > existing
> > +# depfile while fixdep is parsing it
> >  ksym_dep_filter =  
> >   \
> > case "$(1)" in   \
> > -   cc_*_c) $(CPP) $(c_flags) -D__KSYM_DEPS__ $< ;;  \
> > -   as_*_S) $(CPP) $(a_flags) -D__KSYM_DEPS__ $< ;;  \
> > +   cc_*_c) $(CPP) $(filter-out -Wp$(comma)-M%, $(c_flags)) -D__KSYM_DEPS__ 
> > $< ;; \
> > +   as_*_S) $(CPP) $(filter-out -Wp$(comma)-M%, $(a_flags)) -D__KSYM_DEPS__ 
> > $< ;; \
> > cpp_lds_S) : ;;  \
> > *) echo "Don't know how to preprocess $(1)"; false ;;\
> > esac | sed -rn 's/^.*=== __KSYM_(.*) ===.*$$/KSYM_\1/p'
> 
> This makes perfect sense even if I can't reproduce on my side.

I folded the following patch into my tree and pushed it out.

diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include
index 0be7e09ba3..2c14a27e39 100644
--- a/scripts/Kbuild.include
+++ b/scripts/Kbuild.include
@@ -270,14 +270,17 @@ else
 
 # Filter out exported kernel symbol names from the preprocessor output.
 # See also __KSYM_DEPS__ in include/linux/export.h.
+# We disable the depfile generation here, so as not to overwrite the existing
+# depfile while fixdep is parsing it
+flags_nodeps = $(filter-out -Wp$(comma)-M%, $($(1)))
 ksym_dep_filter =\
case "$(1)" in   \
-   cc_*_c) $(CPP) $(c_flags) -D__KSYM_DEPS__ $< ;;  \
-   as_*_S) $(CPP) $(a_flags) -D__KSYM_DEPS__ $< ;;  \
+   cc_*_c) $(CPP) $(call flags_nodeps,c_flags) -D__KSYM_DEPS__ $< ;;\
+   as_*_S) $(CPP) $(call flags_nodeps,a_flags) -D__KSYM_DEPS__ $< ;;\
cpp_lds_S) : ;;  \
-   *) echo "Don't know how to preprocess $(1)"; false ;;\
+   *) echo "Don't know how to preprocess $(1)" >&2; false ;;\
esac | sed -rn 's/^.*=== __KSYM_(.*) ===.*$$/KSYM_\1/p'
-   
+
 cmd_and_fixdep = \
$(echo-cmd) $(cmd_$(1)); \
$(ksym_dep_filter) | \

Nicolas

Re: [PATCH v5 5/8] kbuild: add fine grained build dependencies for exported symbols

[Nicolas Pitre]

On Sat, 5 Mar 2016, Michal Marek wrote:

> On Fri, Mar 04, 2016 at 11:53:53PM +0100, Michal Marek wrote:
> > Dne 4.3.2016 v 23:51 Michal Marek napsal(a):
> > > Dne 4.3.2016 v 06:40 Nicolas Pitre napsal(a):
> > >> +cmd_and_fixdep =
> > >>  \
> > >> +$(echo-cmd) $(cmd_$(1));
> > >>  \
> > >> +$(ksym_dep_filter) |
> > >>  \
> > >> +scripts/basic/fixdep -e $(depfile) $@ '$(make-cmd)' 
> > >>  \
> > >> +> $(dot-target).tmp;
> > >>  \
> > >> +rm -f $(depfile);   
> > >>  \
> > >> +mv -f $(dot-target).tmp $(dot-target).cmd;
> > > 
> > > While trying this, I got a SIGBUS from fixdep once. My theory is that
> > > the depfile is mmap()ed by fixdep and modified by the preprocesor run at
> > > the same time. I could not reproduce this so far (still trying). But if
> > > it's really this race, the fix would be to disable dependency generation
> > > in the preprocessor by passing -Wp,MD,/dev/null or somesuch. But we
> > > never had this problem with genksyms, which is weird. It could as well
> > > be that my build machine's memory is faulty :(.
> > 
> > Actually, genksyms does not ran in parallel. neither before nor after
>   run
> > this patch.
> 
> I reproduced the SIGBUS after a few iterations, and it crashes in
> parse_dep_file(). I'm now testing this
> 
> diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include
> index 0be7e09ba381..4fdd8348acf6 100644
> --- a/scripts/Kbuild.include
> +++ b/scripts/Kbuild.include
> @@ -270,10 +270,12 @@ else
>  
>  # Filter out exported kernel symbol names from the preprocessor output.
>  # See also __KSYM_DEPS__ in include/linux/export.h.
> +# We disable the depfile generation here, so as not to overwrite the existing
> +# depfile while fixdep is parsing it
>  ksym_dep_filter =
> \
>   case "$(1)" in   \
> - cc_*_c) $(CPP) $(c_flags) -D__KSYM_DEPS__ $< ;;  \
> - as_*_S) $(CPP) $(a_flags) -D__KSYM_DEPS__ $< ;;  \
> + cc_*_c) $(CPP) $(filter-out -Wp$(comma)-M%, $(c_flags)) -D__KSYM_DEPS__ 
> $< ;; \
> + as_*_S) $(CPP) $(filter-out -Wp$(comma)-M%, $(a_flags)) -D__KSYM_DEPS__ 
> $< ;; \
>   cpp_lds_S) : ;;  \
>   *) echo "Don't know how to preprocess $(1)"; false ;;\
>   esac | sed -rn 's/^.*=== __KSYM_(.*) ===.*$$/KSYM_\1/p'

This makes perfect sense even if I can't reproduce on my side.

Are you willing to fold this patch in, or do you prefer that I repost?


Nicolas

Re: [PATCH v5 5/8] kbuild: add fine grained build dependencies for exported symbols

[Nicolas Pitre]

On Sat, 5 Mar 2016, Michal Marek wrote:

> On Fri, Mar 04, 2016 at 11:53:53PM +0100, Michal Marek wrote:
> > Dne 4.3.2016 v 23:51 Michal Marek napsal(a):
> > > Dne 4.3.2016 v 06:40 Nicolas Pitre napsal(a):
> > >> +cmd_and_fixdep =
> > >>  \
> > >> +$(echo-cmd) $(cmd_$(1));
> > >>  \
> > >> +$(ksym_dep_filter) |
> > >>  \
> > >> +scripts/basic/fixdep -e $(depfile) $@ '$(make-cmd)' 
> > >>  \
> > >> +> $(dot-target).tmp;
> > >>  \
> > >> +rm -f $(depfile);   
> > >>  \
> > >> +mv -f $(dot-target).tmp $(dot-target).cmd;
> > > 
> > > While trying this, I got a SIGBUS from fixdep once. My theory is that
> > > the depfile is mmap()ed by fixdep and modified by the preprocesor run at
> > > the same time. I could not reproduce this so far (still trying). But if
> > > it's really this race, the fix would be to disable dependency generation
> > > in the preprocessor by passing -Wp,MD,/dev/null or somesuch. But we
> > > never had this problem with genksyms, which is weird. It could as well
> > > be that my build machine's memory is faulty :(.
> > 
> > Actually, genksyms does not ran in parallel. neither before nor after
>   run
> > this patch.
> 
> I reproduced the SIGBUS after a few iterations, and it crashes in
> parse_dep_file(). I'm now testing this
> 
> diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include
> index 0be7e09ba381..4fdd8348acf6 100644
> --- a/scripts/Kbuild.include
> +++ b/scripts/Kbuild.include
> @@ -270,10 +270,12 @@ else
>  
>  # Filter out exported kernel symbol names from the preprocessor output.
>  # See also __KSYM_DEPS__ in include/linux/export.h.
> +# We disable the depfile generation here, so as not to overwrite the existing
> +# depfile while fixdep is parsing it
>  ksym_dep_filter =
> \
>   case "$(1)" in   \
> - cc_*_c) $(CPP) $(c_flags) -D__KSYM_DEPS__ $< ;;  \
> - as_*_S) $(CPP) $(a_flags) -D__KSYM_DEPS__ $< ;;  \
> + cc_*_c) $(CPP) $(filter-out -Wp$(comma)-M%, $(c_flags)) -D__KSYM_DEPS__ 
> $< ;; \
> + as_*_S) $(CPP) $(filter-out -Wp$(comma)-M%, $(a_flags)) -D__KSYM_DEPS__ 
> $< ;; \
>   cpp_lds_S) : ;;  \
>   *) echo "Don't know how to preprocess $(1)"; false ;;\
>   esac | sed -rn 's/^.*=== __KSYM_(.*) ===.*$$/KSYM_\1/p'

This makes perfect sense even if I can't reproduce on my side.

Are you willing to fold this patch in, or do you prefer that I repost?


Nicolas

Re: [PATCH v5 5/8] kbuild: add fine grained build dependencies for exported symbols

[Michal Marek]

On Fri, Mar 04, 2016 at 11:53:53PM +0100, Michal Marek wrote:
> Dne 4.3.2016 v 23:51 Michal Marek napsal(a):
> > Dne 4.3.2016 v 06:40 Nicolas Pitre napsal(a):
> >> +cmd_and_fixdep =  
> >>\
> >> +  $(echo-cmd) $(cmd_$(1)); \
> >> +  $(ksym_dep_filter) | \
> >> +  scripts/basic/fixdep -e $(depfile) $@ '$(make-cmd)'  \
> >> +  > $(dot-target).tmp; \
> >> +  rm -f $(depfile);\
> >> +  mv -f $(dot-target).tmp $(dot-target).cmd;
> > 
> > While trying this, I got a SIGBUS from fixdep once. My theory is that
> > the depfile is mmap()ed by fixdep and modified by the preprocesor run at
> > the same time. I could not reproduce this so far (still trying). But if
> > it's really this race, the fix would be to disable dependency generation
> > in the preprocessor by passing -Wp,MD,/dev/null or somesuch. But we
> > never had this problem with genksyms, which is weird. It could as well
> > be that my build machine's memory is faulty :(.
> 
> Actually, genksyms does not ran in parallel. neither before nor after
  run
> this patch.

I reproduced the SIGBUS after a few iterations, and it crashes in
parse_dep_file(). I'm now testing this

diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include
index 0be7e09ba381..4fdd8348acf6 100644
--- a/scripts/Kbuild.include
+++ b/scripts/Kbuild.include
@@ -270,10 +270,12 @@ else
 
 # Filter out exported kernel symbol names from the preprocessor output.
 # See also __KSYM_DEPS__ in include/linux/export.h.
+# We disable the depfile generation here, so as not to overwrite the existing
+# depfile while fixdep is parsing it
 ksym_dep_filter =\
case "$(1)" in   \
-   cc_*_c) $(CPP) $(c_flags) -D__KSYM_DEPS__ $< ;;  \
-   as_*_S) $(CPP) $(a_flags) -D__KSYM_DEPS__ $< ;;  \
+   cc_*_c) $(CPP) $(filter-out -Wp$(comma)-M%, $(c_flags)) -D__KSYM_DEPS__ 
$< ;; \
+   as_*_S) $(CPP) $(filter-out -Wp$(comma)-M%, $(a_flags)) -D__KSYM_DEPS__ 
$< ;; \
cpp_lds_S) : ;;  \
*) echo "Don't know how to preprocess $(1)"; false ;;\
esac | sed -rn 's/^.*=== __KSYM_(.*) ===.*$$/KSYM_\1/p'

Michal

Re: [PATCH v5 5/8] kbuild: add fine grained build dependencies for exported symbols

[Michal Marek]

On Fri, Mar 04, 2016 at 11:53:53PM +0100, Michal Marek wrote:
> Dne 4.3.2016 v 23:51 Michal Marek napsal(a):
> > Dne 4.3.2016 v 06:40 Nicolas Pitre napsal(a):
> >> +cmd_and_fixdep =  
> >>\
> >> +  $(echo-cmd) $(cmd_$(1)); \
> >> +  $(ksym_dep_filter) | \
> >> +  scripts/basic/fixdep -e $(depfile) $@ '$(make-cmd)'  \
> >> +  > $(dot-target).tmp; \
> >> +  rm -f $(depfile);\
> >> +  mv -f $(dot-target).tmp $(dot-target).cmd;
> > 
> > While trying this, I got a SIGBUS from fixdep once. My theory is that
> > the depfile is mmap()ed by fixdep and modified by the preprocesor run at
> > the same time. I could not reproduce this so far (still trying). But if
> > it's really this race, the fix would be to disable dependency generation
> > in the preprocessor by passing -Wp,MD,/dev/null or somesuch. But we
> > never had this problem with genksyms, which is weird. It could as well
> > be that my build machine's memory is faulty :(.
> 
> Actually, genksyms does not ran in parallel. neither before nor after
  run
> this patch.

I reproduced the SIGBUS after a few iterations, and it crashes in
parse_dep_file(). I'm now testing this

diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include
index 0be7e09ba381..4fdd8348acf6 100644
--- a/scripts/Kbuild.include
+++ b/scripts/Kbuild.include
@@ -270,10 +270,12 @@ else
 
 # Filter out exported kernel symbol names from the preprocessor output.
 # See also __KSYM_DEPS__ in include/linux/export.h.
+# We disable the depfile generation here, so as not to overwrite the existing
+# depfile while fixdep is parsing it
 ksym_dep_filter =\
case "$(1)" in   \
-   cc_*_c) $(CPP) $(c_flags) -D__KSYM_DEPS__ $< ;;  \
-   as_*_S) $(CPP) $(a_flags) -D__KSYM_DEPS__ $< ;;  \
+   cc_*_c) $(CPP) $(filter-out -Wp$(comma)-M%, $(c_flags)) -D__KSYM_DEPS__ 
$< ;; \
+   as_*_S) $(CPP) $(filter-out -Wp$(comma)-M%, $(a_flags)) -D__KSYM_DEPS__ 
$< ;; \
cpp_lds_S) : ;;  \
*) echo "Don't know how to preprocess $(1)"; false ;;\
esac | sed -rn 's/^.*=== __KSYM_(.*) ===.*$$/KSYM_\1/p'

Michal

Re: [PATCH v5 5/8] kbuild: add fine grained build dependencies for exported symbols

[Michal Marek]

Dne 4.3.2016 v 23:51 Michal Marek napsal(a):
> Dne 4.3.2016 v 06:40 Nicolas Pitre napsal(a):
>> +cmd_and_fixdep =
>>  \
>> +$(echo-cmd) $(cmd_$(1)); \
>> +$(ksym_dep_filter) | \
>> +scripts/basic/fixdep -e $(depfile) $@ '$(make-cmd)'  \
>> +> $(dot-target).tmp; \
>> +rm -f $(depfile);\
>> +mv -f $(dot-target).tmp $(dot-target).cmd;
> 
> While trying this, I got a SIGBUS from fixdep once. My theory is that
> the depfile is mmap()ed by fixdep and modified by the preprocesor run at
> the same time. I could not reproduce this so far (still trying). But if
> it's really this race, the fix would be to disable dependency generation
> in the preprocessor by passing -Wp,MD,/dev/null or somesuch. But we
> never had this problem with genksyms, which is weird. It could as well
> be that my build machine's memory is faulty :(.

Actually, genksyms does not ran in parallel. neither before nor after
this patch.

Michal

Re: [PATCH v5 5/8] kbuild: add fine grained build dependencies for exported symbols

[Michal Marek]

Dne 4.3.2016 v 23:51 Michal Marek napsal(a):
> Dne 4.3.2016 v 06:40 Nicolas Pitre napsal(a):
>> +cmd_and_fixdep =
>>  \
>> +$(echo-cmd) $(cmd_$(1)); \
>> +$(ksym_dep_filter) | \
>> +scripts/basic/fixdep -e $(depfile) $@ '$(make-cmd)'  \
>> +> $(dot-target).tmp; \
>> +rm -f $(depfile);\
>> +mv -f $(dot-target).tmp $(dot-target).cmd;
> 
> While trying this, I got a SIGBUS from fixdep once. My theory is that
> the depfile is mmap()ed by fixdep and modified by the preprocesor run at
> the same time. I could not reproduce this so far (still trying). But if
> it's really this race, the fix would be to disable dependency generation
> in the preprocessor by passing -Wp,MD,/dev/null or somesuch. But we
> never had this problem with genksyms, which is weird. It could as well
> be that my build machine's memory is faulty :(.

Actually, genksyms does not ran in parallel. neither before nor after
this patch.

Michal

Re: [PATCH v5 5/8] kbuild: add fine grained build dependencies for exported symbols

[Michal Marek]

Dne 4.3.2016 v 06:40 Nicolas Pitre napsal(a):
> +cmd_and_fixdep = 
> \
> + $(echo-cmd) $(cmd_$(1)); \
> + $(ksym_dep_filter) | \
> + scripts/basic/fixdep -e $(depfile) $@ '$(make-cmd)'  \
> + > $(dot-target).tmp; \
> + rm -f $(depfile);\
> + mv -f $(dot-target).tmp $(dot-target).cmd;

While trying this, I got a SIGBUS from fixdep once. My theory is that
the depfile is mmap()ed by fixdep and modified by the preprocesor run at
the same time. I could not reproduce this so far (still trying). But if
it's really this race, the fix would be to disable dependency generation
in the preprocessor by passing -Wp,MD,/dev/null or somesuch. But we
never had this problem with genksyms, which is weird. It could as well
be that my build machine's memory is faulty :(.

Michal

Re: [PATCH v5 5/8] kbuild: add fine grained build dependencies for exported symbols

[Michal Marek]

Dne 4.3.2016 v 06:40 Nicolas Pitre napsal(a):
> +cmd_and_fixdep = 
> \
> + $(echo-cmd) $(cmd_$(1)); \
> + $(ksym_dep_filter) | \
> + scripts/basic/fixdep -e $(depfile) $@ '$(make-cmd)'  \
> + > $(dot-target).tmp; \
> + rm -f $(depfile);\
> + mv -f $(dot-target).tmp $(dot-target).cmd;

While trying this, I got a SIGBUS from fixdep once. My theory is that
the depfile is mmap()ed by fixdep and modified by the preprocesor run at
the same time. I could not reproduce this so far (still trying). But if
it's really this race, the fix would be to disable dependency generation
in the preprocessor by passing -Wp,MD,/dev/null or somesuch. But we
never had this problem with genksyms, which is weird. It could as well
be that my build machine's memory is faulty :(.

Michal

[PATCH v5 5/8] kbuild: add fine grained build dependencies for exported symbols

[Nicolas Pitre]

Like with kconfig options, we now have the ability to compile in and
out individual EXPORT_SYMBOL() declarations based on the content of
include/generated/autoksyms.h.  However we don't want the entire
world to be rebuilt whenever that file is touched.

Let's apply the same build dependency trick used for CONFIG_* symbols
where the time stamp of empty files whose paths matching those symbols
is used to trigger fine grained rebuilds. In our case the key is the
symbol name passed to EXPORT_SYMBOL().

However, unlike config options, we cannot just use fixdep to parse
the source code for EXPORT_SYMBOL(ksym) because several variants exist
and parsing them all in a separate tool, and keeping it in synch, is
not trivially maintainable.  Furthermore, there are variants such as

EXPORT_SYMBOL_GPL(pci_user_read_config_##size);

that are instanciated via a macro for which we can't easily determine
the actual exported symbol name(s) short of actually running the
preprocessor on them.

Storing the symbol name string in a special ELF section doesn't work
for targets that output assembly or preprocessed source.

So the best way is really to leverage the preprocessor by having it
output actual symbol names anchored by a special sequence that can be
easily filtered out. Then the list of symbols is simply fed to fixdep
to be merged with the other dependencies.

That implies the preprocessor is executed twice for each source file.
A previous attempt relied on a warning pragma for each EXPORT_SYMBOL()
instance that was filtered apart from stderr by the build system with
a sed script during the actual compilation pass. Unfortunately the
preprocessor/compiler diagnostic output isn't stable between versions
and this solution, although more efficient, was deemed too fragile.

Because of the lowercasing performed by fixdep, there might be name
collisions triggering spurious rebuilds for similar symbols. But this
shouldn't be a big issue in practice. (This is the case for CONFIG_*
symbols and I didn't want to be different here, whatever the original
reason for doing so.)

To avoid needless build overhead, the exported symbol name gathering is
performed only when CONFIG_TRIM_UNUSED_KSYMS is selected.

Signed-off-by: Nicolas Pitre 
Acked-by: Rusty Russell 
---
 include/linux/export.h | 13 -
 scripts/Kbuild.include | 24 
 scripts/basic/fixdep.c |  1 +
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/include/linux/export.h b/include/linux/export.h
index 77afdb2a25..2f9ccbe6a6 100644
--- a/include/linux/export.h
+++ b/include/linux/export.h
@@ -65,7 +65,18 @@ extern struct module __this_module;
__attribute__((section("___ksymtab" sec "+" #sym), unused)) \
= { (unsigned long), __kstrtab_##sym }
 
-#ifdef CONFIG_TRIM_UNUSED_KSYMS
+#if defined(__KSYM_DEPS__)
+
+/*
+ * For fine grained build dependencies, we want to tell the build system
+ * about each possible exported symbol even if they're not actually exported.
+ * We use a string pattern that is unlikely to be valid code that the build
+ * system filters out from the preprocessor output (see ksym_dep_filter
+ * in scripts/Kbuild.include).
+ */
+#define __EXPORT_SYMBOL(sym, sec)  === __KSYM_##sym ===
+
+#elif defined(CONFIG_TRIM_UNUSED_KSYMS)
 
 #include 
 #include 
diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include
index 8a257fa663..0be7e09ba3 100644
--- a/scripts/Kbuild.include
+++ b/scripts/Kbuild.include
@@ -258,12 +258,36 @@ if_changed_dep = $(if $(strip $(any-prereq) $(arg-check) 
),  \
@set -e; \
$(cmd_and_fixdep))
 
+ifndef CONFIG_TRIM_UNUSED_KSYMS
+
 cmd_and_fixdep = \
$(echo-cmd) $(cmd_$(1)); \
scripts/basic/fixdep $(depfile) $@ '$(make-cmd)' > $(dot-target).tmp;\
rm -f $(depfile);\
mv -f $(dot-target).tmp $(dot-target).cmd;
 
+else
+
+# Filter out exported kernel symbol names from the preprocessor output.
+# See also __KSYM_DEPS__ in include/linux/export.h.
+ksym_dep_filter =\
+   case "$(1)" in   \
+   cc_*_c) $(CPP) $(c_flags) -D__KSYM_DEPS__ $< ;;  \
+   as_*_S) $(CPP) $(a_flags) -D__KSYM_DEPS__ $< ;;  \
+   cpp_lds_S) : ;;  \
+   *) echo "Don't know how to preprocess $(1)"; false ;;\
+   esac | sed -rn 's/^.*=== __KSYM_(.*) ===.*$$/KSYM_\1/p'
+   
+cmd_and_fixdep = \
+   $(echo-cmd) $(cmd_$(1)); \
+   

[PATCH v5 5/8] kbuild: add fine grained build dependencies for exported symbols

[Nicolas Pitre]

Like with kconfig options, we now have the ability to compile in and
out individual EXPORT_SYMBOL() declarations based on the content of
include/generated/autoksyms.h.  However we don't want the entire
world to be rebuilt whenever that file is touched.

Let's apply the same build dependency trick used for CONFIG_* symbols
where the time stamp of empty files whose paths matching those symbols
is used to trigger fine grained rebuilds. In our case the key is the
symbol name passed to EXPORT_SYMBOL().

However, unlike config options, we cannot just use fixdep to parse
the source code for EXPORT_SYMBOL(ksym) because several variants exist
and parsing them all in a separate tool, and keeping it in synch, is
not trivially maintainable.  Furthermore, there are variants such as

EXPORT_SYMBOL_GPL(pci_user_read_config_##size);

that are instanciated via a macro for which we can't easily determine
the actual exported symbol name(s) short of actually running the
preprocessor on them.

Storing the symbol name string in a special ELF section doesn't work
for targets that output assembly or preprocessed source.

So the best way is really to leverage the preprocessor by having it
output actual symbol names anchored by a special sequence that can be
easily filtered out. Then the list of symbols is simply fed to fixdep
to be merged with the other dependencies.

That implies the preprocessor is executed twice for each source file.
A previous attempt relied on a warning pragma for each EXPORT_SYMBOL()
instance that was filtered apart from stderr by the build system with
a sed script during the actual compilation pass. Unfortunately the
preprocessor/compiler diagnostic output isn't stable between versions
and this solution, although more efficient, was deemed too fragile.

Because of the lowercasing performed by fixdep, there might be name
collisions triggering spurious rebuilds for similar symbols. But this
shouldn't be a big issue in practice. (This is the case for CONFIG_*
symbols and I didn't want to be different here, whatever the original
reason for doing so.)

To avoid needless build overhead, the exported symbol name gathering is
performed only when CONFIG_TRIM_UNUSED_KSYMS is selected.

Signed-off-by: Nicolas Pitre 
Acked-by: Rusty Russell 
---
 include/linux/export.h | 13 -
 scripts/Kbuild.include | 24 
 scripts/basic/fixdep.c |  1 +
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/include/linux/export.h b/include/linux/export.h
index 77afdb2a25..2f9ccbe6a6 100644
--- a/include/linux/export.h
+++ b/include/linux/export.h
@@ -65,7 +65,18 @@ extern struct module __this_module;
__attribute__((section("___ksymtab" sec "+" #sym), unused)) \
= { (unsigned long), __kstrtab_##sym }
 
-#ifdef CONFIG_TRIM_UNUSED_KSYMS
+#if defined(__KSYM_DEPS__)
+
+/*
+ * For fine grained build dependencies, we want to tell the build system
+ * about each possible exported symbol even if they're not actually exported.
+ * We use a string pattern that is unlikely to be valid code that the build
+ * system filters out from the preprocessor output (see ksym_dep_filter
+ * in scripts/Kbuild.include).
+ */
+#define __EXPORT_SYMBOL(sym, sec)  === __KSYM_##sym ===
+
+#elif defined(CONFIG_TRIM_UNUSED_KSYMS)
 
 #include 
 #include 
diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include
index 8a257fa663..0be7e09ba3 100644
--- a/scripts/Kbuild.include
+++ b/scripts/Kbuild.include
@@ -258,12 +258,36 @@ if_changed_dep = $(if $(strip $(any-prereq) $(arg-check) 
),  \
@set -e; \
$(cmd_and_fixdep))
 
+ifndef CONFIG_TRIM_UNUSED_KSYMS
+
 cmd_and_fixdep = \
$(echo-cmd) $(cmd_$(1)); \
scripts/basic/fixdep $(depfile) $@ '$(make-cmd)' > $(dot-target).tmp;\
rm -f $(depfile);\
mv -f $(dot-target).tmp $(dot-target).cmd;
 
+else
+
+# Filter out exported kernel symbol names from the preprocessor output.
+# See also __KSYM_DEPS__ in include/linux/export.h.
+ksym_dep_filter =\
+   case "$(1)" in   \
+   cc_*_c) $(CPP) $(c_flags) -D__KSYM_DEPS__ $< ;;  \
+   as_*_S) $(CPP) $(a_flags) -D__KSYM_DEPS__ $< ;;  \
+   cpp_lds_S) : ;;  \
+   *) echo "Don't know how to preprocess $(1)"; false ;;\
+   esac | sed -rn 's/^.*=== __KSYM_(.*) ===.*$$/KSYM_\1/p'
+   
+cmd_and_fixdep = \
+   $(echo-cmd) $(cmd_$(1)); \
+   $(ksym_dep_filter) |