Re: [gentoo-portage-dev] [PATCH 3/4] portage.package.ebuild.doebuild: Override DISTDIR unconditionally, #612972

[Zac Medico]

On 03/18/2017 02:16 PM, Michał Górny wrote:
> On sob, 2017-03-18 at 13:45 -0700, Zac Medico wrote:
>> On Sat, Mar 18, 2017 at 1:12 PM, Zac Medico  wrote:
>>> On Sat, Mar 18, 2017 at 12:57 PM, Zac Medico  wrote:
 On Sat, Mar 18, 2017 at 12:04 PM, Michał Górny  wrote:
> Ensure that DISTDIR is always defined to the path to the shadow
> directory. This ensures that PMS rules for consistent value are
> followed, and that no global scope calls should be able to access
> the distfile directory. This also ensures that global-scope assignments
> (e.g. in PATCHES) do not work around the shadow directory.
> ---
>  pym/portage/package/ebuild/doebuild.py   | 9 +
>  pym/portage/package/ebuild/prepare_build_dirs.py | 6 ++
>  2 files changed, 7 insertions(+), 8 deletions(-)
>
> diff --git a/pym/portage/package/ebuild/doebuild.py 
> b/pym/portage/package/ebuild/doebuild.py
> index 15e4abb48..8efc08334 100644
> --- a/pym/portage/package/ebuild/doebuild.py
> +++ b/pym/portage/package/ebuild/doebuild.py
> @@ -348,7 +348,8 @@ def doebuild_environment(myebuild, mydo, myroot=None, 
> settings=None,
>
> mysettings["PORTDIR"] = os.path.realpath(mysettings["PORTDIR"])
> mysettings.pop("PORTDIR_OVERLAY", None)
> -   mysettings["DISTDIR"] = os.path.realpath(mysettings["DISTDIR"])
> +   if "PORTAGE_ACTUAL_DISTDIR" not in mysettings:
> +   mysettings["PORTAGE_ACTUAL_DISTDIR"] = 
> os.path.realpath(mysettings["DISTDIR"])
> mysettings["RPMDIR"]  = os.path.realpath(mysettings["RPMDIR"])
>
> mysettings["ECLASSDIR"]   = mysettings["PORTDIR"]+"/eclass"
> @@ -390,6 +391,7 @@ def doebuild_environment(myebuild, mydo, myroot=None, 
> settings=None,
> mysettings["WORKDIR"] = 
> os.path.join(mysettings["PORTAGE_BUILDDIR"], "work")
> mysettings["D"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], 
> "image") + os.sep
> mysettings["T"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], 
> "temp")
> +   mysettings["DISTDIR"] = 
> os.path.join(settings["PORTAGE_BUILDDIR"], "distdir")
> mysettings["FILESDIR"] = 
> os.path.join(settings["PORTAGE_BUILDDIR"], "files")

 The EbuildFetcher already_fetched and _prefetch_size_ok methods use
 DISTDIR after doebuild_environment has been called, so they need to be
 updated to use PORTAGE_ACTUAL_DISTDIR.
>>>
>>> Also, this code in doebuild uses DISTDIR before _prepare_fake_distdir
>>> gets called called:
>>>
>>> for x in alist:
>>> writemsg_stdout(">>> Checking %s's mtime...\n" % x)
>>> try:
>>> x_st = os.stat(os.path.join(
>>> mysettings["DISTDIR"], x))
>>>
>>> Also, bin/ebuild calls doebuild_environment before doebuild, which
>>> means that this doebuild code also needs to use PORTAGE_ACTUAL_DISTDIR
>>> if it's in mysettings:
>>>
>>> mf = repo_config.load_manifest(pkgdir, mysettings["DISTDIR"])
>>
>> We should probably implement the fake DISTDIR setting inside the
>> config environ method, so that we can eliminate all of this
>> error-prone DISTDIR/PORTAGE_ACTUAL_DISTDIR stuff.
> 
> You're the expert here. I'm just the person who hammers it till it seems
> to work semi-reliably, and curse on whoever designed it ;-P. Would you
> be able to take it from here? I barely understand what you're talking
> about.

Every ebuild phase gets its environment from the config environ method, so
adding this code to the end of the environ method will perform the
necessary translation for all ebuild phases:

diff --git a/pym/portage/package/ebuild/config.py 
b/pym/portage/package/ebuild/config.py
index ef29afe..4c642a7 100644
--- a/pym/portage/package/ebuild/config.py
+++ b/pym/portage/package/ebuild/config.py
@@ -2816,6 +2816,15 @@ class config(object):
else:
raise AssertionError("C locale did not 
pass the test!")
 
+   try:
+   builddir = mydict["PORTAGE_BUILDDIR"]
+   distdir = mydict["DISTDIR"]
+   except KeyError:
+   pass
+   else:
+   mydict["PORTAGE_ACTUAL_DISTDIR"] = distdir
+   mydict["DISTDIR"] = os.path.join(builddir, "distdir")
+
return mydict
 
def thirdpartymirrors(self):


-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH 3/4] portage.package.ebuild.doebuild: Override DISTDIR unconditionally, #612972

[Michał Górny]

On sob, 2017-03-18 at 13:45 -0700, Zac Medico wrote:
> On Sat, Mar 18, 2017 at 1:12 PM, Zac Medico  wrote:
> > On Sat, Mar 18, 2017 at 12:57 PM, Zac Medico  wrote:
> > > On Sat, Mar 18, 2017 at 12:04 PM, Michał Górny  wrote:
> > > > Ensure that DISTDIR is always defined to the path to the shadow
> > > > directory. This ensures that PMS rules for consistent value are
> > > > followed, and that no global scope calls should be able to access
> > > > the distfile directory. This also ensures that global-scope assignments
> > > > (e.g. in PATCHES) do not work around the shadow directory.
> > > > ---
> > > >  pym/portage/package/ebuild/doebuild.py   | 9 +
> > > >  pym/portage/package/ebuild/prepare_build_dirs.py | 6 ++
> > > >  2 files changed, 7 insertions(+), 8 deletions(-)
> > > > 
> > > > diff --git a/pym/portage/package/ebuild/doebuild.py 
> > > > b/pym/portage/package/ebuild/doebuild.py
> > > > index 15e4abb48..8efc08334 100644
> > > > --- a/pym/portage/package/ebuild/doebuild.py
> > > > +++ b/pym/portage/package/ebuild/doebuild.py
> > > > @@ -348,7 +348,8 @@ def doebuild_environment(myebuild, mydo, 
> > > > myroot=None, settings=None,
> > > > 
> > > > mysettings["PORTDIR"] = os.path.realpath(mysettings["PORTDIR"])
> > > > mysettings.pop("PORTDIR_OVERLAY", None)
> > > > -   mysettings["DISTDIR"] = os.path.realpath(mysettings["DISTDIR"])
> > > > +   if "PORTAGE_ACTUAL_DISTDIR" not in mysettings:
> > > > +   mysettings["PORTAGE_ACTUAL_DISTDIR"] = 
> > > > os.path.realpath(mysettings["DISTDIR"])
> > > > mysettings["RPMDIR"]  = os.path.realpath(mysettings["RPMDIR"])
> > > > 
> > > > mysettings["ECLASSDIR"]   = mysettings["PORTDIR"]+"/eclass"
> > > > @@ -390,6 +391,7 @@ def doebuild_environment(myebuild, mydo, 
> > > > myroot=None, settings=None,
> > > > mysettings["WORKDIR"] = 
> > > > os.path.join(mysettings["PORTAGE_BUILDDIR"], "work")
> > > > mysettings["D"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], 
> > > > "image") + os.sep
> > > > mysettings["T"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], 
> > > > "temp")
> > > > +   mysettings["DISTDIR"] = 
> > > > os.path.join(settings["PORTAGE_BUILDDIR"], "distdir")
> > > > mysettings["FILESDIR"] = 
> > > > os.path.join(settings["PORTAGE_BUILDDIR"], "files")
> > > 
> > > The EbuildFetcher already_fetched and _prefetch_size_ok methods use
> > > DISTDIR after doebuild_environment has been called, so they need to be
> > > updated to use PORTAGE_ACTUAL_DISTDIR.
> > 
> > Also, this code in doebuild uses DISTDIR before _prepare_fake_distdir
> > gets called called:
> > 
> > for x in alist:
> > writemsg_stdout(">>> Checking %s's mtime...\n" % x)
> > try:
> > x_st = os.stat(os.path.join(
> > mysettings["DISTDIR"], x))
> > 
> > Also, bin/ebuild calls doebuild_environment before doebuild, which
> > means that this doebuild code also needs to use PORTAGE_ACTUAL_DISTDIR
> > if it's in mysettings:
> > 
> > mf = repo_config.load_manifest(pkgdir, mysettings["DISTDIR"])
> 
> We should probably implement the fake DISTDIR setting inside the
> config environ method, so that we can eliminate all of this
> error-prone DISTDIR/PORTAGE_ACTUAL_DISTDIR stuff.

You're the expert here. I'm just the person who hammers it till it seems
to work semi-reliably, and curse on whoever designed it ;-P. Would you
be able to take it from here? I barely understand what you're talking
about.

-- 
Best regards,
Michał Górny


signature.asc
Description: This is a digitally signed message part

Re: [gentoo-portage-dev] [PATCH 3/4] portage.package.ebuild.doebuild: Override DISTDIR unconditionally, #612972

[Zac Medico]

On Sat, Mar 18, 2017 at 1:12 PM, Zac Medico  wrote:
> On Sat, Mar 18, 2017 at 12:57 PM, Zac Medico  wrote:
>> On Sat, Mar 18, 2017 at 12:04 PM, Michał Górny  wrote:
>>> Ensure that DISTDIR is always defined to the path to the shadow
>>> directory. This ensures that PMS rules for consistent value are
>>> followed, and that no global scope calls should be able to access
>>> the distfile directory. This also ensures that global-scope assignments
>>> (e.g. in PATCHES) do not work around the shadow directory.
>>> ---
>>>  pym/portage/package/ebuild/doebuild.py   | 9 +
>>>  pym/portage/package/ebuild/prepare_build_dirs.py | 6 ++
>>>  2 files changed, 7 insertions(+), 8 deletions(-)
>>>
>>> diff --git a/pym/portage/package/ebuild/doebuild.py 
>>> b/pym/portage/package/ebuild/doebuild.py
>>> index 15e4abb48..8efc08334 100644
>>> --- a/pym/portage/package/ebuild/doebuild.py
>>> +++ b/pym/portage/package/ebuild/doebuild.py
>>> @@ -348,7 +348,8 @@ def doebuild_environment(myebuild, mydo, myroot=None, 
>>> settings=None,
>>>
>>> mysettings["PORTDIR"] = os.path.realpath(mysettings["PORTDIR"])
>>> mysettings.pop("PORTDIR_OVERLAY", None)
>>> -   mysettings["DISTDIR"] = os.path.realpath(mysettings["DISTDIR"])
>>> +   if "PORTAGE_ACTUAL_DISTDIR" not in mysettings:
>>> +   mysettings["PORTAGE_ACTUAL_DISTDIR"] = 
>>> os.path.realpath(mysettings["DISTDIR"])
>>> mysettings["RPMDIR"]  = os.path.realpath(mysettings["RPMDIR"])
>>>
>>> mysettings["ECLASSDIR"]   = mysettings["PORTDIR"]+"/eclass"
>>> @@ -390,6 +391,7 @@ def doebuild_environment(myebuild, mydo, myroot=None, 
>>> settings=None,
>>> mysettings["WORKDIR"] = 
>>> os.path.join(mysettings["PORTAGE_BUILDDIR"], "work")
>>> mysettings["D"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], 
>>> "image") + os.sep
>>> mysettings["T"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], 
>>> "temp")
>>> +   mysettings["DISTDIR"] = os.path.join(settings["PORTAGE_BUILDDIR"], 
>>> "distdir")
>>> mysettings["FILESDIR"] = os.path.join(settings["PORTAGE_BUILDDIR"], 
>>> "files")
>>
>> The EbuildFetcher already_fetched and _prefetch_size_ok methods use
>> DISTDIR after doebuild_environment has been called, so they need to be
>> updated to use PORTAGE_ACTUAL_DISTDIR.
>
> Also, this code in doebuild uses DISTDIR before _prepare_fake_distdir
> gets called called:
>
> for x in alist:
> writemsg_stdout(">>> Checking %s's mtime...\n" % x)
> try:
> x_st = os.stat(os.path.join(
> mysettings["DISTDIR"], x))
>
> Also, bin/ebuild calls doebuild_environment before doebuild, which
> means that this doebuild code also needs to use PORTAGE_ACTUAL_DISTDIR
> if it's in mysettings:
>
> mf = repo_config.load_manifest(pkgdir, mysettings["DISTDIR"])

We should probably implement the fake DISTDIR setting inside the
config environ method, so that we can eliminate all of this
error-prone DISTDIR/PORTAGE_ACTUAL_DISTDIR stuff.
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH 3/4] portage.package.ebuild.doebuild: Override DISTDIR unconditionally, #612972

[Zac Medico]

On Sat, Mar 18, 2017 at 12:57 PM, Zac Medico  wrote:
> On Sat, Mar 18, 2017 at 12:04 PM, Michał Górny  wrote:
>> Ensure that DISTDIR is always defined to the path to the shadow
>> directory. This ensures that PMS rules for consistent value are
>> followed, and that no global scope calls should be able to access
>> the distfile directory. This also ensures that global-scope assignments
>> (e.g. in PATCHES) do not work around the shadow directory.
>> ---
>>  pym/portage/package/ebuild/doebuild.py   | 9 +
>>  pym/portage/package/ebuild/prepare_build_dirs.py | 6 ++
>>  2 files changed, 7 insertions(+), 8 deletions(-)
>>
>> diff --git a/pym/portage/package/ebuild/doebuild.py 
>> b/pym/portage/package/ebuild/doebuild.py
>> index 15e4abb48..8efc08334 100644
>> --- a/pym/portage/package/ebuild/doebuild.py
>> +++ b/pym/portage/package/ebuild/doebuild.py
>> @@ -348,7 +348,8 @@ def doebuild_environment(myebuild, mydo, myroot=None, 
>> settings=None,
>>
>> mysettings["PORTDIR"] = os.path.realpath(mysettings["PORTDIR"])
>> mysettings.pop("PORTDIR_OVERLAY", None)
>> -   mysettings["DISTDIR"] = os.path.realpath(mysettings["DISTDIR"])
>> +   if "PORTAGE_ACTUAL_DISTDIR" not in mysettings:
>> +   mysettings["PORTAGE_ACTUAL_DISTDIR"] = 
>> os.path.realpath(mysettings["DISTDIR"])
>> mysettings["RPMDIR"]  = os.path.realpath(mysettings["RPMDIR"])
>>
>> mysettings["ECLASSDIR"]   = mysettings["PORTDIR"]+"/eclass"
>> @@ -390,6 +391,7 @@ def doebuild_environment(myebuild, mydo, myroot=None, 
>> settings=None,
>> mysettings["WORKDIR"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], 
>> "work")
>> mysettings["D"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], 
>> "image") + os.sep
>> mysettings["T"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], 
>> "temp")
>> +   mysettings["DISTDIR"] = os.path.join(settings["PORTAGE_BUILDDIR"], 
>> "distdir")
>> mysettings["FILESDIR"] = os.path.join(settings["PORTAGE_BUILDDIR"], 
>> "files")
>
> The EbuildFetcher already_fetched and _prefetch_size_ok methods use
> DISTDIR after doebuild_environment has been called, so they need to be
> updated to use PORTAGE_ACTUAL_DISTDIR.

Also, this code in doebuild uses DISTDIR before _prepare_fake_distdir
gets called called:

for x in alist:
writemsg_stdout(">>> Checking %s's mtime...\n" % x)
try:
x_st = os.stat(os.path.join(
mysettings["DISTDIR"], x))

Also, bin/ebuild calls doebuild_environment before doebuild, which
means that this doebuild code also needs to use PORTAGE_ACTUAL_DISTDIR
if it's in mysettings:

mf = repo_config.load_manifest(pkgdir, mysettings["DISTDIR"])
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH 3/4] portage.package.ebuild.doebuild: Override DISTDIR unconditionally, #612972

[Zac Medico]

On Sat, Mar 18, 2017 at 12:04 PM, Michał Górny  wrote:
> Ensure that DISTDIR is always defined to the path to the shadow
> directory. This ensures that PMS rules for consistent value are
> followed, and that no global scope calls should be able to access
> the distfile directory. This also ensures that global-scope assignments
> (e.g. in PATCHES) do not work around the shadow directory.
> ---
>  pym/portage/package/ebuild/doebuild.py   | 9 +
>  pym/portage/package/ebuild/prepare_build_dirs.py | 6 ++
>  2 files changed, 7 insertions(+), 8 deletions(-)
>
> diff --git a/pym/portage/package/ebuild/doebuild.py 
> b/pym/portage/package/ebuild/doebuild.py
> index 15e4abb48..8efc08334 100644
> --- a/pym/portage/package/ebuild/doebuild.py
> +++ b/pym/portage/package/ebuild/doebuild.py
> @@ -348,7 +348,8 @@ def doebuild_environment(myebuild, mydo, myroot=None, 
> settings=None,
>
> mysettings["PORTDIR"] = os.path.realpath(mysettings["PORTDIR"])
> mysettings.pop("PORTDIR_OVERLAY", None)
> -   mysettings["DISTDIR"] = os.path.realpath(mysettings["DISTDIR"])
> +   if "PORTAGE_ACTUAL_DISTDIR" not in mysettings:
> +   mysettings["PORTAGE_ACTUAL_DISTDIR"] = 
> os.path.realpath(mysettings["DISTDIR"])
> mysettings["RPMDIR"]  = os.path.realpath(mysettings["RPMDIR"])
>
> mysettings["ECLASSDIR"]   = mysettings["PORTDIR"]+"/eclass"
> @@ -390,6 +391,7 @@ def doebuild_environment(myebuild, mydo, myroot=None, 
> settings=None,
> mysettings["WORKDIR"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], 
> "work")
> mysettings["D"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], 
> "image") + os.sep
> mysettings["T"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], "temp")
> +   mysettings["DISTDIR"] = os.path.join(settings["PORTAGE_BUILDDIR"], 
> "distdir")
> mysettings["FILESDIR"] = os.path.join(settings["PORTAGE_BUILDDIR"], 
> "files")

The EbuildFetcher already_fetched and _prefetch_size_ok methods use
DISTDIR after doebuild_environment has been called, so they need to be
updated to use PORTAGE_ACTUAL_DISTDIR.
-- 
Thanks,
Zac

[gentoo-portage-dev] [PATCH 3/4] portage.package.ebuild.doebuild: Override DISTDIR unconditionally, #612972

[Michał Górny]

Ensure that DISTDIR is always defined to the path to the shadow
directory. This ensures that PMS rules for consistent value are
followed, and that no global scope calls should be able to access
the distfile directory. This also ensures that global-scope assignments
(e.g. in PATCHES) do not work around the shadow directory.
---
 pym/portage/package/ebuild/doebuild.py   | 9 +
 pym/portage/package/ebuild/prepare_build_dirs.py | 6 ++
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/pym/portage/package/ebuild/doebuild.py 
b/pym/portage/package/ebuild/doebuild.py
index 15e4abb48..8efc08334 100644
--- a/pym/portage/package/ebuild/doebuild.py
+++ b/pym/portage/package/ebuild/doebuild.py
@@ -348,7 +348,8 @@ def doebuild_environment(myebuild, mydo, myroot=None, 
settings=None,
 
mysettings["PORTDIR"] = os.path.realpath(mysettings["PORTDIR"])
mysettings.pop("PORTDIR_OVERLAY", None)
-   mysettings["DISTDIR"] = os.path.realpath(mysettings["DISTDIR"])
+   if "PORTAGE_ACTUAL_DISTDIR" not in mysettings:
+   mysettings["PORTAGE_ACTUAL_DISTDIR"] = 
os.path.realpath(mysettings["DISTDIR"])
mysettings["RPMDIR"]  = os.path.realpath(mysettings["RPMDIR"])
 
mysettings["ECLASSDIR"]   = mysettings["PORTDIR"]+"/eclass"
@@ -390,6 +391,7 @@ def doebuild_environment(myebuild, mydo, myroot=None, 
settings=None,
mysettings["WORKDIR"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], 
"work")
mysettings["D"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], "image") 
+ os.sep
mysettings["T"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], "temp")
+   mysettings["DISTDIR"] = os.path.join(settings["PORTAGE_BUILDDIR"], 
"distdir")
mysettings["FILESDIR"] = os.path.join(settings["PORTAGE_BUILDDIR"], 
"files")
 
# Prefix forward compatability
@@ -1207,9 +1209,8 @@ def doebuild(myebuild, mydo, _unused=DeprecationWarning, 
settings=None, debug=0,
mysettings.pop("REPLACING_VERSIONS", None)
 
# Make sure that DISTDIR is restored to it's normal value 
before we return!
-   if "PORTAGE_ACTUAL_DISTDIR" in mysettings:
-   mysettings["DISTDIR"] = 
mysettings["PORTAGE_ACTUAL_DISTDIR"]
-   del mysettings["PORTAGE_ACTUAL_DISTDIR"]
+   mysettings["DISTDIR"] = mysettings["PORTAGE_ACTUAL_DISTDIR"]
+   del mysettings["PORTAGE_ACTUAL_DISTDIR"]
 
if logfile and not returnpid:
try:
diff --git a/pym/portage/package/ebuild/prepare_build_dirs.py 
b/pym/portage/package/ebuild/prepare_build_dirs.py
index 0fa43ba59..6528d550e 100644
--- a/pym/portage/package/ebuild/prepare_build_dirs.py
+++ b/pym/portage/package/ebuild/prepare_build_dirs.py
@@ -411,10 +411,8 @@ def _prepare_fake_filesdir(settings):
os.symlink(real_filesdir, symlink_path)
 
 def _prepare_fake_distdir(settings, alist):
-   orig_distdir = settings["DISTDIR"]
-   settings["PORTAGE_ACTUAL_DISTDIR"] = orig_distdir
-   edpath = settings["DISTDIR"] = \
-   os.path.join(settings["PORTAGE_BUILDDIR"], "distdir")
+   orig_distdir = settings["PORTAGE_ACTUAL_DISTDIR"]
+   edpath = settings["DISTDIR"]
portage.util.ensure_dirs(edpath, gid=portage_gid, mode=0o755)
 
# Remove any unexpected files or directories.
-- 
2.12.0