Re: [gentoo-portage-dev] [PATCH] Escape percent-signs in filename when fetching from mirrors

2020-05-31 Thread Mike Gilbert 
On Sun, May 31, 2020 at 4:20 PM Zac Medico  wrote:
>
> On 5/30/20 8:26 PM, Mike Gilbert wrote:
> > Bug: https://bugs.gentoo.org/719810
> > Signed-off-by: Mike Gilbert 
> > ---
> >  lib/portage/package/ebuild/fetch.py | 9 +++--
> >  1 file changed, 7 insertions(+), 2 deletions(-)
> >
> > diff --git a/lib/portage/package/ebuild/fetch.py 
> > b/lib/portage/package/ebuild/fetch.py
> > index 28e7caf53..47c3ad28f 100644
> > --- a/lib/portage/package/ebuild/fetch.py
> > +++ b/lib/portage/package/ebuild/fetch.py
> > @@ -26,6 +26,11 @@ try:
> >  except ImportError:
> >   from urlparse import urlparse
> >
> > +try:
> > + from urllib.parse import quote as urlquote
> > +except ImportError:
> > + from urllib import quote as urlquote
> > +
> >  import portage
> >  portage.proxy.lazyimport.lazyimport(globals(),
> >   'portage.package.ebuild.config:check_config_instance,config',
> > @@ -351,7 +356,7 @@ _size_suffix_map = {
> >
> >  class FlatLayout(object):
> >   def get_path(self, filename):
> > - return filename
> > + return urlquote(filename)
> >
> >   def get_filenames(self, distdir):
> >   for dirpath, dirnames, filenames in os.walk(distdir,
> > @@ -382,7 +387,7 @@ class FilenameHashLayout(object):
> >   c = c // 4
> >   ret += fnhash[:c] + '/'
> >   fnhash = fnhash[c:]
> > - return ret + filename
> > + return ret + urlquote(filename)
> >
> >   def get_filenames(self, distdir):
> >   pattern = ''
> >
>
> Please go ahead an merge, since SRC_URI arrows allow use to neglect
> unquoting as discussed in
> https://archives.gentoo.org/gentoo-portage-dev/message/176d5e6f38b346be760d05f9c6e72e02.

I ended up reverting this; I did not realize emirrordist shared the
same code. A new patch is on its way.

Re: [gentoo-portage-dev] [PATCH] Escape percent-signs in filename when fetching from mirrors

2020-05-31 Thread Zac Medico 
On 5/30/20 8:26 PM, Mike Gilbert wrote:
> Bug: https://bugs.gentoo.org/719810
> Signed-off-by: Mike Gilbert 
> ---
>  lib/portage/package/ebuild/fetch.py | 9 +++--
>  1 file changed, 7 insertions(+), 2 deletions(-)
> 
> diff --git a/lib/portage/package/ebuild/fetch.py 
> b/lib/portage/package/ebuild/fetch.py
> index 28e7caf53..47c3ad28f 100644
> --- a/lib/portage/package/ebuild/fetch.py
> +++ b/lib/portage/package/ebuild/fetch.py
> @@ -26,6 +26,11 @@ try:
>  except ImportError:
>   from urlparse import urlparse
>  
> +try:
> + from urllib.parse import quote as urlquote
> +except ImportError:
> + from urllib import quote as urlquote
> +
>  import portage
>  portage.proxy.lazyimport.lazyimport(globals(),
>   'portage.package.ebuild.config:check_config_instance,config',
> @@ -351,7 +356,7 @@ _size_suffix_map = {
>  
>  class FlatLayout(object):
>   def get_path(self, filename):
> - return filename
> + return urlquote(filename)
>  
>   def get_filenames(self, distdir):
>   for dirpath, dirnames, filenames in os.walk(distdir,
> @@ -382,7 +387,7 @@ class FilenameHashLayout(object):
>   c = c // 4
>   ret += fnhash[:c] + '/'
>   fnhash = fnhash[c:]
> - return ret + filename
> + return ret + urlquote(filename)
>  
>   def get_filenames(self, distdir):
>   pattern = ''
> 

Please go ahead an merge, since SRC_URI arrows allow use to neglect
unquoting as discussed in
https://archives.gentoo.org/gentoo-portage-dev/message/176d5e6f38b346be760d05f9c6e72e02.
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature

Re: [gentoo-portage-dev] [PATCH] Escape percent-signs in filename when fetching from mirrors

2020-05-31 Thread Ulrich Mueller 
> On Sun, 31 May 2020, Zac Medico wrote:

> In order to ensure that the filename will not have an extra level of quoting
> in the mirror path, we'll have to do something like this wherever we
> translate a uri in SRC_URI to a filename:

> diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
> index 08af17bcd..99d8ab720 100644
> --- a/lib/portage/dbapi/porttree.py
> +++ b/lib/portage/dbapi/porttree.py
> @@ -1547,7 +1547,7 @@ def _parse_uri_map(cpv, metadata, use=None):
> myuris.pop()
> distfile = myuris.pop()
> else:
> -   distfile = os.path.basename(uri)
> +   distfile = unquote(os.path.basename(uri))
> if not distfile:
> raise portage.exception.InvalidDependString(
> ("getFetchMap(): '%s' SRC_URI has no 
> file " + \

I think that unescaping is correct, but you'll need a subsequent sanity
check to make sure that the resulting filename doesn't contain any
characters that aren't allowed in the Manifest.

Namely, "excluding the backwards slash (\) and characters classified as
control characters or as whitespace" (GLEP 74).

Ulrich


signature.asc
Description: PGP signature

Re: [gentoo-portage-dev] [PATCH] Escape percent-signs in filename when fetching from mirrors

2020-05-30 Thread Zac Medico 
On 5/30/20 8:26 PM, Mike Gilbert wrote:
> Bug: https://bugs.gentoo.org/719810
> Signed-off-by: Mike Gilbert 
> ---
>  lib/portage/package/ebuild/fetch.py | 9 +++--
>  1 file changed, 7 insertions(+), 2 deletions(-)
> 
> diff --git a/lib/portage/package/ebuild/fetch.py 
> b/lib/portage/package/ebuild/fetch.py
> index 28e7caf53..47c3ad28f 100644
> --- a/lib/portage/package/ebuild/fetch.py
> +++ b/lib/portage/package/ebuild/fetch.py
> @@ -26,6 +26,11 @@ try:
>  except ImportError:
>   from urlparse import urlparse
>  
> +try:
> + from urllib.parse import quote as urlquote
> +except ImportError:
> + from urllib import quote as urlquote
> +
>  import portage
>  portage.proxy.lazyimport.lazyimport(globals(),
>   'portage.package.ebuild.config:check_config_instance,config',
> @@ -351,7 +356,7 @@ _size_suffix_map = {
>  
>  class FlatLayout(object):
>   def get_path(self, filename):
> - return filename
> + return urlquote(filename)
>  
>   def get_filenames(self, distdir):
>   for dirpath, dirnames, filenames in os.walk(distdir,
> @@ -382,7 +387,7 @@ class FilenameHashLayout(object):
>   c = c // 4
>   ret += fnhash[:c] + '/'
>   fnhash = fnhash[c:]
> - return ret + filename
> + return ret + urlquote(filename)
>  
>   def get_filenames(self, distdir):
>   pattern = ''
> 

In order to ensure that the filename will not have an extra level of quoting
in the mirror path, we'll have to do something like this wherever we
translate a uri in SRC_URI to a filename:

diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 08af17bcd..99d8ab720 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -1547,7 +1547,7 @@ def _parse_uri_map(cpv, metadata, use=None):
myuris.pop()
distfile = myuris.pop()
else:
-   distfile = os.path.basename(uri)
+   distfile = unquote(os.path.basename(uri))
if not distfile:
raise portage.exception.InvalidDependString(
("getFetchMap(): '%s' SRC_URI has no 
file " + \

-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature

[gentoo-portage-dev] [PATCH] Escape percent-signs in filename when fetching from mirrors

2020-05-30 Thread Mike Gilbert 
Bug: https://bugs.gentoo.org/719810
Signed-off-by: Mike Gilbert 
---
 lib/portage/package/ebuild/fetch.py | 9 +++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/lib/portage/package/ebuild/fetch.py 
b/lib/portage/package/ebuild/fetch.py
index 28e7caf53..47c3ad28f 100644
--- a/lib/portage/package/ebuild/fetch.py
+++ b/lib/portage/package/ebuild/fetch.py
@@ -26,6 +26,11 @@ try:
 except ImportError:
from urlparse import urlparse
 
+try:
+   from urllib.parse import quote as urlquote
+except ImportError:
+   from urllib import quote as urlquote
+
 import portage
 portage.proxy.lazyimport.lazyimport(globals(),
'portage.package.ebuild.config:check_config_instance,config',
@@ -351,7 +356,7 @@ _size_suffix_map = {
 
 class FlatLayout(object):
def get_path(self, filename):
-   return filename
+   return urlquote(filename)
 
def get_filenames(self, distdir):
for dirpath, dirnames, filenames in os.walk(distdir,
@@ -382,7 +387,7 @@ class FilenameHashLayout(object):
c = c // 4
ret += fnhash[:c] + '/'
fnhash = fnhash[c:]
-   return ret + filename
+   return ret + urlquote(filename)
 
def get_filenames(self, distdir):
pattern = ''
-- 
2.27.0.rc2