This is an automated email from the git hooks/post-receive script.

lunar pushed a commit to branch pu/reproducible_builds
in repository dh-python.

commit 78c479a3558c5cc6d2e5f567d0c64f5d42bb9d14
Author: Piotr Ożarowski <pi...@debian.org>
Date:   Mon Jul 1 22:45:12 2013 +0200

    move dh_python{2,3}'s scan method into dhpython.fs.Scan class
    
    ... and make it easier to overload it (and read! :)
---
 dh_python2           | 243 ++++-------------------------------------
 dh_python3           | 216 +++---------------------------------
 dhpython/__init__.py |   6 +-
 dhpython/fs.py       | 301 +++++++++++++++++++++++++++++++++++++++++++++++++--
 4 files changed, 331 insertions(+), 435 deletions(-)

diff --git a/dh_python2 b/dh_python2
index 83af4cd..664b199 100755
--- a/dh_python2
+++ b/dh_python2
@@ -27,22 +27,17 @@ import re
 import sys
 from filecmp import dircmp, cmpfiles, cmp as fcmp
 from optparse import OptionParser, SUPPRESS_HELP
-from os.path import isabs, isdir, islink, exists, join, normpath, realpath,\
-    split
-from shutil import rmtree, copy as fcopy
-from stat import ST_MODE, S_IXUSR, S_IXGRP, S_IXOTH
+from os.path import isabs, isdir, islink, exists, join, normpath, realpath
+from shutil import copy as fcopy
 sys.path.insert(1, '/usr/share/dh-python/')
 from dhpython.debhelper import DebHelper
 from dhpython.depends import Dependencies
 from dhpython.interpreter import Interpreter
-from dhpython.fs import fix_locations
+from dhpython.fs import fix_locations, Scan
 from dhpython.version import supported, default, VersionRange, \
     get_requested_versions
 from dhpython.pydist import validate as validate_pydist
-from dhpython.tools import relative_symlink,\
-    fix_shebang,\
-    so2pyver, clean_egg_name,\
-    parse_ns, remove_ns,\
+from dhpython.tools import relative_symlink, so2pyver, parse_ns, remove_ns,\
     pyinstall, pyremove
 from dhpython.option import Option
 
@@ -55,6 +50,22 @@ DEFAULT = default('cpython2')
 SUPPORTED = supported('cpython2')
 
 
+class Scanner(Scan):
+    def handle_ext(self, fpath):
+        so_version = so2pyver(fpath)
+        if so_version:
+            path, fn = fpath.rsplit('/', 1)
+            if self.current_pub_version:
+                if self.current_pub_version != so_version:
+                    log.error('extension linked to libpython%s '
+                              'and shipped in python%s\'s dist-'
+                              'packages: %s',
+                              so_version, self.current_pub_version, fn)
+                log.warn('public extension linked with '
+                         'libpython%s: %s', so_version, fn)
+            return so_version
+
+
 ### SHARING FILES ##############################################
 def share(package, stats, options):
     """Move files to /usr/share/pyshared/ if possible."""
@@ -248,215 +259,6 @@ def share_2x(dir1, dir2, dc=None):
         share_2x(join(dir1, dn), join(dir2, dn), dc)
 
 
-### PACKAGE DETAILS ############################################
-def scan(package, dname=None, options=None):
-    """Gather statistics about Python files in given package."""
-    r = {'requires.txt': set(),
-         'nsp.txt': set(),
-         'shebangs': set(),
-         'public_vers': set(),
-         'private_dirs': {},
-         'compile': False,
-         'ext': set()}
-
-    dbg_package = package.endswith('-dbg')
-    interpreter = Interpreter('python', debug=dbg_package)
-
-    if not dname:
-        proot = "debian/%s" % package
-        if dname is False:
-            private_to_check = []
-        else:
-            private_to_check = [i % package for i in
-                                ('usr/lib/%s', 'usr/lib/games/%s',
-                                'usr/share/%s', 'usr/share/games/%s')]
-    else:
-        # scan private directory *only*
-        dname = dname.strip('/')
-        proot = join('debian', package, dname)
-        private_to_check = [dname]
-
-    for root, dirs, file_names in os.walk(proot):
-        if interpreter.should_ignore(root):
-            del dirs[:]
-            continue
-
-        bin_dir = private_dir = None
-        version = interpreter.parse_public_version(root)
-        public_dir = bool(version)
-        if public_dir:
-            if root.endswith('-packages'):
-                r['public_vers'].add(version)
-        else:
-            # TODO: find a way to specify Python version private
-            # extension was build for
-            version = False
-            for i in private_to_check:
-                if root.startswith(join('debian', package, i)):
-                    private_dir = '/' + i
-                    break
-            else:  # i.e. not public_dir and not private_dir
-                if len(root.split('/', 6)) < 6 and \
-                        root.endswith(('/sbin', '/bin', '/usr/games')):
-                    # /(s)bin or /usr/(s)bin or /usr/games
-                    bin_dir = root
-
-        for name in dirs:
-            if name == '__pycache__':
-                rmtree(join(root, name))
-                dirs.remove(name)
-                continue
-            # handle EGG related data (.egg-info dirs)
-            if name.endswith('.egg-info'):
-                if dbg_package and options.clean_dbg_pkg:
-                    rmtree(join(root, name))
-                    dirs.remove(name)
-                    continue
-                clean_name = clean_egg_name(name)
-                if clean_name != name:
-                    if exists(join(root, clean_name)):
-                        log.info('removing %s (%s is already available)', 
name, clean_name)
-                        rmtree(join(root, name))
-                        dirs[dirs.index(name)] = clean_name
-                    else:
-                        log.info('renaming %s to %s', name, clean_name)
-                        os.rename(join(root, name), join(root, clean_name))
-                        dirs[dirs.index(name)] = clean_name
-        if root.endswith('.egg-info'):
-            if 'requires.txt' in file_names:
-                r['requires.txt'].add(join(root, 'requires.txt'))
-            if 'namespace_packages.txt' in file_names:
-                r['nsp.txt'].add(join(root, 'namespace_packages.txt'))
-            if 'SOURCES.txt' in file_names:
-                file_names.remove('SOURCES.txt')
-                os.remove(join(root, 'SOURCES.txt'))
-            continue
-
-        # check files
-        for fn in sorted(file_names):
-            # sorted() to make sure .so files are handled before .so.foo
-            fpath = join(root, fn)
-            if not exists(fpath):
-                # could be removed while handling .so symlinks
-                if islink(fpath) and '.so.' in split(fpath)[-1]:
-                    # dangling symlink to (now removed/renamed) .so file
-                    # which wasn't removed yet (see test3's quux.so.0)
-                    log.info('removing symlink: %s', fpath)
-                    os.remove(fpath)
-                continue
-            fext = fn.rsplit('.', 1)[-1]
-            if fext in ('pyc', 'pyo'):
-                os.remove(fpath)
-                continue
-            if public_dir:
-                if fext == 'so' and islink(fpath):
-                    dstfpath = fpath
-                    links = set()
-                    while islink(dstfpath):
-                        links.add(dstfpath)
-                        dstfpath = join(root, os.readlink(dstfpath))
-                    if exists(dstfpath) and '.so.' in split(dstfpath)[-1]:
-                        # rename .so.$FOO symlinks, remove other ones
-                        for lpath in links:
-                            log.info('removing symlink: %s', lpath)
-                            os.remove(lpath)
-                        log.info('renaming %s to %s', dstfpath, fn)
-                        os.rename(dstfpath, fpath)
-                if dbg_package and options.clean_dbg_pkg and \
-                        fext not in ('so', 'h'):
-                    os.remove(join(root, fn))
-                    continue
-                # assume all extensions were build for cPython
-                if fext == 'so':
-                    new_fn = interpreter.check_extname(fn, version)
-                    if new_fn:
-                        new_fpath = join(root, new_fn)
-                        if exists(new_fpath):
-                            log.warn('destination file exist, '
-                                     'cannot rename %s to %s', fn, new_fn)
-                        else:
-                            log.warn('renaming %s to %s', fn, new_fn)
-                            os.rename(fpath, new_fpath)
-
-            elif private_dir:
-                if exists(fpath) and fext != 'so':
-                    mode = os.stat(fpath)[ST_MODE]
-                    if mode & S_IXUSR or mode & S_IXGRP or mode & S_IXOTH:
-                        if (options.no_shebang_rewrite or
-                            fix_shebang(fpath, options.shebang)) and \
-                                not options.ignore_shebangs:
-                            try:
-                                res = Interpreter.from_file(fpath)
-                            except Exception as e:
-                                log.debug('cannot parse shebang %s: %s', 
fpath, e)
-                            else:
-                                r['private_dirs'].setdefault(private_dir, {})\
-                                    .setdefault('shebangs', set()).add(res)
-
-            if public_dir or private_dir:
-                if fext == 'so':
-                    so_version = so2pyver(join(root, fn))
-                    if so_version:
-                        if public_dir:
-                            if version != so_version:
-                                log.error('extension linked to libpython%s '
-                                          'and shipped in python%s\'s dist-'
-                                          'packages: %s',
-                                          so_version, version, fn)
-                                exit(7)
-                            log.warn('public extension linked with '
-                                     'libpython%s: %s', so_version, fn)
-                        elif not version:
-                            version = so_version
-
-                    (r if public_dir else
-                     r['private_dirs'].setdefault(private_dir, {}))\
-                        .setdefault('ext', set()).add(version)
-                    continue
-                elif fext == 'py':
-                    (r if public_dir else
-                     r['private_dirs'].setdefault(private_dir, {})
-                     )['compile'] = True
-                    continue
-
-            # .egg-info files
-            if fn.endswith('.egg-info'):
-                clean_name = clean_egg_name(fn)
-                if clean_name != fn:
-                    if exists(join(root, clean_name)):
-                        log.info('removing %s (%s is already available)',
-                                 fn, clean_name)
-                        os.remove(join(root, fn))
-                    else:
-                        log.info('renaming %s to %s', fn, clean_name)
-                        os.rename(join(root, fn), join(root, clean_name))
-                continue
-            # search for scripts in bin dirs
-            if bin_dir:
-                if (options.no_shebang_rewrite or
-                    fix_shebang(fpath, options.shebang)) and \
-                        not options.ignore_shebangs:
-                    try:
-                        res = Interpreter.from_file(fpath)
-                    except Exception as e:
-                        log.debug('cannot parse shebang %s: %s', fpath, e)
-                    else:
-                        r['shebangs'].add(res)
-
-    if dbg_package and options.clean_dbg_pkg:
-        # remove empty directories in -dbg packages
-        proot = proot + '/usr/lib'
-        for root, dirs, file_names in os.walk(proot, topdown=False):
-            if '-packages/' in root and not file_names:
-                try:
-                    os.rmdir(root)
-                except Exception:
-                    pass
-
-    log.debug("package %s details = %s", package, r)
-    return r
-
-
 ################################################################
 def main():
     usage = '%prog -p PACKAGE [-V [X.Y][-][A.B]] DIR [-X REGEXPR]\n'
@@ -605,14 +407,13 @@ def main():
                 log.error("%s.pyremove: %s", package, err)
                 exit(5)
             fix_locations(package, interpreter, SUPPORTED)
-        stats = scan(package, private_dir, options)
+        stats = Scanner(interpreter, package, private_dir, options).result
         if not private_dir:
             share(package, stats, options)
             pyshared_dir = "debian/%s/usr/share/pyshared/" % package
             if not stats['public_vers'] and exists(pyshared_dir):
                 create_public_links(pyshared_dir, options.vrange)
-                stats['public_vers'] = get_requested_versions('cpython2', 
options.vrange)
-                stats['compile'] = True
+                stats = Scanner(interpreter, package, private_dir, 
options).result
 
         dependencies = Dependencies(package, 'cpython2')
         dependencies.parse(stats, options)
diff --git a/dh_python3 b/dh_python3
index 0c829c6..3a06b2c 100755
--- a/dh_python3
+++ b/dh_python3
@@ -26,17 +26,15 @@ import os
 import re
 import sys
 from optparse import OptionParser, SUPPRESS_HELP
-from os.path import islink, exists, join, split
-from shutil import rmtree, copy as fcopy
-from stat import ST_MODE, S_IXUSR, S_IXGRP, S_IXOTH
+from os.path import exists, join
+from shutil import copy as fcopy
 sys.path.insert(1, '/usr/share/dh-python/')
 from dhpython.debhelper import DebHelper
 from dhpython.depends import Dependencies
 from dhpython.interpreter import Interpreter, EXTFILE_RE
 from dhpython.version import supported, default, Version, VersionRange
 from dhpython.pydist import validate as validate_pydist
-from dhpython.fs import fix_locations
-from dhpython.tools import fix_shebang, clean_egg_name
+from dhpython.fs import fix_locations, Scan
 from dhpython.option import Option
 
 # initialize script
@@ -48,201 +46,21 @@ DEFAULT = default('cpython3')
 SUPPORTED = supported('cpython3')
 
 
-### PACKAGE DETAILS ############################################
-def scan(package, dname=None, options=None):
-    """Gather statistics about Python files in given package."""
-    r = {'requires.txt': set(),
-         'shebangs': set(),
-         'private_dirs': {},
-         'compile': False,
-         'ext': set()}
+class Scanner(Scan):
+    def handle_ext(self, fpath):
+        path, fname = fpath.rsplit('/', 1)
+        tagver = EXTFILE_RE.search(fname)
+        if tagver is None:
+            # yeah, python3.1 is not covered, but we don't want to
+            # mess with non-Python libraries, don't we?
+            return
+        tagver = tagver.groupdict()['ver']
+        if tagver is None:
+            return
+        tagver = Version("%s.%s" % (tagver[0], tagver[1]))
+        return tagver
 
-    dbg_package = package.endswith('-dbg')
-    interpreter = Interpreter('python', impl='cpython3', debug=dbg_package)
 
-    if not dname:
-        proot = "debian/%s" % package
-        if dname is False:
-            private_to_check = []
-        else:
-            private_to_check = [i % package for i in
-                                ('usr/lib/%s', 'usr/lib/games/%s',
-                                'usr/share/%s', 'usr/share/games/%s')]
-    else:
-        # scan private directory *only*
-        dname = dname.strip('/')
-        proot = join('debian', package, dname)
-        private_to_check = [dname]
-
-    for root, dirs, file_names in os.walk(proot):
-        if interpreter.should_ignore(root):
-            del dirs[:]
-            continue
-
-        bin_dir = private_dir = None
-        version = interpreter.parse_public_version(root)
-        public_dir = bool(version)
-        if not public_dir:
-            for i in private_to_check:
-                if root.startswith(join('debian', package, i)):
-                    private_dir = '/' + i
-                    break
-            else:  # i.e. not public_dir and not private_dir
-                if len(root.split('/', 6)) < 6 and \
-                        root.endswith(('/sbin', '/bin', '/usr/games')):
-                    # /(s)bin or /usr/(s)bin or /usr/games
-                    bin_dir = root
-
-        for name in dirs:
-            if name == '__pycache__':
-                rmtree(join(root, name))
-                dirs.remove(name)
-                continue
-            # handle EGG related data (.egg-info dirs)
-            if name.endswith('.egg-info'):
-                if dbg_package and options.clean_dbg_pkg:
-                    rmtree(join(root, name))
-                    dirs.remove(name)
-                    continue
-                clean_name = clean_egg_name(name)
-                if clean_name != name:
-                    if exists(join(root, clean_name)):
-                        log.info('removing %s (%s is already available)', 
name, clean_name)
-                        rmtree(join(root, name))
-                        dirs[dirs.index(name)] = clean_name
-                    else:
-                        log.info('renaming %s to %s', name, clean_name)
-                        os.rename(join(root, name), join(root, clean_name))
-                        dirs[dirs.index(name)] = clean_name
-        if root.endswith('.egg-info'):
-            if 'requires.txt' in file_names:
-                r['requires.txt'].add(join(root, 'requires.txt'))
-            if 'SOURCES.txt' in file_names:
-                file_names.remove('SOURCES.txt')
-                os.remove(join(root, 'SOURCES.txt'))
-            continue
-
-        # check files
-        for fn in sorted(file_names):
-            # sorted() to make sure .so files are handled before .so.foo
-            fpath = join(root, fn)
-            if not exists(fpath):
-                # could be removed while handling .so symlinks
-                if islink(fpath) and '.so.' in split(fpath)[-1]:
-                    # dangling symlink to (now removed/renamed) .so file
-                    # which wasn't removed yet (see test3's quux.so.0)
-                    log.info('removing symlink: %s', fpath)
-                    os.remove(fpath)
-                continue
-            fext = fn.rsplit('.', 1)[-1]
-            if fext in ('pyc', 'pyo'):
-                os.remove(fpath)
-                continue
-            if public_dir:
-                if fext == 'so' and islink(fpath):
-                    dstfpath = fpath
-                    links = set()
-                    while islink(dstfpath):
-                        links.add(dstfpath)
-                        dstfpath = join(root, os.readlink(dstfpath))
-                    if exists(dstfpath) and '.so.' in split(dstfpath)[-1]:
-                        # rename .so.$FOO symlinks, remove other ones
-                        for lpath in links:
-                            log.info('removing symlink: %s', lpath)
-                            os.remove(lpath)
-                        log.info('renaming %s to %s', dstfpath, fn)
-                        os.rename(dstfpath, fpath)
-                if dbg_package and options.clean_dbg_pkg and \
-                        fext not in ('so', 'h'):
-                    os.remove(join(root, fn))
-                    continue
-                # assume all extensions were build for cPython
-                if fext == 'so':
-                    new_fn = interpreter.check_extname(fn, version)
-                    if new_fn:
-                        new_fpath = join(root, new_fn)
-                        if exists(new_fpath):
-                            log.warn('destination file exist, '
-                                     'cannot rename %s to %s', fn, new_fn)
-                        else:
-                            log.warn('renaming %s to %s', fn, new_fn)
-                            os.rename(fpath, new_fpath)
-
-            elif private_dir:
-                if exists(fpath) and fext != 'so':
-                    mode = os.stat(fpath)[ST_MODE]
-                    if mode & S_IXUSR or mode & S_IXGRP or mode & S_IXOTH:
-                        if (options.no_shebang_rewrite or
-                            fix_shebang(fpath, options.shebang)) and \
-                                not options.ignore_shebangs:
-                            try:
-                                res = Interpreter.from_file(fpath)
-                            except Exception as e:
-                                log.debug('cannot parse shebang %s: %s', 
fpath, e)
-                            else:
-                                r['private_dirs'].setdefault(private_dir, {})\
-                                    .setdefault('shebangs', set()).add(res)
-
-            if public_dir or private_dir:
-                if fext == 'so':
-                    tagver = EXTFILE_RE.search(fn)
-                    if tagver is None:
-                        # yeah, python3.1 is not covered, but we don't want to
-                        # mess with non-Python libraries, don't we?
-                        continue
-                    tagver = tagver.groupdict()['ver']
-                    if tagver is None:
-                        continue
-                    tagver = Version("%s.%s" % (tagver[0], tagver[1]))
-                    (r if public_dir else
-                     r['private_dirs'].setdefault(private_dir, {}))\
-                        .setdefault('ext', set()).add(tagver)
-                    continue
-                elif fext == 'py':
-                    (r if public_dir else
-                     r['private_dirs'].setdefault(private_dir, {})
-                     )['compile'] = True
-                    continue
-
-            # .egg-info files
-            if fn.endswith('.egg-info'):
-                clean_name = clean_egg_name(fn)
-                if clean_name != fn:
-                    if exists(join(root, clean_name)):
-                        log.info('removing %s (%s is already available)',
-                                 fn, clean_name)
-                        os.remove(join(root, fn))
-                    else:
-                        log.info('renaming %s to %s', fn, clean_name)
-                        os.rename(join(root, fn), join(root, clean_name))
-                continue
-            # search for scripts in bin dirs
-            if bin_dir:
-                if (options.no_shebang_rewrite or
-                    fix_shebang(fpath, options.shebang)) and \
-                        not options.ignore_shebangs:
-                    try:
-                        res = Interpreter.from_file(fpath)
-                    except Exception as e:
-                        log.debug('cannot parse shebang %s: %s', fpath, e)
-                    else:
-                        r['shebangs'].add(res)
-
-    if dbg_package and options.clean_dbg_pkg:
-        # remove empty directories in -dbg packages
-        proot = proot + '/usr/lib'
-        for root, dirs, file_names in os.walk(proot, topdown=False):
-            if '-packages/' in root and not file_names:
-                try:
-                    os.rmdir(root)
-                except Exception:
-                    pass
-
-    log.debug("package %s details = %s", package, r)
-    return r
-
-
-################################################################
 def main():
     usage = '%prog -p PACKAGE [-V [X.Y][-][A.B]] DIR [-X REGEXPR]\n'
     parser = OptionParser(usage, version='%prog DEVELV', option_class=Option)
@@ -349,7 +167,7 @@ def main():
 
         if not private_dir:
             fix_locations(package, interpreter, SUPPORTED)
-        stats = scan(package, private_dir, options)
+        stats = Scanner(interpreter, package, private_dir, options).result
 
         dependencies = Dependencies(package, 'cpython3')
         dependencies.parse(stats, options)
diff --git a/dhpython/__init__.py b/dhpython/__init__.py
index cd6bcf5..39fa8b7 100644
--- a/dhpython/__init__.py
+++ b/dhpython/__init__.py
@@ -30,9 +30,9 @@ MINPYCDEP = {'cpython2': 'python (>= 2.6.6-3)',
              'pypy': 'pypy'}
 
 PUBLIC_DIR_RE = {
-    'cpython2': re.compile(r'.*?/usr/lib/python(2\.\d)/(site|dist)-packages'),
-    'cpython3': re.compile(r'.*?/usr/lib/python(3(?:\.\d+)?)/dist-packages'),
-    'pypy': re.compile(r'.*?/usr/lib/pypy/dist-packages')}
+    'cpython2': re.compile(r'.*?/usr/lib/python(2\.\d)(?:/|$)'),
+    'cpython3': re.compile(r'.*?/usr/lib/python(3(?:\.\d+)?)(?:/|$)'),
+    'pypy': re.compile(r'.*?/usr/lib/pypy(?:/|$)')}
 
 INTERPRETER_DIR_TPLS = {
     'cpython2': r'.*/python2\.\d/',
diff --git a/dhpython/fs.py b/dhpython/fs.py
index b7ddf86..05b9bb2 100644
--- a/dhpython/fs.py
+++ b/dhpython/fs.py
@@ -19,9 +19,14 @@
 # THE SOFTWARE.
 
 import logging
+import os
+import re
 from filecmp import cmp as cmpfile
-from os import listdir, remove, renames, rmdir
-from os.path import exists, isdir, join
+from os.path import exists, isdir, islink, join, split
+from shutil import rmtree
+from stat import ST_MODE, S_IXUSR, S_IXGRP, S_IXOTH
+from dhpython.tools import fix_shebang, clean_egg_name
+from dhpython.interpreter import Interpreter
 
 log = logging.getLogger('dhpython')
 
@@ -38,8 +43,8 @@ def fix_locations(package, interpreter, versions):
                 log.debug('moving files from %s to %s', srcdir, dstdir)
                 share_files(srcdir, dstdir, interpreter)
                 parent_dir = '/'.join(srcdir.split('/')[:-1])
-                if exists(parent_dir) and not listdir(parent_dir):
-                    rmdir(parent_dir)
+                if exists(parent_dir) and not os.listdir(parent_dir):
+                    os.rmdir(parent_dir)
 
         # do the same with debug locations
         dstdir = interpreter.sitedir(package, gdb=True)
@@ -48,13 +53,13 @@ def fix_locations(package, interpreter, versions):
                 log.debug('moving files from %s to %s', srcdir, dstdir)
                 share_files(srcdir, dstdir, interpreter)
                 parent_dir = '/'.join(srcdir.split('/')[:-1])
-                if exists(parent_dir) and not listdir(parent_dir):
-                    rmdir(parent_dir)
+                if exists(parent_dir) and not os.listdir(parent_dir):
+                    os.rmdir(parent_dir)
 
 
 def share_files(srcdir, dstdir, interpreter):
     """Try to move as many files from srcdir to dstdir as possible."""
-    for i in listdir(srcdir):
+    for i in os.listdir(srcdir):
         fpath1 = join(srcdir, i)
         if i.rsplit('.', 1)[-1] == 'so':
             # try to rename extension here as well (in :meth:`scan` info about
@@ -71,18 +76,290 @@ def share_files(srcdir, dstdir, interpreter):
                              'cannot rename %s to %s', fpath1_orig, fpath1)
                 else:
                     log.warn('renaming %s to %s', fpath1_orig, fpath1)
-                    renames(fpath1_orig, fpath1)
+                    os.renames(fpath1_orig, fpath1)
         fpath2 = join(dstdir, i)
         if not exists(fpath2):
-            renames(fpath1, fpath2)
+            os.renames(fpath1, fpath2)
             continue
         if isdir(fpath1):
             share_files(fpath1, fpath2, interpreter)
         elif cmpfile(fpath1, fpath2, shallow=False):
-            remove(fpath1)
+            os.remove(fpath1)
         # XXX: check symlinks
 
-    if exists(srcdir) and not listdir(srcdir):
-        rmdir(srcdir)
+    if exists(srcdir) and not os.listdir(srcdir):
+        os.rmdir(srcdir)
 
 
+class Scan:
+    UNWANTED_DIRS = re.compile('.*/__pycache__/.*')
+    UNWANTED_FILES = re.compile('.*\.py[co]')
+
+    def __init__(self, interpreter, package, dpath=None, options=None):
+        self.interpreter = interpreter
+        self.impl = interpreter.impl
+
+        self.package = package
+
+        if not dpath:
+            self.proot = "debian/%s" % self.package
+        else:
+            dpath = dpath.strip('/')
+            self.proot = join('debian', self.package, dpath)
+        self.dpath = dpath
+        del dpath
+
+        self.options = options
+        self.result = {'requires.txt': set(),
+                       'nsp.txt': set(),
+                       'shebangs': set(),
+                       'public_vers': set(),
+                       'private_dirs': {},
+                       'compile': False,
+                       'ext_vers': set()}
+
+        for root, dirs, file_names in os.walk(self.proot):
+            if interpreter.should_ignore(root):
+                del dirs[:]
+                continue
+
+            self.current_private_dir = None
+            self.current_pub_version = version = 
interpreter.parse_public_version(root)
+            if self.current_pub_version:  # i.e. a public site-packages 
directory
+                if root.endswith('-packages'):
+                    self.result['public_vers'].add(version)
+            else:
+                self.current_private_dir = self.check_private_dir(root)
+                if not self.current_private_dir:
+                    # i.e. not a public dir and not a private dir
+                    is_bin_dir = self.is_bin_dir(root)
+                    if is_bin_dir:
+                        self.handle_bin_dir(root, file_names)
+                    else:  # not a public, private or bin directory
+                        # continue with a subdirectory
+                        continue
+
+            for name in dirs:
+                dpath = join(root, name)
+                if self.is_unwanted_dir(dpath):
+                    rmtree(dpath)
+                    dirs.remove(name)
+                    continue
+
+            if self.is_egg_dir(root):
+                self.handle_egg_dir(root, file_names)
+                continue
+
+            # check files
+            for fn in sorted(file_names):
+                # sorted() to make sure .so files are handled before .so.foo
+                fpath = join(root, fn)
+
+                if self.is_unwanted_file(fpath):
+                    log.debug('removing unwanted: %s', fpath)
+                    os.remove(fpath)
+                    continue
+
+                if self.is_egg_file(fpath):
+                    self.handle_egg_file(fpath)
+                    continue
+
+                if not exists(fpath):
+                    # possibly removed while handling .so symlinks
+                    if islink(fpath) and '.so.' in split(fpath)[-1]:
+                        # dangling symlink to (now removed/renamed) .so file
+                        # which wasn't removed yet (see test203's quux.so.0)
+                        log.info('removing dangling symlink: %s', fpath)
+                        os.remove(fpath)
+                    continue
+
+                fext = fn.rsplit('.', 1)[-1]
+                if fext == 'so':
+                    fpath = self.rename_ext(fpath)
+                    ver = self.handle_ext(fpath)
+                    if ver:
+                        self.current_result.setdefault('ext_vers', 
set()).add(ver)
+                    else:
+                        self.current_result.setdefault('ext_no_version', 
set()).add(fpath)
+
+                if self.current_private_dir:
+                    if exists(fpath) and fext != 'so':
+                        mode = os.stat(fpath)[ST_MODE]
+                        if mode & S_IXUSR or mode & S_IXGRP or mode & S_IXOTH:
+                            if (options.no_shebang_rewrite or
+                                fix_shebang(fpath, self.options.shebang)) and \
+                                    not self.options.ignore_shebangs:
+                                try:
+                                    res = Interpreter.from_file(fpath)
+                                except Exception as e:
+                                    log.debug('cannot parse shebang %s: %s', 
fpath, e)
+                                else:
+                                    self.current_result.setdefault('shebangs', 
set()).add(res)
+
+                if fext == 'py' and self.handle_public_module(fpath) is not 
False:
+                    self.current_result['compile'] = True
+
+        log.debug("package %s details = %s", package, self.result)
+
+    @property
+    def current_result(self):
+        if self.current_private_dir:
+            return 
self.result['private_dirs'].setdefault(self.current_private_dir, {})
+        return self.result
+
+    def is_unwanted_dir(self, dpath):
+        return self.__class__.UNWANTED_DIRS.match(dpath)
+
+    def is_unwanted_file(self, fpath):
+        if self.__class__.UNWANTED_FILES.match(fpath):
+            return True
+        if self.current_pub_version and self.is_dbg_package\
+                and self.options.clean_dbg_pkg\
+                and fpath.rsplit('.', 1)[1] not in ('so', 'h'):
+            return True
+
+    @property
+    def private_dirs_to_check(self):
+        if self.dpath:
+            # scan private directory *only*
+            return [self.dpath]
+
+        if self.dpath is False:
+            result = []
+        else:
+            result = [i % self.package for i in (
+                      'usr/lib/%s',
+                      'usr/lib/games/%s',
+                      'usr/share/%s',
+                      'usr/share/games/%s')]
+        return result
+
+    @property
+    def is_dbg_package(self):
+        #return self.interpreter.debug
+        return self.package.endswith('-dbg')
+
+    def check_private_dir(self, dpath):
+        """Return private dir's root if it's a private dir."""
+        for i in self.private_dirs_to_check:
+            if dpath.startswith(join('debian', self.package, i)):
+                return '/' + i
+
+    def rename_ext(self, fpath):
+        """Add multiarch triplet, etc. Return new name.
+
+        This method is invoked for all .so files in public or private 
directories.
+        """
+        path, fname = fpath.rsplit('/', 1)
+        if islink(fpath):
+            dstfpath = fpath
+            links = set()
+            while islink(dstfpath):
+                links.add(dstfpath)
+                dstfpath = join(path, os.readlink(dstfpath))
+            if exists(dstfpath) and '.so.' in split(dstfpath)[-1]:
+                # rename .so.$FOO symlinks, remove other ones
+                for lpath in links:
+                    log.info('removing symlink: %s', lpath)
+                    os.remove(lpath)
+                log.info('renaming %s to %s', dstfpath, fname)
+                os.rename(dstfpath, fpath)
+
+        new_fn = self.interpreter.check_extname(fname, 
self.current_pub_version)
+        if new_fn:
+            # TODO: what about symlinks pointing to this file
+            new_fpath = join(path, new_fn)
+            if exists(new_fpath):
+                log.warn('destination file exist, '
+                         'cannot rename %s to %s', fname, new_fn)
+            else:
+                log.warn('renaming %s to %s', fname, new_fn)
+                os.rename(fpath, new_fpath)
+            return new_fpath
+        return fpath
+
+    def handle_ext(self, fpath):
+        """Handle .so file, return its version if detected."""
+
+    def handle_public_module(self, fpath):
+        pass
+
+    def is_bin_dir(self, dpath):
+        """Check if dir is one from PATH ones."""
+        # dname = debian/packagename/usr/games
+        spath = dpath.strip('/').split('/', 4)
+        if len(spath) > 4:
+            return False  # assume bin directories don't have subdirectories
+        if dpath.endswith(('/sbin', '/bin', '/usr/games')):
+            # /(s)bin or /usr/(s)bin or /usr/games
+            return True
+
+    def handle_bin_dir(self, dpath, file_names):
+        if self.options.no_shebang_rewrite or self.options.ignore_shebangs:
+            return
+        for fn in file_names:
+            fpath = join(dpath, fn)
+            if fix_shebang(fpath, self.options.shebang):
+                try:
+                    res = Interpreter.from_file(fpath)
+                except Exception as e:
+                    log.debug('cannot parse shebang %s: %s', fpath, e)
+                else:
+                    self.result['shebangs'].add(res)
+
+    def is_egg_dir(self, dname):
+        """Check if given directory contains egg-info."""
+        return dname.endswith('.egg-info')
+
+    def handle_egg_dir(self, dpath, file_names):
+        path, dname = dpath.rsplit('/', 1)
+        if self.is_dbg_package and self.options.clean_dbg_pkg:
+            rmtree(dpath)
+            return
+
+        clean_name = clean_egg_name(dname)
+        if clean_name != dname:
+            if exists(join(path, clean_name)):
+                log.info('removing %s (%s is already available)', dname, 
clean_name)
+                rmtree(dpath)
+                return
+            else:
+                log.info('renaming %s to %s', dname, clean_name)
+                os.rename(dpath, join(path, clean_name))
+                dname = clean_name
+                dpath = join(path, dname)
+        if file_names:
+            if 'requires.txt' in file_names:
+                self.result['requires.txt'].add(join(dpath, 'requires.txt'))
+            if 'namespace_packages.txt' in file_names:
+                self.result['nsp.txt'].add(join(dpath, 
'namespace_packages.txt'))
+            if 'SOURCES.txt' in file_names:
+                os.remove(join(dpath, 'SOURCES.txt'))
+                file_names.remove('SOURCES.txt')
+
+    def is_egg_file(self, fpath):
+        """Check if given file contains egg-info."""
+        return fpath.endswith('.egg-info')
+
+    def handle_egg_file(self, fpath):
+        root, name = fpath.rsplit('/', 1)
+        clean_name = clean_egg_name(name)
+        if clean_name != name:
+            if exists(join(root, clean_name)):
+                log.info('removing %s (%s is already available)',
+                         name, clean_name)
+                os.remove(fpath)
+            else:
+                log.info('renaming %s to %s', name, clean_name)
+                os.rename(fpath, join(root, clean_name))
+
+    def cleanup(self):
+        if self.is_dbg_package and self.options.clean_dbg_pkg:
+            # remove empty directories in -dbg packages
+            proot = self.proot + '/usr/lib'
+            for root, dirs, file_names in os.walk(proot, topdown=False):
+                if '-packages/' in root and not file_names:
+                    try:
+                        os.rmdir(root)
+                    except Exception:
+                        pass

-- 
Alioth's /usr/local/bin/git-commit-notice on 
/srv/git.debian.org/git/reproducible/dh-python.git

_______________________________________________
Reproducible-builds mailing list
Reproducible-builds@lists.alioth.debian.org
http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/reproducible-builds

Reply via email to