On Tue, 31 Jan 2012 14:37:02 +0100, Florian Friesdorf <[email protected]> wrote: > (..) > My goals would be: > > - achieving pureness for python packages
Attached is a patchset to achieve pureness for python packages and handle dependencies via pth files. While I'm still working on some issues it would be great to get feedback on it. @Cillian: I used your distutils.cfg idea to create a offline version of distutils used only during installations. An installed setuptools is then able again to download things. Also available via github: https://github.com/chaoflow/nixpkgs/tree/python
From 7c33e0041bb837c17d80705b90bc728b77d02cb8 Mon Sep 17 00:00:00 2001 From: Florian Friesdorf <[email protected]> Date: Wed, 15 Feb 2012 01:55:40 +0100 Subject: [PATCH 01/17] python site.py that loads pth files anywhere on PYTHONPATH This file is normally created by easy_install / distutils, but removed and packaged separately to avoid collisions and allow for more use cases. --- pkgs/development/python-modules/site/default.nix | 33 +++++++++ pkgs/development/python-modules/site/site.py | 82 ++++++++++++++++++++++ pkgs/top-level/python-packages.nix | 6 ++ 3 files changed, 121 insertions(+), 0 deletions(-) create mode 100644 pkgs/development/python-modules/site/default.nix create mode 100644 pkgs/development/python-modules/site/site.py diff --git a/pkgs/development/python-modules/site/default.nix b/pkgs/development/python-modules/site/default.nix new file mode 100644 index 0000000..98a1485 --- /dev/null +++ b/pkgs/development/python-modules/site/default.nix @@ -0,0 +1,33 @@ +{ stdenv, python }: + +stdenv.mkDerivation rec { + name = "site-1.0"; + + unpackPhase = "true"; + + buildInputs = [ python ]; + + installPhase = + '' + dst=$out/lib/${python.libPrefix}/site-packages + mkdir -p $dst + cat ${./site.py} >> $dst/site.py + ''; + + meta = { + description = "Enable processing of pth files anywhere in PYTHONPATH"; + longDescription = '' + This file is normally created by easy_install / distutils in + site-packages and overrides python's default site.py. It adds + all parts of PYTHONPATH as site directories, which means pth + files are processed in them. We remove the normally created + site.py's and package it separately instead as it would cause + collisions. + + For each module we have a pth file listing the module and all + its dependencies and we include python-site into the + PYTHONPATH of wrapped python programs so they can find their + dependencies. + ''; + }; +} diff --git a/pkgs/development/python-modules/site/site.py b/pkgs/development/python-modules/site/site.py new file mode 100644 index 0000000..b233222 --- /dev/null +++ b/pkgs/development/python-modules/site/site.py @@ -0,0 +1,82 @@ +def __boot(): + import sys, imp, os, os.path + PYTHONPATH = os.environ.get('PYTHONPATH') + if PYTHONPATH is None or (sys.platform=='win32' and not PYTHONPATH): + PYTHONPATH = [] + else: + PYTHONPATH = PYTHONPATH.split(os.pathsep) + + pic = getattr(sys,'path_importer_cache',{}) + stdpath = sys.path[len(PYTHONPATH):] + mydir = os.path.dirname(__file__) + #print "searching",stdpath,sys.path + + for item in stdpath: + if item==mydir or not item: + continue # skip if current dir. on Windows, or my own directory + importer = pic.get(item) + if importer is not None: + loader = importer.find_module('site') + if loader is not None: + # This should actually reload the current module + loader.load_module('site') + break + else: + try: + stream, path, descr = imp.find_module('site',[item]) + except ImportError: + continue + if stream is None: + continue + try: + # This should actually reload the current module + imp.load_module('site',stream,path,descr) + finally: + stream.close() + break + else: + raise ImportError("Couldn't find the real 'site' module") + + #print "loaded", __file__ + + known_paths = dict([(makepath(item)[1],1) for item in sys.path]) # 2.2 comp + + oldpos = getattr(sys,'__egginsert',0) # save old insertion position + sys.__egginsert = 0 # and reset the current one + + for item in PYTHONPATH: + addsitedir(item) + + sys.__egginsert += oldpos # restore effective old position + + d,nd = makepath(stdpath[0]) + insert_at = None + new_path = [] + + for item in sys.path: + p,np = makepath(item) + + if np==nd and insert_at is None: + # We've hit the first 'system' path entry, so added entries go here + insert_at = len(new_path) + + if np in known_paths or insert_at is None: + new_path.append(item) + else: + # new path after the insert point, back-insert it + new_path.insert(insert_at, item) + insert_at += 1 + + sys.path[:] = new_path + +if __name__=='site': + __boot() + del __boot + + + + + + + + diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 77b10c1..5d62689 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -19,6 +19,12 @@ let pythonPackages = python.modules // rec { }; + site = import ../development/python-modules/site { + inherit (pkgs) stdenv; + inherit python; + }; + + ipython = import ../shells/ipython { inherit (pkgs) stdenv fetchurl; inherit buildPythonPackage pythonPackages; -- 1.7.8
From 4768d31ceb1fc80cdad8b6270057fc8cd01be239 Mon Sep 17 00:00:00 2001 From: Florian Friesdorf <[email protected]> Date: Wed, 15 Feb 2012 01:37:59 +0100 Subject: [PATCH 02/17] pth file with deps, only current package's scripts, no colliding files --- .../development/python-modules/generic/default.nix | 28 ++++++++++++++++++++ 1 files changed, 28 insertions(+), 0 deletions(-) diff --git a/pkgs/development/python-modules/generic/default.nix b/pkgs/development/python-modules/generic/default.nix index 4614917..a6b7148 100644 --- a/pkgs/development/python-modules/generic/default.nix +++ b/pkgs/development/python-modules/generic/default.nix @@ -17,9 +17,33 @@ # pollute the user environment. pythonPath ? [] + # distutils registers dependencies in .pth (good) but also creates + # console_scripts for dependencies in bin/ (bad). easy_install + # creates no scripts for dependencies (good) but does not register + # them in pth neither (bad) - the combination gives us a pth with + # dependencies and scripts only for the package we are currently + # installing. , installCommand ? '' + # install the current package with easy_install.pth including dependencies + python setup.py install --prefix="$out" + + # remove console_scripts again, because they were created for deps, too + rm -Rf "$out"/bin + + # run easy_install to generate scripts for the current package, + # it won't reinstall easy_install --prefix="$out" . + + # move colliding easy_install.pth to specifically named one + mv "$out/lib/${python.libPrefix}/site-packages/"{easy-install.pth,${name}.pth} + + # These cause collisions and our output is not a site anyway + # If you need a site, install python-site + rm -f "$out/lib/${python.libPrefix}/site-packages/"site.py* + + # If setuptools is a dependency, it is included in $(name}.pth + rm -f "$out/lib/${python.libPrefix}/site-packages/setuptools.pth" '' , buildPhase ? "true" @@ -67,6 +91,10 @@ python.stdenv.mkDerivation (attrs // { # dependencies in the user environment (since Python modules don't # have something like an RPATH, so the only way to find the # dependencies is to have them in the PYTHONPATH variable). + # + # XXX: this is not needed for things to work (pth pulls in deps) + # but would be nice to have anyway - However, python deps end up + # in propagated-build-native-inputs if test -e $out/nix-support/propagated-build-inputs; then ln -s $out/nix-support/propagated-build-inputs $out/nix-support/propagated-user-env-packages fi -- 1.7.8
From 7c50ea52bc11c49b420ed417edfb56fc5969f50e Mon Sep 17 00:00:00 2001 From: Florian Friesdorf <[email protected]> Date: Wed, 15 Feb 2012 01:59:29 +0100 Subject: [PATCH 03/17] clean setuptools from colliding files - it is not using buildPythonPkg --- .../python-modules/setuptools/default.nix | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/pkgs/development/python-modules/setuptools/default.nix b/pkgs/development/python-modules/setuptools/default.nix index 50b73aa..97e77a2 100644 --- a/pkgs/development/python-modules/setuptools/default.nix +++ b/pkgs/development/python-modules/setuptools/default.nix @@ -18,6 +18,9 @@ stdenv.mkDerivation rec { mkdir -p $dst PYTHONPATH=$dst:$PYTHONPATH python setup.py install --prefix=$out + # remove generic files, leave setuptools.pth + rm $dst/site.py* + rm $dst/easy-install.pth wrapPythonPrograms ''; -- 1.7.8
From 52a386a7b530c208aa0c121ceab29d0d199111de Mon Sep 17 00:00:00 2001 From: Florian Friesdorf <[email protected]> Date: Thu, 16 Feb 2012 17:27:25 +0100 Subject: [PATCH 04/17] include site for python wrappers to enable deps via pth files --- .../development/python-modules/generic/default.nix | 5 +++-- .../python-modules/setuptools/default.nix | 6 +++--- pkgs/top-level/python-packages.nix | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/pkgs/development/python-modules/generic/default.nix b/pkgs/development/python-modules/generic/default.nix index a6b7148..98f1c06 100644 --- a/pkgs/development/python-modules/generic/default.nix +++ b/pkgs/development/python-modules/generic/default.nix @@ -3,7 +3,7 @@ (http://pypi.python.org/pypi/setuptools/), which represents a large number of Python packages nowadays. */ -{ python, setuptools, wrapPython, lib }: +{ python, setuptools, wrapPython, lib, site }: { name, namePrefix ? "python-" @@ -69,7 +69,8 @@ python.stdenv.mkDerivation (attrs // { buildInputs = [ python wrapPython setuptools ] ++ buildInputs ++ pythonPath; - pythonPath = [ setuptools] ++ pythonPath; + # XXX: I think setuptools is not needed here + pythonPath = [ setuptools site ] ++ pythonPath; # XXX: Should we run `easy_install --always-unzip'? It doesn't seem # to have a noticeable impact on small scripts. diff --git a/pkgs/development/python-modules/setuptools/default.nix b/pkgs/development/python-modules/setuptools/default.nix index 97e77a2..c478c25 100644 --- a/pkgs/development/python-modules/setuptools/default.nix +++ b/pkgs/development/python-modules/setuptools/default.nix @@ -1,4 +1,4 @@ -{ stdenv, fetchurl, python, wrapPython }: +{ stdenv, fetchurl, python, wrapPython, site }: stdenv.mkDerivation rec { name = "setuptools-0.6c11"; @@ -8,7 +8,7 @@ stdenv.mkDerivation rec { sha256 = "1lx1hwxkhipyh206bgl90ddnfcnb68bzcvyawczbf833fadyl3v3"; }; - buildInputs = [ python wrapPython ]; + buildInputs = [ python wrapPython site ]; buildPhase = "python setup.py build --build-base $out"; @@ -21,7 +21,7 @@ stdenv.mkDerivation rec { # remove generic files, leave setuptools.pth rm $dst/site.py* rm $dst/easy-install.pth - wrapPythonPrograms + pythonPath=${site} wrapPythonPrograms ''; doCheck = false; # doesn't work with Python 2.7 diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 5d62689..358f346 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -9,13 +9,13 @@ let pythonPackages = python.modules // rec { buildPythonPackage = import ../development/python-modules/generic { inherit (pkgs) lib; - inherit python wrapPython setuptools; + inherit python wrapPython setuptools site; }; setuptools = import ../development/python-modules/setuptools { inherit (pkgs) stdenv fetchurl; - inherit python wrapPython; + inherit python wrapPython site; }; -- 1.7.8
From 7cfd8b81aec3c823e8b4e7d9233b6e9fd58c36ac Mon Sep 17 00:00:00 2001 From: Florian Friesdorf <[email protected]> Date: Thu, 16 Feb 2012 17:58:20 +0100 Subject: [PATCH 05/17] prevent distutils during module install from downloading and load pth files --- .../development/python-modules/generic/default.nix | 11 +++++++++- .../python-modules/offline-distutils/default.nix | 22 ++++++++++++++++++++ pkgs/top-level/python-packages.nix | 6 ++++- 3 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 pkgs/development/python-modules/offline-distutils/default.nix diff --git a/pkgs/development/python-modules/generic/default.nix b/pkgs/development/python-modules/generic/default.nix index 98f1c06..7ad052f 100644 --- a/pkgs/development/python-modules/generic/default.nix +++ b/pkgs/development/python-modules/generic/default.nix @@ -3,7 +3,7 @@ (http://pypi.python.org/pypi/setuptools/), which represents a large number of Python packages nowadays. */ -{ python, setuptools, wrapPython, lib, site }: +{ python, setuptools, wrapPython, lib, site, offlineDistutils }: { name, namePrefix ? "python-" @@ -69,6 +69,15 @@ python.stdenv.mkDerivation (attrs // { buildInputs = [ python wrapPython setuptools ] ++ buildInputs ++ pythonPath; + configurePhase = '' + # do not allow distutils to make downloads, whatever install command is used + export PYTHONPATH="${setuptools}/lib/${python.libPrefix}:$PYTHONPATH" + export PYTHONPATH="${offlineDistutils}/lib/${python.libPrefix}:$PYTHONPATH" + + # enable pth files for dependencies + export PYTHONPATH="${site}/lib/${python.libPrefix}/site-packages:$PYTHONPATH" + ''; + # XXX: I think setuptools is not needed here pythonPath = [ setuptools site ] ++ pythonPath; diff --git a/pkgs/development/python-modules/offline-distutils/default.nix b/pkgs/development/python-modules/offline-distutils/default.nix new file mode 100644 index 0000000..5fff742 --- /dev/null +++ b/pkgs/development/python-modules/offline-distutils/default.nix @@ -0,0 +1,22 @@ +# Used during module installation to prevent easy_install and python +# setup.py install/test from downloading + +{ stdenv, python }: + +stdenv.mkDerivation { + name = "python-offline-distutils-${python.version}"; + + buildInputs = [ python ]; + + unpackPhase = "true"; + installPhase = '' + dst="$out/lib/${python.libPrefix}" + ensureDir $dst + cp -R ${python}/lib/${python.libPrefix}/distutils $dst/ + chmod u+w $dst/distutils + cat <<EOF > $dst/distutils/distutils.cfg +[easy_install] +allow-hosts = None +EOF + ''; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 358f346..f1a3f21 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -9,7 +9,7 @@ let pythonPackages = python.modules // rec { buildPythonPackage = import ../development/python-modules/generic { inherit (pkgs) lib; - inherit python wrapPython setuptools site; + inherit python wrapPython setuptools site offlineDistutils; }; @@ -24,6 +24,10 @@ let pythonPackages = python.modules // rec { inherit python; }; + offlineDistutils = import ../development/python-modules/offline-distutils { + inherit (pkgs) stdenv; + inherit python; + }; ipython = import ../shells/ipython { inherit (pkgs) stdenv fetchurl; -- 1.7.8
From 5b2407bcfc6cc19b9c0f3785181d291e42f3406b Mon Sep 17 00:00:00 2001 From: Florian Friesdorf <[email protected]> Date: Thu, 16 Feb 2012 17:30:34 +0100 Subject: [PATCH 06/17] remove old now unneeded pythonhome-wrapper --- .../interpreters/python/pythonhome-wrapper.nix | 21 -------------------- pkgs/top-level/all-packages.nix | 2 - 2 files changed, 0 insertions(+), 23 deletions(-) delete mode 100644 pkgs/development/interpreters/python/pythonhome-wrapper.nix diff --git a/pkgs/development/interpreters/python/pythonhome-wrapper.nix b/pkgs/development/interpreters/python/pythonhome-wrapper.nix deleted file mode 100644 index 2afbafe..0000000 --- a/pkgs/development/interpreters/python/pythonhome-wrapper.nix +++ /dev/null @@ -1,21 +0,0 @@ -{ stdenv }: - -stdenv.mkDerivation { - name = "pythonhome-wrapper"; - - unpackPhase = "true"; - installPhase = '' - ensureDir $out/bin - echo ' -#!/bin/sh - -BINDIR=`dirname $0` -PYTHONHOME=`dirname $BINDIR` -PYTHONHOME=`(cd $PYTHONHOME && pwd)` -export PYTHONHOME - -$BINDIR/python "$@" - ' > $out/bin/py - chmod +x $out/bin/py - ''; -} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 3f58d34..91e2d9b 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -2754,8 +2754,6 @@ let python = python27; }; - pythonhomeWrapper = callPackage ../development/interpreters/python/pythonhome-wrapper.nix { }; - pyrex = pyrex095; pyrex095 = callPackage ../development/interpreters/pyrex/0.9.5.nix { }; -- 1.7.8
-- Florian Friesdorf <[email protected]> GPG FPR: 7A13 5EEE 1421 9FC2 108D BAAF 38F8 99A3 0C45 F083 Jabber/XMPP: [email protected] IRC: chaoflow on freenode,ircnet,blafasel,OFTC
pgpAyTJJXQB35.pgp
Description: PGP signature
_______________________________________________ nix-dev mailing list [email protected] http://lists.science.uu.nl/mailman/listinfo/nix-dev
