Hello, ng0 <n...@we.make.ritual.n0.is> writes:
> I think this should not happen with pypi import: > > (inputs > `(("python-certifi==2016.2.28" > ,python-certifi==2016.2.28) > ("python-dateutil==2.5.3" > ,python-dateutil==2.5.3) > ("python-flask-babel==0.11.1" > ,python-flask-babel==0.11.1) > ("python-flask==0.11.1" ,python-flask==0.11.1) > ("python-lxml==3.6.0" ,python-lxml==3.6.0) > ("python-ndg-httpsclient==0.4.1" > ,python-ndg-httpsclient==0.4.1) > ("python-pyasn1-modules==0.0.8" > ,python-pyasn1-modules==0.0.8) > ("python-pyasn1==0.1.9" ,python-pyasn1==0.1.9) > ("python-pygments==2.1.3" > ,python-pygments==2.1.3) > ("python-pyopenssl==0.15.1" > ,python-pyopenssl==0.15.1) > ("python-pyyaml==3.11" ,python-pyyaml==3.11) > ("python-requests[socks]==2.10.0" > ,#{python-requests\x5b;socks\x5d;==2.10.0}#) > ("python-setuptools" ,python-setuptools))) > > > I can understand the version numbers, I can also understand the optional > socks building/module of the python-requests, but why does it read like > Gobbledygook? Can't we improve the output here? > > For version numbers, this is not a format which happened recently which > is exclusive for python build system right? This is just bad formated > because of the pypi query. > I will first try and not pin the application to these version numbers, > maybe itjustworks™. > > > To reproduce: "guix import pypi searx" The following patches fix this, and more!
From 54e44b7397f17910d95dbdb233d23e5c97c095aa Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer <maxim.courno...@gmail.com> Date: Thu, 28 Mar 2019 00:26:00 -0400 Subject: [PATCH 1/7] import: pypi: Do not consider requirements.txt files. * guix/import/pypi.scm (guess-requirements): Update comment. [guess-requirements-from-source]: Do not attempt to parse the file requirements.txt. Streamline logic. --- guix/import/pypi.scm | 35 +++++++++++++---------------------- tests/pypi.scm | 23 +++++++++++------------ 2 files changed, 24 insertions(+), 34 deletions(-) diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index 3a20fc4b9b..8269aa61d7 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -206,35 +206,26 @@ cannot determine package dependencies")) (call-with-temporary-directory (lambda (dir) (let* ((pypi-name (string-take dirname (string-rindex dirname #\-))) - (req-files (list (string-append dirname "/requirements.txt") - (string-append dirname "/" pypi-name ".egg-info" - "/requires.txt"))) - (exit-codes (map (lambda (file-name) - (parameterize ((current-error-port (%make-void-port "rw+")) - (current-output-port (%make-void-port "rw+"))) - (system* "tar" "xf" tarball "-C" dir file-name))) - req-files))) - ;; Only one of these files needs to exist. - (if (any zero? exit-codes) - (match (find-files dir) - ((file . _) - (read-requirements file)) - (() - (warning (G_ "No requirements file found.\n")))) + (requires.txt (string-append dirname "/" pypi-name + ".egg-info" "/requires.txt")) + (exit-code (parameterize ((current-error-port (%make-void-port "rw+")) + (current-output-port (%make-void-port "rw+"))) + (system* "tar" "xf" tarball "-C" dir requires.txt)))) + (if (zero? exit-code) + (read-requirements (string-append dir "/" requires.txt)) (begin - (warning (G_ "Failed to extract requirements files\n")) + (warning + (G_ "Failed to extract file: ~a from source.~%") + requires.txt) '()))))) '()))) - ;; First, try to compute the requirements using the wheel, since that is the - ;; most reliable option. If a wheel is not provided for this package, try - ;; getting them by reading either the "requirements.txt" file or the - ;; "requires.txt" from the egg-info directory from the source tarball. Note - ;; that "requirements.txt" is not mandatory, so this is likely to fail. + ;; First, try to compute the requirements using the wheel, else, fallback to + ;; reading the "requires.txt" from the egg-info directory from the source + ;; tarball. (or (guess-requirements-from-wheel) (guess-requirements-from-source))) - (define (compute-inputs source-url wheel-url tarball) "Given the SOURCE-URL of an already downloaded TARBALL, return a list of name/variable pairs describing the required inputs of this package. Also diff --git a/tests/pypi.scm b/tests/pypi.scm index 6daa44a6e7..335be42644 100644 --- a/tests/pypi.scm +++ b/tests/pypi.scm @@ -23,7 +23,7 @@ #:use-module (gcrypt hash) #:use-module (guix tests) #:use-module (guix build-system python) - #:use-module ((guix build utils) #:select (delete-file-recursively which)) + #:use-module ((guix build utils) #:select (delete-file-recursively which mkdir-p)) #:use-module (srfi srfi-64) #:use-module (ice-9 match)) @@ -55,11 +55,10 @@ (define test-source-hash "") -(define test-requirements -"# A comment - # A comment after a space +(define test-requires.txt "\ bar -baz > 13.37") +baz > 13.37 +") (define test-metadata "{ @@ -107,10 +106,10 @@ baz > 13.37") (match url ("https://example.com/foo-1.0.0.tar.gz" (begin - (mkdir "foo-1.0.0") - (with-output-to-file "foo-1.0.0/requirements.txt" + (mkdir-p "foo-1.0.0/foo.egg-info/") + (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt" (lambda () - (display test-requirements))) + (display test-requires.txt))) (system* "tar" "czvf" file-name "foo-1.0.0/") (delete-file-recursively "foo-1.0.0") (set! test-source-hash @@ -157,11 +156,11 @@ baz > 13.37") (lambda (url file-name) (match url ("https://example.com/foo-1.0.0.tar.gz" - (begin - (mkdir "foo-1.0.0") - (with-output-to-file "foo-1.0.0/requirements.txt" + (begin + (mkdir-p "foo-1.0.0/foo.egg-info/") + (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt" (lambda () - (display test-requirements))) + (display test-requires.txt))) (system* "tar" "czvf" file-name "foo-1.0.0/") (delete-file-recursively "foo-1.0.0") (set! test-source-hash -- 2.20.1
From 5f79b0502f62bd1dacc8ea143c1dbd9ef7cfc29d Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer <maxim.courno...@gmail.com> Date: Thu, 28 Mar 2019 00:26:00 -0400 Subject: [PATCH 2/7] import: pypi: Do not parse optional requirements from source. * guix/import/pypi.scm: Export PARSE-REQUIRES.TXT. (guess-requirements): Move the READ-REQUIREMENTS procedure to the top level, and rename it to PARSE-REQUIRES.TXT. Move the CLEAN-REQUIREMENT and COMMENT? functions inside the READ-REQUIREMENTS procedure. (parse-requires.txt): Add a SECTION-HEADER? predicate, and use it to prevent parsing optional requirements. * tests/pypi.scm (test-requires-with-sections): New variable. ("parse-requires.txt, with sections"): New test. ("pypi->guix-package"): Mute tar output to stdout. --- guix/import/pypi.scm | 76 +++++++++++++++++++++++++++----------------- tests/pypi.scm | 21 ++++++++++-- 2 files changed, 65 insertions(+), 32 deletions(-) diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index 8269aa61d7..91e987e9f1 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -47,7 +47,8 @@ #:use-module (guix upstream) #:use-module ((guix licenses) #:prefix license:) #:use-module (guix build-system python) - #:export (guix-package->pypi-name + #:export (parse-requires.txt + guix-package->pypi-name pypi-recursive-import pypi->guix-package %pypi-updater)) @@ -117,6 +118,49 @@ package definition." ((package-inputs ...) `((propagated-inputs (,'quasiquote ,package-inputs)))))) +(define (clean-requirement s) + ;; Given a requirement LINE, as can be found in a setuptools requires.txt + ;; file, remove everything other than the actual name of the required + ;; package, and return it. + (string-take s (or (string-index s (lambda (chr) + (member chr '(#\space #\> #\= #\<)))) + (string-length s)))) + +(define (parse-requires.txt requires.txt) + "Given REQUIRES.TXT, a Setuptools requires.txt file, return a list of +requirement names." + ;; This is a very incomplete parser, which job is to select the non-optional + ;; dependencies and strip them out of any version information. + ;; Alternatively, we could implement a PEG parser with the (ice-9 peg) + ;; library and the requirements grammar defined by PEP-0508 + ;; (https://www.python.org/dev/peps/pep-0508/). + + (define (comment? line) + ;; Return #t if the given LINE is a comment, #f otherwise. + (eq? (string-ref (string-trim line) 0) #\#)) + + (define (section-header? line) + ;; Return #t if the given LINE is a section header, #f otherwise. + (let ((trimmed-line (string-trim line))) + (and (not (string-null? trimmed-line)) + (eq? (string-ref trimmed-line 0) #\[)))) + + (call-with-input-file requires.txt + (lambda (port) + (let loop ((result '())) + (let ((line (read-line port))) + ;; Stop when a section is encountered, as sections contains optional + ;; (extra) requirements. Non-optional requirements must appear + ;; before any section is defined. + (if (or (eof-object? line) (section-header? line)) + (reverse result) + (cond + ((or (string-null? line) (comment? line)) + (loop result)) + (else + (loop (cons (clean-requirement line) + result)))))))))) + (define (guess-requirements source-url wheel-url tarball) "Given SOURCE-URL, WHEEL-URL and a TARBALL of the package, return a list of the required packages specified in the requirements.txt file. TARBALL will @@ -139,34 +183,6 @@ be extracted in a temporary directory." cannot determine package dependencies")) #f))))) - (define (clean-requirement s) - ;; Given a requirement LINE, as can be found in a Python requirements.txt - ;; file, remove everything other than the actual name of the required - ;; package, and return it. - (string-take s - (or (string-index s (lambda (chr) (member chr '(#\space #\> #\= #\<)))) - (string-length s)))) - - (define (comment? line) - ;; Return #t if the given LINE is a comment, #f otherwise. - (eq? (string-ref (string-trim line) 0) #\#)) - - (define (read-requirements requirements-file) - ;; Given REQUIREMENTS-FILE, a Python requirements.txt file, return a list - ;; of name/variable pairs describing the requirements. - (call-with-input-file requirements-file - (lambda (port) - (let loop ((result '())) - (let ((line (read-line port))) - (if (eof-object? line) - result - (cond - ((or (string-null? line) (comment? line)) - (loop result)) - (else - (loop (cons (clean-requirement line) - result)))))))))) - (define (read-wheel-metadata wheel-archive) ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's ;; requirements. @@ -212,7 +228,7 @@ cannot determine package dependencies")) (current-output-port (%make-void-port "rw+"))) (system* "tar" "xf" tarball "-C" dir requires.txt)))) (if (zero? exit-code) - (read-requirements (string-append dir "/" requires.txt)) + (parse-requires.txt (string-append dir "/" requires.txt)) (begin (warning (G_ "Failed to extract file: ~a from source.~%") diff --git a/tests/pypi.scm b/tests/pypi.scm index 335be42644..e4b7142311 100644 --- a/tests/pypi.scm +++ b/tests/pypi.scm @@ -60,6 +60,15 @@ bar baz > 13.37 ") +(define test-requires-with-sections "\ +# A comment +foo ~= 3 +bar != 2 + +[test] +pytest (>=2.5.0) +") + (define test-metadata "{ \"run_requires\": [ @@ -99,6 +108,12 @@ baz > 13.37 (uri (list "https://bitheap.org/cram/cram-0.7.tar.gz" (pypi-uri "cram" "0.7")))))))) +(test-equal "parse-requires.txt, with sections" + '("foo" "bar") + (mock ((ice-9 ports) call-with-input-file + call-with-input-string) + (parse-requires.txt test-requires-with-sections))) + (test-assert "pypi->guix-package" ;; Replace network resources with sample data. (mock ((guix import utils) url-fetch @@ -110,7 +125,8 @@ baz > 13.37 (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt" (lambda () (display test-requires.txt))) - (system* "tar" "czvf" file-name "foo-1.0.0/") + (parameterize ((current-output-port (%make-void-port "rw+"))) + (system* "tar" "czvf" file-name "foo-1.0.0/")) (delete-file-recursively "foo-1.0.0") (set! test-source-hash (call-with-input-file file-name port-sha256)))) @@ -161,7 +177,8 @@ baz > 13.37 (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt" (lambda () (display test-requires.txt))) - (system* "tar" "czvf" file-name "foo-1.0.0/") + (parameterize ((current-output-port (%make-void-port "rw+"))) + (system* "tar" "czvf" file-name "foo-1.0.0/")) (delete-file-recursively "foo-1.0.0") (set! test-source-hash (call-with-input-file file-name port-sha256)))) -- 2.20.1
From 0c62b541a3e8925b5ca31fe55dbe7536cf95151f Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer <maxim.courno...@gmail.com> Date: Thu, 28 Mar 2019 00:26:01 -0400 Subject: [PATCH 3/7] import: pypi: Improve parsing of requirement specifications. The previous solution was fragile and could leave unwanted characters in a requirement name, such as '[' or ']'. Partially fixes issue #33047 (see: https://debbugs.gnu.org/cgi/bugreport.cgi?bug=33047). * guix/import/pypi.scm (use-modules): Export SPECIFICATION->REQUIREMENT-NAME (%requirement-name-regexp): New variable. (clean-requirement): Rename to... (specification->requirement-name): this, which now uses %requirement-name-regexp to select the requirement name from the requirement specification. (parse-requires.txt): Adapt. --- guix/import/pypi.scm | 43 ++++++++++++++++++++++++++++++++++--------- tests/pypi.scm | 12 ++++++++++++ 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index 91e987e9f1..efb5939c78 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -48,6 +48,7 @@ #:use-module ((guix licenses) #:prefix license:) #:use-module (guix build-system python) #:export (parse-requires.txt + specification->requirement-name guix-package->pypi-name pypi-recursive-import pypi->guix-package @@ -118,13 +119,37 @@ package definition." ((package-inputs ...) `((propagated-inputs (,'quasiquote ,package-inputs)))))) -(define (clean-requirement s) - ;; Given a requirement LINE, as can be found in a setuptools requires.txt - ;; file, remove everything other than the actual name of the required - ;; package, and return it. - (string-take s (or (string-index s (lambda (chr) - (member chr '(#\space #\> #\= #\<)))) - (string-length s)))) +(define %requirement-name-regexp + ;; Regexp to match the requirement name in a requirement specification. + + ;; Some grammar, taken from PEP-0508 (see: + ;; https://www.python.org/dev/peps/pep-0508/). + + ;; The unified rule can be expressed as: + ;; specification = wsp* ( url_req | name_req ) wsp* + + ;; where url_req is: + ;; url_req = name wsp* extras? wsp* urlspec wsp+ quoted_marker? + + ;; and where name_req is: + ;; name_req = name wsp* extras? wsp* versionspec? wsp* quoted_marker? + + ;; Thus, we need only matching NAME, which is expressed as: + ;; identifer_end = letterOrDigit | (('-' | '_' | '.' )* letterOrDigit) + ;; identifier = letterOrDigit identifier_end* + ;; name = identifier + (let* ((letter-or-digit "[A-Za-z0-9]") + (identifier-end (string-append "(" letter-or-digit "|" + "[-_.]*" letter-or-digit ")")) + (identifier (string-append "^" letter-or-digit identifier-end "*")) + (name identifier)) + (make-regexp name))) + +(define (specification->requirement-name spec) + "Given a specification SPEC, return the requirement name." + (match:substring + (or (regexp-exec %requirement-name-regexp spec) + (error (G_ "Could not extract requirement name in spec:") spec)))) (define (parse-requires.txt requires.txt) "Given REQUIRES.TXT, a Setuptools requires.txt file, return a list of @@ -158,7 +183,7 @@ requirement names." ((or (string-null? line) (comment? line)) (loop result)) (else - (loop (cons (clean-requirement line) + (loop (cons (specification->requirement-name line) result)))))))))) (define (guess-requirements source-url wheel-url tarball) @@ -200,7 +225,7 @@ cannot determine package dependencies")) (hash-ref (list-ref run_requires 0) "requires") '()))) - (map clean-requirement requirements))))) + (map specification->requirement-name requirements))))) (lambda () (delete-file json-file) (rmdir dirname)))))) diff --git a/tests/pypi.scm b/tests/pypi.scm index e4b7142311..82d6bba8dd 100644 --- a/tests/pypi.scm +++ b/tests/pypi.scm @@ -55,6 +55,14 @@ (define test-source-hash "") +(define test-specifications + '("Fizzy [foo, bar]" + "PickyThing<1.6,>1.9,!=1.9.6,<2.0a0,==2.4c1" + "SomethingWithMarker[foo]>1.0;python_version<\"2.7\"" + "requests [security,tests] >= 2.8.1, == 2.8.* ; python_version < \"2.7\"" + "pip @ https://github.com/pypa/pip/archive/1.3.1.zip#\ +sha1=da9234ee9982d4bbb3c72346a6de940a148ea686")) + (define test-requires.txt "\ bar baz > 13.37 @@ -108,6 +116,10 @@ pytest (>=2.5.0) (uri (list "https://bitheap.org/cram/cram-0.7.tar.gz" (pypi-uri "cram" "0.7")))))))) +(test-equal "specification->requirement-name" + '("Fizzy" "PickyThing" "SomethingWithMarker" "requests" "pip") + (map specification->requirement-name test-specifications)) + (test-equal "parse-requires.txt, with sections" '("foo" "bar") (mock ((ice-9 ports) call-with-input-file -- 2.20.1
From 76e4a3150f8126e0b952c6129b6e1371afba80c0 Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer <maxim.courno...@gmail.com> Date: Thu, 28 Mar 2019 00:26:01 -0400 Subject: [PATCH 4/7] import: pypi: Deduplicate requirements. * guix/import/pypi.scm (parse-requires.txt): Remove potential duplicates. --- guix/import/pypi.scm | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index efb5939c78..a90be67bb0 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -178,7 +178,11 @@ requirement names." ;; (extra) requirements. Non-optional requirements must appear ;; before any section is defined. (if (or (eof-object? line) (section-header? line)) - (reverse result) + ;; Duplicates can occur, since the same requirement can be + ;; listed multiple times with different conditional markers, e.g. + ;; pytest >= 3 ; python_version >= "3.3" + ;; pytest < 3 ; python_version < "3.3" + (reverse (delete-duplicates result)) (cond ((or (string-null? line) (comment? line)) (loop result)) -- 2.20.1
From 73e27235cac1275ba7671fd2364325cf5788cb3c Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer <maxim.courno...@gmail.com> Date: Thu, 28 Mar 2019 00:26:02 -0400 Subject: [PATCH 5/7] import: pypi: Support more types of archives. This change enables the PyPI importer to look for requirements in a source archive of a different type than "tar.gz" or "tar.bz2". * guix/import/pypi.scm: (guess-requirements)[tarball-directory]: Rename to... [archive-root-directory]: this. Use COMPRESSED-FILED? to determine if an archive is supported or not. [guess-requirements-from-source]: Adapt to use the new method, and use unzip to extract ZIP archives. (guess-requirements): Rename the TARBALL argument to ARCHIVE, to denote the archive format is no longer bound specifically to the Tar format. --- guix/import/pypi.scm | 47 ++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index a90be67bb0..8e93653717 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -190,27 +190,24 @@ requirement names." (loop (cons (specification->requirement-name line) result)))))))))) -(define (guess-requirements source-url wheel-url tarball) - "Given SOURCE-URL, WHEEL-URL and a TARBALL of the package, return a list -of the required packages specified in the requirements.txt file. TARBALL will +(define (guess-requirements source-url wheel-url archive) + "Given SOURCE-URL, WHEEL-URL and a ARCHIVE of the package, return a list +of the required packages specified in the requirements.txt file. ARCHIVE will be extracted in a temporary directory." - (define (tarball-directory url) - ;; Given the URL of the package's tarball, return the name of the directory + (define (archive-root-directory url) + ;; Given the URL of the package's archive, return the name of the directory ;; that will be created upon decompressing it. If the filetype is not ;; supported, return #f. - ;; TODO: Support more archive formats. - (let ((basename (substring url (+ 1 (string-rindex url #\/))))) - (cond - ((string-suffix? ".tar.gz" basename) - (string-drop-right basename 7)) - ((string-suffix? ".tar.bz2" basename) - (string-drop-right basename 8)) - (else + (if (compressed-file? url) + (let ((root-directory (file-sans-extension (basename url)))) + (if (string=? "tar" (file-extension root-directory)) + (file-sans-extension root-directory) + root-directory)) (begin - (warning (G_ "Unsupported archive format: \ -cannot determine package dependencies")) - #f))))) + (warning (G_ "Unsupported archive format (~a): \ +cannot determine package dependencies") (file-extension url)) + #f))) (define (read-wheel-metadata wheel-archive) ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's @@ -246,16 +243,20 @@ cannot determine package dependencies")) (define (guess-requirements-from-source) ;; Return the package's requirements by guessing them from the source. - (let ((dirname (tarball-directory source-url))) + (let ((dirname (archive-root-directory source-url)) + (extension (file-extension source-url))) (if (string? dirname) (call-with-temporary-directory (lambda (dir) (let* ((pypi-name (string-take dirname (string-rindex dirname #\-))) (requires.txt (string-append dirname "/" pypi-name ".egg-info" "/requires.txt")) - (exit-code (parameterize ((current-error-port (%make-void-port "rw+")) - (current-output-port (%make-void-port "rw+"))) - (system* "tar" "xf" tarball "-C" dir requires.txt)))) + (exit-code + (parameterize ((current-error-port (%make-void-port "rw+")) + (current-output-port (%make-void-port "rw+"))) + (if (string=? "zip" extension) + (system* "unzip" archive "-d" dir requires.txt) + (system* "tar" "xf" archive "-C" dir requires.txt))))) (if (zero? exit-code) (parse-requires.txt (string-append dir "/" requires.txt)) (begin @@ -271,13 +272,13 @@ cannot determine package dependencies")) (or (guess-requirements-from-wheel) (guess-requirements-from-source))) -(define (compute-inputs source-url wheel-url tarball) - "Given the SOURCE-URL of an already downloaded TARBALL, return a list of +(define (compute-inputs source-url wheel-url archive) + "Given the SOURCE-URL of an already downloaded ARCHIVE, return a list of name/variable pairs describing the required inputs of this package. Also return the unaltered list of upstream dependency names." (let ((dependencies (remove (cut string=? "argparse" <>) - (guess-requirements source-url wheel-url tarball)))) + (guess-requirements source-url wheel-url archive)))) (values (sort (map (lambda (input) (let ((guix-name (python->package-name input))) -- 2.20.1
From fb0547ef225103c0f8355a7eccc41e0d028f6563 Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer <maxim.courno...@gmail.com> Date: Thu, 28 Mar 2019 00:26:03 -0400 Subject: [PATCH 6/7] import: pypi: Parse wheel METADATA instead of metadata.json. With newer Wheel releases, there is no more metadata.json file; the METADATA file should be used instead (see: https://github.com/pypa/wheel/issues/195). This change updates our PyPI importer so that it uses the later. * guix/import/pypi.scm (define-module): Remove unnecessary modules and export the PARSE-WHEEL-METADATA method. (parse-wheel-metadata): Add method. (guess-requirements): Use it. * tests/pypi.scm (test-metadata): Test it. --- guix/import/pypi.scm | 66 +++++++++++++++++++++++++++++--------------- tests/pypi.scm | 60 ++++++++++++++++++++++++++++++---------- 2 files changed, 89 insertions(+), 37 deletions(-) diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index 8e93653717..c520213b6a 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -21,9 +21,7 @@ ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. (define-module (guix import pypi) - #:use-module (ice-9 binary-ports) #:use-module (ice-9 match) - #:use-module (ice-9 pretty-print) #:use-module (ice-9 regex) #:use-module (ice-9 receive) #:use-module ((ice-9 rdelim) #:select (read-line)) @@ -31,9 +29,6 @@ #:use-module (srfi srfi-26) #:use-module (srfi srfi-34) #:use-module (srfi srfi-35) - #:use-module (rnrs bytevectors) - #:use-module (json) - #:use-module (web uri) #:use-module (guix ui) #:use-module (guix utils) #:use-module ((guix build utils) @@ -48,6 +43,7 @@ #:use-module ((guix licenses) #:prefix license:) #:use-module (guix build-system python) #:export (parse-requires.txt + parse-wheel-metadata specification->requirement-name guix-package->pypi-name pypi-recursive-import @@ -190,6 +186,37 @@ requirement names." (loop (cons (specification->requirement-name line) result)))))))))) +(define (parse-wheel-metadata metadata) + "Given METADATA, a Wheel metadata file, return a list of requirement names." + ;; METADATA is a RFC-2822-like, header based file. + + (define (requires-dist-header? line) + ;; Return #t if the given LINE is a Requires-Dist header. + (regexp-match? (string-match "^Requires-Dist: " line))) + + (define (requires-dist-value line) + (string-drop line (string-length "Requires-Dist: "))) + + (define (extra? line) + ;; Return #t if the given LINE is an "extra" requirement. + (regexp-match? (string-match "extra == " line))) + + (call-with-input-file metadata + (lambda (port) + (let loop ((requirements '())) + (let ((line (read-line port))) + ;; Stop at the first 'Provides-Extra' section: the non-optional + ;; requirements appear before the optional ones. + (if (eof-object? line) + (reverse (delete-duplicates requirements)) + (cond + ((and (requires-dist-header? line) (not (extra? line))) + (loop (cons (specification->requirement-name + (requires-dist-value line)) + requirements))) + (else + (loop requirements))))))))) + (define (guess-requirements source-url wheel-url archive) "Given SOURCE-URL, WHEEL-URL and a ARCHIVE of the package, return a list of the required packages specified in the requirements.txt file. ARCHIVE will @@ -211,25 +238,18 @@ cannot determine package dependencies") (file-extension url)) (define (read-wheel-metadata wheel-archive) ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's - ;; requirements. + ;; requirements, or #f if the metadata file contained therein couldn't be + ;; extracted. (let* ((dirname (wheel-url->extracted-directory wheel-url)) - (json-file (string-append dirname "/metadata.json"))) - (and (zero? (system* "unzip" "-q" wheel-archive json-file)) - (dynamic-wind - (const #t) - (lambda () - (call-with-input-file json-file - (lambda (port) - (let* ((metadata (json->scm port)) - (run_requires (hash-ref metadata "run_requires")) - (requirements (if run_requires - (hash-ref (list-ref run_requires 0) - "requires") - '()))) - (map specification->requirement-name requirements))))) - (lambda () - (delete-file json-file) - (rmdir dirname)))))) + (metadata (string-append dirname "/METADATA"))) + (call-with-temporary-directory + (lambda (dir) + (if (zero? (system* "unzip" "-q" wheel-archive "-d" dir metadata)) + (parse-wheel-metadata (string-append dir "/" metadata)) + (begin + (warning + (G_ "Failed to extract file: ~a from wheel.~%") metadata) + #f)))))) (define (guess-requirements-from-wheel) ;; Return the package's requirements using the wheel, or #f if an error diff --git a/tests/pypi.scm b/tests/pypi.scm index 82d6bba8dd..ca8cb5f6de 100644 --- a/tests/pypi.scm +++ b/tests/pypi.scm @@ -21,6 +21,7 @@ #:use-module (guix import pypi) #:use-module (guix base32) #:use-module (gcrypt hash) + #:use-module (guix memoization) #:use-module (guix tests) #:use-module (guix build-system python) #:use-module ((guix build utils) #:select (delete-file-recursively which mkdir-p)) @@ -77,17 +78,33 @@ bar != 2 pytest (>=2.5.0) ") -(define test-metadata - "{ - \"run_requires\": [ - { - \"requires\": [ - \"bar\", - \"baz (>13.37)\" - ] - } - ] -}") +(define test-metadata "\ +Classifier: Programming Language :: Python :: 3.7 +Requires-Dist: baz ~= 3 +Requires-Dist: bar != 2 +Provides-Extra: test +pytest (>=2.5.0) +") + +(define test-metadata-with-extras " +Classifier: Programming Language :: Python :: 3.7 +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +Requires-Dist: wrapt (<2,>=1) +Requires-Dist: bar + +Provides-Extra: dev +Requires-Dist: tox ; extra == 'dev' +Requires-Dist: bumpversion (<1) ; extra == 'dev' +") + +;;; Provides-Extra can appear before Requires-Dist. +(define test-metadata-with-extras-jedi "\ +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +Provides-Extra: testing +Requires-Dist: parso (>=0.3.0) +Provides-Extra: testing +Requires-Dist: pytest (>=3.1.0); extra == 'testing' +") (test-begin "pypi") @@ -126,6 +143,18 @@ pytest (>=2.5.0) call-with-input-string) (parse-requires.txt test-requires-with-sections))) +(test-equal "parse-wheel-metadata, with extras" + '("wrapt" "bar") + (mock ((ice-9 ports) call-with-input-file + call-with-input-string) + (parse-wheel-metadata test-metadata-with-extras))) + +(test-equal "parse-wheel-metadata, with extras - Jedi" + '("parso") + (mock ((ice-9 ports) call-with-input-file + call-with-input-string) + (parse-wheel-metadata test-metadata-with-extras-jedi))) + (test-assert "pypi->guix-package" ;; Replace network resources with sample data. (mock ((guix import utils) url-fetch @@ -188,7 +217,7 @@ pytest (>=2.5.0) (mkdir-p "foo-1.0.0/foo.egg-info/") (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt" (lambda () - (display test-requires.txt))) + (display "wrong data to make sure we're testing wheels "))) (parameterize ((current-output-port (%make-void-port "rw+"))) (system* "tar" "czvf" file-name "foo-1.0.0/")) (delete-file-recursively "foo-1.0.0") @@ -197,13 +226,13 @@ pytest (>=2.5.0) ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" (begin (mkdir "foo-1.0.0.dist-info") - (with-output-to-file "foo-1.0.0.dist-info/metadata.json" + (with-output-to-file "foo-1.0.0.dist-info/METADATA" (lambda () (display test-metadata))) (let ((zip-file (string-append file-name ".zip"))) ;; zip always adds a "zip" extension to the file it creates, ;; so we need to rename it. - (system* "zip" zip-file "foo-1.0.0.dist-info/metadata.json") + (system* "zip" zip-file "foo-1.0.0.dist-info/METADATA") (rename-file zip-file file-name)) (delete-file-recursively "foo-1.0.0.dist-info"))) (_ (error "Unexpected URL: " url))))) @@ -215,6 +244,9 @@ pytest (>=2.5.0) (string-length test-json))) ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f) (_ (error "Unexpected URL: " url))))) + ;; Not clearing the memoization cache here would mean returning the value + ;; computed in the previous test. + (invalidate-memoization! pypi->guix-package) (match (pypi->guix-package "foo") (('package ('name "python-foo") -- 2.20.1
From ea0f24eb7b19c57ebb24ec48ba776b240bccfc99 Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer <maxim.courno...@gmail.com> Date: Thu, 28 Mar 2019 23:12:26 -0400 Subject: [PATCH 7/7] import: pypi: Include optional test inputs as native-inputs. * guix/import/pypi.scm (maybe-inputs): Add INPUT-TYPE argument, and use it. (test-section?): New predicate. (parse-requires.txt): Collect the optional test inputs, and return them as the second element of the returned list. (parse-wheel-metadata): Likewise. (guess-requirements): Adapt, and hide unzip output. (make-pypi-sexp): Likewise, and include the test inputs requirements as native inputs in the returned package expression. * tests/pypi.scm (test-requires.txt): Include a test section in the test-requires.txt data. (test-requires.txt-beaker): New variable. ("parse-requires.txt"): Adapt. ("parse-requires.txt - Beaker"): New test. ("parse-wheel-metadata, with extras"): Adapt. ("parse-wheel-metadata, with extras - Jedi"): Adapt. ("pypi->guix-package, no wheel"): Re-indent, and add the expected native-inputs. ("pypi->guix-package, wheels"): Likewise. ("pypi->guix-package, no usable requirement file."): New test. --- guix/import/pypi.scm | 158 ++++++++++++++++++++++++++++--------------- tests/pypi.scm | 123 +++++++++++++++++++++++++-------- 2 files changed, 199 insertions(+), 82 deletions(-) diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index c520213b6a..f84ad88e44 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -4,6 +4,7 @@ ;;; Copyright © 2015, 2016, 2017 Ludovic Courtès <l...@gnu.org> ;;; Copyright © 2017 Mathieu Othacehe <m.othac...@gmail.com> ;;; Copyright © 2018 Ricardo Wurmus <rek...@elephly.net> +;;; Copyright © 2019 Maxim Cournoyer <maxim.courno...@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -26,6 +27,7 @@ #:use-module (ice-9 receive) #:use-module ((ice-9 rdelim) #:select (read-line)) #:use-module (srfi srfi-1) + #:use-module (srfi srfi-11) #:use-module (srfi srfi-26) #:use-module (srfi srfi-34) #:use-module (srfi srfi-35) @@ -106,14 +108,15 @@ package on PyPI." ((name version _ ...) (string-append name "-" version ".dist-info")))) -(define (maybe-inputs package-inputs) +(define (maybe-inputs package-inputs input-type) "Given a list of PACKAGE-INPUTS, tries to generate the 'inputs' field of a -package definition." +package definition. INPUT-TYPE, a symbol, is used to populate the name of +the input field." (match package-inputs (() '()) ((package-inputs ...) - `((propagated-inputs (,'quasiquote ,package-inputs)))))) + `((,input-type (,'quasiquote ,package-inputs)))))) (define %requirement-name-regexp ;; Regexp to match the requirement name in a requirement specification. @@ -147,11 +150,21 @@ package definition." (or (regexp-exec %requirement-name-regexp spec) (error (G_ "Could not extract requirement name in spec:") spec)))) +(define (test-section? name) + "Return #t if the section name contains 'test' or 'dev'." + (any (cut string-contains-ci name <>) + '("test" "dev"))) + (define (parse-requires.txt requires.txt) - "Given REQUIRES.TXT, a Setuptools requires.txt file, return a list of -requirement names." - ;; This is a very incomplete parser, which job is to select the non-optional - ;; dependencies and strip them out of any version information. + "Given REQUIRES.TXT, a Setuptools requires.txt file, return a pair of requirements. + +The first element of the pair contains the required dependencies while the +second the optional test dependencies. Note that currently, optional, +non-test dependencies are omitted since these can be difficult or expensive to +satisfy." + + ;; This is a very incomplete parser, which job is to read in the requirement + ;; specification lines, and strip them out of any version information. ;; Alternatively, we could implement a PEG parser with the (ice-9 peg) ;; library and the requirements grammar defined by PEP-0508 ;; (https://www.python.org/dev/peps/pep-0508/). @@ -168,57 +181,89 @@ requirement names." (call-with-input-file requires.txt (lambda (port) - (let loop ((result '())) + (let loop ((required-deps '()) + (test-deps '()) + (inside-test-section? #f) + (optional? #f)) (let ((line (read-line port))) - ;; Stop when a section is encountered, as sections contains optional - ;; (extra) requirements. Non-optional requirements must appear - ;; before any section is defined. - (if (or (eof-object? line) (section-header? line)) + (if (eof-object? line) ;; Duplicates can occur, since the same requirement can be ;; listed multiple times with different conditional markers, e.g. ;; pytest >= 3 ; python_version >= "3.3" ;; pytest < 3 ; python_version < "3.3" - (reverse (delete-duplicates result)) + (map (compose reverse delete-duplicates) + (list required-deps test-deps)) (cond ((or (string-null? line) (comment? line)) - (loop result)) - (else + (loop required-deps test-deps inside-test-section? optional?)) + ((section-header? line) + ;; Encountering a section means that all the requirements + ;; listed below are optional. Since we want to pick only the + ;; test dependencies from the optional dependencies, we must + ;; track those separately. + (loop required-deps test-deps (test-section? line) #t)) + (inside-test-section? + (loop required-deps + (cons (specification->requirement-name line) + test-deps) + inside-test-section? optional?)) + ((not optional?) (loop (cons (specification->requirement-name line) - result)))))))))) + required-deps) + test-deps inside-test-section? optional?)) + (optional? + ;; Skip optional items. + (loop required-deps test-deps inside-test-section? optional?)) + (else + (warning (G_ "parse-requires.txt reached an unexpected \ +condition on line ~a~%") line))))))))) (define (parse-wheel-metadata metadata) - "Given METADATA, a Wheel metadata file, return a list of requirement names." + "Given METADATA, a Wheel metadata file, return a pair of requirements. + +The first element of the pair contains the required dependencies while the second the optional +test dependencies. Note that currently, optional, non-test dependencies are +omitted since these can be difficult or expensive to satisfy." ;; METADATA is a RFC-2822-like, header based file. (define (requires-dist-header? line) ;; Return #t if the given LINE is a Requires-Dist header. - (regexp-match? (string-match "^Requires-Dist: " line))) + (string-match "^Requires-Dist: " line)) (define (requires-dist-value line) (string-drop line (string-length "Requires-Dist: "))) (define (extra? line) ;; Return #t if the given LINE is an "extra" requirement. - (regexp-match? (string-match "extra == " line))) + (string-match "extra == '(.*)'" line)) + + (define (test-requirement? line) + (let ((extra-label (match:substring (extra? line) 1))) + (and extra-label (test-section? extra-label)))) (call-with-input-file metadata (lambda (port) - (let loop ((requirements '())) + (let loop ((required-deps '()) + (test-deps '())) (let ((line (read-line port))) - ;; Stop at the first 'Provides-Extra' section: the non-optional - ;; requirements appear before the optional ones. (if (eof-object? line) - (reverse (delete-duplicates requirements)) + (map (compose reverse delete-duplicates) + (list required-deps test-deps)) (cond ((and (requires-dist-header? line) (not (extra? line))) (loop (cons (specification->requirement-name (requires-dist-value line)) - requirements))) + required-deps) + test-deps)) + ((and (requires-dist-header? line) (test-requirement? line)) + (loop required-deps + (cons (specification->requirement-name (requires-dist-value line)) + test-deps))) (else - (loop requirements))))))))) + (loop required-deps test-deps))))))))) ;skip line (define (guess-requirements source-url wheel-url archive) - "Given SOURCE-URL, WHEEL-URL and a ARCHIVE of the package, return a list + "Given SOURCE-URL, WHEEL-URL and an ARCHIVE of the package, return a list of the required packages specified in the requirements.txt file. ARCHIVE will be extracted in a temporary directory." @@ -244,7 +289,10 @@ cannot determine package dependencies") (file-extension url)) (metadata (string-append dirname "/METADATA"))) (call-with-temporary-directory (lambda (dir) - (if (zero? (system* "unzip" "-q" wheel-archive "-d" dir metadata)) + (if (zero? + (parameterize ((current-error-port (%make-void-port "rw+")) + (current-output-port (%make-void-port "rw+"))) + (system* "unzip" wheel-archive "-d" dir metadata))) (parse-wheel-metadata (string-append dir "/" metadata)) (begin (warning @@ -283,32 +331,38 @@ cannot determine package dependencies") (file-extension url)) (warning (G_ "Failed to extract file: ~a from source.~%") requires.txt) - '()))))) - '()))) + (list '() '())))))) + (list '() '())))) ;; First, try to compute the requirements using the wheel, else, fallback to ;; reading the "requires.txt" from the egg-info directory from the source - ;; tarball. + ;; archive. (or (guess-requirements-from-wheel) (guess-requirements-from-source))) (define (compute-inputs source-url wheel-url archive) - "Given the SOURCE-URL of an already downloaded ARCHIVE, return a list of -name/variable pairs describing the required inputs of this package. Also -return the unaltered list of upstream dependency names." - (let ((dependencies - (remove (cut string=? "argparse" <>) - (guess-requirements source-url wheel-url archive)))) - (values (sort - (map (lambda (input) - (let ((guix-name (python->package-name input))) - (list guix-name (list 'unquote (string->symbol guix-name))))) - dependencies) - (lambda args - (match args - (((a _ ...) (b _ ...)) - (string-ci<? a b))))) - dependencies))) + "Given the SOURCE-URL and WHEEL-URL of an already downloaded ARCHIVE, return +a pair of lists, each consisting of a list of name/variable pairs, for the +propagated inputs and the native inputs, respectively." + + (define (strip-argparse deps) + (remove (cut string=? "argparse" <>) deps)) + + (define (requirement->package-name/sort deps) + (sort + (map (lambda (input) + (let ((guix-name (python->package-name input))) + (list guix-name (list 'unquote (string->symbol guix-name))))) + deps) + (lambda args + (match args + (((a _ ...) (b _ ...)) + (string-ci<? a b)))))) + + (define process-requirements + (compose requirement->package-name/sort strip-argparse)) + + (map process-requirements (guess-requirements source-url wheel-url archive))) (define (make-pypi-sexp name version source-url wheel-url home-page synopsis description license) @@ -317,15 +371,13 @@ VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE." (call-with-temporary-output-file (lambda (temp port) (and (url-fetch source-url temp) - (receive (input-package-names upstream-dependency-names) - (compute-inputs source-url wheel-url temp) - (values + (match (compute-inputs source-url wheel-url temp) + ((required-inputs test-inputs) `(package (name ,(python->package-name name)) (version ,version) (source (origin (method url-fetch) - ;; Sometimes 'pypi-uri' doesn't quite work due to mixed ;; cases in NAME, for instance, as is the case with ;; "uwsgi". In that case, fall back to a full URL. @@ -334,12 +386,12 @@ VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE." (base32 ,(guix-hash-url temp))))) (build-system python-build-system) - ,@(maybe-inputs input-package-names) + ,@(maybe-inputs required-inputs 'propagated-inputs) + ,@(maybe-inputs test-inputs 'native-inputs) (home-page ,home-page) (synopsis ,synopsis) (description ,description) - (license ,(license->symbol license))) - upstream-dependency-names)))))) + (license ,(license->symbol license))))))))) (define pypi->guix-package (memoize diff --git a/tests/pypi.scm b/tests/pypi.scm index ca8cb5f6de..aa08e2cb54 100644 --- a/tests/pypi.scm +++ b/tests/pypi.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2014 David Thompson <da...@gnu.org> ;;; Copyright © 2016 Ricardo Wurmus <rek...@elephly.net> +;;; Copyright © 2019 Maxim Cournoyer <maxim.courno...@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -65,11 +66,6 @@ sha1=da9234ee9982d4bbb3c72346a6de940a148ea686")) (define test-requires.txt "\ -bar -baz > 13.37 -") - -(define test-requires-with-sections "\ # A comment foo ~= 3 bar != 2 @@ -78,12 +74,25 @@ bar != 2 pytest (>=2.5.0) ") +;; Beaker contains only optional dependencies. +(define test-requires.txt-beaker "\ +[crypto] +pycryptopp>=0.5.12 + +[cryptography] +cryptography + +[testsuite] +Mock +coverage +") + (define test-metadata "\ Classifier: Programming Language :: Python :: 3.7 Requires-Dist: baz ~= 3 Requires-Dist: bar != 2 Provides-Extra: test -pytest (>=2.5.0) +Requires-Dist: pytest (>=2.5.0) ; extra == 'test' ") (define test-metadata-with-extras " @@ -137,25 +146,31 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing' '("Fizzy" "PickyThing" "SomethingWithMarker" "requests" "pip") (map specification->requirement-name test-specifications)) -(test-equal "parse-requires.txt, with sections" - '("foo" "bar") +(test-equal "parse-requires.txt" + (list '("foo" "bar") '("pytest")) (mock ((ice-9 ports) call-with-input-file call-with-input-string) - (parse-requires.txt test-requires-with-sections))) + (parse-requires.txt test-requires.txt))) + +(test-equal "parse-requires.txt - Beaker" + (list '() '("Mock" "coverage")) + (mock ((ice-9 ports) call-with-input-file + call-with-input-string) + (parse-requires.txt test-requires.txt-beaker))) (test-equal "parse-wheel-metadata, with extras" - '("wrapt" "bar") + (list '("wrapt" "bar") '("tox" "bumpversion")) (mock ((ice-9 ports) call-with-input-file call-with-input-string) (parse-wheel-metadata test-metadata-with-extras))) (test-equal "parse-wheel-metadata, with extras - Jedi" - '("parso") + (list '("parso") '("pytest")) (mock ((ice-9 ports) call-with-input-file call-with-input-string) (parse-wheel-metadata test-metadata-with-extras-jedi))) -(test-assert "pypi->guix-package" +(test-assert "pypi->guix-package, no wheel" ;; Replace network resources with sample data. (mock ((guix import utils) url-fetch (lambda (url file-name) @@ -195,7 +210,10 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing' ('propagated-inputs ('quasiquote (("python-bar" ('unquote 'python-bar)) - ("python-baz" ('unquote 'python-baz))))) + ("python-foo" ('unquote 'python-foo))))) + ('native-inputs + ('quasiquote + (("python-pytest" ('unquote 'python-pytest))))) ('home-page "http://example.com") ('synopsis "summary") ('description "summary") @@ -216,25 +234,25 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing' (begin (mkdir-p "foo-1.0.0/foo.egg-info/") (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt" - (lambda () - (display "wrong data to make sure we're testing wheels "))) + (lambda () + (display "wrong data to make sure we're testing wheels "))) (parameterize ((current-output-port (%make-void-port "rw+"))) (system* "tar" "czvf" file-name "foo-1.0.0/")) - (delete-file-recursively "foo-1.0.0") - (set! test-source-hash - (call-with-input-file file-name port-sha256)))) + (delete-file-recursively "foo-1.0.0") + (set! test-source-hash + (call-with-input-file file-name port-sha256)))) ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" - (begin - (mkdir "foo-1.0.0.dist-info") - (with-output-to-file "foo-1.0.0.dist-info/METADATA" - (lambda () - (display test-metadata))) - (let ((zip-file (string-append file-name ".zip"))) - ;; zip always adds a "zip" extension to the file it creates, - ;; so we need to rename it. - (system* "zip" zip-file "foo-1.0.0.dist-info/METADATA") - (rename-file zip-file file-name)) - (delete-file-recursively "foo-1.0.0.dist-info"))) + (begin + (mkdir "foo-1.0.0.dist-info") + (with-output-to-file "foo-1.0.0.dist-info/METADATA" + (lambda () + (display test-metadata))) + (let ((zip-file (string-append file-name ".zip"))) + ;; zip always adds a "zip" extension to the file it creates, + ;; so we need to rename it. + (system* "zip" "-q" zip-file "foo-1.0.0.dist-info/METADATA") + (rename-file zip-file file-name)) + (delete-file-recursively "foo-1.0.0.dist-info"))) (_ (error "Unexpected URL: " url))))) (mock ((guix http-client) http-fetch (lambda (url . rest) @@ -262,6 +280,9 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing' ('quasiquote (("python-bar" ('unquote 'python-bar)) ("python-baz" ('unquote 'python-baz))))) + ('native-inputs + ('quasiquote + (("python-pytest" ('unquote 'python-pytest))))) ('home-page "http://example.com") ('synopsis "summary") ('description "summary") @@ -272,4 +293,48 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing' (x (pk 'fail x #f)))))) +(test-assert "pypi->guix-package, no usable requirement file." + ;; Replace network resources with sample data. + (mock ((guix import utils) url-fetch + (lambda (url file-name) + (match url + ("https://example.com/foo-1.0.0.tar.gz" + (set! test-source-hash + (call-with-input-file file-name port-sha256)) + #t) + ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #t) + (_ (error "Unexpected URL: " url))))) + (mock ((guix http-client) http-fetch + (lambda (url . rest) + (match url + ("https://pypi.org/pypi/foo/json" + (values (open-input-string test-json) + (string-length test-json))) + ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f) + (_ (error "Unexpected URL: " url))))) + ;; Not clearing the memoization cache here would mean returning the value + ;; computed in the previous test. + (invalidate-memoization! pypi->guix-package) + (match (pypi->guix-package "foo") + (('package + ('name "python-foo") + ('version "1.0.0") + ('source ('origin + ('method 'url-fetch) + ('uri ('pypi-uri "foo" 'version)) + ('sha256 + ('base32 + (? string? hash))))) + ('build-system 'python-build-system) + ('home-page "http://example.com") + ('synopsis "summary") + ('description "summary") + ('license 'license:lgpl2.0)) + (string=? (bytevector->nix-base32-string + test-source-hash) + hash)) + (x + (pk 'fail x #f))) + ))) + (test-end "pypi") -- 2.20.1
Thanks, Maxim
signature.asc
Description: PGP signature