Subject: bug#24450: [PATCHv2] Re: pypi importer outputs strange character series in optional dependency case.

From: Maxim Cournoyer <[email-masked]>
Date: Mon, 20 May 2019 00:05:59 -0400

Hi Ricardo!

Ricardo Wurmus <[email-masked]> writes:

> Hi Maxim,
>
> I would very much like to see your improvements to the pypi importer to
> be merged.  Have you been able to separate the independent changes as
> suggested by Ludo?

I'm thrilled that someone has an interest in this :-)

I took my time, but finally got around to restructure the changes a
bit.  I hope it'll be easier to review this time around!

Thank you!

Maxim

From 54e44b7397f17910d95dbdb233d23e5c97c095aa Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <[email-masked]>
Date: Thu, 28 Mar 2019 00:26:00 -0400
Subject: [PATCH 1/9] import: pypi: Do not consider requirements.txt files.

* guix/import/pypi.scm (guess-requirements): Update comment.
[guess-requirements-from-source]: Do not attempt to parse the file
requirements.txt.  Streamline logic.
---
 guix/import/pypi.scm | 35 +++++++++++++----------------------
 tests/pypi.scm       | 23 +++++++++++------------
 2 files changed, 24 insertions(+), 34 deletions(-)

diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index 3a20fc4b9b..8269aa61d7 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -206,35 +206,26 @@ cannot determine package dependencies"))
           (call-with-temporary-directory
            (lambda (dir)
              (let* ((pypi-name (string-take dirname (string-rindex dirname #\-)))
-                    (req-files (list (string-append dirname "/requirements.txt")
-                                     (string-append dirname "/" pypi-name ".egg-info"
-                                                    "/requires.txt")))
-                    (exit-codes (map (lambda (file-name)
-                                       (parameterize ((current-error-port (%make-void-port "rw+"))
-                                                      (current-output-port (%make-void-port "rw+")))
-                                         (system* "tar" "xf" tarball "-C" dir file-name)))
-                                     req-files)))
-               ;; Only one of these files needs to exist.
-               (if (any zero? exit-codes)
-                   (match (find-files dir)
-                     ((file . _)
-                      (read-requirements file))
-                     (()
-                      (warning (G_ "No requirements file found.\n"))))
+                    (requires.txt (string-append dirname "/" pypi-name
+                                                 ".egg-info" "/requires.txt"))
+                    (exit-code (parameterize ((current-error-port (%make-void-port "rw+"))
+                                              (current-output-port (%make-void-port "rw+")))
+                                 (system* "tar" "xf" tarball "-C" dir requires.txt))))
+               (if (zero? exit-code)
+                   (read-requirements (string-append dir "/" requires.txt))
                    (begin
-                     (warning (G_ "Failed to extract requirements files\n"))
+                     (warning
+                      (G_ "Failed to extract file: ~a from source.~%")
+                      requires.txt)
                      '())))))
           '())))
 
-  ;; First, try to compute the requirements using the wheel, since that is the
-  ;; most reliable option. If a wheel is not provided for this package, try
-  ;; getting them by reading either the "requirements.txt" file or the
-  ;; "requires.txt" from the egg-info directory from the source tarball. Note
-  ;; that "requirements.txt" is not mandatory, so this is likely to fail.
+  ;; First, try to compute the requirements using the wheel, else, fallback to
+  ;; reading the "requires.txt" from the egg-info directory from the source
+  ;; tarball.
   (or (guess-requirements-from-wheel)
       (guess-requirements-from-source)))
 
-
 (define (compute-inputs source-url wheel-url tarball)
   "Given the SOURCE-URL of an already downloaded TARBALL, return a list of
 name/variable pairs describing the required inputs of this package.  Also
diff --git a/tests/pypi.scm b/tests/pypi.scm
index 6daa44a6e7..335be42644 100644
--- a/tests/pypi.scm
+++ b/tests/pypi.scm
@@ -23,7 +23,7 @@
   #:use-module (gcrypt hash)
   #:use-module (guix tests)
   #:use-module (guix build-system python)
-  #:use-module ((guix build utils) #:select (delete-file-recursively which))
+  #:use-module ((guix build utils) #:select (delete-file-recursively which mkdir-p))
   #:use-module (srfi srfi-64)
   #:use-module (ice-9 match))
 
@@ -55,11 +55,10 @@
 (define test-source-hash
   "")
 
-(define test-requirements
-"# A comment
- # A comment after a space
+(define test-requires.txt "\
 bar
-baz > 13.37")
+baz > 13.37
+")
 
 (define test-metadata
   "{
@@ -107,10 +106,10 @@ baz > 13.37")
              (match url
                ("https://example.com/foo-1.0.0.tar.gz"
                 (begin
-                  (mkdir "foo-1.0.0")
-                  (with-output-to-file "foo-1.0.0/requirements.txt"
+                  (mkdir-p "foo-1.0.0/foo.egg-info/")
+                  (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt"
                     (lambda ()
-                      (display test-requirements)))
+                      (display test-requires.txt)))
                   (system* "tar" "czvf" file-name "foo-1.0.0/")
                   (delete-file-recursively "foo-1.0.0")
                   (set! test-source-hash
@@ -157,11 +156,11 @@ baz > 13.37")
          (lambda (url file-name)
            (match url
              ("https://example.com/foo-1.0.0.tar.gz"
-               (begin
-                 (mkdir "foo-1.0.0")
-                 (with-output-to-file "foo-1.0.0/requirements.txt"
+              (begin
+                (mkdir-p "foo-1.0.0/foo.egg-info/")
+                (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt"
                    (lambda ()
-                     (display test-requirements)))
+                     (display test-requires.txt)))
                  (system* "tar" "czvf" file-name "foo-1.0.0/")
                  (delete-file-recursively "foo-1.0.0")
                  (set! test-source-hash
-- 
2.21.0

From 5f79b0502f62bd1dacc8ea143c1dbd9ef7cfc29d Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <[email-masked]>
Date: Thu, 28 Mar 2019 00:26:00 -0400
Subject: [PATCH 2/9] import: pypi: Do not parse optional requirements from
 source.

* guix/import/pypi.scm: Export PARSE-REQUIRES.TXT.
(guess-requirements): Move the READ-REQUIREMENTS procedure to the top level,
and rename it to PARSE-REQUIRES.TXT.  Move the CLEAN-REQUIREMENT and COMMENT?
functions inside the READ-REQUIREMENTS procedure.
(parse-requires.txt): Add a SECTION-HEADER? predicate, and use it to prevent
parsing optional requirements.

* tests/pypi.scm (test-requires-with-sections): New variable.
("parse-requires.txt, with sections"): New test.
("pypi->guix-package"): Mute tar output to stdout.
---
 guix/import/pypi.scm | 76 +++++++++++++++++++++++++++-----------------
 tests/pypi.scm       | 21 ++++++++++--
 2 files changed, 65 insertions(+), 32 deletions(-)

diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index 8269aa61d7..91e987e9f1 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -47,7 +47,8 @@
   #:use-module (guix upstream)
   #:use-module ((guix licenses) #:prefix license:)
   #:use-module (guix build-system python)
-  #:export (guix-package->pypi-name
+  #:export (parse-requires.txt
+            guix-package->pypi-name
             pypi-recursive-import
             pypi->guix-package
             %pypi-updater))
@@ -117,6 +118,49 @@ package definition."
     ((package-inputs ...)
      `((propagated-inputs (,'quasiquote ,package-inputs))))))
 
+(define (clean-requirement s)
+  ;; Given a requirement LINE, as can be found in a setuptools requires.txt
+  ;; file, remove everything other than the actual name of the required
+  ;; package, and return it.
+  (string-take s (or (string-index s (lambda (chr)
+                                       (member chr '(#\space #\> #\= #\<))))
+                     (string-length s))))
+
+(define (parse-requires.txt requires.txt)
+  "Given REQUIRES.TXT, a Setuptools requires.txt file, return a list of
+requirement names."
+  ;; This is a very incomplete parser, which job is to select the non-optional
+  ;; dependencies and strip them out of any version information.
+  ;; Alternatively, we could implement a PEG parser with the (ice-9 peg)
+  ;; library and the requirements grammar defined by PEP-0508
+  ;; (https://www.python.org/dev/peps/pep-0508/).
+
+  (define (comment? line)
+    ;; Return #t if the given LINE is a comment, #f otherwise.
+    (eq? (string-ref (string-trim line) 0) #\#))
+
+  (define (section-header? line)
+    ;; Return #t if the given LINE is a section header, #f otherwise.
+    (let ((trimmed-line (string-trim line)))
+      (and (not (string-null? trimmed-line))
+           (eq? (string-ref trimmed-line 0) #\[))))
+
+  (call-with-input-file requires.txt
+    (lambda (port)
+      (let loop ((result '()))
+        (let ((line (read-line port)))
+          ;; Stop when a section is encountered, as sections contains optional
+          ;; (extra) requirements.  Non-optional requirements must appear
+          ;; before any section is defined.
+          (if (or (eof-object? line) (section-header? line))
+              (reverse result)
+              (cond
+               ((or (string-null? line) (comment? line))
+                (loop result))
+               (else
+                (loop (cons (clean-requirement line)
+                            result))))))))))
+
 (define (guess-requirements source-url wheel-url tarball)
   "Given SOURCE-URL, WHEEL-URL and a TARBALL of the package, return a list
 of the required packages specified in the requirements.txt file.  TARBALL will
@@ -139,34 +183,6 @@ be extracted in a temporary directory."
 cannot determine package dependencies"))
           #f)))))
 
-  (define (clean-requirement s)
-    ;; Given a requirement LINE, as can be found in a Python requirements.txt
-    ;; file, remove everything other than the actual name of the required
-    ;; package, and return it.
-    (string-take s
-      (or (string-index s (lambda (chr) (member chr '(#\space #\> #\= #\<))))
-          (string-length s))))
-
-  (define (comment? line)
-    ;; Return #t if the given LINE is a comment, #f otherwise.
-    (eq? (string-ref (string-trim line) 0) #\#))
-
-  (define (read-requirements requirements-file)
-    ;; Given REQUIREMENTS-FILE, a Python requirements.txt file, return a list
-    ;; of name/variable pairs describing the requirements.
-    (call-with-input-file requirements-file
-      (lambda (port)
-        (let loop ((result '()))
-          (let ((line (read-line port)))
-            (if (eof-object? line)
-                result
-                (cond
-                 ((or (string-null? line) (comment? line))
-                  (loop result))
-                 (else
-                  (loop (cons (clean-requirement line)
-                              result))))))))))
-
   (define (read-wheel-metadata wheel-archive)
     ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's
     ;; requirements.
@@ -212,7 +228,7 @@ cannot determine package dependencies"))
                                               (current-output-port (%make-void-port "rw+")))
                                  (system* "tar" "xf" tarball "-C" dir requires.txt))))
                (if (zero? exit-code)
-                   (read-requirements (string-append dir "/" requires.txt))
+                   (parse-requires.txt (string-append dir "/" requires.txt))
                    (begin
                      (warning
                       (G_ "Failed to extract file: ~a from source.~%")
diff --git a/tests/pypi.scm b/tests/pypi.scm
index 335be42644..e4b7142311 100644
--- a/tests/pypi.scm
+++ b/tests/pypi.scm
@@ -60,6 +60,15 @@ bar
 baz > 13.37
 ")
 
+(define test-requires-with-sections "\
+# A comment
+foo ~= 3
+bar != 2
+
+[test]
+pytest (>=2.5.0)
+")
+
 (define test-metadata
   "{
   \"run_requires\": [
@@ -99,6 +108,12 @@ baz > 13.37
                     (uri (list "https://bitheap.org/cram/cram-0.7.tar.gz"
                                (pypi-uri "cram" "0.7"))))))))
 
+(test-equal "parse-requires.txt, with sections"
+  '("foo" "bar")
+  (mock ((ice-9 ports) call-with-input-file
+         call-with-input-string)
+        (parse-requires.txt test-requires-with-sections)))
+
 (test-assert "pypi->guix-package"
   ;; Replace network resources with sample data.
     (mock ((guix import utils) url-fetch
@@ -110,7 +125,8 @@ baz > 13.37
                   (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt"
                     (lambda ()
                       (display test-requires.txt)))
-                  (system* "tar" "czvf" file-name "foo-1.0.0/")
+                  (parameterize ((current-output-port (%make-void-port "rw+")))
+                    (system* "tar" "czvf" file-name "foo-1.0.0/"))
                   (delete-file-recursively "foo-1.0.0")
                   (set! test-source-hash
                     (call-with-input-file file-name port-sha256))))
@@ -161,7 +177,8 @@ baz > 13.37
                 (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt"
                    (lambda ()
                      (display test-requires.txt)))
-                 (system* "tar" "czvf" file-name "foo-1.0.0/")
+                (parameterize ((current-output-port (%make-void-port "rw+")))
+                  (system* "tar" "czvf" file-name "foo-1.0.0/"))
                  (delete-file-recursively "foo-1.0.0")
                  (set! test-source-hash
                        (call-with-input-file file-name port-sha256))))
-- 
2.21.0

From 0c62b541a3e8925b5ca31fe55dbe7536cf95151f Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <[email-masked]>
Date: Thu, 28 Mar 2019 00:26:01 -0400
Subject: [PATCH 3/9] import: pypi: Improve parsing of requirement
 specifications.

The previous solution was fragile and could leave unwanted characters in a
requirement name, such as '[' or ']'.

Partially fixes issue #33047 (see:
https://debbugs.gnu.org/cgi/bugreport.cgi?bug=33047).

* guix/import/pypi.scm (use-modules): Export SPECIFICATION->REQUIREMENT-NAME
(%requirement-name-regexp): New variable.
(clean-requirement): Rename to...
(specification->requirement-name): this, which now uses
%requirement-name-regexp to select the requirement name from the requirement
specification.
(parse-requires.txt): Adapt.
---
 guix/import/pypi.scm | 43 ++++++++++++++++++++++++++++++++++---------
 tests/pypi.scm       | 12 ++++++++++++
 2 files changed, 46 insertions(+), 9 deletions(-)

diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index 91e987e9f1..efb5939c78 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -48,6 +48,7 @@
   #:use-module ((guix licenses) #:prefix license:)
   #:use-module (guix build-system python)
   #:export (parse-requires.txt
+            specification->requirement-name
             guix-package->pypi-name
             pypi-recursive-import
             pypi->guix-package
@@ -118,13 +119,37 @@ package definition."
     ((package-inputs ...)
      `((propagated-inputs (,'quasiquote ,package-inputs))))))
 
-(define (clean-requirement s)
-  ;; Given a requirement LINE, as can be found in a setuptools requires.txt
-  ;; file, remove everything other than the actual name of the required
-  ;; package, and return it.
-  (string-take s (or (string-index s (lambda (chr)
-                                       (member chr '(#\space #\> #\= #\<))))
-                     (string-length s))))
+(define %requirement-name-regexp
+  ;; Regexp to match the requirement name in a requirement specification.
+
+  ;; Some grammar, taken from PEP-0508 (see:
+  ;; https://www.python.org/dev/peps/pep-0508/).
+
+  ;; The unified rule can be expressed as:
+  ;; specification = wsp* ( url_req | name_req ) wsp*
+
+  ;; where url_req is:
+  ;; url_req = name wsp* extras? wsp* urlspec wsp+ quoted_marker?
+
+  ;; and where name_req is:
+  ;; name_req = name wsp* extras? wsp* versionspec? wsp* quoted_marker?
+
+  ;; Thus, we need only matching NAME, which is expressed as:
+  ;; identifer_end = letterOrDigit | (('-' | '_' | '.' )* letterOrDigit)
+  ;; identifier    = letterOrDigit identifier_end*
+  ;; name          = identifier
+  (let* ((letter-or-digit "[A-Za-z0-9]")
+         (identifier-end (string-append "(" letter-or-digit "|"
+                                        "[-_.]*" letter-or-digit ")"))
+         (identifier (string-append "^" letter-or-digit identifier-end "*"))
+         (name identifier))
+    (make-regexp name)))
+
+(define (specification->requirement-name spec)
+  "Given a specification SPEC, return the requirement name."
+  (match:substring
+   (or (regexp-exec %requirement-name-regexp spec)
+       (error (G_ "Could not extract requirement name in spec:") spec))))
 
 (define (parse-requires.txt requires.txt)
   "Given REQUIRES.TXT, a Setuptools requires.txt file, return a list of
@@ -158,7 +183,7 @@ requirement names."
                ((or (string-null? line) (comment? line))
                 (loop result))
                (else
-                (loop (cons (clean-requirement line)
+                (loop (cons (specification->requirement-name line)
                             result))))))))))
 
 (define (guess-requirements source-url wheel-url tarball)
@@ -200,7 +225,7 @@ cannot determine package dependencies"))
                                             (hash-ref (list-ref run_requires 0)
                                                        "requires")
                                             '())))
-                     (map clean-requirement requirements)))))
+                     (map specification->requirement-name requirements)))))
              (lambda ()
                (delete-file json-file)
                (rmdir dirname))))))
diff --git a/tests/pypi.scm b/tests/pypi.scm
index e4b7142311..82d6bba8dd 100644
--- a/tests/pypi.scm
+++ b/tests/pypi.scm
@@ -55,6 +55,14 @@
 (define test-source-hash
   "")
 
+(define test-specifications
+  '("Fizzy [foo, bar]"
+    "PickyThing<1.6,>1.9,!=1.9.6,<2.0a0,==2.4c1"
+    "SomethingWithMarker[foo]>1.0;python_version<\"2.7\""
+    "requests [security,tests] >= 2.8.1, == 2.8.* ; python_version < \"2.7\""
+    "pip @ https://github.com/pypa/pip/archive/1.3.1.zip#\
+sha1=da9234ee9982d4bbb3c72346a6de940a148ea686"))
+
 (define test-requires.txt "\
 bar
 baz > 13.37
@@ -108,6 +116,10 @@ pytest (>=2.5.0)
                     (uri (list "https://bitheap.org/cram/cram-0.7.tar.gz"
                                (pypi-uri "cram" "0.7"))))))))
 
+(test-equal "specification->requirement-name"
+  '("Fizzy" "PickyThing" "SomethingWithMarker" "requests" "pip")
+  (map specification->requirement-name test-specifications))
+
 (test-equal "parse-requires.txt, with sections"
   '("foo" "bar")
   (mock ((ice-9 ports) call-with-input-file
-- 
2.21.0

From 76e4a3150f8126e0b952c6129b6e1371afba80c0 Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <[email-masked]>
Date: Thu, 28 Mar 2019 00:26:01 -0400
Subject: [PATCH 4/9] import: pypi: Deduplicate requirements.

* guix/import/pypi.scm (parse-requires.txt): Remove potential duplicates.
---
 guix/import/pypi.scm | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index efb5939c78..a90be67bb0 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -178,7 +178,11 @@ requirement names."
           ;; (extra) requirements.  Non-optional requirements must appear
           ;; before any section is defined.
           (if (or (eof-object? line) (section-header? line))
-              (reverse result)
+              ;; Duplicates can occur, since the same requirement can be
+              ;; listed multiple times with different conditional markers, e.g.
+              ;; pytest >= 3 ; python_version >= "3.3"
+              ;; pytest < 3 ; python_version < "3.3"
+              (reverse (delete-duplicates result))
               (cond
                ((or (string-null? line) (comment? line))
                 (loop result))
-- 
2.21.0

From 73e27235cac1275ba7671fd2364325cf5788cb3c Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <[email-masked]>
Date: Thu, 28 Mar 2019 00:26:02 -0400
Subject: [PATCH 5/9] import: pypi: Support more types of archives.

This change enables the PyPI importer to look for requirements in a source
archive of a different type than "tar.gz" or "tar.bz2".

* guix/import/pypi.scm: (guess-requirements)[tarball-directory]: Rename to...
[archive-root-directory]: this. Use COMPRESSED-FILED? to determine if an
archive is supported or not.
[guess-requirements-from-source]: Adapt to use the new method, and use unzip
to extract ZIP archives.
(guess-requirements): Rename the TARBALL argument to ARCHIVE, to denote the
archive format is no longer bound specifically to the Tar format.
---
 guix/import/pypi.scm | 47 ++++++++++++++++++++++----------------------
 1 file changed, 24 insertions(+), 23 deletions(-)

diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index a90be67bb0..8e93653717 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -190,27 +190,24 @@ requirement names."
                 (loop (cons (specification->requirement-name line)
                             result))))))))))
 
-(define (guess-requirements source-url wheel-url tarball)
-  "Given SOURCE-URL, WHEEL-URL and a TARBALL of the package, return a list
-of the required packages specified in the requirements.txt file.  TARBALL will
+(define (guess-requirements source-url wheel-url archive)
+  "Given SOURCE-URL, WHEEL-URL and a ARCHIVE of the package, return a list
+of the required packages specified in the requirements.txt file.  ARCHIVE will
 be extracted in a temporary directory."
 
-  (define (tarball-directory url)
-    ;; Given the URL of the package's tarball, return the name of the directory
+  (define (archive-root-directory url)
+    ;; Given the URL of the package's archive, return the name of the directory
     ;; that will be created upon decompressing it. If the filetype is not
     ;; supported, return #f.
-    ;; TODO: Support more archive formats.
-    (let ((basename (substring url (+ 1 (string-rindex url #\/)))))
-      (cond
-       ((string-suffix? ".tar.gz" basename)
-        (string-drop-right basename 7))
-       ((string-suffix? ".tar.bz2" basename)
-        (string-drop-right basename 8))
-       (else
+    (if (compressed-file? url)
+        (let ((root-directory (file-sans-extension (basename url))))
+          (if (string=? "tar" (file-extension root-directory))
+              (file-sans-extension root-directory)
+              root-directory))
         (begin
-          (warning (G_ "Unsupported archive format: \
-cannot determine package dependencies"))
-          #f)))))
+          (warning (G_ "Unsupported archive format (~a): \
+cannot determine package dependencies") (file-extension url))
+          #f)))
 
   (define (read-wheel-metadata wheel-archive)
     ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's
@@ -246,16 +243,20 @@ cannot determine package dependencies"))
 
   (define (guess-requirements-from-source)
     ;; Return the package's requirements by guessing them from the source.
-    (let ((dirname (tarball-directory source-url)))
+    (let ((dirname (archive-root-directory source-url))
+          (extension (file-extension source-url)))
       (if (string? dirname)
           (call-with-temporary-directory
            (lambda (dir)
              (let* ((pypi-name (string-take dirname (string-rindex dirname #\-)))
                     (requires.txt (string-append dirname "/" pypi-name
                                                  ".egg-info" "/requires.txt"))
-                    (exit-code (parameterize ((current-error-port (%make-void-port "rw+"))
-                                              (current-output-port (%make-void-port "rw+")))
-                                 (system* "tar" "xf" tarball "-C" dir requires.txt))))
+                    (exit-code
+                     (parameterize ((current-error-port (%make-void-port "rw+"))
+                                    (current-output-port (%make-void-port "rw+")))
+                       (if (string=? "zip" extension)
+                           (system* "unzip" archive "-d" dir requires.txt)
+                           (system* "tar" "xf" archive "-C" dir requires.txt)))))
                (if (zero? exit-code)
                    (parse-requires.txt (string-append dir "/" requires.txt))
                    (begin
@@ -271,13 +272,13 @@ cannot determine package dependencies"))
   (or (guess-requirements-from-wheel)
       (guess-requirements-from-source)))
 
-(define (compute-inputs source-url wheel-url tarball)
-  "Given the SOURCE-URL of an already downloaded TARBALL, return a list of
+(define (compute-inputs source-url wheel-url archive)
+  "Given the SOURCE-URL of an already downloaded ARCHIVE, return a list of
 name/variable pairs describing the required inputs of this package.  Also
 return the unaltered list of upstream dependency names."
   (let ((dependencies
          (remove (cut string=? "argparse" <>)
-                 (guess-requirements source-url wheel-url tarball))))
+                 (guess-requirements source-url wheel-url archive))))
     (values (sort
              (map (lambda (input)
                     (let ((guix-name (python->package-name input)))
-- 
2.21.0

From fb0547ef225103c0f8355a7eccc41e0d028f6563 Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <[email-masked]>
Date: Thu, 28 Mar 2019 00:26:03 -0400
Subject: [PATCH 6/9] import: pypi: Parse wheel METADATA instead of
 metadata.json.

With newer Wheel releases, there is no more metadata.json file; the METADATA
file should be used instead (see: https://github.com/pypa/wheel/issues/195).

This change updates our PyPI importer so that it uses the later.

* guix/import/pypi.scm (define-module): Remove unnecessary modules and export
  the PARSE-WHEEL-METADATA method.
(parse-wheel-metadata): Add method.
(guess-requirements): Use it.
* tests/pypi.scm (test-metadata): Test it.
---
 guix/import/pypi.scm | 66 +++++++++++++++++++++++++++++---------------
 tests/pypi.scm       | 60 ++++++++++++++++++++++++++++++----------
 2 files changed, 89 insertions(+), 37 deletions(-)

diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index 8e93653717..c520213b6a 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -21,9 +21,7 @@
 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
 
 (define-module (guix import pypi)
-  #:use-module (ice-9 binary-ports)
   #:use-module (ice-9 match)
-  #:use-module (ice-9 pretty-print)
   #:use-module (ice-9 regex)
   #:use-module (ice-9 receive)
   #:use-module ((ice-9 rdelim) #:select (read-line))
@@ -31,9 +29,6 @@
   #:use-module (srfi srfi-26)
   #:use-module (srfi srfi-34)
   #:use-module (srfi srfi-35)
-  #:use-module (rnrs bytevectors)
-  #:use-module (json)
-  #:use-module (web uri)
   #:use-module (guix ui)
   #:use-module (guix utils)
   #:use-module ((guix build utils)
@@ -48,6 +43,7 @@
   #:use-module ((guix licenses) #:prefix license:)
   #:use-module (guix build-system python)
   #:export (parse-requires.txt
+            parse-wheel-metadata
             specification->requirement-name
             guix-package->pypi-name
             pypi-recursive-import
@@ -190,6 +186,37 @@ requirement names."
                 (loop (cons (specification->requirement-name line)
                             result))))))))))
 
+(define (parse-wheel-metadata metadata)
+  "Given METADATA, a Wheel metadata file, return a list of requirement names."
+  ;; METADATA is a RFC-2822-like, header based file.
+
+  (define (requires-dist-header? line)
+    ;; Return #t if the given LINE is a Requires-Dist header.
+    (regexp-match? (string-match "^Requires-Dist: " line)))
+
+  (define (requires-dist-value line)
+    (string-drop line (string-length "Requires-Dist: ")))
+
+  (define (extra? line)
+    ;; Return #t if the given LINE is an "extra" requirement.
+    (regexp-match? (string-match "extra == " line)))
+
+  (call-with-input-file metadata
+    (lambda (port)
+      (let loop ((requirements '()))
+        (let ((line (read-line port)))
+          ;; Stop at the first 'Provides-Extra' section: the non-optional
+          ;; requirements appear before the optional ones.
+          (if (eof-object? line)
+              (reverse (delete-duplicates requirements))
+              (cond
+               ((and (requires-dist-header? line) (not (extra? line)))
+                (loop (cons (specification->requirement-name
+                             (requires-dist-value line))
+                            requirements)))
+               (else
+                (loop requirements)))))))))
+
 (define (guess-requirements source-url wheel-url archive)
   "Given SOURCE-URL, WHEEL-URL and a ARCHIVE of the package, return a list
 of the required packages specified in the requirements.txt file.  ARCHIVE will
@@ -211,25 +238,18 @@ cannot determine package dependencies") (file-extension url))
 
   (define (read-wheel-metadata wheel-archive)
     ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's
-    ;; requirements.
+    ;; requirements, or #f if the metadata file contained therein couldn't be
+    ;; extracted.
     (let* ((dirname (wheel-url->extracted-directory wheel-url))
-           (json-file (string-append dirname "/metadata.json")))
-      (and (zero? (system* "unzip" "-q" wheel-archive json-file))
-           (dynamic-wind
-             (const #t)
-             (lambda ()
-               (call-with-input-file json-file
-                 (lambda (port)
-                   (let* ((metadata (json->scm port))
-                          (run_requires (hash-ref metadata "run_requires"))
-                          (requirements (if run_requires
-                                            (hash-ref (list-ref run_requires 0)
-                                                       "requires")
-                                            '())))
-                     (map specification->requirement-name requirements)))))
-             (lambda ()
-               (delete-file json-file)
-               (rmdir dirname))))))
+           (metadata (string-append dirname "/METADATA")))
+      (call-with-temporary-directory
+       (lambda (dir)
+         (if (zero? (system* "unzip" "-q" wheel-archive "-d" dir metadata))
+             (parse-wheel-metadata (string-append dir "/" metadata))
+             (begin
+               (warning
+                (G_ "Failed to extract file: ~a from wheel.~%") metadata)
+               #f))))))
 
   (define (guess-requirements-from-wheel)
     ;; Return the package's requirements using the wheel, or #f if an error
diff --git a/tests/pypi.scm b/tests/pypi.scm
index 82d6bba8dd..ca8cb5f6de 100644
--- a/tests/pypi.scm
+++ b/tests/pypi.scm
@@ -21,6 +21,7 @@
   #:use-module (guix import pypi)
   #:use-module (guix base32)
   #:use-module (gcrypt hash)
+  #:use-module (guix memoization)
   #:use-module (guix tests)
   #:use-module (guix build-system python)
   #:use-module ((guix build utils) #:select (delete-file-recursively which mkdir-p))
@@ -77,17 +78,33 @@ bar != 2
 pytest (>=2.5.0)
 ")
 
-(define test-metadata
-  "{
-  \"run_requires\": [
-    {
-      \"requires\": [
-        \"bar\",
-        \"baz (>13.37)\"
-      ]
-    }
-  ]
-}")
+(define test-metadata "\
+Classifier: Programming Language :: Python :: 3.7
+Requires-Dist: baz ~= 3
+Requires-Dist: bar != 2
+Provides-Extra: test
+pytest (>=2.5.0)
+")
+
+(define test-metadata-with-extras "
+Classifier: Programming Language :: Python :: 3.7
+Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
+Requires-Dist: wrapt (<2,>=1)
+Requires-Dist: bar
+
+Provides-Extra: dev
+Requires-Dist: tox ; extra == 'dev'
+Requires-Dist: bumpversion (<1) ; extra == 'dev'
+")
+
+;;; Provides-Extra can appear before Requires-Dist.
+(define test-metadata-with-extras-jedi "\
+Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
+Provides-Extra: testing
+Requires-Dist: parso (>=0.3.0)
+Provides-Extra: testing
+Requires-Dist: pytest (>=3.1.0); extra == 'testing'
+")
 
 (test-begin "pypi")
 
@@ -126,6 +143,18 @@ pytest (>=2.5.0)
          call-with-input-string)
         (parse-requires.txt test-requires-with-sections)))
 
+(test-equal "parse-wheel-metadata, with extras"
+  '("wrapt" "bar")
+  (mock ((ice-9 ports) call-with-input-file
+         call-with-input-string)
+        (parse-wheel-metadata test-metadata-with-extras)))
+
+(test-equal "parse-wheel-metadata, with extras - Jedi"
+  '("parso")
+  (mock ((ice-9 ports) call-with-input-file
+         call-with-input-string)
+        (parse-wheel-metadata test-metadata-with-extras-jedi)))
+
 (test-assert "pypi->guix-package"
   ;; Replace network resources with sample data.
     (mock ((guix import utils) url-fetch
@@ -188,7 +217,7 @@ pytest (>=2.5.0)
                 (mkdir-p "foo-1.0.0/foo.egg-info/")
                 (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt"
                    (lambda ()
-                     (display test-requires.txt)))
+                     (display "wrong data to make sure we're testing wheels ")))
                 (parameterize ((current-output-port (%make-void-port "rw+")))
                   (system* "tar" "czvf" file-name "foo-1.0.0/"))
                  (delete-file-recursively "foo-1.0.0")
@@ -197,13 +226,13 @@ pytest (>=2.5.0)
              ("https://example.com/foo-1.0.0-py2.py3-none-any.whl"
                (begin
                  (mkdir "foo-1.0.0.dist-info")
-                 (with-output-to-file "foo-1.0.0.dist-info/metadata.json"
+                 (with-output-to-file "foo-1.0.0.dist-info/METADATA"
                    (lambda ()
                      (display test-metadata)))
                  (let ((zip-file (string-append file-name ".zip")))
                    ;; zip always adds a "zip" extension to the file it creates,
                    ;; so we need to rename it.
-                   (system* "zip" zip-file "foo-1.0.0.dist-info/metadata.json")
+                   (system* "zip" zip-file "foo-1.0.0.dist-info/METADATA")
                    (rename-file zip-file file-name))
                  (delete-file-recursively "foo-1.0.0.dist-info")))
              (_ (error "Unexpected URL: " url)))))
@@ -215,6 +244,9 @@ pytest (>=2.5.0)
                             (string-length test-json)))
                    ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f)
                    (_ (error "Unexpected URL: " url)))))
+              ;; Not clearing the memoization cache here would mean returning the value
+              ;; computed in the previous test.
+              (invalidate-memoization! pypi->guix-package)
               (match (pypi->guix-package "foo")
                 (('package
                    ('name "python-foo")
-- 
2.21.0

From 37e499d5d5d5f690aa0a065c730e13f6a31dd30d Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <[email-masked]>
Date: Thu, 28 Mar 2019 23:12:26 -0400
Subject: [PATCH 7/9] import: pypi: Include optional test inputs as
 native-inputs.

* guix/import/pypi.scm (maybe-inputs): Add INPUT-TYPE argument, and use it.
(test-section?): New predicate.
(parse-requires.txt): Collect the optional test inputs, and return them as the
second element of the returned list.
(parse-wheel-metadata): Likewise.
(guess-requirements): Adapt, and hide unzip output.
(make-pypi-sexp): Likewise, and include the test inputs requirements as native
inputs in the returned package expression.

* tests/pypi.scm (test-requires.txt): Include a test section in the
test-requires.txt data.
(test-requires.txt-beaker): New variable.
("parse-requires.txt"): Adapt.
("parse-requires.txt - Beaker"): New test.
("parse-wheel-metadata, with extras"): Adapt.
("parse-wheel-metadata, with extras - Jedi"): Adapt.
("pypi->guix-package, no wheel"): Re-indent, and add the expected
native-inputs.
("pypi->guix-package, wheels"): Likewise.
("pypi->guix-package, no usable requirement file."): New test.
---
 guix/import/pypi.scm | 195 ++++++++++++++++++++++++++++---------------
 tests/pypi.scm       | 123 ++++++++++++++++++++-------
 2 files changed, 222 insertions(+), 96 deletions(-)

diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index c520213b6a..099768f0c8 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -4,6 +4,7 @@
 ;;; Copyright © 2015, 2016, 2017 Ludovic Courtès <[email-masked]>
 ;;; Copyright © 2017 Mathieu Othacehe <[email-masked]>
 ;;; Copyright © 2018 Ricardo Wurmus <[email-masked]>
+;;; Copyright © 2019 Maxim Cournoyer <[email-masked]>
 ;;;
 ;;; This file is part of GNU Guix.
 ;;;
@@ -26,6 +27,7 @@
   #:use-module (ice-9 receive)
   #:use-module ((ice-9 rdelim) #:select (read-line))
   #:use-module (srfi srfi-1)
+  #:use-module (srfi srfi-11)
   #:use-module (srfi srfi-26)
   #:use-module (srfi srfi-34)
   #:use-module (srfi srfi-35)
@@ -106,14 +108,15 @@ package on PyPI."
     ((name version _ ...)
      (string-append name "-" version ".dist-info"))))
 
-(define (maybe-inputs package-inputs)
+(define (maybe-inputs package-inputs input-type)
   "Given a list of PACKAGE-INPUTS, tries to generate the 'inputs' field of a
-package definition."
+package definition.  INPUT-TYPE, a symbol, is used to populate the name of
+the input field."
   (match package-inputs
     (()
      '())
     ((package-inputs ...)
-     `((propagated-inputs (,'quasiquote ,package-inputs))))))
+     `((,input-type (,'quasiquote ,package-inputs))))))
 
 (define %requirement-name-regexp
   ;; Regexp to match the requirement name in a requirement specification.
@@ -147,11 +150,21 @@ package definition."
    (or (regexp-exec %requirement-name-regexp spec)
        (error (G_ "Could not extract requirement name in spec:") spec))))
 
+(define (test-section? name)
+  "Return #t if the section name contains 'test' or 'dev'."
+  (any (cut string-contains-ci name <>)
+       '("test" "dev")))
+
 (define (parse-requires.txt requires.txt)
-  "Given REQUIRES.TXT, a Setuptools requires.txt file, return a list of
-requirement names."
-  ;; This is a very incomplete parser, which job is to select the non-optional
-  ;; dependencies and strip them out of any version information.
+  "Given REQUIRES.TXT, a Setuptools requires.txt file, return a pair of requirements.
+
+The first element of the pair contains the required dependencies while the
+second the optional test dependencies.  Note that currently, optional,
+non-test dependencies are omitted since these can be difficult or expensive to
+satisfy."
+
+  ;; This is a very incomplete parser, which job is to read in the requirement
+  ;; specification lines, and strip them out of any version information.
   ;; Alternatively, we could implement a PEG parser with the (ice-9 peg)
   ;; library and the requirements grammar defined by PEP-0508
   ;; (https://www.python.org/dev/peps/pep-0508/).
@@ -168,57 +181,89 @@ requirement names."
 
   (call-with-input-file requires.txt
     (lambda (port)
-      (let loop ((result '()))
+      (let loop ((required-deps '())
+                 (test-deps '())
+                 (inside-test-section? #f)
+                 (optional? #f))
         (let ((line (read-line port)))
-          ;; Stop when a section is encountered, as sections contains optional
-          ;; (extra) requirements.  Non-optional requirements must appear
-          ;; before any section is defined.
-          (if (or (eof-object? line) (section-header? line))
+          (if (eof-object? line)
               ;; Duplicates can occur, since the same requirement can be
               ;; listed multiple times with different conditional markers, e.g.
               ;; pytest >= 3 ; python_version >= "3.3"
               ;; pytest < 3 ; python_version < "3.3"
-              (reverse (delete-duplicates result))
+              (map (compose reverse delete-duplicates)
+                   (list required-deps test-deps))
               (cond
                ((or (string-null? line) (comment? line))
-                (loop result))
-               (else
+                (loop required-deps test-deps inside-test-section? optional?))
+               ((section-header? line)
+                ;; Encountering a section means that all the requirements
+                ;; listed below are optional. Since we want to pick only the
+                ;; test dependencies from the optional dependencies, we must
+                ;; track those separately.
+                (loop required-deps test-deps (test-section? line) #t))
+               (inside-test-section?
+                (loop required-deps
+                      (cons (specification->requirement-name line)
+                            test-deps)
+                      inside-test-section? optional?))
+               ((not optional?)
                 (loop (cons (specification->requirement-name line)
-                            result))))))))))
+                            required-deps)
+                      test-deps inside-test-section? optional?))
+               (optional?
+                ;; Skip optional items.
+                (loop required-deps test-deps inside-test-section? optional?))
+               (else
+                (warning (G_ "parse-requires.txt reached an unexpected \
+condition on line ~a~%") line)))))))))
 
 (define (parse-wheel-metadata metadata)
-  "Given METADATA, a Wheel metadata file, return a list of requirement names."
+  "Given METADATA, a Wheel metadata file, return a pair of requirements.
+
+The first element of the pair contains the required dependencies while the second the optional
+test dependencies.  Note that currently, optional, non-test dependencies are
+omitted since these can be difficult or expensive to satisfy."
   ;; METADATA is a RFC-2822-like, header based file.
 
   (define (requires-dist-header? line)
     ;; Return #t if the given LINE is a Requires-Dist header.
-    (regexp-match? (string-match "^Requires-Dist: " line)))
+    (string-match "^Requires-Dist: " line))
 
   (define (requires-dist-value line)
     (string-drop line (string-length "Requires-Dist: ")))
 
   (define (extra? line)
     ;; Return #t if the given LINE is an "extra" requirement.
-    (regexp-match? (string-match "extra == " line)))
+    (string-match "extra == '(.*)'" line))
+
+  (define (test-requirement? line)
+    (let ((extra-label (match:substring (extra? line) 1)))
+      (and extra-label (test-section? extra-label))))
 
   (call-with-input-file metadata
     (lambda (port)
-      (let loop ((requirements '()))
+      (let loop ((required-deps '())
+                 (test-deps '()))
         (let ((line (read-line port)))
-          ;; Stop at the first 'Provides-Extra' section: the non-optional
-          ;; requirements appear before the optional ones.
           (if (eof-object? line)
-              (reverse (delete-duplicates requirements))
+              (map (compose reverse delete-duplicates)
+                   (list required-deps test-deps))
               (cond
                ((and (requires-dist-header? line) (not (extra? line)))
                 (loop (cons (specification->requirement-name
                              (requires-dist-value line))
-                            requirements)))
+                            required-deps)
+                      test-deps))
+               ((and (requires-dist-header? line) (test-requirement? line))
+                (loop required-deps
+                      (cons (specification->requirement-name (requires-dist-value line))
+                            test-deps)))
                (else
-                (loop requirements)))))))))
+                (loop required-deps test-deps))))))))) ;skip line
 
 (define (guess-requirements source-url wheel-url archive)
-  "Given SOURCE-URL, WHEEL-URL and a ARCHIVE of the package, return a list
+  "Given SOURCE-URL, WHEEL-URL and an ARCHIVE of the package, return a list
 of the required packages specified in the requirements.txt file.  ARCHIVE will
 be extracted in a temporary directory."
 
@@ -244,7 +289,10 @@ cannot determine package dependencies") (file-extension url))
            (metadata (string-append dirname "/METADATA")))
       (call-with-temporary-directory
        (lambda (dir)
-         (if (zero? (system* "unzip" "-q" wheel-archive "-d" dir metadata))
+         (if (zero?
+              (parameterize ((current-error-port (%make-void-port "rw+"))
+                             (current-output-port (%make-void-port "rw+")))
+                (system* "unzip" wheel-archive "-d" dir metadata)))
              (parse-wheel-metadata (string-append dir "/" metadata))
              (begin
                (warning
@@ -283,32 +331,41 @@ cannot determine package dependencies") (file-extension url))
                      (warning
                       (G_ "Failed to extract file: ~a from source.~%")
                       requires.txt)
-                     '())))))
-          '())))
+                     (list '() '()))))))
+          (list '() '()))))
 
   ;; First, try to compute the requirements using the wheel, else, fallback to
   ;; reading the "requires.txt" from the egg-info directory from the source
-  ;; tarball.
+  ;; archive.
   (or (guess-requirements-from-wheel)
       (guess-requirements-from-source)))
 
 (define (compute-inputs source-url wheel-url archive)
-  "Given the SOURCE-URL of an already downloaded ARCHIVE, return a list of
-name/variable pairs describing the required inputs of this package.  Also
+  "Given the SOURCE-URL and WHEEL-URL of an already downloaded ARCHIVE, return
+a pair of lists, each consisting of a list of name/variable pairs, for the
+propagated inputs and the native inputs, respectively.  Also
 return the unaltered list of upstream dependency names."
-  (let ((dependencies
-         (remove (cut string=? "argparse" <>)
-                 (guess-requirements source-url wheel-url archive))))
-    (values (sort
-             (map (lambda (input)
-                    (let ((guix-name (python->package-name input)))
-                      (list guix-name (list 'unquote (string->symbol guix-name)))))
-                  dependencies)
-             (lambda args
-               (match args
-                 (((a _ ...) (b _ ...))
-                  (string-ci<? a b)))))
-            dependencies)))
+
+  (define (strip-argparse deps)
+    (remove (cut string=? "argparse" <>) deps))
+
+  (define (requirement->package-name/sort deps)
+    (sort
+     (map (lambda (input)
+            (let ((guix-name (python->package-name input)))
+              (list guix-name (list 'unquote (string->symbol guix-name)))))
+          deps)
+     (lambda args
+       (match args
+         (((a _ ...) (b _ ...))
+          (string-ci<? a b))))))
+
+  (define process-requirements
+    (compose requirement->package-name/sort strip-argparse))
+
+  (let ((dependencies (guess-requirements source-url wheel-url archive)))
+    (values (map process-requirements dependencies)
+            (concatenate dependencies))))
 
 (define (make-pypi-sexp name version source-url wheel-url home-page synopsis
                         description license)
@@ -317,29 +374,33 @@ VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE."
   (call-with-temporary-output-file
    (lambda (temp port)
      (and (url-fetch source-url temp)
-          (receive (input-package-names upstream-dependency-names)
+          (receive (guix-dependencies upstream-dependencies)
               (compute-inputs source-url wheel-url temp)
-            (values
-             `(package
-                (name ,(python->package-name name))
-                (version ,version)
-                (source (origin
-                          (method url-fetch)
-
-                          ;; Sometimes 'pypi-uri' doesn't quite work due to mixed
-                          ;; cases in NAME, for instance, as is the case with
-                          ;; "uwsgi".  In that case, fall back to a full URL.
-                          (uri (pypi-uri ,(string-downcase name) version))
-                          (sha256
-                           (base32
-                            ,(guix-hash-url temp)))))
-                (build-system python-build-system)
-                ,@(maybe-inputs input-package-names)
-                (home-page ,home-page)
-                (synopsis ,synopsis)
-                (description ,description)
-                (license ,(license->symbol license)))
-             upstream-dependency-names))))))
+            (match guix-dependencies
+              ((required-inputs test-inputs)
+               (values
+                `(package
+                   (name ,(python->package-name name))
+                   (version ,version)
+                   (source (origin
+                             (method url-fetch)
+                             ;; Sometimes 'pypi-uri' doesn't quite work due to mixed
+                             ;; cases in NAME, for instance, as is the case with
+                             ;; "uwsgi".  In that case, fall back to a full URL.
+                             (uri (pypi-uri ,(string-downcase name) version))
+                             (sha256
+                              (base32
+                               ,(guix-hash-url temp)))))
+                   (build-system python-build-system)
+                   ,@(maybe-inputs required-inputs 'propagated-inputs)
+                   ,@(maybe-inputs test-inputs 'native-inputs)
+                   (home-page ,home-page)
+                   (synopsis ,synopsis)
+                   (description ,description)
+                   (license ,(license->symbol license)))
+                ;; Flatten the nested lists and return the upstream
+                ;; dependencies.
+                upstream-dependencies))))))))
 
 (define pypi->guix-package
   (memoize
diff --git a/tests/pypi.scm b/tests/pypi.scm
index ca8cb5f6de..aa08e2cb54 100644
--- a/tests/pypi.scm
+++ b/tests/pypi.scm
@@ -1,6 +1,7 @@
 ;;; GNU Guix --- Functional package management for GNU
 ;;; Copyright © 2014 David Thompson <[email-masked]>
 ;;; Copyright © 2016 Ricardo Wurmus <[email-masked]>
+;;; Copyright © 2019 Maxim Cournoyer <[email-masked]>
 ;;;
 ;;; This file is part of GNU Guix.
 ;;;
@@ -65,11 +66,6 @@
 sha1=da9234ee9982d4bbb3c72346a6de940a148ea686"))
 
 (define test-requires.txt "\
-bar
-baz > 13.37
-")
-
-(define test-requires-with-sections "\
 # A comment
 foo ~= 3
 bar != 2
@@ -78,12 +74,25 @@ bar != 2
 pytest (>=2.5.0)
 ")
 
+;; Beaker contains only optional dependencies.
+(define test-requires.txt-beaker "\
+[crypto]
+pycryptopp>=0.5.12
+
+[cryptography]
+cryptography
+
+[testsuite]
+Mock
+coverage
+")
+
 (define test-metadata "\
 Classifier: Programming Language :: Python :: 3.7
 Requires-Dist: baz ~= 3
 Requires-Dist: bar != 2
 Provides-Extra: test
-pytest (>=2.5.0)
+Requires-Dist: pytest (>=2.5.0) ; extra == 'test'
 ")
 
 (define test-metadata-with-extras "
@@ -137,25 +146,31 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing'
   '("Fizzy" "PickyThing" "SomethingWithMarker" "requests" "pip")
   (map specification->requirement-name test-specifications))
 
-(test-equal "parse-requires.txt, with sections"
-  '("foo" "bar")
+(test-equal "parse-requires.txt"
+  (list '("foo" "bar") '("pytest"))
   (mock ((ice-9 ports) call-with-input-file
          call-with-input-string)
-        (parse-requires.txt test-requires-with-sections)))
+        (parse-requires.txt test-requires.txt)))
+
+(test-equal "parse-requires.txt - Beaker"
+  (list '() '("Mock" "coverage"))
+  (mock ((ice-9 ports) call-with-input-file
+         call-with-input-string)
+        (parse-requires.txt test-requires.txt-beaker)))
 
 (test-equal "parse-wheel-metadata, with extras"
-  '("wrapt" "bar")
+  (list '("wrapt" "bar") '("tox" "bumpversion"))
   (mock ((ice-9 ports) call-with-input-file
          call-with-input-string)
         (parse-wheel-metadata test-metadata-with-extras)))
 
 (test-equal "parse-wheel-metadata, with extras - Jedi"
-  '("parso")
+  (list '("parso") '("pytest"))
   (mock ((ice-9 ports) call-with-input-file
          call-with-input-string)
         (parse-wheel-metadata test-metadata-with-extras-jedi)))
 
-(test-assert "pypi->guix-package"
+(test-assert "pypi->guix-package, no wheel"
   ;; Replace network resources with sample data.
     (mock ((guix import utils) url-fetch
            (lambda (url file-name)
@@ -195,7 +210,10 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing'
                      ('propagated-inputs
                       ('quasiquote
                        (("python-bar" ('unquote 'python-bar))
-                        ("python-baz" ('unquote 'python-baz)))))
+                        ("python-foo" ('unquote 'python-foo)))))
+                     ('native-inputs
+                      ('quasiquote
+                       (("python-pytest" ('unquote 'python-pytest)))))
                      ('home-page "http://example.com")
                      ('synopsis "summary")
                      ('description "summary")
@@ -216,25 +234,25 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing'
               (begin
                 (mkdir-p "foo-1.0.0/foo.egg-info/")
                 (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt"
-                   (lambda ()
-                     (display "wrong data to make sure we're testing wheels ")))
+                  (lambda ()
+                    (display "wrong data to make sure we're testing wheels ")))
                 (parameterize ((current-output-port (%make-void-port "rw+")))
                   (system* "tar" "czvf" file-name "foo-1.0.0/"))
-                 (delete-file-recursively "foo-1.0.0")
-                 (set! test-source-hash
-                       (call-with-input-file file-name port-sha256))))
+                (delete-file-recursively "foo-1.0.0")
+                (set! test-source-hash
+                  (call-with-input-file file-name port-sha256))))
              ("https://example.com/foo-1.0.0-py2.py3-none-any.whl"
-               (begin
-                 (mkdir "foo-1.0.0.dist-info")
-                 (with-output-to-file "foo-1.0.0.dist-info/METADATA"
-                   (lambda ()
-                     (display test-metadata)))
-                 (let ((zip-file (string-append file-name ".zip")))
-                   ;; zip always adds a "zip" extension to the file it creates,
-                   ;; so we need to rename it.
-                   (system* "zip" zip-file "foo-1.0.0.dist-info/METADATA")
-                   (rename-file zip-file file-name))
-                 (delete-file-recursively "foo-1.0.0.dist-info")))
+              (begin
+                (mkdir "foo-1.0.0.dist-info")
+                (with-output-to-file "foo-1.0.0.dist-info/METADATA"
+                  (lambda ()
+                    (display test-metadata)))
+                (let ((zip-file (string-append file-name ".zip")))
+                  ;; zip always adds a "zip" extension to the file it creates,
+                  ;; so we need to rename it.
+                  (system* "zip" "-q" zip-file "foo-1.0.0.dist-info/METADATA")
+                  (rename-file zip-file file-name))
+                (delete-file-recursively "foo-1.0.0.dist-info")))
              (_ (error "Unexpected URL: " url)))))
         (mock ((guix http-client) http-fetch
                (lambda (url . rest)
@@ -262,6 +280,9 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing'
                     ('quasiquote
                      (("python-bar" ('unquote 'python-bar))
                       ("python-baz" ('unquote 'python-baz)))))
+                   ('native-inputs
+                    ('quasiquote
+                     (("python-pytest" ('unquote 'python-pytest)))))
                    ('home-page "http://example.com")
                    ('synopsis "summary")
                    ('description "summary")
@@ -272,4 +293,48 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing'
                 (x
                  (pk 'fail x #f))))))
 
+(test-assert "pypi->guix-package, no usable requirement file."
+  ;; Replace network resources with sample data.
+  (mock ((guix import utils) url-fetch
+         (lambda (url file-name)
+           (match url
+             ("https://example.com/foo-1.0.0.tar.gz"
+              (set! test-source-hash
+                (call-with-input-file file-name port-sha256))
+              #t)
+             ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #t)
+             (_ (error "Unexpected URL: " url)))))
+        (mock ((guix http-client) http-fetch
+               (lambda (url . rest)
+                 (match url
+                   ("https://pypi.org/pypi/foo/json"
+                    (values (open-input-string test-json)
+                            (string-length test-json)))
+                   ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f)
+                   (_ (error "Unexpected URL: " url)))))
+              ;; Not clearing the memoization cache here would mean returning the value
+              ;; computed in the previous test.
+              (invalidate-memoization! pypi->guix-package)
+              (match (pypi->guix-package "foo")
+                (('package
+                   ('name "python-foo")
+                   ('version "1.0.0")
+                   ('source ('origin
+                              ('method 'url-fetch)
+                              ('uri ('pypi-uri "foo" 'version))
+                              ('sha256
+                               ('base32
+                                (? string? hash)))))
+                   ('build-system 'python-build-system)
+                   ('home-page "http://example.com")
+                   ('synopsis "summary")
+                   ('description "summary")
+                   ('license 'license:lgpl2.0))
+                 (string=? (bytevector->nix-base32-string
+                            test-source-hash)
+                           hash))
+                (x
+                 (pk 'fail x #f)))
+              )))
+
 (test-end "pypi")
-- 
2.21.0

From cfde6e09f8f8c692fe252d76ed27e8c50a9e5377 Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <[email-masked]>
Date: Sat, 30 Mar 2019 23:13:26 -0400
Subject: [PATCH 8/9] import: pypi: Scan source archive to find requires.txt
 file.

* guix/import/pypi.scm (use-modules): Use invoke from (guix build utils).
(guess-requirements)[archive-root-directory]: Remove procedure.
[guess-requirements-from-wheel]: Re-ident.
[guess-requirements-from-source]: Search for the requires.txt file in the
archive instead of using a static, expected location.
* tests/pypi.scm ("pypi->guix-package, no wheel"): Mock the requires.txt at a
non-standard location to test the new feature.
("pypi->guix-package, no usable requirement file."): Adapt.
---
 guix/import/pypi.scm | 65 ++++++++++++++++++--------------------------
 tests/pypi.scm       | 17 +++++++-----
 2 files changed, 37 insertions(+), 45 deletions(-)

diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index 099768f0c8..a2ce14b192 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -36,7 +36,8 @@
   #:use-module ((guix build utils)
                 #:select ((package-name->name+version
                            . hyphen-package-name->name+version)
-                          find-files))
+                          find-files
+                          invoke))
   #:use-module (guix import utils)
   #:use-module ((guix download) #:prefix download:)
   #:use-module (guix import json)
@@ -267,19 +268,6 @@ omitted since these can be difficult or expensive to satisfy."
 of the required packages specified in the requirements.txt file.  ARCHIVE will
 be extracted in a temporary directory."
 
-  (define (archive-root-directory url)
-    ;; Given the URL of the package's archive, return the name of the directory
-    ;; that will be created upon decompressing it. If the filetype is not
-    ;; supported, return #f.
-    (if (compressed-file? url)
-        (let ((root-directory (file-sans-extension (basename url))))
-          (if (string=? "tar" (file-extension root-directory))
-              (file-sans-extension root-directory)
-              root-directory))
-        (begin
-          (warning (G_ "Unsupported archive format (~a): \
-cannot determine package dependencies") (file-extension url))
-          #f)))
 
   (define (read-wheel-metadata wheel-archive)
     ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's
@@ -305,33 +293,34 @@ cannot determine package dependencies") (file-extension url))
     (call-with-temporary-output-file
      (lambda (temp port)
        (if wheel-url
-         (and (url-fetch wheel-url temp)
-              (read-wheel-metadata temp))
-         #f))))
+           (and (url-fetch wheel-url temp)
+                (read-wheel-metadata temp))
+           #f))))
 
   (define (guess-requirements-from-source)
     ;; Return the package's requirements by guessing them from the source.
-    (let ((dirname (archive-root-directory source-url))
-          (extension (file-extension source-url)))
-      (if (string? dirname)
-          (call-with-temporary-directory
-           (lambda (dir)
-             (let* ((pypi-name (string-take dirname (string-rindex dirname #\-)))
-                    (requires.txt (string-append dirname "/" pypi-name
-                                                 ".egg-info" "/requires.txt"))
-                    (exit-code
-                     (parameterize ((current-error-port (%make-void-port "rw+"))
-                                    (current-output-port (%make-void-port "rw+")))
-                       (if (string=? "zip" extension)
-                           (system* "unzip" archive "-d" dir requires.txt)
-                           (system* "tar" "xf" archive "-C" dir requires.txt)))))
-               (if (zero? exit-code)
-                   (parse-requires.txt (string-append dir "/" requires.txt))
-                   (begin
-                     (warning
-                      (G_ "Failed to extract file: ~a from source.~%")
-                      requires.txt)
-                     (list '() '()))))))
+    (if (compressed-file? source-url)
+        (call-with-temporary-directory
+         (lambda (dir)
+           (parameterize ((current-error-port (%make-void-port "rw+"))
+                          (current-output-port (%make-void-port "rw+")))
+             (if (string=? "zip" (file-extension source-url))
+                 (invoke "unzip" archive "-d" dir)
+                 (invoke "tar" "xf" archive "-C" dir)))
+           (let ((requires.txt-files
+                  (find-files dir (lambda (abs-file-name _)
+		                    (string-match "\\.egg-info/requires.txt$"
+                                                  abs-file-name)))))
+             (if (> (length requires.txt-files) 0)
+                 (begin
+                   (parse-requires.txt (first requires.txt-files)))
+                 (begin (warning (G_ "Cannot guess requirements from source archive:\
+ no requires.txt file found.~%"))
+                        (list '() '()))))))
+        (begin
+          (warning (G_ "Unsupported archive format; \
+cannot determine package dependencies from source archive: ~a~%")
+                   (basename source-url))
           (list '() '()))))
 
   ;; First, try to compute the requirements using the wheel, else, fallback to
diff --git a/tests/pypi.scm b/tests/pypi.scm
index aa08e2cb54..ad188df16c 100644
--- a/tests/pypi.scm
+++ b/tests/pypi.scm
@@ -177,8 +177,9 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing'
              (match url
                ("https://example.com/foo-1.0.0.tar.gz"
                 (begin
-                  (mkdir-p "foo-1.0.0/foo.egg-info/")
-                  (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt"
+                  ;; Unusual requires.txt location should still be found.
+                  (mkdir-p "foo-1.0.0/src/bizarre.egg-info")
+                  (with-output-to-file "foo-1.0.0/src/bizarre.egg-info/requires.txt"
                     (lambda ()
                       (display test-requires.txt)))
                   (parameterize ((current-output-port (%make-void-port "rw+")))
@@ -299,10 +300,13 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing'
          (lambda (url file-name)
            (match url
              ("https://example.com/foo-1.0.0.tar.gz"
+              (mkdir-p "foo-1.0.0/foo.egg-info/")
+              (parameterize ((current-output-port (%make-void-port "rw+")))
+                (system* "tar" "czvf" file-name "foo-1.0.0/"))
+              (delete-file-recursively "foo-1.0.0")
               (set! test-source-hash
-                (call-with-input-file file-name port-sha256))
-              #t)
-             ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #t)
+                (call-with-input-file file-name port-sha256)))
+             ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f)
              (_ (error "Unexpected URL: " url)))))
         (mock ((guix http-client) http-fetch
                (lambda (url . rest)
@@ -334,7 +338,6 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing'
                             test-source-hash)
                            hash))
                 (x
-                 (pk 'fail x #f)))
-              )))
+                 (pk 'fail x #f))))))
 
 (test-end "pypi")
-- 
2.21.0

From 1290f9d1f0d594fdd4723d76b94116be25da9dd5 Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <[email-masked]>
Date: Sat, 30 Mar 2019 20:27:35 -0400
Subject: [PATCH 9/9] import: pypi: Preserve package name case when forming
 pypi-uri.

Fixes issue: #33046.

* guix/build-system/python.scm (pypi-uri): Update the host URI to
"files.pythonhosted.org".
* guix/import/pypi.scm (make-pypi-sexp): Preserve the package name case when
the source URL calls for it.
---
 guix/build-system/python.scm |  2 +-
 guix/import/pypi.scm         | 23 ++++++++++++++---------
 2 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/guix/build-system/python.scm b/guix/build-system/python.scm
index b753940bad..e39c06528e 100644
--- a/guix/build-system/python.scm
+++ b/guix/build-system/python.scm
@@ -50,7 +50,7 @@
   "Return a URI string for the Python package hosted on the Python Package
 Index (PyPI) corresponding to NAME and VERSION.  EXTENSION is the file name
 extension, such as '.tar.gz'."
-  (string-append "https://pypi.org/packages/source/"
+  (string-append "https://files.pythonhosted.org/packages/source/"
                  (string-take name 1) "/" name "/"
                  name "-" version extension))
 
diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index a2ce14b192..fecf95d0a7 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -371,15 +371,20 @@ VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE."
                 `(package
                    (name ,(python->package-name name))
                    (version ,version)
-                   (source (origin
-                             (method url-fetch)
-                             ;; Sometimes 'pypi-uri' doesn't quite work due to mixed
-                             ;; cases in NAME, for instance, as is the case with
-                             ;; "uwsgi".  In that case, fall back to a full URL.
-                             (uri (pypi-uri ,(string-downcase name) version))
-                             (sha256
-                              (base32
-                               ,(guix-hash-url temp)))))
+                   (source
+                    (origin
+                      (method url-fetch)
+                      ;; PyPI URL are case sensitive, but sometimes a project
+                      ;; named using mixed case has a URL using lower case, so
+                      ;; we must work around this inconsistency.  For actual
+                      ;; examples, compare the URLs of the "Deprecated" and
+                      ;; "uWSGI" PyPI packages.
+                      (uri ,(if (string-contains source-url name)
+                                `(pypi-uri ,name version)
+                                `(pypi-uri ,(string-downcase name) version)))
+                      (sha256
+                       (base32
+                        ,(guix-hash-url temp)))))
                    (build-system python-build-system)
                    ,@(maybe-inputs required-inputs 'propagated-inputs)
                    ,@(maybe-inputs test-inputs 'native-inputs)
-- 
2.21.0