Subject: bug#24450: [PATCH] bug#24450: pypi importer outputs strange character series in optional dependency case.

From: Maxim Cournoyer <[email-masked]>
Date: Sun, 31 Mar 2019 10:40:34 -0400

Hello!

I've yet another commit to add on the top of the current stack of
patches already sent. This one makes it so that the source archive is
searched for the '[...].egg-info/requires.txt' file rather than check
the more common, fixed location.

This makes the importer more useful, as some packages such as
robotframework-sshlibrary use a "src" directory in their hierarchy which
would cause the previous scheme to not find requires.txt.

From cfde6e09f8f8c692fe252d76ed27e8c50a9e5377 Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <[email-masked]>
Date: Sat, 30 Mar 2019 23:13:26 -0400
Subject: [PATCH] import: pypi: Scan source archive to find requires.txt file.

* guix/import/pypi.scm (use-modules): Use invoke from (guix build utils).
(guess-requirements)[archive-root-directory]: Remove procedure.
[guess-requirements-from-wheel]: Re-ident.
[guess-requirements-from-source]: Search for the requires.txt file in the
archive instead of using a static, expected location.
* tests/pypi.scm ("pypi->guix-package, no wheel"): Mock the requires.txt at a
non-standard location to test the new feature.
("pypi->guix-package, no usable requirement file."): Adapt.
---
 guix/import/pypi.scm | 65 ++++++++++++++++++--------------------------
 tests/pypi.scm       | 17 +++++++-----
 2 files changed, 37 insertions(+), 45 deletions(-)

diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index 099768f0c8..a2ce14b192 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -36,7 +36,8 @@
   #:use-module ((guix build utils)
                 #:select ((package-name->name+version
                            . hyphen-package-name->name+version)
-                          find-files))
+                          find-files
+                          invoke))
   #:use-module (guix import utils)
   #:use-module ((guix download) #:prefix download:)
   #:use-module (guix import json)
@@ -267,19 +268,6 @@ omitted since these can be difficult or expensive to satisfy."
 of the required packages specified in the requirements.txt file.  ARCHIVE will
 be extracted in a temporary directory."
 
-  (define (archive-root-directory url)
-    ;; Given the URL of the package's archive, return the name of the directory
-    ;; that will be created upon decompressing it. If the filetype is not
-    ;; supported, return #f.
-    (if (compressed-file? url)
-        (let ((root-directory (file-sans-extension (basename url))))
-          (if (string=? "tar" (file-extension root-directory))
-              (file-sans-extension root-directory)
-              root-directory))
-        (begin
-          (warning (G_ "Unsupported archive format (~a): \
-cannot determine package dependencies") (file-extension url))
-          #f)))
 
   (define (read-wheel-metadata wheel-archive)
     ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's
@@ -305,33 +293,34 @@ cannot determine package dependencies") (file-extension url))
     (call-with-temporary-output-file
      (lambda (temp port)
        (if wheel-url
-         (and (url-fetch wheel-url temp)
-              (read-wheel-metadata temp))
-         #f))))
+           (and (url-fetch wheel-url temp)
+                (read-wheel-metadata temp))
+           #f))))
 
   (define (guess-requirements-from-source)
     ;; Return the package's requirements by guessing them from the source.
-    (let ((dirname (archive-root-directory source-url))
-          (extension (file-extension source-url)))
-      (if (string? dirname)
-          (call-with-temporary-directory
-           (lambda (dir)
-             (let* ((pypi-name (string-take dirname (string-rindex dirname #\-)))
-                    (requires.txt (string-append dirname "/" pypi-name
-                                                 ".egg-info" "/requires.txt"))
-                    (exit-code
-                     (parameterize ((current-error-port (%make-void-port "rw+"))
-                                    (current-output-port (%make-void-port "rw+")))
-                       (if (string=? "zip" extension)
-                           (system* "unzip" archive "-d" dir requires.txt)
-                           (system* "tar" "xf" archive "-C" dir requires.txt)))))
-               (if (zero? exit-code)
-                   (parse-requires.txt (string-append dir "/" requires.txt))
-                   (begin
-                     (warning
-                      (G_ "Failed to extract file: ~a from source.~%")
-                      requires.txt)
-                     (list '() '()))))))
+    (if (compressed-file? source-url)
+        (call-with-temporary-directory
+         (lambda (dir)
+           (parameterize ((current-error-port (%make-void-port "rw+"))
+                          (current-output-port (%make-void-port "rw+")))
+             (if (string=? "zip" (file-extension source-url))
+                 (invoke "unzip" archive "-d" dir)
+                 (invoke "tar" "xf" archive "-C" dir)))
+           (let ((requires.txt-files
+                  (find-files dir (lambda (abs-file-name _)
+		                    (string-match "\\.egg-info/requires.txt$"
+                                                  abs-file-name)))))
+             (if (> (length requires.txt-files) 0)
+                 (begin
+                   (parse-requires.txt (first requires.txt-files)))
+                 (begin (warning (G_ "Cannot guess requirements from source archive:\
+ no requires.txt file found.~%"))
+                        (list '() '()))))))
+        (begin
+          (warning (G_ "Unsupported archive format; \
+cannot determine package dependencies from source archive: ~a~%")
+                   (basename source-url))
           (list '() '()))))
 
   ;; First, try to compute the requirements using the wheel, else, fallback to
diff --git a/tests/pypi.scm b/tests/pypi.scm
index aa08e2cb54..ad188df16c 100644
--- a/tests/pypi.scm
+++ b/tests/pypi.scm
@@ -177,8 +177,9 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing'
              (match url
                ("https://example.com/foo-1.0.0.tar.gz"
                 (begin
-                  (mkdir-p "foo-1.0.0/foo.egg-info/")
-                  (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt"
+                  ;; Unusual requires.txt location should still be found.
+                  (mkdir-p "foo-1.0.0/src/bizarre.egg-info")
+                  (with-output-to-file "foo-1.0.0/src/bizarre.egg-info/requires.txt"
                     (lambda ()
                       (display test-requires.txt)))
                   (parameterize ((current-output-port (%make-void-port "rw+")))
@@ -299,10 +300,13 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing'
          (lambda (url file-name)
            (match url
              ("https://example.com/foo-1.0.0.tar.gz"
+              (mkdir-p "foo-1.0.0/foo.egg-info/")
+              (parameterize ((current-output-port (%make-void-port "rw+")))
+                (system* "tar" "czvf" file-name "foo-1.0.0/"))
+              (delete-file-recursively "foo-1.0.0")
               (set! test-source-hash
-                (call-with-input-file file-name port-sha256))
-              #t)
-             ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #t)
+                (call-with-input-file file-name port-sha256)))
+             ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f)
              (_ (error "Unexpected URL: " url)))))
         (mock ((guix http-client) http-fetch
                (lambda (url . rest)
@@ -334,7 +338,6 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing'
                             test-source-hash)
                            hash))
                 (x
-                 (pk 'fail x #f)))
-              )))
+                 (pk 'fail x #f))))))
 
 (test-end "pypi")
-- 
2.20.1

Thanks,

Maxim