Subject: [bug#46848] Questions regarding Python packaging

From: Lars-Dominik Braun <[email-masked]>
Date: Tue, 6 Jul 2021 14:16:03 +0200

Hi again,

> No, try
> 
> 	git clone https://github.com/pypa/pep517.git
> 	cd pep517
> 	pip wheel --use-pep517 -v .
> 
> which has no setup.py and uses flit instead. pip can build it, because
> it supports PEP 517-based builds. As I said, if we decide to keep it
> bundled with our python package, there’s no good reason to choose
> pypa-build.
I now have a proof of concept for a pip-based python-build-system, see
1st patch. The changeset is quite small, but it does not handle testing
at all right now.

Note that alot of setuptools-based packages lack a dependency on
python-wheel (2nd patch; pardon the whitespace errors), which is
required when using pyproject.toml (and also declared there, but our
Python importer cannot read that file yet). But – as expected – that’s
really the only issue I’ve encountered while rebuilding alot of Python
packages so far.

Cheers,
Lars

>From a7c6750917f5dc2e1eaf34520f7e6b0e3d5e0d3c Mon Sep 17 00:00:00 2001
From: Lars-Dominik Braun <[email-masked]>
Date: Tue, 6 Jul 2021 14:13:51 +0200
Subject: [PATCH 1/2] dirty: build: Build Python packages using pip.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* guix/build/python-build-system.scm …
---
 guix/build/python-build-system.scm | 109 +++++++++++------------------
 1 file changed, 42 insertions(+), 67 deletions(-)

diff --git a/guix/build/python-build-system.scm b/guix/build/python-build-system.scm
index 8ade1d5911..e4b63e131e 100644
--- a/guix/build/python-build-system.scm
+++ b/guix/build/python-build-system.scm
@@ -109,30 +109,6 @@
 ;; and the scripts defined in entry-points will always be created.

-(define setuptools-shim
-  ;; Run setup.py with "setuptools" being imported, which will patch
-  ;; "distutils". This is needed for packages using "distutils" instead of
-  ;; "setuptools" since the former does not understand the
-  ;; "--single-version-externally-managed" flag.
-  ;; Python code taken from pip 9.0.1 pip/utils/setuptools_build.py
-  (string-append
-   "import setuptools, tokenize;__file__='setup.py';"
-   "f=getattr(tokenize, 'open', open)(__file__);"
-   "code=f.read().replace('\\r\\n', '\\n');"
-   "f.close();"
-   "exec(compile(code, __file__, 'exec'))"))
-
-(define (call-setuppy command params use-setuptools?)
-  (if (file-exists? "setup.py")
-      (begin
-         (format #t "running \"python setup.py\" with command ~s and parameters ~s~%"
-                command params)
-         (if use-setuptools?
-             (apply invoke "python" "-c" setuptools-shim
-                    command params)
-             (apply invoke "python" "./setup.py" command params)))
-      (error "no setup.py found")))
-
 (define* (sanity-check #:key tests? inputs outputs #:allow-other-keys)
   "Ensure packages depending on this package via setuptools work properly,
 their advertised endpoints work and their top level modules are importable
@@ -142,26 +118,6 @@ without errors."
     (with-directory-excursion "/tmp"
       (invoke "python" sanity-check.py (site-packages inputs outputs)))))
 
-(define* (build #:key use-setuptools? #:allow-other-keys)
-  "Build a given Python package."
-  (call-setuppy "build" '() use-setuptools?)
-  #t)
-
-(define* (check #:key tests? test-target use-setuptools? #:allow-other-keys)
-  "Run the test suite of a given Python package."
-  (if tests?
-      ;; Running `setup.py test` creates an additional .egg-info directory in
-      ;; build/lib in some cases, e.g. if the source is in a sub-directory
-      ;; (given with `package_dir`). This will by copied to the output, too,
-      ;; so we need to remove.
-      (let ((before (find-files "build" "\\.egg-info$" #:directories? #t)))
-        (call-setuppy test-target '() use-setuptools?)
-        (let* ((after (find-files "build" "\\.egg-info$" #:directories? #t))
-               (inter (lset-difference string=? after before)))
-          (for-each delete-file-recursively inter)))
-      (format #t "test suite not run~%"))
-  #t)
-
 (define (python-version python)
   (let* ((version     (last (string-split python #\-)))
          (components  (string-split version #\.))
@@ -195,31 +151,40 @@ running checks after installing the package."
                                 "/bin:"
                                 (getenv "PATH"))))
 
-(define* (install #:key inputs outputs (configure-flags '()) use-setuptools?
+;; Some packages expect 'build and 'check exist. If they don’t replacing them
+;; or adding phases before/after will fail. Preserve them as dummy-phases.
+(define* (build #:key outputs (configure-flags '()) use-setuptools?
+                  #:allow-other-keys)
+    #t)
+
+(define* (check #:key outputs (configure-flags '()) use-setuptools?
+                  #:allow-other-keys)
+    #t)
+
+(define* (install #:key outputs (configure-flags '()) use-setuptools?
                   #:allow-other-keys)
   "Install a given Python package."
-  (let* ((out (python-output outputs))
-         (python (assoc-ref inputs "python"))
-         (major-minor (map string->number
-                           (take (string-split (python-version python) #\.) 2)))
-         (<3.7? (match major-minor
-                   ((major minor)
-                    (or (< major 3) (and (= major 3) (< minor 7))))))
-         (params (append (list (string-append "--prefix=" out)
-                               "--no-compile")
-                         (if use-setuptools?
-                             ;; distutils does not accept these flags
-                             (list "--single-version-externally-managed"
-                                   "--root=/")
-                             '())
-                         configure-flags)))
-    (call-setuppy "install" params use-setuptools?)
-    ;; Rather than produce potentially non-reproducible .pyc files on Pythons
-    ;; older than 3.7, whose 'compileall' module lacks the
-    ;; '--invalidation-mode' option, do not generate any.
-    (unless <3.7?
-      (invoke "python" "-m" "compileall" "--invalidation-mode=unchecked-hash"
-              out))))
+  (let* ((out (assoc-ref outputs "out")))
+    (setenv "HOME" "/tmp") ; silence warning
+    (invoke "pip" "install"
+            (string-append "--prefix=" out)
+            "--root=/"
+            ;; Hide pip’s own version check. Nothing we can do anyway.
+            "--disable-pip-version-check"
+            ;; Do not search and install dependencies.
+            "--no-deps"
+            ;; Do not search PyPi.
+            "--no-index"
+            ;; Don’t reinstall build dependencies into virtal environent,
+            ;; instead use local ones. Otherwise build process will fail
+            ;; finding dependencies.
+            "--no-build-isolation"
+            ;; Ignore installed packages, important for packages bundled by
+            ;; Python, like setuptools.
+            "--ignore-installed"
+            ;; Library is in the current directory.
+            ".")
+    #t))
 
 (define* (wrap #:key inputs outputs #:allow-other-keys)
   (define (list-of-files dir)
@@ -267,6 +232,15 @@ installed with setuptools."
                  (utime file early-1980 early-1980))
                #t))))
 
+;; python-wheel respects this virable and passes an invalid early timestamp to
+;; Python’s zip module if not redefined to a later date.
+(define* (set-SOURCE-DATE-EPOCH #:rest _)
+  "Set the 'SOURCE_DATE_EPOCH' environment variable.  This is used by tools
+that incorporate timestamps as a way to tell them to use a fixed timestamp.
+See https://reproducible-builds.org/specs/source-date-epoch/."
+  (setenv "SOURCE_DATE_EPOCH" "315619200")
+  #t)
+
 (define* (enable-bytecode-determinism #:rest _)
   "Improve determinism of pyc files."
   ;; Use deterministic hashes for strings, bytes, and datetime objects.
@@ -297,6 +271,7 @@ by Cython."
       enable-bytecode-determinism)
     (add-after 'enable-bytecode-determinism 'ensure-no-cythonized-files
       ensure-no-cythonized-files)
+    (replace 'set-SOURCE-DATE-EPOCH set-SOURCE-DATE-EPOCH)
     (delete 'bootstrap)
     (delete 'configure)                 ;not needed
     (replace 'build build)
-- 
2.31.1

>From 7e6d12aac228f369a1f09870529dc003a8ad356e Mon Sep 17 00:00:00 2001
From: Lars-Dominik Braun <[email-masked]>
Date: Tue, 6 Jul 2021 14:14:21 +0200
Subject: [PATCH 2/2] dirty: Fix build errors.

---
 gnu/packages/build-tools.scm   |  2 ++
 gnu/packages/check.scm         | 16 ++++++++----
 gnu/packages/python-build.scm  | 47 ++++++++++++++++++++++++++++++++++
 gnu/packages/python-crypto.scm |  9 ++++---
 gnu/packages/python-xyz.scm    | 44 ++++++++++++++++---------------
 gnu/packages/time.scm          |  4 ++-
 6 files changed, 92 insertions(+), 30 deletions(-)

diff --git a/gnu/packages/build-tools.scm b/gnu/packages/build-tools.scm
index dc2411801c..6c29fb9bdc 100644
--- a/gnu/packages/build-tools.scm
+++ b/gnu/packages/build-tools.scm
@@ -50,6 +50,7 @@
   #:use-module (gnu packages pretty-print)
   #:use-module (gnu packages protobuf)
   #:use-module (gnu packages python)
+  #:use-module (gnu packages python-build)
   #:use-module (gnu packages python-crypto)
   #:use-module (gnu packages python-web)
   #:use-module (gnu packages python-xyz)
@@ -237,6 +238,7 @@ files and generates build instructions for the Ninja build system.")
                   (delete 'wrap))))
     (inputs `(("ninja" ,ninja)))
     (propagated-inputs `(("python" ,python)))
+    (native-inputs `(("python-wheel" ,python-wheel)))
     (home-page "https://mesonbuild.com/")
     (synopsis "Build system designed to be fast and user-friendly")
     (description
diff --git a/gnu/packages/check.scm b/gnu/packages/check.scm
index 11117b88e4..12f17c41c6 100644
--- a/gnu/packages/check.scm
+++ b/gnu/packages/check.scm
@@ -975,6 +975,7 @@ standard library.")
        ("python-mock" ,python-mock)
        ("python-pytest" ,python-pytest-bootstrap)
        ("python-setuptools-scm" ,python-setuptools-scm)
+       ("python-wheel" ,python-wheel)
        ("python-toml" ,python-toml)
        ("python-xmlschema" ,python-xmlschema)))
     (home-page "https://docs.pytest.org/en/latest/")
@@ -1028,7 +1029,8 @@ and many external plugins.")
     (name "python-pytest-bootstrap")
     (native-inputs `(("python-iniconfig" ,python-iniconfig)
                      ("python-setuptools-scm" ,python-setuptools-scm)
-                     ("python-toml" ,python-toml)))
+                     ("python-toml" ,python-toml)
+                     ("python-wheel" ,python-wheel)))
     (arguments `(#:tests? #f))
     (properties `((python2-variant . ,(delay python2-pytest-bootstrap))))))
 
@@ -1113,7 +1115,8 @@ supports coverage of subprocesses.")
                           (format #t "test suite not run~%"))
                       #t)))))
     (native-inputs
-     `(("python-setuptools-scm" ,python-setuptools-scm)))
+     `(("python-setuptools-scm" ,python-setuptools-scm)
+	   ("python-wheel" ,python-wheel)))
     (home-page "https://github.com/pytest-dev/pytest-runner")
     (synopsis "Invoke py.test as a distutils command")
     (description
@@ -1236,7 +1239,8 @@ same arguments.")
               (when tests?
                 (invoke "py.test" "-v")))))))
     (native-inputs
-     `(("python-setuptools-scm" ,python-setuptools-scm)))
+     `(("python-setuptools-scm" ,python-setuptools-scm)
+	   ("python-wheel" ,python-wheel)))
     (propagated-inputs
      `(("python-execnet" ,python-execnet)
        ("python-pytest" ,python-pytest)
@@ -1302,7 +1306,8 @@ timeout has been exceeded.")
     (build-system python-build-system)
     (native-inputs
      `(("python-pytest" ,python-pytest)
-       ("python-setuptools-scm" ,python-setuptools-scm)))
+       ("python-setuptools-scm" ,python-setuptools-scm)
+	   ("python-wheel" ,python-wheel)))
     (home-page
      "https://github.com/pytest-dev/pytest-forked")
     (synopsis
@@ -2486,7 +2491,8 @@ create data based on random numbers and yet remain repeatable.")
     (build-system python-build-system)
     (native-inputs
      `(("python-mock" ,python-mock)
-       ("python-pytest" ,python-pytest)))
+       ("python-pytest" ,python-pytest)
+	   ("python-wheel" ,python-wheel)))
     (propagated-inputs
      `(("python-six" ,python-six)
        ("python-dateutil" ,python-dateutil)))
diff --git a/gnu/packages/python-build.scm b/gnu/packages/python-build.scm
index 140629ca43..913d1b04ce 100644
--- a/gnu/packages/python-build.scm
+++ b/gnu/packages/python-build.scm
@@ -89,6 +89,52 @@ installed with a newer @code{pip} or with wheel's own command line utility.")
 Language (TOML) configuration files.")
     (license license:expat)))
 
+(define-public python-pytoml
+  (package
+    (name "python-pytoml")
+    (version "0.1.21")
+    (source
+     (origin
+       (method url-fetch)
+       (uri (pypi-uri "pytoml" version))
+       (sha256
+        (base32
+         "1rv1byiw82k7mj6aprcrqi2vdabs801y97xhfnrz7kxds34ggv4f"))))
+    (build-system python-build-system)
+    (home-page "https://github.com/avakar/pytoml")
+    (synopsis "Parser for TOML")
+    (description "This package provides a Python parser for TOML-0.4.0.")
+    (license license:expat)))
+
+(define-public python-flit-core
+  (package
+    (name "python-flit-core")
+    (version "3.2.0")
+    (source
+      (origin
+        (method url-fetch)
+        (uri (pypi-uri "flit-core" version))
+        (sha256
+          (base32
+            "0cclv7v4cmzi457bzqsx9ig8ir1ha3ip8h1kx8qfy95wbmfg51zz"))))
+    (build-system python-build-system)
+	(arguments
+     `(#:phases
+       (modify-phases %standard-phases
+         ;; Requirements refer to a specific version of dependencies,
+         ;; which are too old. So we patch to refer to any later version.
+         (add-after 'unpack 'use-local-sourcetree
+           (lambda _
+             (setenv "GUIX_PYTHONPATH" (string-append (getenv "GUIX_PYTHONPATH") ":" (getcwd))))))))
+    (propagated-inputs
+      `(("python-toml" ,python-toml)))
+    (home-page "https://github.com/takluyver/flit")
+    (synopsis
+      "Distribution-building parts of Flit. See flit package for more information")
+    (description
+      "Distribution-building parts of Flit. See flit package for more information")
+    (license #f)))
+
 (define-public python-pep517-bootstrap
   (hidden-package
    (package
@@ -104,6 +150,7 @@ Language (TOML) configuration files.")
      (build-system python-build-system)
      (arguments
       `(#:tests? #f))                     ;to avoid circular dependencies
+	 (native-inputs `(("python-flit-core" ,python-flit-core)))
      (propagated-inputs
       `(("python-toml" ,python-toml)
         ("python-wheel" ,python-wheel)))
diff --git a/gnu/packages/python-crypto.scm b/gnu/packages/python-crypto.scm
index 39b194b25e..e40a60a9ca 100644
--- a/gnu/packages/python-crypto.scm
+++ b/gnu/packages/python-crypto.scm
@@ -124,7 +124,8 @@ with what is used by the Bitcoin network.")
     (build-system python-build-system)
     (native-inputs
      `(("python-pycparser" ,python-pycparser)
-       ("python-pytest" ,python-pytest)))
+       ("python-pytest" ,python-pytest)
+	   ("python-wheel" ,python-wheel)))
     (propagated-inputs
      `(("python-cffi" ,python-cffi)
        ("python-six" ,python-six)))
@@ -544,7 +545,8 @@ is used by the Requests library to verify HTTPS requests.")
        ("python-hypothesis" ,python-hypothesis)
        ("python-pretend" ,python-pretend)
        ("python-pytz" ,python-pytz)
-       ("python-pytest" ,python-pytest)))
+       ("python-pytest" ,python-pytest)
+	   ("python-wheel" ,python-wheel)))
     (home-page "https://github.com/pyca/cryptography")
     (synopsis "Cryptographic recipes and primitives for Python")
     (description
@@ -611,7 +613,8 @@ message digests and key derivation functions.")
      `(("libfaketime" ,libfaketime)
        ("python-flaky" ,python-flaky)
        ("python-pretend" ,python-pretend)
-       ("python-pytest" ,python-pytest)))
+       ("python-pytest" ,python-pytest)
+	   ("python-wheel" ,python-wheel)))
     (home-page "https://github.com/pyca/pyopenssl")
     (synopsis "Python wrapper module around the OpenSSL library")
     (description
diff --git a/gnu/packages/python-xyz.scm b/gnu/packages/python-xyz.scm
index 7a2b9eecc6..59dbe45a4f 100644
--- a/gnu/packages/python-xyz.scm
+++ b/gnu/packages/python-xyz.scm
@@ -2307,6 +2307,7 @@ and is not compatible with JSON.")
              (invoke "python" "bootstrap.py" "build/scons" "DEVELOPER=guix")
              (chdir "build/scons")
              #t)))))
+    (native-inputs `(("python-wheel" ,python-wheel)))
     (home-page "https://scons.org/")
     (synopsis "Software construction tool written in Python")
     (description
@@ -3161,7 +3162,8 @@ e.g. filters, callbacks and errbacks can all be promises.")
     (native-inputs
      `(("python-mock" ,python-mock)
        ("python-pytest" ,python-pytest)
-       ("python-setuptools-scm" ,python-setuptools-scm)))
+       ("python-setuptools-scm" ,python-setuptools-scm)
+	   ("python-wheel" ,python-wheel)))
     (propagated-inputs
      `(("python-appdirs" ,python-appdirs)
        ("python-distlib" ,python-distlib/next)
@@ -6498,6 +6500,7 @@ child application and control it as if a human were typing commands.")
                (base32
                 "0ahlrxxkx2xhmxskx57gc96w3bdndflxx30304ihvm7ds136nny8"))))
     (build-system python-build-system)
+    (native-inputs `(("python-wheel" ,python-wheel)))
     (home-page "https://github.com/pypa/setuptools_scm/")
     (synopsis "Manage Python package versions in SCM metadata")
     (description
@@ -8284,7 +8287,8 @@ PEP 8.")
     (native-inputs
      `(("python-mock" ,python-mock)
        ("python-pytest" ,python-pytest)
-       ("python-testpath" ,python-testpath)))
+       ("python-testpath" ,python-testpath)
+       ,@(package-native-inputs python-pep517-bootstrap)))
     (properties `((python2-variant . ,(delay python2-pep517))))))
 
 ;; Skip the tests so we don't create a cyclical dependency with pytest.
@@ -8752,6 +8756,7 @@ output.")
         (sha256
           (base32 "1p9p7mn8x2j9psc4jxab98897v4i9s4fliyfw8rp8v4bx1n7pjj2"))))
     (build-system python-build-system)
+    (native-inputs `(("python-poetry-core" ,python-poetry-core)))
     (home-page "https://github.com/sdispater/crashtest")
     (synopsis "Manage Python errors with ease")
     (description
@@ -9891,20 +9896,7 @@ Jupyter Notebook format and Python APIs for working with notebooks.")
         (base32
          "0lc4si3xb7hza424414rdqdc3vng3kcrph8jbvjqb32spqddf3f7"))))
     (build-system python-build-system)
-    ;; The package does not come with a setup.py file, so we have to generate
-    ;; one ourselves.
-    (arguments
-     `(#:tests? #f
-       #:phases
-       (modify-phases %standard-phases
-         (add-after 'unpack 'create-setup.py
-           (lambda _
-             (call-with-output-file "setup.py"
-               (lambda (port)
-                 (format port "\
-from setuptools import setup
-setup(name='entrypoints', version='~a', py_modules=['entrypoints'])
-" ,version))))))))
+    (native-inputs `(("python-flit" ,python-flit)))
     (home-page "https://github.com/takluyver/entrypoints")
     (synopsis "Discover and load entry points from installed Python packages")
     (description "Entry points are a way for Python packages to advertise
@@ -11176,7 +11168,8 @@ library as well as on the command line.")
        "1c35qyhvy27q9ih9n899f3h4sdnpgq027dbiilly2qb5cvgarchm"))))
    (build-system python-build-system)
    (native-inputs
-    `(("python-setuptools-scm" ,python-setuptools-scm)))
+    `(("python-setuptools-scm" ,python-setuptools-scm)
+      ("python-wheel" ,python-wheel)))
    (synopsis "Plugin and hook calling mechanism for Python")
    (description "Pluggy is an extraction of the plugin manager as used by
 Pytest but stripped of Pytest specific details.")
@@ -14657,7 +14650,8 @@ than during a preprocessing step).")
            "17xbrgi23l87yg6h0qcknssp2q812miiy33qw6v45v5gx0jwv5xh"))))
     (build-system python-build-system)
     (propagated-inputs
-     `(("python-setuptools-scm" ,python-setuptools-scm)))
+     `(("python-setuptools-scm" ,python-setuptools-scm)
+	   ("python-wheel" ,python-wheel)))
     (home-page "https://github.com/marcelm/xopen/")
     (synopsis "Open compressed files transparently")
     (description "This module provides an @code{xopen} function that works like
@@ -15194,7 +15188,8 @@ and bit flag values.")
                     (lambda _
                       (invoke "pytest"))))))
     (native-inputs
-     `(("python-coverage" ,python-coverage)
+     `(("python-wheel" ,python-wheel)
+       ("python-coverage" ,python-coverage)
        ("python-hypothesis" ,python-hypothesis)
        ("python-pympler" ,python-pympler)
        ("python-pytest" ,python-pytest)
@@ -15215,7 +15210,7 @@ protocols.")
   (package
     (inherit python-attrs)
     (name "python-attrs-bootstrap")
-    (native-inputs `())
+    (native-inputs `(("python-wheel" ,python-wheel)))
     (arguments `(#:tests? #f))))
 
 (define-public python2-attrs-bootstrap
@@ -15606,6 +15601,7 @@ in other versions.")
         (base32
          "0ckzngs3scaa1mcfmsi1w40a1l8cxxnncscrxzjjwjyisx8z0fmw"))))
     (build-system python-build-system)
+    (native-inputs `(("python-wheel" ,python-wheel)))
     (home-page "https://github.com/RonnyPfannschmidt/iniconfig")
     (synopsis "Simple INI-file parser")
     (description "The @code{iniconfig} package provides a small and simple
@@ -20784,7 +20780,8 @@ register custom encoders and decoders.")
     (native-inputs
      `(("double-conversion" ,double-conversion)
        ("python-setuptools-scm" ,python-setuptools-scm)
-       ("python-pytest" ,python-pytest)))
+       ("python-pytest" ,python-pytest)
+	   ("python-wheel" ,python-wheel)))
     (home-page "https://github.com/ultrajson/ultrajson")
     (synopsis "Ultra fast JSON encoder and decoder for Python")
     (description
@@ -20980,6 +20977,11 @@ the syntactic logic to configure and launch jobs in an execution environment.")
     (build-system python-build-system)
     (arguments
      `(#:tests? #f)) ; XXX: Check requires network access.
+	(propagated-inputs
+	  `(("python-docutils" ,python-docutils)
+		("python-pytoml" ,python-pytoml)
+		("python-requests" ,python-requests)
+		("python-flit-core" ,python-flit-core)))
     (home-page "https://flit.readthedocs.io/")
     (synopsis
      "Simple packaging tool for simple packages")
diff --git a/gnu/packages/time.scm b/gnu/packages/time.scm
index 9b006a5438..60a0584202 100644
--- a/gnu/packages/time.scm
+++ b/gnu/packages/time.scm
@@ -46,6 +46,7 @@
   #:use-module (gnu packages compression)
   #:use-module (gnu packages perl)
   #:use-module (gnu packages python)
+  #:use-module (gnu packages python-build)
   #:use-module (gnu packages python-xyz))
 
 (define-public time
@@ -194,7 +195,8 @@ Pendulum instances.")
     (native-inputs
      `(("python-pytest" ,python-pytest)
        ("python-pytest-cov" ,python-pytest-cov)
-       ("python-setuptools-scm" ,python-setuptools-scm)))
+       ("python-setuptools-scm" ,python-setuptools-scm)
+	   ("python-wheel" ,python-wheel)))
     (propagated-inputs
      `(("python-six" ,python-six)))
     (home-page "https://dateutil.readthedocs.io/en/stable/")
-- 
2.31.1