Subject: [bug#58587] [PATCH 00/14] Introducing pyproject-build-system.

From: Marius Bakke <[email-masked]>
Date: Thu, 20 Oct 2022 01:11:49 +0200

zimoun <[email-masked]> skriver:

> I would add a note, for instance, something like that:

[...]

Good idea, although I don't like to duplicate so much information from
the build system description.  Here is my attempt:

diff --git a/doc/contributing.texi b/doc/contributing.texi
index b9260267fa..d12daa7ec3 100644
--- a/doc/contributing.texi
+++ b/doc/contributing.texi
@@ -786,6 +786,21 @@ for instance, the module python-dateutil is packaged under the names
 starts with @code{py} (e.g.@: @code{pytz}), we keep it and prefix it as
 described above.
 
+@quotation Note
+Currently there are two different build systems for Python packages in Guix:
+@var{python-build-system} and @var{pyproject-build-system}.  For the
+longest time, Python packages were built from an informally specified
+@file{setup.py} file.  That worked amazingly well, considering Pythons
+success, but was difficult to build tooling around.  As a result, a host
+of alternative build systems emerged and the community eventually settled on a
+@url{https://peps.python.org/pep-0517/, formal standard} for specifying build
+requirements.  @var{pyproject-build-system} is Guix's implementation of this
+standard.  It is considered ``experimental'' in that it does not yet support
+all the various PEP-517 @emph{build backends}, but you are encouraged to try
+it for new Python packages and report any problems.  It will eventually be
+deprecated and merged into @var{python-build-system}.
+@end quotation
+
 @subsubsection Specifying Dependencies
 @cindex inputs, for Python packages

>> +@defvr {Scheme Variable} pyproject-build-system
>> +This is a variable exported by @code{guix build-system pyproject}.  It
>> +is a reimplementation of @code{python-build-system} designed around
>                              ^
> s/code/var

Fixed both instances, thanks.  :-)

I also made some substantial changes to the build system:

* Moved 'python-toolchain' into python.scm which seems to work fine.
* Inherit from python-build-system where possible.
* Hint about deprecating 'add-installed-pythonpath', which has always
  annoyed me :-) I think a 'with-installed-pythonpath' would be much
  more ergonomic and plan to submit that later.
* Most importantly, use three ;;; for module commentary.

Patch below:

From e765fbe232ddc57dd8ac2622a30736c5483bbf54 Mon Sep 17 00:00:00 2001
From: Marius Bakke <[email-masked]>
Date: Wed, 19 Oct 2022 23:10:24 +0200
Subject: [PATCH] fixup pyproject

---
 gnu/local.mk                          |   1 -
 gnu/packages/python-commencement.scm  |  64 ---------
 gnu/packages/python.scm               |  39 +++++-
 guix/build-system/pyproject.scm       |   2 +-
 guix/build/pyproject-build-system.scm | 180 ++++++++------------------
 5 files changed, 95 insertions(+), 191 deletions(-)
 delete mode 100644 gnu/packages/python-commencement.scm

diff --git a/gnu/local.mk b/gnu/local.mk
index b6e0d262be..bf598cec8b 100644
--- a/gnu/local.mk
+++ b/gnu/local.mk
@@ -511,7 +511,6 @@ GNU_SYSTEM_MODULES =				\
   %D%/packages/python-build.scm			\
   %D%/packages/python-check.scm			\
   %D%/packages/python-compression.scm		\
-  %D%/packages/python-commencement.scm		\
   %D%/packages/python-crypto.scm		\
   %D%/packages/python-science.scm		\
   %D%/packages/python-web.scm			\
diff --git a/gnu/packages/python-commencement.scm b/gnu/packages/python-commencement.scm
deleted file mode 100644
index 011ba2c38d..0000000000
--- a/gnu/packages/python-commencement.scm
+++ /dev/null
@@ -1,64 +0,0 @@
-;;; GNU Guix --- Functional package management for GNU
-;;; Copyright © 2021 Lars-Dominik Braun <[email-masked]>
-;;; Copyright © 2022 Marius Bakke <[email-masked]>
-;;;
-;;; This file is part of GNU Guix.
-;;;
-;;; GNU Guix is free software; you can redistribute it and/or modify it
-;;; under the terms of the GNU General Public License as published by
-;;; the Free Software Foundation; either version 3 of the License, or (at
-;;; your option) any later version.
-;;;
-;;; GNU Guix is distributed in the hope that it will be useful, but
-;;; WITHOUT ANY WARRANTY; without even the implied warranty of
-;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-;;; GNU General Public License for more details.
-;;;
-;;; You should have received a copy of the GNU General Public License
-;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
-
-(define-module (gnu packages python-commencement)
-  #:use-module ((guix licenses) #:prefix license:)
-  #:use-module (guix packages)
-  #:use-module (guix gexp)
-  #:use-module (guix build-system trivial)
-  #:use-module (gnu packages)
-  #:use-module (gnu packages python)
-  #:use-module (gnu packages python-build)
-  #:use-module (srfi srfi-1)
-  #:use-module (srfi srfi-26))
-
-(define-public python-toolchain
-  (package
-    (name "python-toolchain")
-    (version (package-version python))
-    (source #f)
-    (build-system trivial-build-system)
-    (arguments
-     (list #:modules '((guix build union))
-           #:builder
-           #~(begin
-               (use-modules (ice-9 match)
-                            (srfi srfi-1)
-                            (guix build union))
-               (union-build #$output
-                            (filter-map (match-lambda
-                                          ((_ . directory) directory))
-                                        %build-inputs)))))
-    (inputs
-     (list python-sans-pip-wrapper
-           python-pypa-build
-           python-pip
-           python-setuptools
-           python-wheel))
-    (native-search-paths
-     (package-native-search-paths python))
-    (search-paths
-     (package-search-paths python))
-    (license (package-license python))
-    (synopsis "Python toolchain")
-    (description
-     "Python toolchain including Python itself, setuptools and pip.  Use this
-package if you need a minimal Python toolchain instead of just the
-interpreter.")
-    (home-page (package-home-page python))))
diff --git a/gnu/packages/python.scm b/gnu/packages/python.scm
index 953044f55f..f55e5a16da 100644
--- a/gnu/packages/python.scm
+++ b/gnu/packages/python.scm
@@ -26,7 +26,7 @@
 ;;; Copyright © 2016, 2017 Nikita <[email-masked]>
 ;;; Copyright © 2016 Dylan Jeffers <sapientech@[email-masked]>
 ;;; Copyright © 2016 David Craven <[email-masked]>
-;;; Copyright © 2016, 2017, 2018, 2019, 2020, 2021 Marius Bakke <[email-masked]>
+;;; Copyright © 2016-2022 Marius Bakke <[email-masked]>
 ;;; Copyright © 2016, 2017 Stefan Reichör <[email-masked]>
 ;;; Copyright © 2016, 2017 Alex Vong <[email-masked]>
 ;;; Copyright © 2016, 2017, 2018 Arun Isaac <[email-masked]>
@@ -60,6 +60,7 @@
 ;;; Copyright © 2020, 2021 Greg Hogan <[email-masked]>
 ;;; Copyright © 2022 Philip McGrath <[email-masked]>
 ;;; Copyright © 2022 jgart <[email-masked]>
+;;; Copyright © 2021 Lars-Dominik Braun <[email-masked]>
 ;;;
 ;;; This file is part of GNU Guix.
 ;;;
@@ -87,6 +88,7 @@ (define-module (gnu packages python)
   #:use-module (gnu packages hurd)
   #:use-module (gnu packages libffi)
   #:use-module (gnu packages pkg-config)
+  #:use-module (gnu packages python-build)
   #:use-module (gnu packages readline)
   #:use-module (gnu packages sqlite)
   #:use-module (gnu packages tcl)
@@ -687,6 +689,41 @@ (define-public python-sans-pip
 (define-public python-sans-pip-wrapper
   (wrap-python3 python-sans-pip))
 
+(define-public python-toolchain
+  (package
+    (name "python-toolchain")
+    (version (package-version python))
+    (source #f)
+    (build-system trivial-build-system)
+    (arguments
+     (list #:modules '((guix build union))
+           #:builder
+           #~(begin
+               (use-modules (ice-9 match)
+                            (srfi srfi-1)
+                            (guix build union))
+               (union-build #$output
+                            (filter-map (match-lambda
+                                          ((_ . directory) directory))
+                                        %build-inputs)))))
+    (inputs
+     (list python-sans-pip-wrapper
+           python-pip
+           python-pypa-build
+           python-setuptools
+           python-wheel))
+    (native-search-paths
+     (package-native-search-paths python))
+    (search-paths
+     (package-search-paths python))
+    (license (package-license python))
+    (synopsis "Python toolchain")
+    (description
+     "Python toolchain including Python itself, setuptools and pip.  Use this
+package if you need a minimal Python toolchain instead of just the
+interpreter.")
+    (home-page (package-home-page python))))
+
 (define-public micropython
   (package
     (name "micropython")
diff --git a/guix/build-system/pyproject.scm b/guix/build-system/pyproject.scm
index 8e320529cc..d6b727ab5d 100644
--- a/guix/build-system/pyproject.scm
+++ b/guix/build-system/pyproject.scm
@@ -56,7 +56,7 @@ (define %pyproject-build-system-modules
 (define (default-python)
   "Return the default Python package."
   ;; Lazily resolve the binding to avoid a circular dependency.
-  (let ((python (resolve-interface '(gnu packages python-commencement))))
+  (let ((python (resolve-interface '(gnu packages python))))
     (module-ref python 'python-toolchain)))
 
 (define sanity-check.py
diff --git a/guix/build/pyproject-build-system.scm b/guix/build/pyproject-build-system.scm
index 141d4d849f..db317e10e5 100644
--- a/guix/build/pyproject-build-system.scm
+++ b/guix/build/pyproject-build-system.scm
@@ -32,7 +32,7 @@
 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
 
 (define-module (guix build pyproject-build-system)
-  #:use-module ((guix build gnu-build-system) #:prefix gnu:)
+  #:use-module ((guix build python-build-system) #:prefix python:)
   #:use-module (guix build utils)
   #:use-module (guix build json)
   #:use-module (ice-9 match)
@@ -50,31 +50,42 @@ (define-module (guix build pyproject-build-system)
             python-version
             pyproject-build))
 
-;; Commentary:
-;;
-;; PEP 517-compatible build system for Python packages.
-;;
-;; PEP 517 mandates the use of a TOML file called pyproject.toml at the
-;; project root, describing build and runtime dependencies, as well as the
-;; build system, which can be different from setuptools. This module uses
-;; that file to extract the build system used and call its wheel-building
-;; entry point build_wheel (see 'build). setuptools’ wheel builder is
-;; used as a fallback if either no pyproject.toml exists or it does not
-;; declare a build-system. It supports config_settings through the
-;; standard #:configure-flags argument.
-;;
-;; This wheel, which is just a ZIP file with a file structure defined
-;; by PEP 427 (https://www.python.org/dev/peps/pep-0427/), is then unpacked
-;; and its contents are moved to the appropriate locations in 'install.
-;;
-;; Then entry points, as defined by the PyPa Entry Point Specification
-;; (https://packaging.python.org/specifications/entry-points/) are read
-;; from a file called entry_points.txt in the package’s site-packages
-;; subdirectory and scripts are written to bin/. These are not part of a
-;; wheel and expected to be created by the installing utility.
-;;
-;; Caveats:
-;; - There is no support for in-tree build backends.
+;;; Commentary:
+;;;
+;;; PEP 517-compatible build system for Python packages.
+;;;
+;;; PEP 517 mandates the use of a TOML file called pyproject.toml at the
+;;; project root, describing build and runtime dependencies, as well as the
+;;; build system, which can be different from setuptools. This module uses
+;;; that file to extract the build system used and call its wheel-building
+;;; entry point build_wheel (see 'build). setuptools’ wheel builder is
+;;; used as a fallback if either no pyproject.toml exists or it does not
+;;; declare a build-system. It supports config_settings through the
+;;; standard #:configure-flags argument.
+;;;
+;;; This wheel, which is just a ZIP file with a file structure defined
+;;; by PEP 427 (https://www.python.org/dev/peps/pep-0427/), is then unpacked
+;;; and its contents are moved to the appropriate locations in 'install.
+;;;
+;;; Then entry points, as defined by the PyPa Entry Point Specification
+;;; (https://packaging.python.org/specifications/entry-points/) are read
+;;; from a file called entry_points.txt in the package’s site-packages
+;;; subdirectory and scripts are written to bin/. These are not part of a
+;;; wheel and expected to be created by the installing utility.
+;;;
+;;; Caveats:
+;;; - There is no support for in-tree build backends.
+;;;
+;;; Code:
+;;;
+
+;; Re-export these variables from python-build-system as many packages
+;; rely on these.
+(define python-version python:python-version)
+(define site-packages python:site-packages)
+;; XXX: This procedure is not very functional, now might be a good time
+;; to deprecate it.
+(define add-installed-pythonpath python:add-installed-pythonpath)
 
 ;; Base error type.
 (define-condition-type &python-build-error &error
@@ -179,39 +190,6 @@ (define* (check #:key inputs outputs tests? test-backend test-flags #:allow-othe
           (else (raise (condition (&test-system-not-found))))))
       (format #t "test suite not run~%")))
 
-(define (python-version python)
-  (let* ((version     (last (string-split python #\-)))
-         (components  (string-split version #\.))
-         (major+minor (take components 2)))
-    (string-join major+minor ".")))
-
-(define (python-output outputs)
-  "Return the path of the python output, if there is one, or fall-back to out."
-  (or (assoc-ref outputs "python")
-      (assoc-ref outputs "out")))
-
-(define (site-packages inputs outputs)
-  "Return the path of the current output's Python site-package."
-  (let* ((out (python-output outputs))
-         (python (assoc-ref inputs "python")))
-    (string-append out "/lib/python" (python-version python) "/site-packages")))
-
-(define (add-installed-pythonpath inputs outputs)
-  "Prepend the site-package of OUTPUT to GUIX_PYTHONPATH.  This is useful when
-running checks after installing the package."
-  (setenv "GUIX_PYTHONPATH" (string-append (site-packages inputs outputs) ":"
-                                           (getenv "GUIX_PYTHONPATH"))))
-
-(define* (add-install-to-pythonpath #:key inputs outputs #:allow-other-keys)
-  "A phase that just wraps the 'add-installed-pythonpath' procedure."
-  (add-installed-pythonpath inputs outputs))
-
-(define* (add-install-to-path #:key outputs #:allow-other-keys)
-  "Adding Python scripts to PATH is also often useful in tests."
-  (setenv "PATH" (string-append (assoc-ref outputs "out")
-                                "/bin:"
-                                (getenv "PATH"))))
-
 (define* (install #:key inputs outputs (configure-flags '()) #:allow-other-keys)
   "Install a wheel file according to PEP 427"
   ;; See https://www.python.org/dev/peps/pep-0427/#installing-a-wheel-distribution-1-0-py32-none-any-whl
@@ -301,9 +279,20 @@ (define (list-directories base predicate)
 
 (define* (compile-bytecode #:key inputs outputs (configure-flags '()) #:allow-other-keys)
   "Compile installed byte-code in site-packages."
-  (let ((site-dir (site-packages inputs outputs)))
-    (invoke "python" "-m" "compileall" "--invalidation-mode=unchecked-hash"
-            site-dir)))
+  (let* ((site-dir (site-packages inputs outputs))
+         (python (assoc-ref inputs "python"))
+         (major-minor (map string->number
+                           (take (string-split (python-version python) #\.) 2)))
+         (<3.7? (match major-minor
+                  ((major minor)
+                   (or (< major 3) (and (= major 3) (< minor 7)))))))
+    (if <3.7?
+        ;; These versions don’t have the hash invalidation modes and do
+        ;; not produce reproducible bytecode files.
+        (format #t "Skipping bytecode compilation for Python version ~a < 3.7~%"
+                (python-version python))
+        (invoke "python" "-m" "compileall" "--invalidation-mode=unchecked-hash"
+                site-dir))))
 
 (define* (create-entrypoints #:key inputs outputs (configure-flags '()) #:allow-other-keys)
   "Implement Entry Points Specification
@@ -371,34 +360,6 @@ (define (create-script path name module function)
           (entry-points.txt->entry-points f)))
       entry-point-files)))
 
-(define* (wrap #:key inputs outputs search-paths #:allow-other-keys)
-  (define (list-of-files dir)
-    (find-files dir (lambda (file stat)
-                      (and (eq? 'regular (stat:type stat))
-                           (not (wrapped-program? file))))))
-
-  (define bindirs
-    (append-map (match-lambda
-                  ((_ . dir)
-                   (list (string-append dir "/bin")
-                         (string-append dir "/sbin"))))
-                outputs))
-
-  ;; Do not require "bash" to be present in the package inputs
-  ;; even when there is nothing to wrap.
-  ;; Also, calculate (sh) only once to prevent some I/O.
-  (define %sh (delay (search-input-file inputs "bin/bash")))
-  (define (sh) (force %sh))
-
-  (let* ((var `("GUIX_PYTHONPATH" prefix
-                ,(search-path-as-string->list
-                  (or (getenv "GUIX_PYTHONPATH") "")))))
-    (for-each (lambda (dir)
-                (let ((files (list-of-files dir)))
-                  (for-each (cut wrap-program <> #:sh (sh) var)
-                            files)))
-              bindirs)))
-
 (define* (set-SOURCE-DATE-EPOCH #:rest _)
   "Set the 'SOURCE_DATE_EPOCH' environment variable.  This is used by tools
 that incorporate timestamps as a way to tell them to use a fixed timestamp.
@@ -407,54 +368,25 @@ (define* (set-SOURCE-DATE-EPOCH #:rest _)
   ;; not support timestamps before 1980.
   (setenv "SOURCE_DATE_EPOCH" "315619200"))
 
-(define* (enable-bytecode-determinism #:rest _)
-  "Improve determinism of pyc files."
-  ;; Use deterministic hashes for strings, bytes, and datetime objects.
-  (setenv "PYTHONHASHSEED" "0")
-  ;; Prevent Python from creating .pyc files when loading modules (such as
-  ;; when running a test suite).
-  (setenv "PYTHONDONTWRITEBYTECODE" "1"))
-
-(define* (ensure-no-cythonized-files #:rest _)
-  "Check the source code for @code{.c} files which may have been pre-generated
-by Cython."
-  (for-each
-    (lambda (file)
-      (let ((generated-file
-              (string-append (string-drop-right file 3) "c")))
-        (when (file-exists? generated-file)
-          (format #t "Possible Cythonized file found: ~a~%" generated-file))))
-    (find-files "." "\\.pyx$")))
-
 (define %standard-phases
   ;; The build phase only builds C extensions and copies the Python sources,
   ;; while the install phase copies then byte-compiles the sources to the
   ;; prefix directory.  The check phase is moved after the installation phase
   ;; to ease testing the built package.
-  (modify-phases gnu:%standard-phases
-    (add-after 'unpack 'enable-bytecode-determinism
-      enable-bytecode-determinism)
-    (add-after 'enable-bytecode-determinism 'ensure-no-cythonized-files
-      ensure-no-cythonized-files)
+  (modify-phases python:%standard-phases
     (replace 'set-SOURCE-DATE-EPOCH set-SOURCE-DATE-EPOCH)
-    (delete 'bootstrap)
-    (delete 'configure)                 ;not needed
     (replace 'build build)
-    (delete 'check)                     ;moved after the install phase
     (replace 'install install)
-    (add-after 'install 'add-install-to-pythonpath add-install-to-pythonpath)
-    (add-after 'add-install-to-pythonpath 'add-install-to-path
-      add-install-to-path)
-    (add-after 'add-install-to-path 'wrap wrap)
-    ;; must be before tests, so they can use installed packages’ entry points.
+    (delete 'check)
+    ;; Must be before tests, so they can use installed packages’ entry points.
     (add-before 'wrap 'create-entrypoints create-entrypoints)
     (add-after 'wrap 'check check)
-    (add-after 'check 'sanity-check sanity-check)
+    (replace 'sanity-check sanity-check)
     (add-before 'check 'compile-bytecode compile-bytecode)))
 
 (define* (pyproject-build #:key inputs (phases %standard-phases)
                           #:allow-other-keys #:rest args)
   "Build the given Python package, applying all of PHASES in order."
-  (apply gnu:gnu-build #:inputs inputs #:phases phases args))
+  (apply python:python-build #:inputs inputs #:phases phases args))
 
 ;;; pyproject-build-system.scm ends here
-- 
2.38.0