Subject: [PATCH] Upgrade pypy3
From: Lars-Dominik Braun <[email-masked]>
To: [email-masked]
Date: Sun, 15 Nov 2020 13:39:11 +0100
List-Id: <debbugs-submit.debbugs.gnu.org>

Hi,

the first patch in this series upgrades pypy3 to 7.3.2 and the second
adjusts several aspects of the package.

Lars

From 0669431bc4ee103cf159f5f1d64684e6db8d6bb5 Mon Sep 17 00:00:00 2001
From: Lars-Dominik Braun <[email-masked]>
Date: Sat, 17 Oct 2020 13:46:21 +0200
Subject: [PATCH 1/2] gnu: pypy3: Update to 7.3.2.

* gnu/packages/python.scm (pypy3): Update to 7.3.2.
---
 gnu/packages/python.scm | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/gnu/packages/python.scm b/gnu/packages/python.scm
index 43704bccae..8ef8ae2e1d 100644
--- a/gnu/packages/python.scm
+++ b/gnu/packages/python.scm
@@ -688,14 +688,14 @@ ease from the desktop to a microcontroller or embedded system.")
 (define-public pypy3
   (package
     (name "pypy3")
-    (version "7.3.1")
+    (version "7.3.2")
     (source (origin
               (method url-fetch)
-              (uri (string-append "https://bitbucket.org/pypy/pypy/downloads/" ;
+              (uri (string-append "https://downloads.python.org/pypy/"
                                   "pypy3.6-v" version "-src.tar.bz2"))
               (sha256
                (base32
-                "10zsk8jby8j6visk5mzikpb1cidvz27qq4pfpa26jv53klic6b0c"))
+                "03f1fdw6yk2mypa9pbmgk26r8y1hhmw801l6g36zry9zsvz7aqgx"))
               (patches (search-patches "pypy3-7.3.1-fix-tests.patch"))))
     (build-system gnu-build-system)
     (native-inputs
-- 
2.26.2

From caf5e4bf9acd10f91de2e6a9c60185475144a1b9 Mon Sep 17 00:00:00 2001
From: Lars-Dominik Braun <[email-masked]>
Date: Sun, 15 Nov 2020 10:54:26 +0100
Subject: [PATCH 2/2] gnu: pypy3: Various package fixes.

* gnu/packages/python.scm (pypy3) [patches]: Add new patch.
[inputs]: Remove bash-minimal and add nss-certs.
[native-inputs]: Remove nss-certs here.
[arguments]: Use gdbm compat library, add 2to3 binary.
[native-search-paths]: Add search path.
* gnu/packages/patches/pypy3-7.3.1-ssl-paths.patch: New file.
* gnu/local.mk: Add it.
---
 gnu/local.mk                                  |   1 +
 .../patches/pypy3-7.3.1-ssl-paths.patch       |  41 +++++++
 gnu/packages/python.scm                       | 109 ++++++++++++------
 3 files changed, 116 insertions(+), 35 deletions(-)
 create mode 100644 gnu/packages/patches/pypy3-7.3.1-ssl-paths.patch

diff --git a/gnu/local.mk b/gnu/local.mk
index 91a3295e75..b644391a11 100644
--- a/gnu/local.mk
+++ b/gnu/local.mk
@@ -1536,6 +1536,7 @@ dist_patch_DATA =						\
   %D%/packages/patches/python-unittest2-remove-argparse.patch	\
   %D%/packages/patches/python-waitress-fix-tests.patch		\
   %D%/packages/patches/pypy3-7.3.1-fix-tests.patch		\
+  %D%/packages/patches/pypy3-7.3.1-ssl-paths.patch		\
   %D%/packages/patches/qemu-build-info-manual.patch		\
   %D%/packages/patches/qemu-glibc-2.27.patch 			\
   %D%/packages/patches/qrcodegen-cpp-make-install.patch		\
diff --git a/gnu/packages/patches/pypy3-7.3.1-ssl-paths.patch b/gnu/packages/patches/pypy3-7.3.1-ssl-paths.patch
new file mode 100644
index 0000000000..d21133b4ae
--- /dev/null
+++ b/gnu/packages/patches/pypy3-7.3.1-ssl-paths.patch
@@ -0,0 +1,41 @@
+Fix default certificate search path, still allowing the user to override it
+with environment variables.
+
+--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
++++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+@@ -1679,20 +1679,9 @@ def get_default_verify_paths():
+      https://golang.org/src/crypto/x509/root_linux.go (for the files)
+     '''
+     certFiles = [
+-        "/etc/ssl/certs/ca-certificates.crt",                # Debian/Ubuntu/Gentoo etc.
+-        "/etc/pki/tls/certs/ca-bundle.crt",                  # Fedora/RHEL 6
+-        "/etc/ssl/ca-bundle.pem",                            # OpenSUSE
+-        "/etc/pki/tls/cacert.pem",                           # OpenELEC
+-        "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", # CentOS/RHEL 7
+-        "/etc/ssl/cert.pem",                                 # Alpine Linux
+     ]
+     certDirectories = [
+-        "/etc/ssl/certs",               # SLES10/SLES11
+-        "/system/etc/security/cacerts", # Android
+-        "/usr/local/share/certs",       # FreeBSD
+-        "/etc/pki/tls/certs",           # Fedora/RHEL
+-        "/etc/openssl/certs",           # NetBSD
+-        "/var/ssl/certs",               # AIX
++        "@GUIX_CERT_PATH@",
+     ]
+ 
+     # optimization: reuse the values from a local varaible
+@@ -1707,9 +1696,10 @@ def get_default_verify_paths():
+     ofile = _cstr_decode_fs(lib.X509_get_default_cert_file())
+     odir = _cstr_decode_fs(lib.X509_get_default_cert_dir())
+ 
+-    if os.path.exists(ofile) and os.path.exists(odir):
+-        get_default_verify_paths.retval = (ofile_env, ofile, odir_env, odir)
+-        return get_default_verify_paths.retval
++    if not os.path.exists(ofile):
++        ofile = None
++    if not os.path.exists(odir):
++        odir = None
+ 
+     # OpenSSL didn't supply the goods. Try some other options
+     for f in certFiles:
diff --git a/gnu/packages/python.scm b/gnu/packages/python.scm
index 8ef8ae2e1d..c0bd3335e3 100644
--- a/gnu/packages/python.scm
+++ b/gnu/packages/python.scm
@@ -696,7 +696,8 @@ ease from the desktop to a microcontroller or embedded system.")
               (sha256
                (base32
                 "03f1fdw6yk2mypa9pbmgk26r8y1hhmw801l6g36zry9zsvz7aqgx"))
-              (patches (search-patches "pypy3-7.3.1-fix-tests.patch"))))
+              (patches (search-patches "pypy3-7.3.1-fix-tests.patch"
+                                       "pypy3-7.3.1-ssl-paths.patch"))))
     (build-system gnu-build-system)
     (native-inputs
      `(("python-2" ,python-2)
@@ -704,13 +705,13 @@ ease from the desktop to a microcontroller or embedded system.")
        ("tar" ,tar)                     ; Required for package.py
        ("python2-pycparser" ,python2-pycparser)
        ("python2-hypothesis" ,python2-hypothesis)
-       ("nss-certs" ,nss-certs)         ; For ssl tests
        ("gzip" ,gzip)))
     (inputs
      `(("libffi" ,libffi)
        ("zlib" ,zlib)
        ("ncurses" ,ncurses)
        ("openssl" ,openssl)
+       ("nss-certs" ,nss-certs)         ; For ssl module
        ("expat" ,expat)
        ("bzip2" ,bzip2)
        ("sqlite" ,sqlite)
@@ -718,10 +719,9 @@ ease from the desktop to a microcontroller or embedded system.")
        ("tcl" ,tcl)
        ("tk" ,tk)
        ("glibc" ,glibc)
-       ("bash-minimal" ,bash-minimal)   ; Used as /bin/sh
        ("xz" ,xz)))                     ; liblzma
     (arguments
-     `(#:tests? #f     ;FIXME: Disabled for now, there are many tests failing.
+     `(#:tests? #f ;FIXME: Disabled for now, there are many tests failing.
        #:modules ((ice-9 ftw) (ice-9 match)
                   (guix build utils) (guix build gnu-build-system))
        #:phases (modify-phases %standard-phases
@@ -750,6 +750,11 @@ ease from the desktop to a microcontroller or embedded system.")
                       (substitute* '("lib_pypy/_curses_build.py")
                         ;; Find curses
                         (("/usr/local") (assoc-ref inputs "ncurses")))
+                      (substitute* '("lib_pypy/_dbm.py")
+                        ;; Use gdbm compat library, so we don’t need to pull in bdb
+                        (("ctypes.util.find_library\\('db'\\)")
+                         (string-append "'" (assoc-ref inputs "gdbm")
+                                        "/lib/libgdbm_compat.so'")))
                       (substitute* '("lib_pypy/_sqlite3_build.py")
                         ;; Always use search paths
                         (("sys\\.platform\\.startswith\\('freebsd'\\)") "True")
@@ -761,11 +766,18 @@ ease from the desktop to a microcontroller or embedded system.")
                                         "/lib/libsqlite3.so.0'")))
                       (substitute* '("lib-python/3/subprocess.py")
                         ;; Fix shell path
-                        (("/bin/sh")
-                         (string-append (assoc-ref inputs "bash-minimal") "/bin/sh")))
+                        (("/bin/sh") (which "sh")))
                       (substitute* '("lib-python/3/distutils/unixccompiler.py")
                         ;; gcc-toolchain does not provide symlink cc -> gcc
                         (("\"cc\"") "\"gcc\""))
+                      (substitute* '("lib_pypy/_cffi_ssl/_stdssl/__init__.py")
+                        ;; Add nss-certs to default certificate search path,
+                        ;; otherwise every packages has to specify nss-certs and
+                        ;; openssl as input to set the proper env variables.
+                        ;; Depends on -ssl-paths.patch.
+                        (("@GUIX_CERT_PATH@")
+                         (string-append (assoc-ref inputs "nss-certs")
+                                        "/etc/ssl/certs")))
                       #t))
                   (add-after
                       'unpack 'set-source-file-times-to-1980
@@ -785,7 +797,8 @@ ease from the desktop to a microcontroller or embedded system.")
                                 (string-append "--make-jobs="
                                                (number->string (parallel-job-count)))
                                 "-Ojit"
-                                "targetpypystandalone"))
+                                "targetpypystandalone"
+                                "--allworkingmodules"))
                       ;; Build c modules and package everything, so tests work.
                       (with-directory-excursion "pypy/tool/release"
                         (unsetenv "PYTHONPATH") ; Do not use the system’s python libs:
@@ -793,7 +806,12 @@ ease from the desktop to a microcontroller or embedded system.")
                                         ; attribute 'IntFlag'
                         (invoke "python2" "package.py"
                                 "--archive-name" "pypy-dist"
-                                "--builddir" (getcwd)))))
+                                "--builddir" (getcwd))
+                        ;; install pip and setuptools into the dist directory.
+                        ;; XXX: Breaks virtualenv, because it does not set +w
+                        ;; on files copied from the store.
+                        ;(invoke "pypy-dist/bin/pypy3" "-m" "ensurepip")
+                        )))
                   (replace 'check
                     (lambda* (#:key tests? #:allow-other-keys)
                       (if tests?
@@ -811,32 +829,54 @@ ease from the desktop to a microcontroller or embedded system.")
                       #t))
                   (replace 'install
                     (lambda* (#:key inputs outputs #:allow-other-keys)
-                      (with-directory-excursion "pypy/tool/release"
-                        ;; Delete test data.
-                        (for-each
-                         (lambda (x)
-                           (delete-file-recursively (string-append
-                                                     "pypy-dist/lib-python/3/" x)))
-                         '("tkinter/test"
-                           "test"
-                           "sqlite3/test"
-                           "lib2to3/tests"
-                           "idlelib/idle_test"
-                           "distutils/tests"
-                           "ctypes/test"
-                           "unittest/test"))
-                        ;; Patch shebang referencing python2
-                        (substitute* '("pypy-dist/lib-python/3/cgi.py"
-                                       "pypy-dist/lib-python/3/encodings/rot_13.py")
-                          (("#!.+/bin/python")
-                           (string-append "#!" (assoc-ref outputs "out") "/bin/pypy3")))
-                        (with-fluids ((%default-port-encoding "ISO-8859-1"))
-                          (substitute* '("pypy-dist/lib_pypy/_md5.py"
-                                         "pypy-dist/lib_pypy/_sha1.py")
-                            (("#!.+/bin/python")
-                             (string-append "#!" (assoc-ref outputs "out") "/bin/pypy3"))))
-                        (copy-recursively "pypy-dist" (assoc-ref outputs "out")))
-                      #t)))))
+                      (let* ((out (assoc-ref outputs "out"))
+                             (bin-pypy3 (string-append out "/bin/pypy3"))
+                             (shebang-match-python "#!.+/bin/python")
+                             (shebang-pypy3 (string-append "#!" bin-pypy3))
+                             (dist-dir "pypy/tool/release/pypy-dist"))
+                        (with-directory-excursion dist-dir
+                          ;; Delete test data.
+                          (for-each
+                           (lambda (x)
+                             (delete-file-recursively (string-append
+                                                       "lib-python/3/" x)))
+                           '("tkinter/test"
+                             "test"
+                             "sqlite3/test"
+                             "lib2to3/tests"
+                             "idlelib/idle_test"
+                             "distutils/tests"
+                             "ctypes/test"
+                             "unittest/test"))
+                          ;; Patch shebang referencing python2
+                          (substitute* '("lib-python/3/cgi.py"
+                                         "lib-python/3/encodings/rot_13.py")
+                            ((shebang-match-python) shebang-pypy3))
+                          (with-fluids ((%default-port-encoding "ISO-8859-1"))
+                                       (substitute* '("lib_pypy/_md5.py"
+                                                      "lib_pypy/_sha1.py")
+                                         ((shebang-match-python) shebang-pypy3))))
+                        (copy-recursively dist-dir out)
+                        ;; Make sure pypy3 is callable as python/python3, so we
+                        ;; don’t have to patch every single package.
+                        (symlink bin-pypy3 (string-append out "/bin/python"))
+                        (symlink bin-pypy3 (string-append out "/bin/python3"))
+                        ;; 2to3 is missing from pypy3, create it.
+                        (let ((2to3 (string-append out "/bin/2to3")))
+                          (call-with-output-file 2to3
+                            (lambda (port)
+                              (format port "#!~a~%" (string-append out "/bin/pypy3"))
+                              (format port "
+import sys
+from lib2to3.main import main
+
+sys.exit(main('lib2to3.fixes'))")))
+                          (chmod 2to3 #o755))
+                        #t))))))
+    (native-search-paths
+     (list (search-path-specification
+            (variable "PYTHONPATH")
+            (files '("lib/pypy3.6/site-packages")))))
     (home-page "https://www.pypy.org/")
     (synopsis "Python implementation with just-in-time compilation")
     (description "PyPy is a faster, alternative implementation of the Python
@@ -849,4 +889,3 @@ Python code natively, including C extensions.")
                    license:bsd-3 ; lib_pypy/cffi/_pycparser/ply/
                    (license:non-copyleft
                     "http://www.unicode.org/copyright.html")))))
-
-- 
2.26.2