Subject: Re: [bug#38619] Byte compilation of Emacs autoloads

From: Maxim Cournoyer <[email-masked]>
Date: Thu, 27 Feb 2020 16:53:52 -0500

Hello Pierre,

Pierre Neidhardt <[email-masked]> writes:

I've now tested this change with the extensive manifest (attached).

I've built two profiles, like this:

(before change)
./pre-inst-env guix package -m emacs-pkg.txt -p /tmp/emacs-ref

(after change)
./pre-inst-env guix package -m emacs-pkg.txt -p /tmp/emacs-byte-compiled

Then proceeded to benchmark them like:

--8<---------------cut here---------------start------------->8---
env -i sh -c "source /tmp/emacs-ref/etc/profile && time emacs --batch --eval '()'"
--8<---------------cut here---------------end--------------->8---

Which returned:

--8<---------------cut here---------------start------------->8---
Loading /gnu/store/ipfaivbqjxfnda970w558vpihd84lqwi-profile/share/emacs/site-lisp/treepy-autoloads.el (source)...
Loading /gnu/store/ipfaivbqjxfnda970w558vpihd84lqwi-profile/share/emacs/site-lisp/ts-autoloads.el (source)...
Loading /gnu/store/ipfaivbqjxfnda970w558vpihd84lqwi-profile/share/emacs/site-lisp/tuareg-autoloads.el (source)...
Eager macro-expansion failure: (file-missing "Searching for program" "No such file or directory" "git")
Searching for program: No such file or directory, git

real    0m0.406s
user    0m0.361s
sys     0m0.044s
--8<---------------cut here---------------end--------------->8---

Now with the patches here:
--8<---------------cut here---------------start------------->8---
Loading /gnu/store/l09fascj5xjhbgi3nzc1bhrprdy44y9g-profile/share/emacs/site-lisp/treepy-autoloads...
Loading /gnu/store/l09fascj5xjhbgi3nzc1bhrprdy44y9g-profile/share/emacs/site-lisp/ts-autoloads...
Loading /gnu/store/l09fascj5xjhbgi3nzc1bhrprdy44y9g-profile/share/emacs/site-lisp/tuareg-autoloads.el (source)...
Eager macro-expansion failure: (file-missing "Searching for program" "No such file or directory" "git")
Searching for program: No such file or directory, git

real    0m0.287s
user    0m0.251s
sys     0m0.036s
--8<---------------cut here---------------end--------------->8---

That's a decent speedup :-)

I'll push this to master shortly.

Maxim

From 7ee79cdaa04a2531adf64cdac38c565774e24da5 Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <[email-masked]>
Date: Thu, 5 Dec 2019 13:40:32 +0900
Subject: [PATCH 1/7] gnu: emacs: Byte compile the site-lisp directory.

* gnu/packages/emacs.scm (emacs): Use the (gnu build emacs-utils) module.
Byte compile the site-lisp directory as part of the 'install-site-start'
phase.
(emacs-minimal, emacs-xwidgets, emacs-no-x, emacs-no-x-toolkit): Reset modules
to those of the gnu-build-system.
---
 gnu/packages/emacs.scm | 43 ++++++++++++++++++++++++++++++++++++++----
 1 file changed, 39 insertions(+), 4 deletions(-)

diff --git a/gnu/packages/emacs.scm b/gnu/packages/emacs.scm
index b26c928e87..18c8549cb9 100644
--- a/gnu/packages/emacs.scm
+++ b/gnu/packages/emacs.scm
@@ -129,6 +129,11 @@
      `(#:tests? #f                      ; no check target
        #:configure-flags (list "--with-modules"
                                "--disable-build-details")
+       #:modules ((guix build emacs-utils)
+                  (guix build glib-or-gtk-build-system)
+                  (guix build utils))
+       #:imported-modules ((guix build emacs-utils)
+                           ,@%glib-or-gtk-build-system-modules)
        #:phases
        (modify-phases %standard-phases
          (add-before 'configure 'fix-/bin/pwd
@@ -145,7 +150,8 @@
            ;; automatically found.
            (lambda* (#:key inputs outputs #:allow-other-keys)
              (let* ((out      (assoc-ref outputs "out"))
-                    (lisp-dir (string-append out "/share/emacs/site-lisp")))
+                    (lisp-dir (string-append out "/share/emacs/site-lisp"))
+                    (emacs (string-append out "/bin/emacs")))
                (copy-file (assoc-ref inputs "guix-emacs.el")
                           (string-append lisp-dir "/guix-emacs.el"))
                (with-output-to-file (string-append lisp-dir "/site-start.el")
@@ -158,6 +164,9 @@
                ;; share/emacs/site-lisp union when added to EMACSLOADPATH,
                ;; which leads to conflicts.
                (delete-file (string-append lisp-dir "/subdirs.el"))
+               ;; Byte compile the site-start files.
+               (parameterize ((%emacs emacs))
+                 (emacs-byte-compile-directory lisp-dir))
                #t))))))
     (inputs
      `(("gnutls" ,gnutls)
@@ -326,6 +335,12 @@ languages.")
     (build-system gnu-build-system)
     (arguments
      (substitute-keyword-arguments (package-arguments emacs)
+       ((#:modules _)
+        `((guix build emacs-utils)
+          ,@%gnu-build-system-modules))
+       ((#:imported-modules _)
+        `((guix build emacs-utils)
+          ,@%gnu-build-system-modules))
        ((#:phases phases)
         `(modify-phases ,phases
            (delete 'install-site-start)))
@@ -345,6 +360,12 @@ editor (with xwidgets support)")
     (build-system gnu-build-system)
     (arguments
      (substitute-keyword-arguments (package-arguments emacs)
+       ((#:modules _)
+        `((guix build emacs-utils)
+          ,@%gnu-build-system-modules))
+       ((#:imported-modules _)
+        `((guix build emacs-utils)
+          ,@%gnu-build-system-modules))
        ((#:configure-flags flags ''())
         `(cons "--with-xwidgets" ,flags))))
     (inputs
@@ -358,6 +379,14 @@ editor (with xwidgets support)")
     (synopsis "The extensible, customizable, self-documenting text
 editor (console only)")
     (build-system gnu-build-system)
+    (arguments
+     (substitute-keyword-arguments (package-arguments emacs)
+       ((#:modules _)
+        `((guix build emacs-utils)
+          ,@%gnu-build-system-modules))
+       ((#:imported-modules _)
+        `((guix build emacs-utils)
+          ,@%gnu-build-system-modules))))
     (inputs (fold alist-delete
                   (package-inputs emacs)
                   '("libx11" "gtk+" "libxft" "libtiff" "giflib" "libjpeg"
@@ -376,9 +405,15 @@ editor (without an X toolkit)" )
     (inputs (append `(("inotify-tools" ,inotify-tools))
                     (alist-delete "gtk+" (package-inputs emacs))))
     (arguments
-     `(,@(substitute-keyword-arguments (package-arguments emacs)
-           ((#:configure-flags cf)
-            `(cons "--with-x-toolkit=no" ,cf)))))))
+     (substitute-keyword-arguments (package-arguments emacs)
+       ((#:modules _)
+        `((guix build emacs-utils)
+          ,@%gnu-build-system-modules))
+       ((#:imported-modules _)
+        `((guix build emacs-utils)
+          ,@%gnu-build-system-modules))
+       ((#:configure-flags cf)
+        `(cons "--with-x-toolkit=no" ,cf))))))
 
 (define-public guile-emacs
   (let ((commit "41120e0f595b16387eebfbf731fff70481de1b4b")
-- 
2.25.0

From 17c0eb0499196600ceb516ad67b010c281e1b188 Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <[email-masked]>
Date: Fri, 6 Dec 2019 15:12:32 +0900
Subject: [PATCH 2/7] emacs-build-system: Byte compile the autoload files.

* guix/build/emacs-build-system.scm (enable-autoloads-compilation)
(validate-compiled-autoloads): Add procedures.
(%standard-phases): Register the new procedures.
* gnu/packages/aux-files/emacs/guix-emacs.el (guix-emacs-find-autoloads):
Remove duplicates in the list of autoload files found.
* guix/build/emacs-utils.scm (expr->string): Add procedure.
(emacs-batch-eval, emacs-batch-edit-file): Use it.
---
 gnu/packages/aux-files/emacs/guix-emacs.el |  5 +++--
 guix/build/emacs-build-system.scm          | 22 ++++++++++++++++++++--
 guix/build/emacs-utils.scm                 | 10 ++++++++--
 3 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/gnu/packages/aux-files/emacs/guix-emacs.el b/gnu/packages/aux-files/emacs/guix-emacs.el
index 05fc9709b6..25a87ee52d 100644
--- a/gnu/packages/aux-files/emacs/guix-emacs.el
+++ b/gnu/packages/aux-files/emacs/guix-emacs.el
@@ -35,8 +35,9 @@
   "Return a list of Emacs 'autoloads' files in DIRECTORY.
 The files in the list do not have extensions (.el, .elc)."
   ;; `directory-files' doesn't honor group in regexp.
-  (mapcar #'file-name-sans-extension
-          (directory-files directory 'full-name guix-emacs-autoloads-regexp)))
+  (delete-dups (mapcar #'file-name-sans-extension
+                       (directory-files directory 'full-name
+                                        guix-emacs-autoloads-regexp))))
 
 ;;;###autoload
 (defun guix-emacs-autoload-packages ()
diff --git a/guix/build/emacs-build-system.scm b/guix/build/emacs-build-system.scm
index 09de244993..219310cf08 100644
--- a/guix/build/emacs-build-system.scm
+++ b/guix/build/emacs-build-system.scm
@@ -225,6 +225,21 @@ parallel. PARALLEL-TESTS? is ignored when using a non-make TEST-COMMAND."
     (parameterize ((%emacs emacs))
       (emacs-generate-autoloads elpa-name site-lisp))))
 
+(define* (enable-autoloads-compilation #:key outputs #:allow-other-keys)
+  "Remove the NO-BYTE-COMPILATION local variable embedded in the generated
+autoload files."
+  (let* ((out (assoc-ref outputs "out"))
+         (autoloads (find-files out "-autoloads.el$")))
+    (substitute* autoloads
+      ((";; no-byte-compile.*") ""))
+    #t))
+
+(define* (validate-compiled-autoloads #:key outputs #:allow-other-keys)
+  "Verify whether the byte compiled autoloads load fine."
+  (let* ((out (assoc-ref outputs "out"))
+         (autoloads (find-files out "-autoloads.elc$")))
+    (emacs-batch-eval (format #f "(mapc #'load '~s)" autoloads))))
+
 (define (emacs-package? name)
   "Check if NAME correspond to the name of an Emacs package."
   (string-prefix? "emacs-" name))
@@ -253,10 +268,13 @@ second hyphen.  This corresponds to 'name-version' as used in ELPA packages."
     (replace 'check check)
     (replace 'install install)
     (add-after 'install 'make-autoloads make-autoloads)
-    (add-after 'make-autoloads 'patch-el-files patch-el-files)
+    (add-after 'make-autoloads 'enable-autoloads-compilation
+      enable-autoloads-compilation)
+    (add-after 'enable-autoloads-compilation 'patch-el-files patch-el-files)
     ;; The .el files are byte compiled directly in the store.
     (add-after 'patch-el-files 'build build)
-    (add-after 'build 'move-doc move-doc)))
+    (add-after 'build 'validate-compiled-autoloads validate-compiled-autoloads)
+    (add-after 'validate-compiled-autoloads 'move-doc move-doc)))
 
 (define* (emacs-build #:key inputs (phases %standard-phases)
                       #:allow-other-keys #:rest args)
diff --git a/guix/build/emacs-utils.scm b/guix/build/emacs-utils.scm
index 885fd0a217..ab64e3714c 100644
--- a/guix/build/emacs-utils.scm
+++ b/guix/build/emacs-utils.scm
@@ -41,16 +41,22 @@
   ;; The `emacs' command.
   (make-parameter "emacs"))
 
+(define (expr->string expr)
+  "Converts EXPR, an expression, into a string."
+  (if (string? expr)
+      expr
+      (format #f "~s" expr)))
+
 (define (emacs-batch-eval expr)
   "Run Emacs in batch mode, and execute the elisp code EXPR."
   (invoke (%emacs) "--quick" "--batch"
-          (format #f "--eval=~S" expr)))
+          (string-append "--eval=" (expr->string expr))))
 
 (define (emacs-batch-edit-file file expr)
   "Load FILE in Emacs using batch mode, and execute the elisp code EXPR."
   (invoke (%emacs) "--quick" "--batch"
           (string-append "--visit=" file)
-          (format #f "--eval=~S" expr)))
+          (string-append "--eval=" (expr->string expr))))
 
 (define (emacs-batch-disable-compilation file)
   (emacs-batch-edit-file file
-- 
2.25.0

From 8f4044b385252b914feff6584ac683e16935006e Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <[email-masked]>
Date: Thu, 27 Feb 2020 14:20:58 -0500
Subject: [PATCH 3/7] gnu: emacs: Fix comment.

* gnu/packages/emacs.scm (emacs): Fix comment.
---
 gnu/packages/emacs.scm | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/gnu/packages/emacs.scm b/gnu/packages/emacs.scm
index 18c8549cb9..fd9c7fa875 100644
--- a/gnu/packages/emacs.scm
+++ b/gnu/packages/emacs.scm
@@ -144,10 +144,8 @@
                 "pwd"))
              #t))
          (add-after 'install 'install-site-start
-           ;; Use 'guix-emacs' in "site-start.el".  This way, Emacs packages
-           ;; provided by Guix and installed in
-           ;; ~/.guix-profile/share/emacs/site-lisp/guix.d/PACKAGE-VERSION are
-           ;; automatically found.
+           ;; Use 'guix-emacs' in "site-start.el", which is used autoload the
+           ;; Elisp packages found in EMACSLOADPATH.
            (lambda* (#:key inputs outputs #:allow-other-keys)
              (let* ((out      (assoc-ref outputs "out"))
                     (lisp-dir (string-append out "/share/emacs/site-lisp"))
-- 
2.25.0

From 9a91d7018ea0d2749f88014c7dd0d1a4bfb99fba Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <[email-masked]>
Date: Sat, 7 Dec 2019 02:47:25 +0900
Subject: [PATCH 4/7] gnu: emacs-cl-generic: Disable byte compilation of its
 autoload file.

*gnu/packages/emacs-xyz.scm (emacs-cl-generic)[phases]
{enable-autoloads-compilation}: Delete.
---
 gnu/packages/emacs-xyz.scm | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/gnu/packages/emacs-xyz.scm b/gnu/packages/emacs-xyz.scm
index 76b9746f69..7235ef611a 100644
--- a/gnu/packages/emacs-xyz.scm
+++ b/gnu/packages/emacs-xyz.scm
@@ -12066,6 +12066,10 @@ match and total match information in the mode-line in various search modes.")
         (base32
          "0vb338bhjpsnrf60qgxny4z5rjrnifahnrv9axd4shay89d894zq"))))
     (build-system emacs-build-system)
+    ;; Byte compilation of the autoload file fails.
+    (arguments
+     `(#:phases (modify-phases %standard-phases
+                  (delete 'enable-autoloads-compilation))))
     (home-page "https://elpa.gnu.org/packages/seq.html")
     (synopsis
      "Forward @code{cl-generic} compatibility for Emacs before version 25")
-- 
2.25.0

From 070c392b7af00b054ee4fe86548d621fbba8d1bb Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <[email-masked]>
Date: Thu, 27 Feb 2020 16:00:58 -0500
Subject: [PATCH 5/7] gnu: emacs-sly-macrostep: Remove the
 enable-autoloads-compilation phase.

* gnu/packages/emacs-xyz.scm (emacs-sly-macrostep)[phases]
{enable-autoloads-compilation}: Remove.
---
 gnu/packages/emacs-xyz.scm | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/gnu/packages/emacs-xyz.scm b/gnu/packages/emacs-xyz.scm
index 7235ef611a..9bb636f75e 100644
--- a/gnu/packages/emacs-xyz.scm
+++ b/gnu/packages/emacs-xyz.scm
@@ -7567,7 +7567,8 @@ file.")
          #:phases
          ;; The package provides autoloads.
          (modify-phases %standard-phases
-           (delete 'make-autoloads))))
+           (delete 'make-autoloads)
+           (delete 'enable-autoloads-compilation))))
       (synopsis "Expand Common Lisp macros inside source files with SLY")
       (description
        "@command{sly-macrostep} is a SLY contrib for expanding CL macros right
-- 
2.25.0

From 2e6c80424943ce8a14fe901dfc5db4afeda64006 Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <[email-masked]>
Date: Thu, 27 Feb 2020 16:03:33 -0500
Subject: [PATCH 6/7] gnu: emacs-sly-quicklisp: Remove the
 enable-autoloads-compilation phase.

* gnu/packages/emacs-xyz.scm (emacs-sly-quicklisp)[phases]
{enable-autoloads-compilation}: Remove.
---
 gnu/packages/emacs-xyz.scm | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/gnu/packages/emacs-xyz.scm b/gnu/packages/emacs-xyz.scm
index 9bb636f75e..faf3a55ba1 100644
--- a/gnu/packages/emacs-xyz.scm
+++ b/gnu/packages/emacs-xyz.scm
@@ -7476,7 +7476,8 @@ xref, etc...) are still available, but with better integration.")
          #:phases
          ;; The package provides autoloads.
          (modify-phases %standard-phases
-           (delete 'make-autoloads))))
+           (delete 'make-autoloads)
+           (delete 'enable-autoloads-compilation))))
       (synopsis "Quicklisp support for SLY")
       (description
        "@command{sly-quicklisp} is an external contrib for SLY that provides a
-- 
2.25.0

From 8e6dfcf318d3f88ae131b737483c1b3bd0139f2e Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <[email-masked]>
Date: Thu, 27 Feb 2020 16:07:06 -0500
Subject: [PATCH 7/7] gnu: emacs-sly-named-readtables: Disable byte compilation
 of its autoload file.

*gnu/packages/emacs-xyz.scm (emacs-sly-named-readtables)[phases]
{enable-autoloads-compilation}: Delete.
---
 gnu/packages/emacs-xyz.scm | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/gnu/packages/emacs-xyz.scm b/gnu/packages/emacs-xyz.scm
index faf3a55ba1..ca8392c896 100644
--- a/gnu/packages/emacs-xyz.scm
+++ b/gnu/packages/emacs-xyz.scm
@@ -7534,7 +7534,10 @@ additional support for working with ASDF projects.")
       (propagated-inputs
        `(("emacs-sly" ,emacs-sly)))
       (arguments
-       '(#:include (cons* "\\.lisp$" "\\.asd$" %default-include)))
+       '(#:include (cons* "\\.lisp$" "\\.asd$" %default-include)
+         #:phases (modify-phases %standard-phases
+                    ;; Byte compilation of the autoload file fails.
+                    (delete 'enable-autoloads-compilation))))
       (synopsis "Named-readtables support for SLY")
       (description
        "@command{sly-named-readtables} is an external contrib for SLY that
-- 
2.25.0