Subject: bug#78995: [PATCH] ;;;autoload-expand for special macros

From: "J.D. Smith" <[email-masked]>
Date: Tue, 15 Jul 2025 23:35:54 -0400

Here is an update on progress.  The attached is working reasonably well,
reproducing lisp/loaddefs.el with a few changes (see attached).  I went
with (declare ((autoload . macro-expand))) on the notion that there
might be other autoload-related declarations in the future, but open to
suggestion.

I've now fully eliminated list 1 in favor of declare statements, but
also had to handle the case of _aliased_ macros, e.g. all the
[easy-mmode]-define-global[ized]-minor-mode yada yada.

There are a couple things I need to check on.

1. Non-repeatable loaddefs:

When you re-run make and it needs to regenerate lisp/loaddefs.el, it
appears easy-mmode is _not_ loaded by whatever emacs process conducts
that subsequent autoload-generation round.  Consequently,
`define-minor-mode' isn't loaded, so calls to it produce only partial
autoload entries (via, I believe, list 2).  Maybe that's why list 2
duplicates entries from list 1, as a backup against this non-repeatable
build.  But it seems strange to have different packages loaded on
different passes of the same Makefile target.  You can confirm this by:

% mv lisp/loaddefs.el lisp/loaddefs_orig.el
% make
% diff -u lisp/loaddefs_orig.el lisp/loaddefs.el

I'm not clear if this has anything to do with this patch, or could even
be a pre-existing build bug.  Once you are in this situation, you
basically have to `make bootstrap' to get the full and complete autoload
statements to appear again in loaddefs.

2. Some files are now being loaded during autoloads generation:

I noticed a few files among bundled lisp packages that now get loaded,
because they autoload undefined macros:

   loaddefs-gen: loading file frameset (cl-defun)
   Loading frameset...
     INFO     Scraping 1565 files for loaddefs...15% 
   loaddefs-gen: loading file ede/cpp-root (defclass)
   Loading ede/cpp-root...
     INFO     Scraping 1565 files for loaddefs...25% 
     INFO     Scraping 1565 files for loaddefs...47% 
     INFO     Scraping 1565 files for loaddefs...64% 
   loaddefs-gen: loading file tramp-adb (tramp--with-startup)
   Loading tramp-adb...
     INFO     Scraping 1565 files for loaddefs...80% 
   loaddefs-gen: loading file grep (define-compilation-mode)
   Loading grep...

These mostly seem to be innocuous in that after loading them, it becomes
clear that these particular autoloaded macros don't request
macro-expansion, so nothing different happens.  But it does represent a
few new "internal" load calls during autoload generation.  I think list
1 is now "doing more work" because of this pre-load mechanism.

Thoughts and suggestions very welcome.

diff --git a/lisp/emacs-lisp/byte-run.el b/lisp/emacs-lisp/byte-run.el
index 366423904db..d2781021003 100644
--- a/lisp/emacs-lisp/byte-run.el
+++ b/lisp/emacs-lisp/byte-run.el
@@ -271,6 +271,12 @@ 'byte-run--set-debug
 	    (list 'put (list 'quote name)
 		  ''edebug-form-spec (list 'quote spec)))))
 
+(defalias 'byte-run--set-autoload
+  #'(lambda (name _args spec)
+      (list 'progn :autoload-end
+            (list 'function-put (list 'quote name)
+	          ''autoload (list 'quote spec)))))
+
 (defalias 'byte-run--set-no-font-lock-keyword
   #'(lambda (name _args val)
       (list 'function-put (list 'quote name)
@@ -350,8 +356,10 @@ macro-declarations-alist
   (cons
    (list 'debug #'byte-run--set-debug)
    (cons
-    (list 'no-font-lock-keyword #'byte-run--set-no-font-lock-keyword)
-    defun-declarations-alist))
+    (list 'autoload #'byte-run--set-autoload)
+    (cons
+     (list 'no-font-lock-keyword #'byte-run--set-no-font-lock-keyword)
+     defun-declarations-alist)))
   "List associating properties of macros to their macro expansion.
 Each element of the list takes the form (PROP FUN) where FUN is a function.
 For each (PROP . VALUES) in a macro's declaration, the FUN corresponding
@@ -397,6 +405,7 @@ 'defmacro
            (if declarations
 	       (cons 'prog1 (cons def (car declarations)))
 	     def))))))
+(function-put 'defmacro 'autoload 'macro-expand) ; Since we cannot `declare' it
 
 ;; Now that we defined defmacro we can use it!
 (defmacro defun (name arglist &rest body)
@@ -409,7 +418,7 @@ defun
 The return value is undefined.
 
 \(fn NAME ARGLIST [DOCSTRING] [DECL] [INTERACTIVE] BODY...)"
-  (declare (doc-string 3) (indent 2))
+  (declare (doc-string 3) (indent 2) (autoload macro-expand))
   (or name (error "Cannot define '%s' as a function" name))
   (if (null
        (and (listp arglist)
diff --git a/lisp/emacs-lisp/cl-generic.el b/lisp/emacs-lisp/cl-generic.el
index 3aa26fba3c3..1546da08ec2 100644
--- a/lisp/emacs-lisp/cl-generic.el
+++ b/lisp/emacs-lisp/cl-generic.el
@@ -258,7 +258,8 @@ cl-defgeneric
                                cl--generic-edebug-make-name in:method]
                               lambda-doc
                               def-body)]]
-             def-body)))
+             def-body))
+           (autoload macro-expand))
   (let* ((doc (if (stringp (car-safe options-and-methods))
                   (pop options-and-methods)))
          (declarations nil)
diff --git a/lisp/emacs-lisp/cl-macs.el b/lisp/emacs-lisp/cl-macs.el
index 2a0a9e5c6de..27db30f414e 100644
--- a/lisp/emacs-lisp/cl-macs.el
+++ b/lisp/emacs-lisp/cl-macs.el
@@ -398,7 +398,8 @@ cl-defun
                      [&optional ("interactive" interactive)]
                      def-body))
            (doc-string 3)
-           (indent 2))
+           (indent 2)
+           (autoload macro-expand))
   `(defun ,name ,@(cl--transform-lambda (cons args body) name)))
 
 ;;;###autoload
@@ -416,7 +417,8 @@ cl-iter-defun
                      [&optional ("interactive" interactive)]
                      def-body))
            (doc-string 3)
-           (indent 2))
+           (indent 2)
+           (autoload macro-expand))
   (require 'generator)
   `(iter-defun ,name ,@(cl--transform-lambda (cons args body) name)))
 
@@ -475,7 +477,8 @@ cl-defmacro
   (declare (debug
             (&define name cl-macro-list cl-declarations-or-string def-body))
            (doc-string 3)
-           (indent 2))
+           (indent 2)
+           (autoload macro-expand))
   `(defmacro ,name ,@(cl--transform-lambda (cons args body) name)))
 
 (def-edebug-elem-spec 'cl-lambda-expr
@@ -3011,7 +3014,8 @@ cl-defstruct
                         sexp])]
              [&optional stringp]
              ;; All the above is for the following def-form.
-             &rest &or symbolp (symbolp &optional def-form &rest sexp))))
+             &rest &or symbolp (symbolp &optional def-form &rest sexp)))
+           (autoload macro-expand))
   (let* ((name (if (consp struct) (car struct) struct))
 	 (warning nil)
 	 (opts (cdr-safe struct))
diff --git a/lisp/emacs-lisp/easy-mmode.el b/lisp/emacs-lisp/easy-mmode.el
index f53db48f0b7..cd36a56a89a 100644
--- a/lisp/emacs-lisp/easy-mmode.el
+++ b/lisp/emacs-lisp/easy-mmode.el
@@ -222,10 +222,11 @@ define-minor-mode
            (indent defun)
            (debug (&define name string-or-null-p
 			   [&optional [&not keywordp] sexp
-			    &optional [&not keywordp] sexp
-			    &optional [&not keywordp] sexp]
+			              &optional [&not keywordp] sexp
+			              &optional [&not keywordp] sexp]
 			   [&rest [keywordp sexp]]
-			   def-body)))
+			   def-body))
+           (autoload macro-expand))
 
   (let* ((last-message (make-symbol "last-message"))
          (mode-name (symbol-name mode))
@@ -487,7 +488,7 @@ define-globalized-minor-mode
 on if the hook has explicitly disabled it.
 
 \(fn GLOBAL-MODE MODE TURN-ON [KEY VALUE]... BODY...)"
-  (declare (doc-string 2) (indent defun))
+  (declare (doc-string 2) (indent defun) (autoload macro-expand))
   (let* ((global-mode-name (symbol-name global-mode))
 	 (mode-name (symbol-name mode))
 	 (pretty-name (easy-mmode-pretty-mode-name mode))
diff --git a/lisp/emacs-lisp/generator.el b/lisp/emacs-lisp/generator.el
index 344e11e245e..0770a66f78d 100644
--- a/lisp/emacs-lisp/generator.el
+++ b/lisp/emacs-lisp/generator.el
@@ -675,7 +675,8 @@ iter-defun
 of values.  Callers can retrieve each value using `iter-next'."
   (declare (indent defun)
            (debug (&define name lambda-list lambda-doc &rest sexp))
-           (doc-string 3))
+           (doc-string 3)
+           (autoload macro-expand))
   (cl-assert lexical-binding)
   (let* ((parsed-body (macroexp-parse-body body))
          (declarations (car parsed-body))
diff --git a/lisp/emacs-lisp/inline.el b/lisp/emacs-lisp/inline.el
index c015e2b57d0..ac0476ce826 100644
--- a/lisp/emacs-lisp/inline.el
+++ b/lisp/emacs-lisp/inline.el
@@ -135,7 +135,7 @@ define-inline
 This is like `defmacro', but has several advantages.
 See Info node `(elisp)Defining Functions' for more details."
   ;; FIXME: How can this work with CL arglists?
-  (declare (indent defun) (debug defun) (doc-string 3))
+  (declare (indent defun) (debug defun) (doc-string 3) (autoload macro-expand))
   (let ((doc (if (stringp (car-safe body)) (list (pop body))))
         (declares (if (eq (car-safe (car-safe body)) 'declare) (pop body)))
         (cm-name (intern (format "%s--inliner" name)))
diff --git a/lisp/emacs-lisp/loaddefs-gen.el b/lisp/emacs-lisp/loaddefs-gen.el
index 0f136df1fe2..b080fae0f40 100644
--- a/lisp/emacs-lisp/loaddefs-gen.el
+++ b/lisp/emacs-lisp/loaddefs-gen.el
@@ -147,7 +147,9 @@ loaddefs-generate--make-autoload
 Returns nil if FORM is not a special autoload form (i.e. a function definition
 or macro definition or a defcustom).
 If EXPANSION is non-nil, we're processing the macro expansion of an
-expression, in which case we want to handle forms differently."
+expression, in which case we want to handle forms differently.  Note
+that macros can request expansion by including `(autoload macro-expand)'
+among their `declare' forms."
   (let ((car (car-safe form)) expand)
     (cond
      ((and expansion (eq car 'defalias))
@@ -196,21 +198,31 @@ loaddefs-generate--make-autoload
                                       (cdr form)))))
           (when exps (cons 'progn exps)))))
 
-     ;; For complex cases, try again on the macro-expansion.
-     ((and (memq car '(easy-mmode-define-global-mode define-global-minor-mode
-                       define-globalized-minor-mode defun defmacro
-		       easy-mmode-define-minor-mode define-minor-mode
-                       define-inline cl-defun cl-defmacro cl-defgeneric
-                       cl-defstruct pcase-defmacro iter-defun cl-iter-defun))
-           (macrop car)
-	   (setq expand (let ((load-true-file-name file)
-                              (load-file-name file))
-                          (macroexpand form)))
-	   (memq (car expand) '(progn prog1 defalias)))
+     ;; For macros which request it, try again on their expansion.
+     ((progn
+        ;; If the car is unknown, we load the file first to give
+        ;; packages a chance to define their macros.
+        (unless (or (null car) (macrop car) (functionp car) (special-form-p car)
+                    (alist-get file load-history))
+          (let ((load-path (cons (file-name-directory file) load-path)))
+            (message "loaddefs-gen: loading file %s (%s)" file car)
+            (condition-case e (load file)
+              (error (warn "loaddefs-gen: load error\n\t%s" e)))))
+        (and (macrop car)
+	     (eq 'macro-expand ; a macro can declare (autoload macro-expand)
+		 (or (get car 'autoload)
+		     (get (car-safe (last (function-alias-p car))) 'autoload)))
+	     (setq expand (let ((load-true-file-name file)
+				(load-file-name file))
+			    (macroexpand form)))
+	     (not (eq car (car expand)))))
       ;; Recurse on the expansion.
       (loaddefs-generate--make-autoload expand file 'expansion))
 
-     ;; For special function-like operators, use the `autoload' function.
+     ;; For special function-like operators, use the `autoload'
+     ;; function.
+     ;; TODO: some macros can be removed from here once they set
+     ;;   (declare (... (autoload macro-expand) ...)
      ((memq car '(define-skeleton define-derived-mode
                    define-compilation-mode define-generic-mode
 		   easy-mmode-define-global-mode define-global-minor-mode
diff --git a/lisp/emacs-lisp/pcase.el b/lisp/emacs-lisp/pcase.el
index 73554fd66fd..c116cece684 100644
--- a/lisp/emacs-lisp/pcase.el
+++ b/lisp/emacs-lisp/pcase.el
@@ -537,7 +537,7 @@ pcase-defmacro
 By convention, DOC should use \"EXPVAL\" to stand
 for the result of evaluating EXP (first arg to `pcase').
 \n(fn NAME ARGS [DOC] &rest BODY...)"
-  (declare (indent 2) (debug defun) (doc-string 3))
+  (declare (indent 2) (debug defun) (doc-string 3) (autoload macro-expand))
   ;; Add the function via `fsym', so that an autoload cookie placed
   ;; on a pcase-defmacro will cause the macro to be loaded on demand.
   (let ((fsym (intern (format "%s--pcase-macroexpander" name)))
--- lisp/loaddefs_old.el	2025-07-15 17:56:40
+++ lisp/loaddefs.el	2025-07-15 23:16:42
@@ -5212,6 +5212,8 @@
 
 The mode's hook is called both when the mode is enabled and when it is
 disabled.
+
+\\{compilation-shell-minor-mode-map}
 
 (fn &optional ARG)" t)
 (autoload 'compilation-minor-mode "compile" "\
@@ -5234,6 +5236,8 @@
 
 The mode's hook is called both when the mode is enabled and when it is
 disabled.
+
+\\{compilation-minor-mode-map}
 
 (fn &optional ARG)" t)
 (autoload 'compilation-next-error-function "compile" "\
@@ -8813,7 +8817,7 @@
 A call with prefix PREFIX reads the symbol to insert from the minibuffer with
 completion.
 
-(fn PREFIX)" '("P"))
+(fn PREFIX)" t)
 (autoload 'ebrowse-tags-loop-continue "ebrowse" "\
 Repeat last operation on files in tree.
 FIRST-TIME non-nil means this is not a repetition, but the first time.
@@ -10549,7 +10553,7 @@
 and defers to `erc-compute-port', `erc-compute-user', and
 `erc-compute-nick' for those respective parameters.
 
-(fn &key SERVER PORT NICK USER PASSWORD FULL-NAME ID)" '((let ((erc--display-context `((erc-interactive-display . erc) ,@erc--display-context))) (erc-select-read-args))))
+(fn &key SERVER PORT NICK USER PASSWORD FULL-NAME ID)" t)
 (defalias 'erc-select #'erc)
 (autoload 'erc-tls "erc" "\
 Connect to an IRC server over a TLS-encrypted connection.
@@ -10572,7 +10576,7 @@
 node `(erc) Connecting' for a fuller description of the various
 parameters, like ID.
 
-(fn &key SERVER PORT NICK USER PASSWORD FULL-NAME CLIENT-CERTIFICATE ID)" '((let ((erc-default-port erc-default-port-tls) (erc--display-context `((erc-interactive-display . erc-tls) ,@erc--display-context))) (erc-select-read-args))))
+(fn &key SERVER PORT NICK USER PASSWORD FULL-NAME CLIENT-CERTIFICATE ID)" t)
 (autoload 'erc-handle-irc-url "erc" "\
 Use ERC to IRC on HOST:PORT in CHANNEL.
 If ERC is already connected to HOST:PORT, simply /join CHANNEL.
@@ -10804,7 +10808,9 @@
 If NAME is already defined as a test and Emacs is running
 in batch mode, an error is signaled.
 
-(fn NAME () [DOCSTRING] [:expected-result RESULT-TYPE] [:tags \\='(TAG...)] BODY...)" nil 'macro)
+(fn NAME () [DOCSTRING] [:expected-result RESULT-TYPE] [:tags \\='(TAG...)] BODY...)" nil t)
+(function-put 'ert-deftest 'doc-string-elt 3)
+(function-put 'ert-deftest 'lisp-indent-function 2)
 (autoload 'ert-run-tests-batch "ert" "\
 Run the tests specified by SELECTOR, printing results to the terminal.
 
@@ -13170,6 +13176,8 @@
 
 The mode's hook is called both when the mode is enabled and when it is
 disabled.
+
+\\{flymake-mode-map}
 
 (fn &optional ARG)" t)
 (autoload 'flymake-mode-on "flymake" "\
@@ -16874,7 +16882,8 @@
 change its definition, you should explicitly call
 `ibuffer-recompile-formats'.
 
-(fn SYMBOL (&key NAME INLINE PROPS SUMMARIZER) &rest BODY)" nil 'macro)
+(fn SYMBOL (&key NAME INLINE PROPS SUMMARIZER) &rest BODY)" nil t)
+(function-put 'define-ibuffer-column 'lisp-indent-function 'defun)
 (autoload 'define-ibuffer-sorter "ibuf-macs" "\
 Define a method of sorting named NAME.
 DOCUMENTATION is the documentation of the function, which will be called
@@ -16885,7 +16894,9 @@
 buffer object, and `b' bound to another.  BODY should return a non-nil
 value if and only if `a' is \"less than\" `b'.
 
-(fn NAME DOCUMENTATION (&key DESCRIPTION) &rest BODY)" nil 'macro)
+(fn NAME DOCUMENTATION (&key DESCRIPTION) &rest BODY)" nil t)
+(function-put 'define-ibuffer-sorter 'lisp-indent-function 1)
+(function-put 'define-ibuffer-sorter 'doc-string-elt 2)
 (autoload 'define-ibuffer-op "ibuf-macs" "\
 Generate a function which operates on a buffer.
 OP becomes the name of the function; if it doesn't begin with
@@ -16924,7 +16935,9 @@
 marked buffer.  BODY is evaluated with `buf' bound to the
 buffer object.
 
-(fn OP ARGS DOCUMENTATION (&key INTERACTIVE MARK MODIFIER-P DANGEROUS OPSTRING ACTIVE-OPSTRING BEFORE AFTER COMPLEX) &rest BODY)" nil 'macro)
+(fn OP ARGS DOCUMENTATION (&key INTERACTIVE MARK MODIFIER-P DANGEROUS OPSTRING ACTIVE-OPSTRING BEFORE AFTER COMPLEX) &rest BODY)" nil t)
+(function-put 'define-ibuffer-op 'lisp-indent-function 2)
+(function-put 'define-ibuffer-op 'doc-string-elt 3)
 (autoload 'define-ibuffer-filter "ibuf-macs" "\
 Define a filter named NAME.
 DOCUMENTATION is the documentation of the function.
@@ -16939,7 +16952,9 @@
 will be evaluated with BUF bound to the buffer object, and QUALIFIER
 bound to the current value of the filter.
 
-(fn NAME DOCUMENTATION (&key READER DESCRIPTION) &rest BODY)" nil 'macro)
+(fn NAME DOCUMENTATION (&key READER DESCRIPTION) &rest BODY)" nil t)
+(function-put 'define-ibuffer-filter 'lisp-indent-function 2)
+(function-put 'define-ibuffer-filter 'doc-string-elt 2)
 (register-definition-prefixes "ibuf-macs" '("ibuffer-"))
 
 
@@ -19797,8 +19812,6 @@
 ;;; Generated autoloads from emacs-lisp/loaddefs-gen.el
 
 (put 'autoload-compute-prefixes 'safe-local-variable #'booleanp)
-(put 'autoload-expand-macros 'safe-local-variable #'listp)
-(put 'autoload-expand-macros 'safe-local-variable #'listp)
 (put 'generated-autoload-file 'safe-local-variable 'stringp)
 (put 'generated-autoload-load-name 'safe-local-variable 'stringp)
 (autoload 'loaddefs-generate "loaddefs-gen" "\
@@ -27069,6 +27082,8 @@
 
 The mode's hook is called both when the mode is enabled and when it is
 disabled.
+
+\\{rectangle-mark-mode-map}
 
 (fn &optional ARG)" t)
 (register-definition-prefixes "rect" '("apply-on-rectangle" "clear-rectangle-line" "delete-" "extract-rectangle-" "killed-rectangle" "ope" "rectangle-" "spaces-string" "string-rectangle-"))