Subject: Re: bug#78240: 30.1; xoauth2 authentication fails in gnus imap mail fetching

From: Robert Pluim <[email-masked]>
Date: Thu, 31 Jul 2025 17:21:56 +0200

>>>>> On Sun, 04 May 2025 16:43:28 -0700, Xiyue Deng <[email-masked]> said:

    Xiyue> I have only tested using Gnus with nnimap (see documentation at [1] or
    Xiyue> [2]) where you want to add `(nnimap-authenticator xoauth2)' to your
    Xiyue> `nnimap' settings.  I haven't tested with `mail-sources', which seems to
    Xiyue> work differently compared to nnimap, and hence the implementation may be
    Xiyue> missing.  I probably need to understand how `mail-sources' works and
    Xiyue> extend the plugin accordingly, which could take a while.

Hi Xiyue,

I implemented auth-source lookup for imap.el (patch below). Would
there be any chance to see if you could extend your package to support
xoauth2 for imap.el?

It should be enough to add xoauth2 to `imap-authenticators', and
`imap-xoauth2-auth-p' and `imap-xoauth2-auth' to
`imap-authenticator-alist' (and their implementations, of course).

Robert
-- 

>From 19901eda8b693937861d93856322f4b73a3dc231 Mon Sep 17 00:00:00 2001
From: Robert Pluim <[email-masked]>
Date: Mon, 28 Jul 2025 11:11:29 +0200
Subject: [PATCH] Add auth-source support to imap.el

* lisp/net/imap.el: Require auth-source.
(imap-use-auth-source): New user option.
(auth-source-creation-prompts): New defvar.
(imap-credentials): New function, performs 'auth-source' search.
(imap-authenticate): Call 'imap-credentials' if no user is
specified and 'imap-use-auth-source' is non-nil.

* etc/NEWS: Announce the feature.
---
 etc/NEWS         |  12 +++++
 lisp/net/imap.el | 119 +++++++++++++++++++++++++++++++++--------------
 2 files changed, 96 insertions(+), 35 deletions(-)

diff --git a/etc/NEWS b/etc/NEWS
index b790c7e318c..61eadc63106 100644
--- a/etc/NEWS
+++ b/etc/NEWS
@@ -1338,6 +1338,18 @@ replies.
 
 ** Imap
 
+---
+*** New user option 'imap-use-auth-source'.
+Default t.  Controls whether 'imap-authenticate' will attempt to lookup
+credentials via 'auth-source' (but only when it is called with username
+nil).
+
+---
+*** 'imap-authenticate' can now retrieve credentials via 'auth-source'.
+Calling 'imap-authenticate' with username nil will now perform an
+'auth-source' search in order to obtain the username and the password.
+Customize 'imap-use-auth-source' to nil to prevent this.
+
 ---
 *** 'imap-authenticate' can now use PLAIN authentication.
 "AUTH=PLAIN" support is auto-enabled if the IMAP server supports it.  Pass
diff --git a/lisp/net/imap.el b/lisp/net/imap.el
index f705da317e5..e381b3dc95f 100644
--- a/lisp/net/imap.el
+++ b/lisp/net/imap.el
@@ -139,6 +139,8 @@
 (eval-when-compile (require 'cl-lib))
 (require 'utf7)
 (require 'rfc2104)
+(require 'auth-source)
+
 ;; Hmm... digest-md5 is not part of Emacs.
 ;; FIXME: Should/can we use sasl-digest.el instead?
 (declare-function digest-md5-parse-digest-challenge "ext:digest-md5")
@@ -246,6 +248,13 @@ imap-store-password
   "If non-nil, store session password without prompting."
   :type 'boolean)
 
+(defcustom imap-use-auth-source t
+  "If non-nil, lookup username and password using `auth-source'.
+This only takes effect when the \"user\" passed to `imap-authenticate'
+is nil."
+  :type 'boolean
+  :version "31.1")
+
 ;;; Various variables
 
 (defvar imap-fetch-data-hook nil
@@ -1107,6 +1116,26 @@ imap-ping-server
       (imap-ok-p (imap-send-command-wait "NOOP" buffer))
     (error nil)))
 
+(defvar auth-source-creation-prompts)
+
+;; Stolen from, uhm, "inspired by" nnimap.el.
+;; We don't want to drag in the whole of nnimap.
+(defun imap-credentials (address ports user)
+  (let* ((auth-source-creation-prompts
+          '((user  . "imap: username for %h: ")
+            (secret . "imap: password for %u: ")))
+         (found (nth 0 (auth-source-search :max 1
+                                           :host address
+                                           :port ports
+                                           :user user
+                                           :require '(:user :secret)
+                                           :create t))))
+    (if found
+        (list (plist-get found :user)
+              (auth-info-password found)
+              (plist-get found :save-function))
+      nil)))
+
 (defun imap-authenticate (&optional user passwd buffer)
   "Authenticate to server in BUFFER, using current buffer if nil.
 It uses the authenticator specified when opening the server.
@@ -1117,41 +1146,61 @@ imap-authenticate
 user for a username and/or password."
   (with-current-buffer (or buffer (current-buffer))
     (if (not (eq imap-state 'nonauth))
-	(or (eq imap-state 'auth)
-	    (eq imap-state 'selected)
-	    (eq imap-state 'examine))
-      (make-local-variable 'imap-username)
-      (make-local-variable 'imap-password)
-      (make-local-variable 'imap-last-authenticator)
-      (when user (setq imap-username user))
-      (when passwd (setq imap-password passwd))
-      (if imap-auth
-	  (and (setq imap-last-authenticator
-		     (assq imap-auth imap-authenticator-alist))
-	       (funcall (nth 2 imap-last-authenticator) (current-buffer))
-	       (setq imap-state 'auth))
-	;; Choose authenticator.
-	(let ((auths imap-authenticators)
-	      auth)
-	  (while (setq auth (pop auths))
-	    ;; OK to use authenticator?
-	    (setq imap-last-authenticator
-		  (assq auth imap-authenticator-alist))
-	    (when (funcall (nth 1 imap-last-authenticator) (current-buffer))
-	      (message "imap: Authenticating to `%s' using `%s'..."
-		       imap-server auth)
-	      (setq imap-auth auth)
-	      (if (funcall (nth 2 imap-last-authenticator) (current-buffer))
-		  (progn
-		    (message "imap: Authenticating to `%s' using `%s'...done"
-			     imap-server auth)
-		    ;; set imap-state correctly on successful auth attempt
-		    (setq imap-state 'auth)
-		    ;; stop iterating through the authenticator list
-		    (setq auths nil))
-		(message "imap: Authenticating to `%s' using `%s'...failed"
-			 imap-server auth)))))
-	imap-state))))
+        (or (eq imap-state 'auth)
+            (eq imap-state 'selected)
+            (eq imap-state 'examine))
+      ;; .authinfo can contain symbolic or numeric ports.
+      (let ((ports (cond ((eq imap-port 993)
+                          '(993 "imaps"))
+                         ((eq imap-port 143)
+                          '(143 "imap"))
+                         (t imap-port)))
+            (imap-credentials nil)
+            (passwd-save-function nil))
+        (make-local-variable 'imap-username)
+        (make-local-variable 'imap-password)
+        (make-local-variable 'imap-last-authenticator)
+        (when user (setq imap-username user))
+        (when passwd (setq imap-password passwd))
+        (unless (or user (not imap-use-auth-source))
+                                        ; Don't perform auth-source
+                                        ; lookup if a user was specified
+                                        ; explicitly.
+          (when (setq imap-credentials (imap-credentials imap-server ports user))
+            (setq imap-username (nth 0 imap-credentials)
+                  imap-password (nth 1 imap-credentials)
+                  passwd-save-function (nth 2 imap-credentials))))
+        (if imap-auth                   ; It's a reauth
+            (and (setq imap-last-authenticator
+                       (assq imap-auth imap-authenticator-alist))
+                 (funcall (nth 2 imap-last-authenticator) (current-buffer))
+                 (setq imap-state 'auth))
+          ;; Choose authenticator.
+          (let ((auths imap-authenticators)
+                auth)
+            (while (setq auth (pop auths))
+              ;; OK to use authenticator?
+              (setq imap-last-authenticator
+                    (assq auth imap-authenticator-alist))
+              (when (funcall (nth 1 imap-last-authenticator) (current-buffer))
+                (message "imap: Authenticating to `%s' using `%s'..."
+                         imap-server auth)
+                (setq imap-auth auth)
+                (if (funcall (nth 2 imap-last-authenticator) (current-buffer))
+                    (progn
+                      (message "imap: Authenticating to `%s' using `%s'...done"
+                               imap-server auth)
+                      ;; Save the credentials if a new token was created
+                      ;; via auth-source.
+                      (when (functionp passwd-save-function)
+                        (funcall passwd-save-function))
+                      ;; set imap-state correctly on successful auth attempt
+                      (setq imap-state 'auth)
+                      ;; stop iterating through the authenticator list
+                      (setq auths nil))
+                  (message "imap: Authenticating to `%s' using `%s'...failed"
+                           imap-server auth)))))
+          imap-state)))))
 
 (defun imap-close (&optional buffer)
   "Close connection to server in BUFFER.
-- 
2.39.5