Subject: Re: bug#61916: 30.0.50; url.el; Can not change User-Agent temporarily when url-retrieve

From: "Peder O. Klingenberg" <[email-masked]>
Date: Fri, 18 Jul 2025 15:03:49 +0200

On Fri, 2025-07-18 14:57:58 +0200, Peder O. Klingenberg wrote:

> Attached is a patch that extends the set of dynamic-values-at-call-time
> -> buffer-local-available-at-request-time variables.  With this, your
> test case works as expected, as do my experiments in restclient.

(Except I failed to attach the patch using the debbugs-gnu-pick-commits
command, so here, attached the old-fashioned way)

-- 
...Peder...

>From a077e45a1cd9003f6255b0817d2182a431c7cd59 Mon Sep 17 00:00:00 2001
From: "Peder O. Klingenberg" <[email-masked]>
Date: Fri, 18 Jul 2025 00:11:57 +0200
Subject: [PATCH] Emulate more dynamic bindings in request buffers

Many variables were copied from the dynamic environment to the request
buffer, which allowed them to influence the handling of requests and
responses.  But some were not, notably some of the mime-related
variables, and the user-agent and privacy variables.  This made them
unreliable when dynamically bound around a call to `url-retrieve'; they
would have the desired effect when reusing an existing connection, but
not when url-http opened a new connection.  In the case of reused
connections, the request construction happens in the dynamic scope of
`url-http', but in the case where a fresh connection is needed, request
construction happens outside that dynamic scope.

This commit adds the remaining variables used in request construction to
the set of buffer local variables mirroring the dynamic values from
url-http, and adds a comment describing the mechanism used and how avoid
the pitfall of inconsistent handling of dynamic bindings.

* lisp/url/url-http.el (url-http-extensions-header): New internal-ish
variable.
(url-http-create-request): Use the new variable instead of the global
one.
(url-http-idle-sentinel): Debug-log when the connection closes.
(url-http): Set up more buffer-local variants of dynamic variables in
the buffer used for the request, and add comment describing why this
copying is needed.

(Bug#61916)
---
 lisp/url/url-http.el | 51 ++++++++++++++++++++++++++++++++++++++------
 1 file changed, 44 insertions(+), 7 deletions(-)

diff --git a/lisp/url/url-http.el b/lisp/url/url-http.el
index eb5d5c13722..a230a37e9ff 100644
--- a/lisp/url/url-http.el
+++ b/lisp/url/url-http.el
@@ -56,6 +56,7 @@ url-http-target-url
 (defvar url-http-transfer-encoding)
 (defvar url-show-status)
 (defvar url-http-referer)
+(defvar url-http-extensions-header)
 
 (require 'url-gw)
 (require 'url-parse)
@@ -374,9 +375,9 @@ url-http-create-request
                                 "close" "keep-alive")
              "\r\n"
              ;; HTTP extensions we support
-             (if url-extensions-header
+             (if url-http-extensions-header
                  (format
-                  "Extension: %s\r\n" url-extensions-header))
+                  "Extension: %s\r\n" url-http-extensions-header))
              ;; Who we want to talk to
              (if (/= (url-port url-http-target-url)
                      (url-scheme-get-property
@@ -1020,8 +1021,9 @@ url-http-activate-callback
 ;; )
 
 ;; These unfortunately cannot be macros... please ignore them!
-(defun url-http-idle-sentinel (proc _why)
+(defun url-http-idle-sentinel (proc why)
   "Remove (now defunct) process PROC from the list of open connections."
+  (url-http-debug "url-http-idle-sentinel for process %S: %s" proc (string-trim why))
   (maphash (lambda (key val)
 		(if (memq proc val)
 		    (puthash key (delq proc val) url-http-open-connections)))
@@ -1340,9 +1342,19 @@ url-http
 
 The return value of this function is the retrieval buffer."
   (cl-check-type url url "Need a pre-parsed URL.")
+  ;; The request is handled by asynchronous processes, which are outside
+  ;; the dynamic scope of the caller of url-http (sometimes, sometimes
+  ;; not).  The caller may still desire to bind variables controlling
+  ;; aspects of the request for the duration of this one http request.
+  ;; The async processes operate on a buffer created in this function,
+  ;; so the way to accomplish this goal is to set buffer local copies of
+  ;; the relevant variables to the dynamic values in scope as we create
+  ;; the buffer.  When new variables are added that influence behaviour
+  ;; of requests, they should be added to the handling in this function
+  ;; to make them work reliably without changing their global values.
   (let* (;; (host (url-host (or url-using-proxy url)))
 	 ;; (port (url-port (or url-using-proxy url)))
-	 (nsm-noninteractive (not (url-interactive-p)))
+	 (noninteractive-p (not (url-interactive-p)))
          ;; The following binding is needed in url-open-stream, which
          ;; is called from url-http-find-free-connection.
          (url-current-object url)
@@ -1350,10 +1362,17 @@ url-http
                                                     (url-port url)
                                                     gateway-method))
          (mime-accept-string url-mime-accept-string)
+         (mime-encoding-string url-mime-encoding-string)
+         (mime-charset-string url-mime-charset-string)
+         (mime-language-string url-mime-language-string)
 	 (buffer (or retry-buffer
 		     (generate-new-buffer
                       (format " *http %s:%d*" (url-host url) (url-port url)))))
-         (referer (url-http--encode-string (url-http--get-referer url))))
+         (referer (url-http--encode-string (url-http--get-referer url)))
+         (httpver url-http-version)
+         (httpkeepalive url-http-attempt-keepalives)
+         (user-agent url-user-agent)
+         (privacy-level url-privacy-level))
     (if (not connection)
 	;; Failed to open the connection for some reason
 	(progn
@@ -1389,8 +1408,17 @@ url-http
 		       url-http-no-retry
 		       url-http-connection-opened
                        url-mime-accept-string
+                       url-mime-encoding-string
+                       url-mime-charset-string
+                       url-mime-language-string
 		       url-http-proxy
-                       url-http-referer))
+                       url-http-referer
+                       url-http-version
+                       url-http-attempt-keepalives
+                       url-http-extensions-header
+                       url-user-agent
+                       url-privacy-level
+                       nsm-noninteractive))
 	  (set (make-local-variable var) nil))
 
 	(setq url-http-method (or url-request-method "GET")
@@ -1409,8 +1437,17 @@ url-http
 	      url-http-no-retry retry-buffer
 	      url-http-connection-opened nil
               url-mime-accept-string mime-accept-string
+              url-mime-encoding-string mime-encoding-string
+              url-mime-charset-string mime-charset-string
+              url-mime-language-string mime-language-string
 	      url-http-proxy url-using-proxy
-              url-http-referer referer)
+              url-http-referer referer
+              url-http-version httpver
+              url-http-attempt-keepalives httpkeepalive
+              url-http-extensions-header url-extensions-header
+              url-user-agent user-agent
+              url-privacy-level privacy-level
+              nsm-noninteractive noninteractive-p)
 
 	(set-process-buffer connection buffer)
 	(set-process-filter connection #'url-http-generic-filter)
-- 
2.25.1