Subject: Re: bug#76196: Obsoleting gnus/nnweb.el

From: James Thomas <[email-masked]>
Date: Mon, 02 Jun 2025 13:22:02 +0530

James Thomas writes:

> James Thomas writes:
>
>> Andrew Cohen writes:
>>
>>> I asked on emacs.gnus.general to see if anyone was using nnweb. Although
>>> the number of responses was small, at least one person, James Thomas,
>>> continues to use it. He has, for example, updated the defaults to use
>>> help-gnu-emacs. This looks potentially useful, and if James (copied
>>> here) is willing to work on this I would suggest fixing nnweb rather
>>> than obsoleting it.
>>
>> Please hold on; I'll send in an updated patch, with the article washing
>> also included.
>>
>> (Thanks for linking to this bug)
>
> IDK if this is the right place for it, but here it is:
>
>
>
> The help-gnu-emacs is only illustrative: may be replaced with the name
> of any of the GNU mailing lists. May be tested by:
> G w emacs-help RET <unique string> RET
>
> Further work could involve gnus-summary-refer-thread working with a new
> 'gnus-search-web' for the related messages of a search result. Also,
> with customization, as a search and gnus-refer-article-method for users
> of public-inbox (public-inbox.org).

Here's an updated version:

From 769eb784e7750047daf92918f1b03d1ed25ac8df Mon Sep 17 00:00:00 2001
From: James Thomas <[email-masked]>
Date: Mon, 2 Jun 2025 12:30:32 +0530
Subject: [PATCH] Update nnweb for searching Emacs's lists' archives

Remove the existing 'google' type and replace it with 'gnu', to
search the GNU mailing lists.

* lisp/gnus/nnweb.el (nnweb-type):
(nnweb-type-definition):
(nnweb-search):
(nnweb-google-wash-article):
(nnweb-gnu-wash-article):
(nnweb-google-parse-1):
(nnweb-google-reference):
(nnweb-gnu-reference):
(nnweb-google-create-mapping):
(nnweb-gnu-create-mapping):
(nnweb-google-search):
(nnweb-gnu-search):
(nnweb-google-identity):
(nnweb-gnu-identity):

Make the changes.

* doc/misc/gnus.texi (Foreign Groups):
(Web Searches): Update doc.
---
 doc/misc/gnus.texi |  10 ++-
 lisp/gnus/nnweb.el | 165 +++++++++++++++++++++------------------------
 2 files changed, 82 insertions(+), 93 deletions(-)

diff --git a/doc/misc/gnus.texi b/doc/misc/gnus.texi
index 1eace0d5b5a..c55d73ae14a 100644
--- a/doc/misc/gnus.texi
+++ b/doc/misc/gnus.texi
@@ -2802,12 +2802,11 @@ Foreign Groups
 (@code{gnus-group-make-web-group}).  If you give a prefix to this
 command, make a solid group instead.  You will be prompted for the
 search engine type and the search string.  Valid search engine types
-include @code{google} and @code{dejanews}.
+include @code{help-gnu-emacs} and @code{emacs-devel}.
 @xref{Web Searches}.
 
-If you use the @code{google} search engine, you can limit the search
-to a particular group by using a match string like
-@samp{shaving group:alt.sysadmin.recovery}.
+The query syntax is described at
+@url{https://lists.gnu.org/archive/cgi-bin/namazu.cgi?idxname=help-gnu-emacs}.
 
 @item G R
 @kindex G R @r{(Group)}
@@ -17332,8 +17331,7 @@ Web Searches
 @item nnweb-type
 @vindex nnweb-type
 What search engine type is being used.  The currently supported types
-are @code{google} and @code{dejanews}.  Note that
-@code{dejanews} is an alias to @code{google}.
+are @code{help-gnu-emacs} and @code{emacs-devel}.
 
 @item nnweb-search
 @vindex nnweb-search
diff --git a/lisp/gnus/nnweb.el b/lisp/gnus/nnweb.el
index 9ada2dbc1d7..87c34876c7c 100644
--- a/lisp/gnus/nnweb.el
+++ b/lisp/gnus/nnweb.el
@@ -40,36 +40,45 @@
 (defvoo nnweb-directory (nnheader-concat gnus-directory "nnweb/")
   "Where nnweb will save its files.")
 
-(defvoo nnweb-type 'google
-  "What search engine type is being used.
-The only valid type is currently `google'.")
+(defvoo nnweb-type 'help-gnu-emacs
+  "What search engine type is being used. ")
 
 (defvar nnweb-type-definition
-  '((google
-     (id . "https://www.google.com/groups?as_umsgid=%s&hl=en&dmode=source")
-     (result . "https://groups.google.com/group/%s/msg/%s?dmode=source")
-     (article . nnweb-google-wash-article)
-     (reference . identity)
-     (map . nnweb-google-create-mapping)
-     (search . nnweb-google-search)
-     (address . "https://groups.google.com/groups")
-     (base    . "https://groups.google.com")
-     (identifier . nnweb-google-identity))
+  '((emacs-devel
+     (id . "https://lists.gnu.org/archive/cgi-bin/namazu.cgi?idxname=emacs-devel&sort=score&result=normal&max=20&submit=Search!&query=%%2Bmessage-id%%3A%%3C%s%%3E")
+     (result . "https://lists.gnu.org/archive/html/%s/%s.html")
+     (article . nnweb-gnu-wash-article)
+     (reference . nnweb-gnu-reference)
+     (map . nnweb-gnu-create-mapping)
+     (search . nnweb-gnu-search)
+     (address . "https://lists.gnu.org/archive/cgi-bin/namazu.cgi")
+     (base    . "https://lists.gnu.org")
+     (identifier . nnweb-gnu-identity))
+    (help-gnu-emacs
+     (id . "https://lists.gnu.org/archive/cgi-bin/namazu.cgi?idxname=help-gnu-emacs&sort=score&result=normal&max=20&submit=Search!&query=%%2Bmessage-id%%3A%%3C%s%%3E")
+     (result . "https://lists.gnu.org/archive/html/%s/%s.html")
+     (article . nnweb-gnu-wash-article)
+     (reference . nnweb-gnu-reference)
+     (map . nnweb-gnu-create-mapping)
+     (search . nnweb-gnu-search)
+     (address . "https://lists.gnu.org/archive/cgi-bin/namazu.cgi")
+     (base    . "https://lists.gnu.org")
+     (identifier . nnweb-gnu-identity))
     ;; FIXME: Make obsolete?
     (dejanews ;; alias of google
      (id . "https://www.google.com/groups?as_umsgid=%s&hl=en&dmode=source")
      (result . "https://groups.google.com/group/%s/msg/%s?dmode=source")
-     (article . nnweb-google-wash-article)
+     (article . nnweb-gnu-wash-article)
      (reference . identity)
-     (map . nnweb-google-create-mapping)
-     (search . nnweb-google-search)
+     (map . nnweb-gnu-create-mapping)
+     (search . nnweb-gnu-search)
      (address . "https://groups.google.com/groups")
      (base    . "https://groups.google.com")
-     (identifier . nnweb-google-identity)))
+     (identifier . nnweb-gnu-identity)))
   "Type-definition alist.")
 
 (defvoo nnweb-search nil
-  "Search string to feed to Google.")
+  "Search string to feed.")
 
 (defvoo nnweb-max-hits 999
   "Maximum number of hits to display.")
@@ -281,41 +290,30 @@ nnweb-init
             (current-buffer)))))
 
 ;;;
-;;; groups.google.com
+;;; lists.gnu.org
 ;;;
 
-(defun nnweb-google-wash-article ()
-  ;; We have Google's masked e-mail addresses here.  :-/
-  (let ((case-fold-search t)
-	(start-re "<pre>[\r\n ]*")
-	(end-re "[\r\n ]*</pre>"))
+(defun nnweb-gnu-wash-article ()
+  (let ((case-fold-search t) headers)
     (goto-char (point-min))
-    (if (save-excursion
-	  (or (re-search-forward "The requested message.*could not be found."
-				 nil t)
-	      (not (and (re-search-forward start-re nil t)
-			(re-search-forward end-re nil t)))))
-	;; FIXME: Don't know how to indicate "not found".
-	;; Should this function throw an error?  --rsteib
-	(progn
-	  (gnus-message 3 "Requested article not found")
-	  (erase-buffer))
-      (delete-region (point-min)
-		     (re-search-forward start-re))
+    (with-restriction (point-min) (search-forward "<!--X-Body-of-Message-->" nil t)
       (goto-char (point-min))
-      (delete-region (progn
-		       (re-search-forward end-re)
-		       (match-beginning 0))
-		     (point-max))
-      (mm-url-decode-entities))))
-
-(defun nnweb-google-parse-1 (&optional Message-ID)
+      (while (re-search-forward "<!--X-\\(.*\\) -->" nil t)
+        (setq headers (concat headers (match-string 1) "\n")))
+      (delete-region (point-min) (point-max))
+      (insert headers))
+    (goto-char (point-max))
+    (delete-region (search-backward "<!--X-Body-of-Message-End-->" nil t) (point-max))
+    (mm-url-remove-markup)
+    (mm-url-decode-entities)))
+
+(defun nnweb-gnu-parse-1 (&optional Message-ID)
   "Parse search result in current buffer."
   (let ((i 0)
 	(case-fold-search t)
 	(active (cadr (assoc nnweb-group nnweb-group-alist)))
 	Subject Date Newsgroups From
-	map url mid)
+	map url mid link)
     (unless active
       (push (list nnweb-group (setq active (cons 1 0)))
 	    nnweb-group-alist))
@@ -323,39 +321,34 @@ nnweb-google-parse-1
     (goto-char (point-min))
     (while
 	(re-search-forward
-	 "a +href=\"/group/\\([^>\"]+\\)/browse_thread/[^>]+#\\([0-9a-f]+\\)"
+	 "<a +href=\"/archive/html/\\([^/]+\\)/\\([0-9-]+/msg[0-9]+\\)\\.html\">"
 	 nil t)
-      (setq Newsgroups (match-string-no-properties 1)
-	    ;; Note: Starting with Google Groups 2, `mid' is a Google-internal
-	    ;; ID, not a proper Message-ID.
+      (setq link (match-string-no-properties 0)
+            Newsgroups (match-string-no-properties 1)
+            ;; `mid' is not a proper Message-ID.
 	    mid (match-string-no-properties 2)
 	    url (format
 		 (nnweb-definition 'result) Newsgroups mid))
-      (narrow-to-region (search-forward ">" nil t)
+      (narrow-to-region (point)
 			(search-forward "</a>" nil t))
       (mm-url-remove-markup)
       (mm-url-decode-entities)
+      (replace-string-in-region "\n" "")
       (setq Subject (buffer-string))
       (goto-char (point-max))
       (widen)
       (narrow-to-region (point)
-			(search-forward "</table" nil t))
+			(search-forward link nil t))
 
       (mm-url-remove-markup)
       (mm-url-decode-entities)
       (goto-char (point-max))
       (when
 	  (re-search-backward
- 	   "^\\(?:\\(\\w+\\) \\([0-9]+\\)\\|\\S-+\\)\\(?: \\([0-9]\\{4\\}\\)\\)? by ?\\(.*\\)"
+	   "^Author: \\(.*\\)\nDate: \\(.*\\)"
 	   nil t)
-	(setq Date (if (match-string 1)
-		       (format "%s %s 00:00:00 %s"
-			       (match-string 1)
-			       (match-string 2)
-			       (or (match-string 3)
-				   (format-time-string "%Y")))
-		     (current-time-string)))
-	(setq From (match-string 4)))
+	(setq Date (match-string-no-properties 2))
+	(setq From (match-string-no-properties 1)))
       (widen)
       (cl-incf i)
       (unless (nnweb-get-hashtb url)
@@ -363,73 +356,71 @@ nnweb-google-parse-1
 	 (list
 	  (cl-incf (cdr active))
 	  (make-full-mail-header
-	   (cdr active) (if Newsgroups
-			    (concat  "(" Newsgroups ") " Subject)
-			  Subject)
-	   From Date (or Message-ID mid)
+	   (cdr active) Subject
+	   From Date (or Message-ID (concat Newsgroups "/" mid))
 	   nil 0 0 url))
 	 map)
 	(nnweb-set-hashtb (cadar map) (car map))))
     map))
 
-(defun nnweb-google-reference (id)
-  (let ((map (nnweb-google-parse-1 id)) header)
+(defun nnweb-gnu-reference (id)
+  (let ((map (nnweb-gnu-parse-1 id)) header)
     (setq nnweb-articles
 	  (nconc nnweb-articles map))
     (when (setq header (cadar map))
       (mm-url-insert (mail-header-xref header))
       (caar map))))
 
-(defun nnweb-google-create-mapping ()
+(defun nnweb-gnu-create-mapping ()
   "Perform the search and create a number-to-url alist."
   (with-current-buffer nnweb-buffer
     (erase-buffer)
-    (nnheader-message 7 "Searching google...")
+    (nnheader-message 7 "Searching...")
     (when (funcall (nnweb-definition 'search) nnweb-search)
-	(let ((more t)
-	      (i 0))
+	(let ((more 0)
+	      (i 0)
+              link)
 	  (while more
 	    (setq nnweb-articles
-		  (nconc nnweb-articles (nnweb-google-parse-1)))
+		  (nconc nnweb-articles (nnweb-gnu-parse-1)))
 	    ;; Check if there are more articles to fetch
 	    (goto-char (point-min))
 	    (cl-incf i 100)
 	    (if (or (not (re-search-forward
-			  "<a [^>]+href=\"\n?\\([^>\" \n\t]+\\)[^<]*<img[^>]+src=[^>]+next"
+                          (format "<a +href=\"\\(/archive/cgi-bin/namazu.cgi?[^>]+&amp;whence=%s\\)\"" (+ more 20))
 			  nil t))
 		    (>= i nnweb-max-hits))
 		(setq more nil)
 	      ;; Yup, there are more articles
-	      (setq more (concat (nnweb-definition 'base) (match-string 1)))
+	      (setq more (+ more 20)
+                    link (mm-url-decode-entities-string (match-string 1)))
 	    (when more
 	      (erase-buffer)
-	      (nnheader-message 7 "Searching google...(%d)" i)
-	      (mm-url-insert more))))
+	      (nnheader-message 7 "Searching...(%d)" i)
+	      (mm-url-insert (concat (nnweb-definition 'base) link)))))
 	  ;; Return the articles in the right order.
-	  (nnheader-message 7 "Searching google...done")
+	  (nnheader-message 7 "Searching...done")
 	  (setq nnweb-articles
 		(sort nnweb-articles #'car-less-than-car))))))
 
-(defun nnweb-google-search (search)
+(defun nnweb-gnu-search (search)
   (mm-url-insert
    (concat
     (nnweb-definition 'address)
     "?"
     (mm-url-encode-www-form-urlencoded
-     `(("q" . ,search)
-       ("num" . ,(number-to-string
-		  (min 100 nnweb-max-hits)))
-       ("hq" . "")
-       ("hl" . "en")
-       ("lr" . "")
-       ("safe" . "off")
-       ("sites" . "groups")
-       ("filter" . "0")))))
+     `(("idxname" . ,(symbol-name nnweb-type))
+       ("sort" . "score")
+       ("result" . "normal")
+       ("max" . ,(number-to-string
+		  (min 20 nnweb-max-hits)))
+       ("submit" . "Search!")
+       ("query" . ,search)))))
   t)
 
-(defun nnweb-google-identity (url)
+(defun nnweb-gnu-identity (url)
   "Return a unique identifier based on URL."
-  (if (string-match "selm=\\([^ &>]+\\)" url)
+  (if (string-match "archive/html/\\(.*\\)\\.html" url)
       (match-string 1 url)
     url))
 
-- 
2.49.0

I hope to add thread-referral later.

Regards,
James