Subject: Re: bug#73638: 31.0.50; doc-view: imenu index cannot be made for LaTeX PDFs

From: Visuwesh <[email-masked]>
Date: Tue, 08 Oct 2024 15:14:38 +0530

[திங்கள் அக்டோபர் 07, 2024] Tassilo Horn wrote:

> Visuwesh <[email-masked]> writes:
>
>>> The patch looks good.  But during testing, it seems that the index is
>>> always off by one page, i.e., the index for some section brings me to
>>> page 117 but the section heading is actually on page 118.
>>>
>>> I have that both with the Peter Atkins et al. book you suggested as
>>> well as with own papers which didn't work at all previously due to
>>> #nameddest references.
>>
>> Ugghhh, looks like the page number returned by the JS function is
>> zero-indexed.  Thanks for the catch (and sorry for the many mistakes
>> and hence the back-and-forth).  Should be corrected in the attached
>> patch.
>
> Nope, now I get off-by-many-hundreds errors.  The Imenu entries have the
> page number in parens, right?  If so, I have many references to pages
> that are thrice as large as the actual number of pages, e.g., here some
> parts of the *Completions* buffer for the Atkins book:
>
> FOCUS.1.The.properties.of.gases.(341)
> FOCUS.10.Molecular.symmetry.(4181)
> FOCUS.11.Molecular.spectroscopy.(4481)
> FOCUS.12.Magnetic.resonance.(5181)
> FOCUS.13.Statistical.thermodynamics.(5621)
> FOCUS.14.Molecular.interactions.(6141)
> FOCUS.15.Solids.(6701)
> FOCUS.16.Molecules.in.motio.(7201)
> FOCUS.17.Chemical.kinetics.(7521)
> FOCUS.18.Reaction.dynamics.(8101)
> FOCUS.19.Processes.at.solid.surfaces.(8541)
>
> It's large but doesn't have more than 8000 pages.

I messed up by not considering the precedence of operators.  :-( Fixed
in the attached patch.

>From 39205fc097c30803c077014defaacef9512af902 Mon Sep 17 00:00:00 2001
From: Visuwesh <[email-masked]>
Date: Sun, 6 Oct 2024 18:02:06 +0530
Subject: [PATCH] Make imenu index generation for PDFs more reliable

Do away with parsing the output of "mutool show FILE outline"
since the URI reported in its output may not include the page
number of the heading, and instead may contained "nameddest"
elements which cannot be resolved using "mutool".  Instead, use
a MuPDF JS script to generate the PDF outline allowing to
resolve such URIs.

* lisp/doc-view.el (doc-view--outline-rx): Remove as no longer
needed.
(doc-view--outline): Reflect that outline can be generated for
non-PDF files too.
(doc-view--mutool-pdf-outline-script): Add new variable to hold
the JS script used to generate the outline.
(doc-view--pdf-outline): Use the script.  (bug#73638)
---
 lisp/doc-view.el | 48 ++++++++++++++++++++++++++++++++----------------
 1 file changed, 32 insertions(+), 16 deletions(-)

diff --git a/lisp/doc-view.el b/lisp/doc-view.el
index 446beeafd9f..57a24418616 100644
--- a/lisp/doc-view.el
+++ b/lisp/doc-view.el
@@ -1969,14 +1969,26 @@ doc-view-search-previous-match
 	(doc-view-goto-page (caar (last doc-view--current-search-matches)))))))
 
 ;;;; Imenu support
-(defconst doc-view--outline-rx
-  "[^\t]+\\(\t+\\)\"\\(.+\\)\"\t#\\(?:page=\\)?\\([0-9]+\\)")
-
 (defvar-local doc-view--outline nil
-  "Cached PDF outline, so that it is only computed once per document.
+  "Cached document outline, so that it is only computed once per document.
 It can be the symbol `unavailable' to indicate that outline is
 unavailable for the document.")
 
+(defvar doc-view--mutool-pdf-outline-script
+  "var document = new Document.openDocument(\"%s\", \"application/pdf\");
+var outline = document.loadOutline();
+if(!outline) quit();
+function pp(outl, level){print(\"((level . \" + level + \")\");\
+print(\"(title . \" + repr(outl.title) + \")\");\
+print(\"(page . \" + (document.resolveLink(outl.uri)+1) + \"))\");\
+if(outl.down){for(var i=0; i<outl.down.length; i++){pp(outl.down[i], level+1);}}};
+function run(){print(\"BEGIN(\");\
+for(var i=0; i<outline.length; i++){pp(outline[i], 1);}print(\")\");};
+run()"
+  "JS script to extract the PDF's outline using mutool.
+The script has to be minified to pass it to the REPL.  The \"BEGIN\"
+marker is here to skip past the prompt characters.")
+
 (defun doc-view--pdf-outline (&optional file-name)
   "Return a list describing the outline of FILE-NAME.
 Return a list describing the current file if FILE-NAME is nil.
@@ -1986,21 +1998,25 @@ doc-view--pdf-outline
 structure is extracted by `doc-view--imenu-subtree'."
   (let ((fn (or file-name (buffer-file-name))))
     (when fn
-      (let ((outline nil)
-            (fn (expand-file-name fn)))
-        (with-temp-buffer
-          (unless (eql 0 (call-process doc-view-pdfdraw-program nil
-                                       (current-buffer) nil "show" fn "outline"))
+      (with-temp-buffer
+        (let ((proc (make-process
+                     :name "doc-view-pdf-outline"
+                     :command (list "mutool" "run")
+                     :buffer (current-buffer))))
+          (process-send-string proc (format doc-view--mutool-pdf-outline-script
+                                            (expand-file-name fn)))
+          ;; Need to send this twice for some reason...
+          (process-send-eof)
+          (process-send-eof)
+          (while (accept-process-output proc))
+          (unless (eq (process-status proc) 'exit)
             (setq doc-view--outline 'unavailable)
             (imenu-unavailable-error "Unable to create imenu index using `mutool'"))
           (goto-char (point-min))
-          (while (re-search-forward doc-view--outline-rx nil t)
-            (push `((level . ,(length (match-string 1)))
-                    (title . ,(replace-regexp-in-string "\\\\[rt]" " "
-                                                        (match-string 2)))
-                    (page . ,(string-to-number (match-string 3))))
-                  outline)))
-        (nreverse outline)))))
+          (when (search-forward "BEGIN" nil t)
+            (condition-case nil
+                (read (current-buffer))
+              (end-of-file nil))))))))
 
 (defun doc-view--djvu-outline (&optional file-name)
   "Return a list describing the outline of FILE-NAME.
-- 
2.45.2