Subject: Tree-sitter compatibility report: tags and sources

From: Peter Oliver <[email-masked]>
Date: Fri, 20 Jun 2025 12:56:29 +0100 (BST)

Back in January, there was some discussion on emacs-devel about how the Tree-sitter parser compatibility report generated by admin/tree-sitter/treesit-admin.el could be made more helpful for distribution packagers: https://yhetil.org/emacs-devel/[email-masked]/#t

Following on from that, here are some patches to enhance the report.

-- 
Peter Oliver

From de5d52524db9ba9c6fd6d75d2477d016942a0cab Mon Sep 17 00:00:00 2001
From: Peter Oliver <[email-masked]>
Date: Fri, 3 Jan 2025 12:02:50 +0000
Subject: [PATCH 1/3] New function treesit--language-git-version-tags

* lisp/treesit.el (treesit--language-git-version-tags): New function to
  list version tags for a Tree-sitter parser Git repository.
---
 lisp/treesit.el | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/lisp/treesit.el b/lisp/treesit.el
index 353e991ec20..4f07e137c8c 100644
--- a/lisp/treesit.el
+++ b/lisp/treesit.el
@@ -5153,6 +5153,18 @@ treesit--language-git-revision
       (string-trim (buffer-string)))
      (t nil))))
 
+(defun treesit--language-git-version-tags (repo-dir)
+  "Return a list of Git version tags in REPO-DIR, sorted latest first.
+
+Return the output of \"git tag --list --sort=-version:refname 'v*'\". If
+anything goes wrong, return nil."
+  (with-temp-buffer
+    (cond
+     ((eq 0 (call-process "git" nil t nil "-C" repo-dir "tag"
+                          "--list" "--sort=-version:refname" "v*"))
+      (split-string (buffer-string)))
+     (t nil))))
+
 (defun treesit--language-git-timestamp (repo-dir)
   "Return the commit date in REPO-DIR in UNIX epoch.
 
-- 
2.49.0

From 805e78782d64e7de2be1d3ab25f8f8d2577a3fef Mon Sep 17 00:00:00 2001
From: Peter Oliver <[email-masked]>
Date: Wed, 11 Jun 2025 17:34:53 +0100
Subject: [PATCH 2/3] Show also tagged versions on Tree-sitter compatibility
 report

* admin/tree-sitter/compat-template.html: Accommodate two tables.
* admin/tree-sitter/treesit-admin.el
(treesit-admin--find-latest-compatible-revision): Pass through new arg
REVISION-TYPE.
(treesit-admin--last-compatible-grammar-for-modes): Optionally iterate
over tagged rather than all commits.
(treesit-admin--generate-compatibility-report): Generate a second table,
showing only tagged commits.
---
 admin/tree-sitter/compat-template.html |  10 +-
 admin/tree-sitter/treesit-admin.el     | 234 ++++++++++++++-----------
 2 files changed, 142 insertions(+), 102 deletions(-)

diff --git a/admin/tree-sitter/compat-template.html b/admin/tree-sitter/compat-template.html
index 23b5b55d7b3..3f8163567af 100644
--- a/admin/tree-sitter/compat-template.html
+++ b/admin/tree-sitter/compat-template.html
@@ -16,24 +16,26 @@
           font-weight: bold;
       }
       table {
+          display: inline-block;
           margin: auto;
       }
+      table caption {
+          font-weight: bold;
+      }
       table td {
           padding: 0.5rem 1rem;
           width: 10rem;
           word-break: break-all;
       }
-      .head {
+      .latest {
           background: lightgreen;
       }
     </style>
   </head>
   <body>
     <h1>Emacs tree-sitter grammar compatibility</h1>
-    <p>This is an auto-generated report of the last compatible version for each grammar in each Emacs version.  A <span class="head">green background</span> on the version indicates that the Emacs version is compatible with the latest commit in the upstream grammar repo.</p>
+    <p>This is an auto-generated report of the last compatible version for each grammar in each Emacs version.  A <span class="head">green background</span> on the version indicates that the Emacs version is compatible with the latest commit/tag in the upstream grammar repo.</p>
     <p>This report is generated on ___REPLACE_TIME___.</p>
-    <table>
 ___REPLACE_TABLE___
-    </table>
   </body>
 </html>
diff --git a/admin/tree-sitter/treesit-admin.el b/admin/tree-sitter/treesit-admin.el
index 1bb8b927e5c..4e2e3855f55 100644
--- a/admin/tree-sitter/treesit-admin.el
+++ b/admin/tree-sitter/treesit-admin.el
@@ -315,21 +315,25 @@ treesit-admin--mode-languages
                      settings)))))
 
 (defun treesit-admin--find-latest-compatible-revision
-    (mode language source-alist grammar-dir &optional emacs-executable)
+    (mode language source-alist grammar-dir revision-type
+          &optional emacs-executable)
   "Find the latest revision for LANGUAGE that's compatible with MODE.
 
 MODE, LANGUAGE, SOURCE-ALIST, GRAMMAR-DIR are the same as in
 `treesit-admin--verify-major-mode-queries'.
 
+REVISION-TYPE is `commit' or `tag', to inspect all or only tagged commits
+respectively.
+
 By default, use the Emacs executable that spawned the current Emacs
 session to validate grammars, but if EMACS-EXECUTABLE is non-nil, use it
 instead.
 
 Return a plist of the form
 
-    (:version VERSION :head-version HEAD-VERSION :timestamp TIMESTAMP).
+    (:version VERSION :latest-version LATEST-VERSION :timestamp TIMESTAMP).
 
-HEAD-VERSION is the version of the HEAD, VERSION is the latest
+LATEST-VERSION is the most-recent version, VERSION is the most-recent
 compatible version.  TIMESTAMP is the commit date of VERSION in UNIX
 epoch format."
   (let ((treesit-extra-load-path (list grammar-dir))
@@ -340,7 +344,7 @@ treesit-admin--find-latest-compatible-revision
         (emacs-executable
          (or emacs-executable
              (expand-file-name invocation-name invocation-directory)))
-        head-version version exit-code timestamp)
+        latest-version version exit-code timestamp)
     (when (not recipe)
       (signal 'treesit-error `("Cannot find recipe" ,language)))
     (pcase-let ((`(,url ,revision ,source-dir ,cc ,c++ ,commit)
@@ -349,44 +353,70 @@ treesit-admin--find-latest-compatible-revision
         (treesit--git-clone-repo url revision workdir)
         (when commit
           (treesit--git-checkout-branch workdir commit))
-        (setq head-version (treesit--language-git-revision workdir))
-        (treesit--build-grammar
-         workdir grammar-dir language source-dir cc c++)
-        (while (not (eq exit-code 0))
-          (unless (null exit-code)
-            (treesit--git-checkout-branch workdir "HEAD~")
+        (cond
+         ((eq revision-type 'tag)
+          (cl-dolist (tag (treesit--language-git-version-tags workdir))
+            (unless latest-version
+              (setq latest-version tag))
+            (treesit--git-checkout-branch workdir tag)
             (treesit--build-grammar
-             workdir grammar-dir language source-dir cc c++))
-          (setq version (treesit--language-git-revision workdir))
-          (setq timestamp (treesit--language-git-timestamp workdir))
-          (message "Validateing version %s" version)
-          (setq exit-code
-                (call-process
-                 emacs-executable nil t nil
-                 "-Q" "--batch"
-                 "--eval" (prin1-to-string
-                           `(let ((treesit-extra-load-path
-                                   '(,grammar-dir)))
-                              (load ,treesit-admin-file-name)
-                              (if (treesit-admin--validate-mode-lang
-                                   ',mode ',language)
-                                  (kill-emacs 0)
-                                (kill-emacs -1)))))))))
-    (list :version version :head-version head-version :timestamp timestamp)))
+             workdir grammar-dir language source-dir cc c++)
+            (setq timestamp (treesit--language-git-timestamp workdir))
+            (setq exit-code
+                  (treesit-admin--validate-grammar
+                   emacs-executable mode grammar-dir language tag))
+            (when (eq exit-code 0)
+              (setq version tag)
+              (cl-return))))
+         ((eq revision-type 'commit)
+          (setq latest-version (treesit--language-git-revision workdir))
+          (treesit--build-grammar
+           workdir grammar-dir language source-dir cc c++)
+          (while (not (eq exit-code 0))
+            (unless (null exit-code)
+              (treesit--git-checkout-branch workdir "HEAD~")
+              (treesit--build-grammar
+               workdir grammar-dir language source-dir cc c++))
+            (setq version (treesit--language-git-revision workdir))
+            (setq timestamp (treesit--language-git-timestamp workdir))
+            (setq exit-code
+                  (treesit-admin--validate-grammar
+                   emacs-executable mode grammar-dir language version)))))))
+    (list :version version
+          :latest-version latest-version
+          :timestamp timestamp)))
+
+(defun treesit-admin--validate-grammar
+    (emacs-executable mode grammar-dir language version)
+"Validate VERSION of LANGUAGE in GRAMMAR-DIR for MODE with EMACS-EXECUTABLE."
+  (message "Validating version %s" version)
+  (call-process
+   emacs-executable nil t nil
+   "-Q" "--batch"
+   "--eval" (prin1-to-string
+             `(let ((treesit-extra-load-path
+                     '(,grammar-dir)))
+                (load ,treesit-admin-file-name)
+                (if (treesit-admin--validate-mode-lang
+                     ',mode ',language)
+                    (kill-emacs 0)
+                  (kill-emacs -1))))))
 
 (defun treesit-admin--last-compatible-grammar-for-modes
-    (modes source-alist grammar-dir &optional emacs-executable)
+    (modes source-alist grammar-dir revision-type &optional emacs-executable)
   "Generate an alist listing latest compatible grammar versions.
 
 MODES, SOURCE-ALIST, GRAMMAR-DIR are the same as
 `treesit-admin--verify-major-mode-queries'.  If EMACS-EXECUTABLE is
 non-nil, use it for validating queries.
 
+REVISION-TYPE is as for `treesit-admin--find-latest-compatible-revision'.
+
 Return an alist of an alist of a plist:
 
-    ((MODE . ((LANG . (:version VERSION :head-VERSION HEAD-VERSION)) ...)) ...)
+    ((MODE . ((LANG . (:version VERSION :latest-version LATEST-VERSION)) ...)) ...)
 
-VERSION and HEAD-VERSION in the plist are the same as in
+VERSION and LATEST-VERSION in the plist are the same as in
 `treesit-admin--find-latest-compatible-revision'."
   (mapcar
    (lambda (mode)
@@ -395,7 +425,7 @@ treesit-admin--last-compatible-grammar-for-modes
             (lambda (language)
               (cons language
                     (treesit-admin--find-latest-compatible-revision
-                     mode language source-alist grammar-dir
+                     mode language source-alist grammar-dir revision-type
                      emacs-executable)))
             (treesit-admin--mode-languages mode))))
    modes))
@@ -410,74 +440,82 @@ treesit-admin--generate-compatibility-report
 compiled).
 
 EMACS-EXECUTABLES is a list of Emacs executables to check for."
-  (let ((tables
-         (mapcar
-          (lambda (emacs)
-            (cons (with-temp-buffer
-                    (call-process emacs nil t nil
-                                  "-Q" "--batch"
-                                  "--eval" "(princ emacs-version)")
-                    (buffer-string))
-                  (treesit-admin--last-compatible-grammar-for-modes
-                   modes
-                   (treesit-admin--unversioned-treesit-language-source-alist)
-                   "/tmp/treesit-grammar"
-                   emacs)))
-          emacs-executables))
-        (database (make-hash-table :test #'equal))
-        languages)
-    (dolist (table tables)
-      (dolist (mode-entry (cdr table))
-        (dolist (language-entry (cdr mode-entry))
-          (let* ((lang (car language-entry))
-                 (plist (cdr language-entry))
-                 ;; KEY = (LANG . EMACS-VERSION)
-                 (key (cons lang (car table)))
-                 (existing-plist (gethash key database)))
-            (push lang languages)
-            ;; If there are two major modes that uses LANG, and they
-            ;; have different compatible versions, use the older
-            ;; version.
-            (when (or (not existing-plist)
-                      (< (plist-get plist :timestamp)
-                         (plist-get existing-plist :timestamp)))
-              (puthash key plist database))))))
-    (setq languages (cl-sort (cl-remove-duplicates languages)
-                             (lambda (a b)
-                               (string< (symbol-name a) (symbol-name b)))))
-    ;; Compose HTML table.
-    (with-temp-buffer
-      (insert "<tr><th>Language</th>")
-      (dolist (emacs-version (mapcar #'car tables))
-        (insert (format "<th>%s</th>" emacs-version)))
-      (insert "</tr>\n")
-      (dolist (lang languages)
-        (insert "<tr>")
-        (insert (format "<th>%s</th>" lang))
+  (with-temp-buffer
+    (dolist (revision-type (list 'tag 'commit))
+      (let ((tables
+             (mapcar
+              (lambda (emacs)
+                (cons (with-temp-buffer
+                        (call-process emacs nil t nil
+                                      "-Q" "--batch"
+                                      "--eval" "(princ emacs-version)")
+                        (buffer-string))
+                      (treesit-admin--last-compatible-grammar-for-modes
+                       modes
+                       (treesit-admin--unversioned-treesit-language-source-alist)
+                       "/tmp/treesit-grammar"
+                       revision-type
+                       emacs)))
+              emacs-executables))
+            (database (make-hash-table :test #'equal))
+            languages)
+        (dolist (table tables)
+          (dolist (mode-entry (cdr table))
+            (dolist (language-entry (cdr mode-entry))
+              (let* ((lang (car language-entry))
+                     (plist (cdr language-entry))
+                     ;; KEY = (LANG . EMACS-VERSION)
+                     (key (cons lang (car table)))
+                     (existing-plist (gethash key database)))
+                (push lang languages)
+                ;; If there are two major modes that uses LANG, and they
+                ;; have different compatible versions, use the older
+                ;; version.
+                (when (or (not existing-plist)
+                          (< (plist-get plist :timestamp)
+                             (plist-get existing-plist :timestamp)))
+                  (puthash key plist database))))))
+        (setq languages (cl-sort (cl-remove-duplicates languages)
+                                 (lambda (a b)
+                                   (string< (symbol-name a) (symbol-name b)))))
+        ;; Compose HTML table.
+        (insert "<table>"
+                "<caption>"
+                (cond ((eq revision-type 'tag) "Tagged")
+                      ((eq revision-type 'commit) "All"))
+                " commits</caption>\n")
+        (insert "<tr><th>Language</th>")
         (dolist (emacs-version (mapcar #'car tables))
-          (let* ((key (cons lang emacs-version))
-                 (plist (gethash key database))
-                 (version (plist-get plist :version))
-                 (head-version (plist-get plist :head-version))
-                 (classname
-                  (if (equal version head-version) "head" "")))
-            (if (not plist)
-                (insert "<td></td>")
-              (insert (format "<td class=\"%s\">%s</td>"
-                              classname version)))))
-        (insert "</tr>\n"))
-
-      ;; Compose table with template and write to out file.
-      (let ((time (current-time-string nil t))
-            (table-text (buffer-string)))
-        (erase-buffer)
-        (insert-file-contents treesit-admin--compat-template-file-name)
-        (goto-char (point-min))
-        (search-forward "___REPLACE_TIME___")
-        (replace-match (format "%s UTC" time) t)
-        (search-forward "___REPLACE_TABLE___")
-        (replace-match table-text t)
-        (write-region (point-min) (point-max) out-file)))))
+          (insert (format "<th>%s</th>" emacs-version)))
+        (insert "</tr>\n")
+        (dolist (lang languages)
+          (insert "<tr>")
+          (insert (format "<th>%s</th>" lang))
+          (dolist (emacs-version (mapcar #'car tables))
+            (let* ((key (cons lang emacs-version))
+                   (plist (gethash key database))
+                   (version (plist-get plist :version))
+                   (latest-version (plist-get plist :latest-version))
+                   (classname
+                    (if (equal version latest-version) "latest" "")))
+              (if (not plist)
+                  (insert "<td></td>")
+                (insert (format "<td class=\"%s\">%s</td>"
+                                classname version)))))
+          (insert "</tr>\n"))
+        (insert "</table>\n")))
+
+    ;; Compose table with template and write to out file.
+    (let ((time (current-time-string nil t))
+          (table-text (buffer-string)))
+      (erase-buffer)
+      (insert-file-contents treesit-admin--compat-template-file-name)
+      (goto-char (point-min))
+      (search-forward "___REPLACE_TIME___")
+      (replace-match (format "%s UTC" time) t)
+      (search-forward "___REPLACE_TABLE___")
+      (replace-match table-text t)
+      (write-region (point-min) (point-max) out-file))))
 
 (provide 'treesit-admin)
 
-- 
2.49.0

From 0ef892974012c48272c6455b4504cce19e3192b0 Mon Sep 17 00:00:00 2001
From: Peter Oliver <[email-masked]>
Date: Wed, 11 Jun 2025 18:03:20 +0100
Subject: [PATCH 3/3] Link to Tree-sitter parser sources from compatibility
 report

* admin/tree-sitter/treesit-admin.el
(treesit-admin--generate-compatibility-report): Add a link to the source
URL.
---
 admin/tree-sitter/treesit-admin.el | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/admin/tree-sitter/treesit-admin.el b/admin/tree-sitter/treesit-admin.el
index 4e2e3855f55..4b6f2d3097b 100644
--- a/admin/tree-sitter/treesit-admin.el
+++ b/admin/tree-sitter/treesit-admin.el
@@ -489,8 +489,9 @@ treesit-admin--generate-compatibility-report
           (insert (format "<th>%s</th>" emacs-version)))
         (insert "</tr>\n")
         (dolist (lang languages)
-          (insert "<tr>")
-          (insert (format "<th>%s</th>" lang))
+          (insert (format "<tr><th><a href=\"%s\"><code>%s</code></a></th>"
+                          (nth 1 (assoc lang treesit-language-source-alist))
+                          lang))
           (dolist (emacs-version (mapcar #'car tables))
             (let* ((key (cons lang emacs-version))
                    (plist (gethash key database))
-- 
2.49.0