Subject: Re: bug#78748: 31.0.50; Stale Tree-sitter compatibility report

From: Peter Oliver <[email-masked]>
Date: Wed, 11 Jun 2025 01:56:29 +0100 (BST)

On Tue, 10 Jun 2025, Michael Albinus wrote:

> Peter Oliver <[email-masked]> writes:
>
>> The Tree-sitter parser version compatibility report generated by admin/tree-sitter/treesit-admin.el currently shows stale versions of the parsers.
>
> The same question like in bug#78747: Shouldn't we simply use treesit-language-source-alist?

Yes, now that treesit-language-source-alist doesn’t need to be manually set by the user, that seems like a good idea.

Attached is a patch that does this, addressing both bugs 78747 and 78748.

-- 
Peter Oliver

From a8d29c8073b08989ab01ab41f8e41345e752b27a Mon Sep 17 00:00:00 2001
From: Peter Oliver <[email-masked]>
Date: Wed, 11 Jun 2025 01:41:08 +0100
Subject: [PATCH] Eliminate redundant variable
 treesit-admin--builtin-language-sources

Nowadays, each mode populates treesit-language-source-alist when the
mode is required; we do not need to duplicate the information.

* admin/tree-sitter/treesit-admin.el
(treesit-admin--builtin-language-sources)
(treesit-admin-verify-major-mode-queries)
(treesit-admin--generate-compatibility-report): Eliminate variable
treesit-admin--builtin-language-sources.
(treesit-admin--builtin-modes): Add missing modes.
(treesit-admin--builtin-features): New variable, telling us which
features to require to populate treesit-language-source-alist.
(treesit-admin--populated-treesit-language-source-alist): New function,
to add everything we know to treesit-language-source-alist.
(treesit-admin--unversioned-treesit-language-source-alist): Return a
copy of treesit-language-source-alist with the version requirements
removed.
* test/infra/Dockerfile.emba: Make use of
treesit-admin--populated-treesit-language-source-alist.
---
 admin/tree-sitter/treesit-admin.el | 107 +++++++++++++++++++----------
 test/infra/Dockerfile.emba         |  10 +--
 2 files changed, 72 insertions(+), 45 deletions(-)

diff --git a/admin/tree-sitter/treesit-admin.el b/admin/tree-sitter/treesit-admin.el
index f41c4592039..7a18497fc57 100644
--- a/admin/tree-sitter/treesit-admin.el
+++ b/admin/tree-sitter/treesit-admin.el
@@ -71,45 +71,76 @@ treesit-admin-check-manual-coverage
 
 ;;; Query validation
 
-(defvar treesit-admin--builtin-language-sources
-  '((c "https://github.com/tree-sitter/tree-sitter-c" "v0.23.4")
-    (cpp "https://github.com/tree-sitter/tree-sitter-cpp" "v0.23.4")
-    (cmake "https://github.com/uyha/tree-sitter-cmake" "v0.5.0")
-    (dockerfile "https://github.com/camdencheek/tree-sitter-dockerfile" "v0.2.0")
-    (go "https://github.com/tree-sitter/tree-sitter-go" "v0.23.4")
-    (ruby "https://github.com/tree-sitter/tree-sitter-ruby" "v0.23.1")
-    (javascript "https://github.com/tree-sitter/tree-sitter-javascript" "v0.23.1")
-    (typescript "https://github.com/tree-sitter/tree-sitter-typescript"
-                "v0.23.2" "typescript/src")
-    (tsx "https://github.com/tree-sitter/tree-sitter-typescript"
-         "v0.23.2" "tsx/src")
-    (json "https://github.com/tree-sitter/tree-sitter-json" "v0.24.8")
-    (rust "https://github.com/tree-sitter/tree-sitter-rust" "v0.23.2")
-    (php "https://github.com/tree-sitter/tree-sitter-php"
-         "v0.23.11" "php/src")
-    (css "https://github.com/tree-sitter/tree-sitter-css" "v0.23.1")
-    (phpdoc "https://github.com/claytonrcarter/tree-sitter-phpdoc")
-    (doxygen "https://github.com/tree-sitter-grammars/tree-sitter-doxygen" "v1.1.0")
-    (lua "https://github.com/tree-sitter-grammars/tree-sitter-lua" "v0.3.0")
-    (python "https://github.com/tree-sitter/tree-sitter-python" "v0.23.6")
-    (html "https://github.com/tree-sitter/tree-sitter-html" "v0.23.2")
-    (elixir "https://github.com/elixir-lang/tree-sitter-elixir" "v0.3.3")
-    (heex "https://github.com/phoenixframework/tree-sitter-heex" "v0.7.0")
-    (java "https://github.com/tree-sitter/tree-sitter-java" "v0.23.5")
-    (jsdoc "https://github.com/tree-sitter/tree-sitter-jsdoc" "v0.23.2")
-    (toml "https://github.com/tree-sitter-grammars/tree-sitter-toml" "v0.7.0")
-    (yaml "https://github.com/tree-sitter-grammars/tree-sitter-yaml" "v0.7.0"))
-  "A list of sources for the builtin modes.
-The source information are in the format of
-`treesit-language-source-alist'.  This is for development only.")
-
 (defvar treesit-admin--builtin-modes
-  '( c-ts-mode c++-ts-mode cmake-ts-mode dockerfile-ts-mode
-     go-ts-mode ruby-ts-mode js-ts-mode typescript-ts-mode tsx-ts-mode
-     json-ts-mode rust-ts-mode php-ts-mode css-ts-mode lua-ts-mode
-     html-ts-mode elixir-ts-mode heex-ts-mode java-ts-mode)
+  '(bash-ts-mode
+    c++-ts-mode
+    c-ts-mode
+    cmake-ts-mode
+    csharp-ts-mode
+    css-ts-mode
+    dockerfile-ts-mode
+    elixir-ts-mode
+    go-ts-mode
+    heex-ts-mode
+    html-ts-mode
+    java-ts-mode
+    js-ts-mode
+    json-ts-mode
+    lua-ts-mode
+    markdown-ts-mode
+    php-ts-mode
+    python-ts-mode
+    ruby-ts-mode
+    rust-ts-mode
+    toml-ts-mode
+    tsx-ts-mode
+    typescript-ts-mode)
   "Builtin tree-sitter modes that we check.")
 
+(defvar treesit-admin--builtin-features
+  '(c-ts-mode
+    cmake-ts-mode
+    csharp-mode
+    css-mode
+    dockerfile-ts-mode
+    elixir-ts-mode
+    go-ts-mode
+    heex-ts-mode
+    html-ts-mode
+    java-ts-mode
+    js
+    json-ts-mode
+    lua-ts-mode
+    markdown-ts-mode
+    php-ts-mode
+    python
+    ruby-ts-mode
+    rust-ts-mode
+    sh-script
+    toml-ts-mode
+    treesit-x
+    typescript-ts-mode
+    yaml-ts-mode)
+  "Built in features that modify treesit-language-source-alist when loaded.")
+
+(defun treesit-admin--populated-treesit-language-source-alist ()
+  "Ensure that treesit-language-source-alist is fully populated.
+This is done by `require'ing all of the features that extend it."
+  (dolist (feature treesit-admin--builtin-features)
+    (require feature))
+  treesit-language-source-alist)
+
+(defun treesit-admin--unversioned-treesit-language-source-alist ()
+  "Return a copy of treesit-language-source-alist, with any revisions removed."
+  (mapcar
+   (lambda (source)
+     (if (nthcdr 2 source)
+         (let ((unversioned-source (copy-sequence source)))
+           (setcar (nthcdr 2 unversioned-source) nil)
+           unversioned-source)
+       source))
+   (treesit-admin--populated-treesit-language-source-alist)))
+
 (defun treesit-admin--verify-major-mode-queries (modes source-alist grammar-dir)
   "Verify font-lock queries in MODES.
 
@@ -228,7 +259,7 @@ treesit-admin-verify-major-mode-queries
   (interactive)
   (treesit-admin--verify-major-mode-queries
    treesit-admin--builtin-modes
-   treesit-admin--builtin-language-sources
+   (treesit-admin--populated-treesit-language-source-alist)
    "/tmp/tree-sitter-grammars"))
 
 ;;; Compatibility report
@@ -387,7 +418,7 @@ treesit-admin--generate-compatibility-report
                     (buffer-string))
                   (treesit-admin--last-compatible-grammar-for-modes
                    modes
-                   treesit-admin--builtin-language-sources
+                   (treesit-admin--unversioned-treesit-language-source-alist)
                    "/tmp/treesit-grammar"
                    emacs)))
           emacs-executables))
diff --git a/test/infra/Dockerfile.emba b/test/infra/Dockerfile.emba
index 6be67a95e75..1ecd3161050 100644
--- a/test/infra/Dockerfile.emba
+++ b/test/infra/Dockerfile.emba
@@ -138,12 +138,9 @@ RUN src/emacs -Q --batch \
         (treesit-library-abi-version t) (treesit-library-abi-version))' \
     --eval '(message "\nInstalling grammars\n===================")' \
     --eval '(setq treesit-extra-load-path (list "/root/.emacs.d/tree-sitter"))' \
-    --eval '(dolist (feature (quote (c-ts-mode cmake-ts-mode csharp-mode \
-        dockerfile-ts-mode elixir-ts-mode heex-ts-mode go-ts-mode java-ts-mode \
-        js json-ts-mode lua-ts-mode php-ts-mode python ruby-ts-mode rust-ts-mode \
-        sh-script typescript-ts-mode css-mode html-ts-mode markdown-ts-mode \
-        toml-ts-mode yaml-ts-mode treesit-x))) (require feature))' \
-    --eval '(dolist (lang (mapcar (quote car) treesit-language-source-alist)) \
+    -l admin/tree-sitter/treesit-admin.el \
+    --eval '(dolist (lang (mapcar (quote car) \
+                                  (treesit-admin--populated-treesit-language-source-alist))) \
         (treesit-install-language-grammar lang "/root/.emacs.d/tree-sitter"))' \
     --eval '(message "\ntreesit-language-source-alist")' \
     --eval '(message "=============================")' \
@@ -152,7 +149,6 @@ RUN src/emacs -Q --batch \
     --eval \
         '(dolist (lang (sort (mapcar (quote car) treesit-language-source-alist))) \
          (message "%s ABI version %d" lang (treesit-language-abi-version lang)))' \
-    -l admin/tree-sitter/treesit-admin.el \
     --eval '(setq treesit-admin--builtin-language-sources \
         treesit-language-source-alist)' \
     --eval '(message "\ntreesit-admin-check-manual-coverage")' \
-- 
2.49.0