From debbugs-submit-bounces@debbugs.gnu.org Sat Oct 28 10:36:08 2023 Received: (at submit) by debbugs.gnu.org; 28 Oct 2023 14:36:08 +0000 Received: from localhost ([127.0.0.1]:39337 helo=debbugs.gnu.org) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1qwkQ4-0002OK-34 for submit@debbugs.gnu.org; Sat, 28 Oct 2023 10:36:08 -0400 Received: from lists.gnu.org ([2001:470:142::17]:42544) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1qwkQ1-0002Np-BU for submit@debbugs.gnu.org; Sat, 28 Oct 2023 10:36:07 -0400 Received: from eggs.gnu.org ([2001:470:142:3::10]) by lists.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1qwkPN-0005Fr-Sl for guix-patches@gnu.org; Sat, 28 Oct 2023 10:35:25 -0400 Received: from mira.cbaines.net ([212.71.252.8]) by eggs.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1qwkPH-0004kv-UO for guix-patches@gnu.org; Sat, 28 Oct 2023 10:35:23 -0400 Received: from localhost (unknown [193.38.40.31]) by mira.cbaines.net (Postfix) with ESMTPSA id 910CD27BBE2 for ; Sat, 28 Oct 2023 15:35:15 +0100 (BST) Received: from localhost (localhost [local]) by localhost (OpenSMTPD) with ESMTPA id f5469498 for ; Sat, 28 Oct 2023 14:35:14 +0000 (UTC) From: Christopher Baines To: guix-patches@gnu.org Subject: [PATCH] lint: Speed up the formatting linter. Date: Sat, 28 Oct 2023 15:35:14 +0100 Message-ID: <4499b0c65aa2b2578b1d2efd17cd9f91d97fd2a0.1698503714.git.mail@cbaines.net> X-Mailer: git-send-email 2.41.0 MIME-Version: 1.0 X-Debbugs-Cc: Christopher Baines , Josselin Poiret , Ludovic Courtès , Mathieu Othacehe , Ricardo Wurmus , Simon Tournier , Tobias Geerinckx-Rice Content-Transfer-Encoding: 8bit Received-SPF: pass client-ip=212.71.252.8; envelope-from=mail@cbaines.net; helo=mira.cbaines.net X-Spam_score_int: -18 X-Spam_score: -1.9 X-Spam_bar: - X-Spam_report: (-1.9 / 5.0 requ) BAYES_00=-1.9, SPF_HELO_PASS=-0.001, SPF_PASS=-0.001, UNPARSEABLE_RELAY=0.001 autolearn=ham autolearn_force=no X-Spam_action: no action X-Spam-Score: 0.9 (/) X-Debbugs-Envelope-To: submit X-BeenThere: debbugs-submit@debbugs.gnu.org X-Mailman-Version: 2.1.18 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: debbugs-submit-bounces@debbugs.gnu.org Sender: "Debbugs-submit" X-Spam-Score: -0.1 (/) By storing the bytes to seek to for the start of each line the first time you want to check a package in a file, rather than figuring out where the package starts each time. This cuts down the time to run guix lint -c formatting from 450 seconds to 13 seconds. * guix/lint.scm (report-formatting-issues): If %check-formatting-seek-lookup is a hash table, store vlist's in it to map from a line number to a byte to seek to. (%check-formatting-seek-lookup): New parameter. * guix/scripts/lint.scm (guix-lint): Enable faster formatting linting, when linting all packages. Change-Id: I34e4d3acfbb1e14e026d2e7f712ba8d22b56c147 --- guix/lint.scm | 44 ++++++++++++++++++++++++++++++++++++++++++- guix/scripts/lint.scm | 3 +++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/guix/lint.scm b/guix/lint.scm index 7ccf52dec1..d94b4026c6 100644 --- a/guix/lint.scm +++ b/guix/lint.scm @@ -68,6 +68,7 @@ (define-module (guix lint) svn-multi-reference-user-name svn-multi-reference-password) #:use-module (guix import stackage) + #:use-module (ice-9 vlist) #:use-module (ice-9 match) #:use-module (ice-9 regex) #:use-module (ice-9 format) @@ -109,6 +110,7 @@ (define-module (guix lint) check-license check-vulnerabilities check-for-updates + %check-formatting-seek-lookup check-formatting check-archival check-profile-collisions @@ -1839,6 +1841,40 @@ (define* (report-formatting-issues package file starting-line #:key (reporters %formatting-reporters)) "Report white-space issues in FILE starting from STARTING-LINE, and report them for PACKAGE." + (define (seek-to-line port line) + (let ((offset + (vlist-ref + (or (hash-ref (%check-formatting-seek-lookup) file) + (call-with-input-file file + (lambda (port) + (let* ((buf-length 80) + (buf (make-string buf-length))) + (let loop ((byte-lookup-list '(0))) + (let* ((rv (%read-delimited! "\n" buf #t port)) + (terminator (car rv)) + (nchars (cdr rv))) + (cond + ((eof-object? terminator) + (let ((byte-lookup-vlist + (list->vlist + (reverse byte-lookup-list)))) + (hash-set! (%check-formatting-seek-lookup) + file + byte-lookup-vlist) + byte-lookup-vlist)) + + ((not terminator) + (loop byte-lookup-list)) + + (nchars + (loop (cons + (ftell port) + byte-lookup-list)))))))))) + (- line 1)))) + (set-port-line! port line) + (seek port offset SEEK_SET) + line)) + (define (sexp-last-line port) ;; Return the last line of the sexp read from PORT or an estimate thereof. (define &failure (list 'failure)) @@ -1857,7 +1893,10 @@ (define* (report-formatting-issues package file starting-line (call-with-input-file file (lambda (port) - (let loop ((line-number 1) + (let loop ((line-number + (if (%check-formatting-seek-lookup) + (seek-to-line port starting-line) + 1)) (last-line #f) (warnings '())) (let ((line (read-line port))) @@ -1879,6 +1918,9 @@ (define* (report-formatting-issues package file starting-line (report package line line-number)) reporters))))))))))) +(define %check-formatting-seek-lookup + (make-parameter #f)) + (define (check-formatting package) "Check the formatting of the source code of PACKAGE." (let ((location (package-location package))) diff --git a/guix/scripts/lint.scm b/guix/scripts/lint.scm index ee3de51fb1..219c3b91be 100644 --- a/guix/scripts/lint.scm +++ b/guix/scripts/lint.scm @@ -222,6 +222,9 @@ (define-command (guix-lint . args) (lambda (store) (cond ((null? args) + ;; Enable fast seeking to lines for the check-formatting linter + (%check-formatting-seek-lookup (make-hash-table)) + (fold-packages (lambda (p r) (run-checkers p checkers #:store store)) '())) (else base-commit: c3cf04d05b452fee549bb84b323d056fd30cef45 -- 2.41.0 From debbugs-submit-bounces@debbugs.gnu.org Sun Nov 05 09:36:12 2023 Received: (at 66796) by debbugs.gnu.org; 5 Nov 2023 14:36:12 +0000 Received: from localhost ([127.0.0.1]:36391 helo=debbugs.gnu.org) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1qzeEW-0002B7-4r for submit@debbugs.gnu.org; Sun, 05 Nov 2023 09:36:12 -0500 Received: from eggs.gnu.org ([2001:470:142:3::10]:47448) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1qzeET-0002At-Ui for 66796@debbugs.gnu.org; Sun, 05 Nov 2023 09:36:10 -0500 Received: from fencepost.gnu.org ([2001:470:142:3::e]) by eggs.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1qzeDm-00020G-7o; Sun, 05 Nov 2023 09:35:26 -0500 DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=gnu.org; s=fencepost-gnu-org; h=MIME-Version:Date:References:In-Reply-To:Subject:To: From; bh=vara9dPixm6WTfZTVgnzl+S/PWGuZ8wqVrJ4c5zxIX0=; b=JaA0evJUK6XdsuMwdc/z EmHdMhnowM3pKD+BXK5G6325zgbzHvrjrP3GLCOn910bJnKtgxMyX37+EuLPWeEBAZXpe0FIAgT9v lq8eDaApk2sxGwMnJ88oN9W4OLutoQCmoGSeHDmrewriozLjkxTSH4rY0di2lDx80RW++O9Ff4+JG dQRf+WwtSmwI63JFGPgCFI7nHy+Z6bh4qAiErQqbL+z/5BzjvYMF8Gv1jfBW7eZo6xkLi/bLkxHBJ 9eDPAZg0owh6T4ad3jg71VTuhrZnXArNzSZosVlnZ+jd7S6+PrcVCoXTGdbi6mDAIJhfVD1ahiJqr KdC6yhDnthRLJw==; From: =?utf-8?Q?Ludovic_Court=C3=A8s?= To: Christopher Baines Subject: Re: [bug#66796] [PATCH] lint: Speed up the formatting linter. In-Reply-To: <4499b0c65aa2b2578b1d2efd17cd9f91d97fd2a0.1698503714.git.mail@cbaines.net> (Christopher Baines's message of "Sat, 28 Oct 2023 15:35:14 +0100") References: <4499b0c65aa2b2578b1d2efd17cd9f91d97fd2a0.1698503714.git.mail@cbaines.net> X-URL: http://www.fdn.fr/~lcourtes/ X-Revolutionary-Date: Quintidi 15 Brumaire an 232 de la =?utf-8?Q?R=C3=A9v?= =?utf-8?Q?olution=2C?= jour du Dindon X-PGP-Key-ID: 0x090B11993D9AEBB5 X-PGP-Key: http://www.fdn.fr/~lcourtes/ludovic.asc X-PGP-Fingerprint: 3CE4 6455 8A84 FDC6 9DB4 0CFB 090B 1199 3D9A EBB5 X-OS: x86_64-pc-linux-gnu Date: Sun, 05 Nov 2023 15:35:22 +0100 Message-ID: <87zfzs1d5h.fsf@gnu.org> User-Agent: Gnus/5.13 (Gnus v5.13) MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable X-Spam-Score: -2.3 (--) X-Debbugs-Envelope-To: 66796 Cc: 66796@debbugs.gnu.org, Josselin Poiret , Simon Tournier , Mathieu Othacehe , Tobias Geerinckx-Rice , Ricardo Wurmus , Christopher Baines X-BeenThere: debbugs-submit@debbugs.gnu.org X-Mailman-Version: 2.1.18 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: debbugs-submit-bounces@debbugs.gnu.org Sender: "Debbugs-submit" X-Spam-Score: -3.3 (---) Hi, Christopher Baines skribis: > By storing the bytes to seek to for the start of each line the first time= you > want to check a package in a file, rather than figuring out where the pac= kage > starts each time. > > This cuts down the time to run guix lint -c formatting from 450 seconds t= o 13 > seconds. Excellent! > + (define (seek-to-line port line) > + (let ((offset > + (vlist-ref > + (or (hash-ref (%check-formatting-seek-lookup) file) > + (call-with-input-file file > + (lambda (port) > + (let* ((buf-length 80) > + (buf (make-string buf-length))) > + (let loop ((byte-lookup-list '(0))) > + (let* ((rv (%read-delimited! "\n" buf #t port)) > + (terminator (car rv)) > + (nchars (cdr rv))) > + (cond > + ((eof-object? terminator) > + (let ((byte-lookup-vlist > + (list->vlist > + (reverse byte-lookup-list)))) > + (hash-set! (%check-formatting-seek-lookup) > + file > + byte-lookup-vlist) > + byte-lookup-vlist)) > + > + ((not terminator) > + (loop byte-lookup-list)) > + > + (nchars > + (loop (cons > + (ftell port) > + byte-lookup-list)))))))))) > + (- line 1)))) > + (set-port-line! port line) > + (seek port offset SEEK_SET) > + line)) Two things: (1) it=E2=80=99s a bit hard to read, in part due to long identifiers, and (2) it would be nice to keep this facility orthogonal, outside the checker. As it turns out, a similar facility is available in (guix utils), exposed via =E2=80=98go-to-location=E2=80=99. Would it be possible to use = it here? > + (let loop ((line-number > + (if (%check-formatting-seek-lookup) > + (seek-to-line port starting-line) > + 1)) Answering myself: I guess =E2=80=98seek-to-line=E2=80=99 can be replaced by (go-to-location port starting-line 0). Thanks! Ludo=E2=80=99. From debbugs-submit-bounces@debbugs.gnu.org Sun Nov 05 13:18:59 2023 Received: (at 66796-done) by debbugs.gnu.org; 5 Nov 2023 18:18:59 +0000 Received: from localhost ([127.0.0.1]:38155 helo=debbugs.gnu.org) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1qzhi7-0002jn-0M for submit@debbugs.gnu.org; Sun, 05 Nov 2023 13:18:59 -0500 Received: from mira.cbaines.net ([2a01:7e00:e000:2f8:fd4d:b5c7:13fb:3d27]:60457) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1qzhi4-0002je-RV for 66796-done@debbugs.gnu.org; Sun, 05 Nov 2023 13:18:57 -0500 Received: from localhost (unknown [193.218.17.244]) by mira.cbaines.net (Postfix) with ESMTPSA id 9C14A27BBE2; Sun, 5 Nov 2023 18:18:18 +0000 (GMT) Received: from felis (localhost [127.0.0.1]) by localhost (OpenSMTPD) with ESMTP id ae65e102; Sun, 5 Nov 2023 18:18:17 +0000 (UTC) References: <4499b0c65aa2b2578b1d2efd17cd9f91d97fd2a0.1698503714.git.mail@cbaines.net> <87zfzs1d5h.fsf@gnu.org> User-agent: mu4e 1.10.7; emacs 29.1 From: Christopher Baines To: Ludovic =?utf-8?Q?Court=C3=A8s?= Subject: Re: [bug#66796] [PATCH] lint: Speed up the formatting linter. Date: Sun, 05 Nov 2023 18:17:31 +0000 In-reply-to: <87zfzs1d5h.fsf@gnu.org> Message-ID: <87pm0ojc7q.fsf@cbaines.net> MIME-Version: 1.0 Content-Type: multipart/signed; boundary="=-=-="; micalg=pgp-sha512; protocol="application/pgp-signature" X-Spam-Score: 3.6 (+++) X-Spam-Report: Spam detection software, running on the system "debbugs.gnu.org", has NOT identified this incoming email as spam. The original message has been attached to this so you can view it or label similar future email. If you have any questions, see the administrator of that system for details. Content preview: Ludovic Courtès writes: > Hi, > > Christopher Baines skribis: > >> By storing the bytes to seek to for the start of each line the first time you >> want to check a package in a file, rather than figuring o [...] Content analysis details: (3.6 points, 10.0 required) pts rule name description ---- ---------------------- -------------------------------------------------- 3.6 RCVD_IN_SBL_CSS RBL: Received via a relay in Spamhaus SBL-CSS [193.218.17.244 listed in zen.spamhaus.org] -0.0 SPF_HELO_PASS SPF: HELO matches SPF record -0.0 SPF_PASS SPF: sender matches SPF record X-Debbugs-Envelope-To: 66796-done Cc: 66796-done@debbugs.gnu.org X-BeenThere: debbugs-submit@debbugs.gnu.org X-Mailman-Version: 2.1.18 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: debbugs-submit-bounces@debbugs.gnu.org Sender: "Debbugs-submit" X-Spam-Score: 2.6 (++) X-Spam-Report: Spam detection software, running on the system "debbugs.gnu.org", has NOT identified this incoming email as spam. The original message has been attached to this so you can view it or label similar future email. If you have any questions, see the administrator of that system for details. Content preview: Ludovic Courtès writes: > Hi, > > Christopher Baines skribis: > >> By storing the bytes to seek to for the start of each line the first time you >> want to check a package in a file, rather than figuring o [...] Content analysis details: (2.6 points, 10.0 required) pts rule name description ---- ---------------------- -------------------------------------------------- 3.6 RCVD_IN_SBL_CSS RBL: Received via a relay in Spamhaus SBL-CSS [193.218.17.244 listed in zen.spamhaus.org] -0.0 SPF_HELO_PASS SPF: HELO matches SPF record -0.0 SPF_PASS SPF: sender matches SPF record -1.0 MAILING_LIST_MULTI Multiple indicators imply a widely-seen list manager --=-=-= Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable Ludovic Court=C3=A8s writes: > Hi, > > Christopher Baines skribis: > >> By storing the bytes to seek to for the start of each line the first tim= e you >> want to check a package in a file, rather than figuring out where the pa= ckage >> starts each time. >> >> This cuts down the time to run guix lint -c formatting from 450 seconds = to 13 >> seconds. > > Excellent! > >> + (define (seek-to-line port line) >> + (let ((offset >> + (vlist-ref >> + (or (hash-ref (%check-formatting-seek-lookup) file) >> + (call-with-input-file file >> + (lambda (port) >> + (let* ((buf-length 80) >> + (buf (make-string buf-length))) >> + (let loop ((byte-lookup-list '(0))) >> + (let* ((rv (%read-delimited! "\n" buf #t port)) >> + (terminator (car rv)) >> + (nchars (cdr rv))) >> + (cond >> + ((eof-object? terminator) >> + (let ((byte-lookup-vlist >> + (list->vlist >> + (reverse byte-lookup-list)))) >> + (hash-set! (%check-formatting-seek-lookup) >> + file >> + byte-lookup-vlist) >> + byte-lookup-vlist)) >> + >> + ((not terminator) >> + (loop byte-lookup-list)) >> + >> + (nchars >> + (loop (cons >> + (ftell port) >> + byte-lookup-list)))))))))) >> + (- line 1)))) >> + (set-port-line! port line) >> + (seek port offset SEEK_SET) >> + line)) > > Two things: (1) it=E2=80=99s a bit hard to read, in part due to long > identifiers, and (2) it would be nice to keep this facility orthogonal, > outside the checker. > > As it turns out, a similar facility is available in (guix utils), > exposed via =E2=80=98go-to-location=E2=80=99. Would it be possible to us= e it here? > >> + (let loop ((line-number >> + (if (%check-formatting-seek-lookup) >> + (seek-to-line port starting-line) >> + 1)) > > Answering myself: I guess =E2=80=98seek-to-line=E2=80=99 can be replaced = by > (go-to-location port starting-line 0). Cool, this simplifies the change a lot, I've pushed the amended version to master as aa98a976079101318d53b412fef6c722bb4332c9. Thanks, Chris --=-=-= Content-Type: application/pgp-signature; name="signature.asc" -----BEGIN PGP SIGNATURE----- iQKlBAEBCgCPFiEEPonu50WOcg2XVOCyXiijOwuE9XcFAmVH3GlfFIAAAAAALgAo aXNzdWVyLWZwckBub3RhdGlvbnMub3BlbnBncC5maWZ0aGhvcnNlbWFuLm5ldDNF ODlFRUU3NDU4RTcyMEQ5NzU0RTBCMjVFMjhBMzNCMEI4NEY1NzcRHG1haWxAY2Jh aW5lcy5uZXQACgkQXiijOwuE9Xfz7g/7B5Y5SFSnjIU9G5M+eZqFOz/FQcUSgobp FWfziNHnc3VpCvEd9f//oVFwkGJtDwP/aex9kCmTeaK76itNJsiu0TI2TpaReCcR l/me1agFyjR2j9mpf3cOdr9VFbenQ1ZryuolZxulNNTee4+fRTOKDTqOarDufc4I WjPKGse6BV1qLA4Ppt1f4OtjfwzyRMCes3HzIrxdUAYSAksj4qTJ33Kc7fDIn6jo o4m78JCIMZWDsKIUlW6VOZ6wSLK/R4xZB8nOW2l4d+HQYjniPvujOO7DZzRt6oYj EJNdMSAmjuNKybFhBwzqS7Vs3OAf9UgdM+7mwBbEPyLMlXC238K/ssuVo/885nCe yDklanS2hz4ugVPqWSj9kpy9fVPbQGtbDJXnyWrGY+UaG3Tzbmkcci8L/kDjyFSk 99iSVTunm0oOqZkaWC/MowVVamrtiwrJ3amwalqQpvGdoDbFFan7LtJQohu70rEb y+G2Z6gzlQL2SaRfRiazUhgFdJ5qsvlNwRZu9USRFWQ2SP2wo6ipjKhFy2Zonu5F szQyxWNIuPlowfkDSrncvsUasDL4Tc3EHPQ1LH866EygSaeaqpxduESp9aT0/UmR TT+eRX/bY5aubmTPEci1mhmUZXq998vrl1Pzq2vOnL9NIJvrDibTpsSzdHnZBQD6 zNvPdrBjGxQ= =VFYY -----END PGP SIGNATURE----- --=-=-=-- From unknown Sat Sep 20 01:51:46 2025 Received: (at fakecontrol) by fakecontrolmessage; To: internal_control@debbugs.gnu.org From: Debbugs Internal Request Subject: Internal Control Message-Id: bug archived. Date: Mon, 04 Dec 2023 12:24:11 +0000 User-Agent: Fakemail v42.6.9 # This is a fake control message. # # The action: # bug archived. thanks # This fakemail brought to you by your local debbugs # administrator