From unknown Sun Jun 22 07:36:47 2025 X-Loop: help-debbugs@gnu.org Subject: [bug#63646] [PATCH] substitute: If a server's nar URL is 404, try the next one(s). Resent-From: Ludovic =?UTF-8?Q?Court=C3=A8s?= Original-Sender: "Debbugs-submit" Resent-CC: mail@cbaines.net, dev@jpoiret.xyz, ludo@gnu.org, othacehe@gnu.org, rekado@elephly.net, zimon.toutoune@gmail.com, me@tobias.gr, guix-patches@gnu.org Resent-Date: Mon, 22 May 2023 15:32:01 +0000 Resent-Message-ID: Resent-Sender: help-debbugs@gnu.org X-GNU-PR-Message: report 63646 X-GNU-PR-Package: guix-patches X-GNU-PR-Keywords: patch To: 63646@debbugs.gnu.org Cc: Ludovic =?UTF-8?Q?Court=C3=A8s?= , Christopher Baines , Josselin Poiret , Ludovic =?UTF-8?Q?Court=C3=A8s?= , Mathieu Othacehe , Ricardo Wurmus , Simon Tournier , Tobias Geerinckx-Rice X-Debbugs-Original-To: guix-patches@gnu.org X-Debbugs-Original-Xcc: Christopher Baines , Josselin Poiret , Ludovic =?UTF-8?Q?Court=C3=A8s?= , Mathieu Othacehe , Ricardo Wurmus , Simon Tournier , Tobias Geerinckx-Rice Received: via spool by submit@debbugs.gnu.org id=B.168476946918760 (code B ref -1); Mon, 22 May 2023 15:32:01 +0000 Received: (at submit) by debbugs.gnu.org; 22 May 2023 15:31:09 +0000 Received: from localhost ([127.0.0.1]:35934 helo=debbugs.gnu.org) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1q17V6-0004sV-Kw for submit@debbugs.gnu.org; Mon, 22 May 2023 11:31:09 -0400 Received: from lists.gnu.org ([209.51.188.17]:43202) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1q17V4-0004sN-F8 for submit@debbugs.gnu.org; Mon, 22 May 2023 11:31:07 -0400 Received: from eggs.gnu.org ([2001:470:142:3::10]) by lists.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1q17V3-0007JC-CE for guix-patches@gnu.org; Mon, 22 May 2023 11:31:06 -0400 Received: from fencepost.gnu.org ([2001:470:142:3::e]) by eggs.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1q17V1-0001E5-PC; Mon, 22 May 2023 11:31:04 -0400 DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=gnu.org; s=fencepost-gnu-org; h=MIME-Version:Date:Subject:To:From:in-reply-to: references; bh=W8FgW+p0K19l9XTbOwJpeMnNIILn3UGIhOSd+7VQ6f4=; b=WMKZnyqJO8dQpq ehNwetFFkQANvEyS2EOB5ibdDFU8hFVRlzuJECOyndywjI6tKiPFz38iMvXaYEM10bPSH9qk5MCAl NHx81MVHbamkLOuZeS8H31hZ0Kxal5VI78bt/mm+Utn8ixgCaGqDbYt/eXHPHiNsy0ylIXR+DwpAA cYV+ft79LqFlcjg8erkwN7psQG2tEDjE4vC/IOX2gre10KDf02wViXvIAPGIkw6VnZtjzI+7RNvgm I+Cbe/ec6RzNB7gEO4CFck3EbH5biPeBLF7EzjYdd3CS3i2i/h8v8iXg876GK45InULqX4aDHxE07 QcSMdPHJsAWhkcceLWOw==; Received: from [193.50.110.247] (helo=gnu.org) by fencepost.gnu.org with esmtpsa (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1q17V0-0003C9-65; Mon, 22 May 2023 11:31:02 -0400 From: Ludovic =?UTF-8?Q?Court=C3=A8s?= Date: Mon, 22 May 2023 17:30:50 +0200 Message-Id: X-Mailer: git-send-email 2.40.1 MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Spam-Score: -2.3 (--) X-BeenThere: debbugs-submit@debbugs.gnu.org X-Mailman-Version: 2.1.18 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: debbugs-submit-bounces@debbugs.gnu.org Sender: "Debbugs-submit" X-Spam-Score: -3.3 (---) If a substitute server advertises in its narinfo, for example, both a /zstd and a /lzip URL but the /zstd URL is unreachable, try the /lzip URL. Fixes . * guix/narinfo.scm (narinfo-preferred-uris): New procedure. (narinfo-best-uri): Rebase on top of it. * guix/scripts/substitute.scm (download-nar)[try-fetch]: New procedure. Use 'narinfo-preferred-uris' and 'try-fetch' to attempt all the URLs of NARINFO. * tests/substitute.scm (request-substitution): Remove 'parameterize'. Delete DESTINATION. ("substitute, preferred nar URL is 404, other is 200"): New test. --- guix/narinfo.scm | 21 +++++++++++++++++---- guix/scripts/substitute.scm | 33 ++++++++++++++++++++++----------- tests/substitute.scm | 36 +++++++++++++++++++++++++++++++----- 3 files changed, 70 insertions(+), 20 deletions(-) diff --git a/guix/narinfo.scm b/guix/narinfo.scm index 741c7ad406..a149d9a901 100644 --- a/guix/narinfo.scm +++ b/guix/narinfo.scm @@ -54,6 +54,7 @@ (define-module (guix narinfo) narinfo-hash-algorithm+value narinfo-hash->sha256 + narinfo-preferred-uris narinfo-best-uri valid-narinfo? @@ -309,9 +310,11 @@ (define (decompresses-faster? compression1 compression2) ("gzip" (string=? compression2 "lzip")) (_ #f))) -(define* (narinfo-best-uri narinfo #:key fast-decompression?) - "Select the \"best\" URI to download NARINFO's nar, and return three values: -the URI, its compression method (a string), and the compressed file size. +(define* (narinfo-preferred-uris narinfo #:key fast-decompression?) + "Return the sorted list of \"preferred\" nar URIs from NARINFO (preferred +comes first) where each entry is a tuple containing: the URI, its compression +method (a string), and the compressed file size. + When FAST-DECOMPRESSION? is true, prefer substitutes with faster decompression (typically zstd) rather than substitutes with a higher compression ratio (typically lzip)." @@ -343,6 +346,16 @@ (define* (narinfo-best-uri narinfo #:key fast-decompression?) ((uri2 compression2 . _) (decompresses-faster? compression2 compression1)))))) - (match (sort choices (if fast-decompression? (negate speedstring uri))))) - (let ((uri compression file-size - (narinfo-best-uri narinfo - #:fast-decompression? - %prefer-fast-decompression?))) - (unless print-build-trace? - (format (current-error-port) - (G_ "Downloading ~a...~%") (uri->string uri))) + (define (try-fetch choices) + (match choices + (((uri compression file-size) rest ...) + (guard (c ((and (pair? rest) (network-error? c)) + (warning (G_ "download from '~a' failed, trying next URL~%") + (uri->string uri)) + (try-fetch rest))) + (let ((port download-size (fetch uri))) + (unless print-build-trace? + (format (current-error-port) + (G_ "Downloading ~a...~%") (uri->string uri))) + (values port uri compression download-size)))) + (() + (leave (G_ "no valid nar URLs for ~a at ~a~%") + (narinfo-path narinfo) + (narinfo-uri-base narinfo))))) - (let* ((raw download-size - ;; 'guix publish' without '--cache' doesn't specify a - ;; Content-Length, so DOWNLOAD-SIZE is #f in this case. - (fetch uri)) + (let ((choices (narinfo-preferred-uris narinfo + #:fast-decompression? + %prefer-fast-decompression?))) + ;; 'guix publish' without '--cache' doesn't specify a Content-Length, so + ;; DOWNLOAD-SIZE is #f in this case. + (let* ((raw uri compression download-size (try-fetch choices)) (progress (let* ((dl-size (or download-size (and (equal? compression "none") diff --git a/tests/substitute.scm b/tests/substitute.scm index 9032a50268..8df3938b59 100644 --- a/tests/substitute.scm +++ b/tests/substitute.scm @@ -64,11 +64,11 @@ (define-syntax-rule (test-quit name error-rx exp) (define (request-substitution item destination) "Run 'guix substitute --substitute' to fetch ITEM to DESTINATION." - (parameterize ((guix-warning-port (current-error-port))) - (with-input-from-string (string-append "substitute " item " " - destination "\n") - (lambda () - (guix-substitute "--substitute"))))) + (false-if-exception (delete-file destination)) + (with-input-from-string (string-append "substitute " item " " + destination "\n") + (lambda () + (guix-substitute "--substitute")))) (define %public-key ;; This key is known to be in the ACL by default. @@ -613,6 +613,32 @@ (define-syntax-rule (with-narinfo* narinfo directory body ...) (lambda () (false-if-exception (delete-file "substitute-retrieved"))))))))))) +(test-equal "substitute, preferred nar URL is 404, other is 200" + "Substitutable data." + (with-narinfo* (string-append %narinfo "Signature: " (signature-field %narinfo)) + %main-substitute-directory + + (with-http-server `((200 ,(string-append %narinfo "Signature: " + (signature-field %narinfo) + "\n" + "URL: example.nar.lz\n" + "Compression: lzip\n")) + (404 "Sorry, nar.lz is missing!") + (200 ,(call-with-input-file + (string-append %main-substitute-directory + "/example.nar") + get-bytevector-all))) + (dynamic-wind + (const #t) + (lambda () + (parameterize ((substitute-urls (list (%local-url)))) + (request-substitution (string-append (%store-prefix) + "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo") + "substitute-retrieved")) + (call-with-input-file "substitute-retrieved" get-string-all)) + (lambda () + (false-if-exception (delete-file "substitute-retrieved"))))))) + (test-quit "substitute, narinfo is available but nar is missing" "failed to find alternative substitute" (with-narinfo* base-commit: dff1689bb37e5303868584d3f1d7a33cbcb7f51e -- 2.40.1 From unknown Sun Jun 22 07:36:47 2025 X-Loop: help-debbugs@gnu.org Subject: [bug#63646] [PATCH] substitute: If a server's nar URL is 404, try the next one(s). Resent-From: Ludovic =?UTF-8?Q?Court=C3=A8s?= Original-Sender: "Debbugs-submit" Resent-CC: guix-patches@gnu.org Resent-Date: Mon, 22 May 2023 15:46:02 +0000 Resent-Message-ID: Resent-Sender: help-debbugs@gnu.org X-GNU-PR-Message: followup 63646 X-GNU-PR-Package: guix-patches X-GNU-PR-Keywords: patch To: 63646@debbugs.gnu.org Cc: Josselin Poiret , Tobias Geerinckx-Rice , Simon Tournier , Mathieu Othacehe , Christopher Baines , Ricardo Wurmus Received: via spool by 63646-submit@debbugs.gnu.org id=B63646.168477034020249 (code B ref 63646); Mon, 22 May 2023 15:46:02 +0000 Received: (at 63646) by debbugs.gnu.org; 22 May 2023 15:45:40 +0000 Received: from localhost ([127.0.0.1]:35994 helo=debbugs.gnu.org) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1q17j9-0005GX-TU for submit@debbugs.gnu.org; Mon, 22 May 2023 11:45:40 -0400 Received: from eggs.gnu.org ([209.51.188.92]:36838) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1q17j7-0005GJ-Q8 for 63646@debbugs.gnu.org; Mon, 22 May 2023 11:45:38 -0400 Received: from fencepost.gnu.org ([2001:470:142:3::e]) by eggs.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1q17j0-0004Ss-Lv; Mon, 22 May 2023 11:45:30 -0400 DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=gnu.org; s=fencepost-gnu-org; h=MIME-Version:In-Reply-To:Date:References:Subject:To: From; bh=jNGDR7jAhZT1/H2LdJMZCjKtyNMIxzu14LgJZEUFzmw=; b=QOtBR7v20b/OdOKJaOtL eUQrtowVRm4UPSvX5j8bew3Lx1TdY8cbfJoXBYt01uirqBdDM79SFp9e7Rr2YJCqVljR7+QnHB6c9 kwokEspHARbPF8i3Jeyz06liWeZKhIbeU+Br0+7Zco3DcAWh3RuEidyCUaNfV6UNyyy6Lp3h+NzT2 HNAgDeSibRt8E3x+Dipx0W0nkXl3h6yYOSXN8lX8pIoX9YmmLDcsFkaErySrC+H6AYwetmBJ/1wyj iYRc+R7fxUNBUvK8TBN/qeW8l5/hrGlnXtiaAQreNAcjgMR66Wq+regBv1rMyIfZwDdpHuNw+qf5w 7R8adxxXeu1bvA==; Received: from [193.50.110.247] (helo=ribbon) by fencepost.gnu.org with esmtpsa (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1q17j0-0002kE-7L; Mon, 22 May 2023 11:45:30 -0400 From: Ludovic =?UTF-8?Q?Court=C3=A8s?= References: Date: Mon, 22 May 2023 17:45:27 +0200 In-Reply-To: ("Ludovic =?UTF-8?Q?Court=C3=A8s?="'s message of "Mon, 22 May 2023 17:30:50 +0200") Message-ID: <87v8gkv088.fsf@gnu.org> User-Agent: Gnus/5.13 (Gnus v5.13) Emacs/28.2 (gnu/linux) MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable X-Spam-Score: -2.3 (--) X-BeenThere: debbugs-submit@debbugs.gnu.org X-Mailman-Version: 2.1.18 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: debbugs-submit-bounces@debbugs.gnu.org Sender: "Debbugs-submit" X-Spam-Score: -3.3 (---) Ludovic Court=C3=A8s skribis: > + (define (try-fetch choices) > + (match choices > + (((uri compression file-size) rest ...) > + (guard (c ((and (pair? rest) (network-error? c)) > + (warning (G_ "download from '~a' failed, trying next U= RL~%") > + (uri->string uri)) I realized we can change =E2=80=98network-error?=E2=80=99 to =E2=80=98http-= get-error?=E2=80=99 above. Otherwise, we could find ourselves trying several nar URLs on the same server when the error is ETIMEDOUT or ECONNREFUSED, which would be a waste of time. Ludo=E2=80=99. From debbugs-submit-bounces@debbugs.gnu.org Mon May 22 12:00:45 2023 Received: (at control) by debbugs.gnu.org; 22 May 2023 16:00:46 +0000 Received: from localhost ([127.0.0.1]:36024 helo=debbugs.gnu.org) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1q17xl-0005tE-Js for submit@debbugs.gnu.org; Mon, 22 May 2023 12:00:45 -0400 Received: from eggs.gnu.org ([209.51.188.92]:40028) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1q17xh-0005sx-Lx for control@debbugs.gnu.org; Mon, 22 May 2023 12:00:44 -0400 Received: from fencepost.gnu.org ([2001:470:142:3::e]) by eggs.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1q17xb-00088Z-9m for control@debbugs.gnu.org; Mon, 22 May 2023 12:00:35 -0400 DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=gnu.org; s=fencepost-gnu-org; h=MIME-version:Subject:From:To:Date:in-reply-to: references; bh=4zECToVhrywzvp+oETONgrTvPkYgWSnwH+ezRWrJLFI=; b=ENNHPHqrOk3ixo QJp6j0qUeeeasLcBaZomwsDNTVhvkVe9v7rIbq48GUa/pJDz0kJFFvd8/cQhVj9VWqvNxzZA3wm27 63+dNVtz8McsPGzB7JwV/BohJQg2QajCFNpbriXbMWCQ2eaoaryKl+H4m9KkjiAVjEftpnFLUDzP5 d0v8QGgoMsH8oYGGKGNG2XXy8PJ/6weKNnjWOxXDwLauPuHWEXWzj757p5PBszxoe0VyGuYWlDwyS TdqSybVSSAt1mGUWZenfiqw1szuarCHzZuoIVDCummcx5MzZHTObmxUKVTZc/xMfZbtduhdr26Cu0 W3OS0SQd5ChtNG37/6Cw==; Received: from [193.50.110.247] (helo=ribbon) by fencepost.gnu.org with esmtpsa (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1q17xR-0001C5-TB for control@debbugs.gnu.org; Mon, 22 May 2023 12:00:34 -0400 Date: Mon, 22 May 2023 18:00:23 +0200 Message-Id: <87ttw4uzjc.fsf@gnu.org> To: control@debbugs.gnu.org From: =?utf-8?Q?Ludovic_Court=C3=A8s?= Subject: control message for bug #63646 MIME-version: 1.0 Content-type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit X-Spam-Score: -2.3 (--) X-Debbugs-Envelope-To: control X-BeenThere: debbugs-submit@debbugs.gnu.org X-Mailman-Version: 2.1.18 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: debbugs-submit-bounces@debbugs.gnu.org Sender: "Debbugs-submit" X-Spam-Score: -3.3 (---) severity 63646 important quit From unknown Sun Jun 22 07:36:47 2025 MIME-Version: 1.0 X-Mailer: MIME-tools 5.505 (Entity 5.505) X-Loop: help-debbugs@gnu.org From: help-debbugs@gnu.org (GNU bug Tracking System) To: Ludovic =?UTF-8?Q?Court=C3=A8s?= Subject: bug#63646: closed (Re: bug#63646: [PATCH] substitute: If a server's nar URL is 404, try the next one(s).) Message-ID: References: <87cz2ieq78.fsf_-_@gnu.org> X-Gnu-PR-Message: they-closed 63646 X-Gnu-PR-Package: guix-patches X-Gnu-PR-Keywords: patch Reply-To: 63646@debbugs.gnu.org Date: Mon, 29 May 2023 22:20:02 +0000 Content-Type: multipart/mixed; boundary="----------=_1685398802-8768-1" This is a multi-part message in MIME format... ------------=_1685398802-8768-1 Content-Disposition: inline Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Your bug report #63646: [PATCH] substitute: If a server's nar URL is 404, try the next one(= s). which was filed against the guix-patches package, has been closed. The explanation is attached below, along with your original report. If you require more details, please reply to 63646@debbugs.gnu.org. --=20 63646: https://debbugs.gnu.org/cgi/bugreport.cgi?bug=3D63646 GNU Bug Tracking System Contact help-debbugs@gnu.org with problems ------------=_1685398802-8768-1 Content-Type: message/rfc822 Content-Disposition: inline Content-Transfer-Encoding: 7bit Received: (at 63646-done) by debbugs.gnu.org; 29 May 2023 22:19:19 +0000 Received: from localhost ([127.0.0.1]:59396 helo=debbugs.gnu.org) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1q3lCx-0002Fi-CI for submit@debbugs.gnu.org; Mon, 29 May 2023 18:19:19 -0400 Received: from eggs.gnu.org ([209.51.188.92]:35700) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1q3lCu-0002FO-GJ; Mon, 29 May 2023 18:19:17 -0400 Received: from fencepost.gnu.org ([2001:470:142:3::e]) by eggs.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1q3lCo-0007cX-H6; Mon, 29 May 2023 18:19:10 -0400 DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=gnu.org; s=fencepost-gnu-org; h=MIME-Version:In-Reply-To:Date:References:Subject:To: From; bh=7kHK5jBS14tAPNWAqOSbiIfo0evyafEB5mRID8IBZvM=; b=iuaH5gdgi3fVM5j5xGeI tJjqHMItEgxHGWR+NXsUSuxBcI/h4jBDJQjCwyUC80RKsR1ewrBvbCqpurHNvc4R3EBmY1wTzzOXg sSP/LBLRvFU2oYl76XQme2h7S5sSNVTb1B+L3J6Rs0wWlm6/a3jRe37E3teFEwEz6WnRrsAnrH2hJ d0V7oXgcLYbflit9Bw8h8yjK1+H6HHsjdjPTxJ5oLf9gIaRW270WkL0zsQSvGbGsmaTsGOYtxnSF3 /wL5o6iJfpe4n0xBSdxIwCxf+Z/XJdgZnmCWs0PMIbLSo0czKvupdi63lhe1dogb852BRbAwvdVKt sWiIctCrF3Fouw==; Received: from 91-160-117-201.subs.proxad.net ([91.160.117.201] helo=ribbon) by fencepost.gnu.org with esmtpsa (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1q3lCo-0006iC-33; Mon, 29 May 2023 18:19:10 -0400 From: =?utf-8?Q?Ludovic_Court=C3=A8s?= To: 63646-done@debbugs.gnu.org, 63634-done@debbugs.gnu.org Subject: Re: bug#63646: [PATCH] substitute: If a server's nar URL is 404, try the next one(s). References: <87v8gkv088.fsf@gnu.org> Date: Tue, 30 May 2023 00:19:07 +0200 In-Reply-To: <87v8gkv088.fsf@gnu.org> ("Ludovic =?utf-8?Q?Court=C3=A8s=22'?= =?utf-8?Q?s?= message of "Mon, 22 May 2023 17:45:27 +0200") Message-ID: <87cz2ieq78.fsf_-_@gnu.org> User-Agent: Gnus/5.13 (Gnus v5.13) Emacs/28.2 (gnu/linux) MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable X-Spam-Score: -2.3 (--) X-Debbugs-Envelope-To: 63646-done Cc: Josselin Poiret , Christopher Baines , Simon Tournier , Mathieu Othacehe , Tobias Geerinckx-Rice , Ricardo Wurmus X-BeenThere: debbugs-submit@debbugs.gnu.org X-Mailman-Version: 2.1.18 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: debbugs-submit-bounces@debbugs.gnu.org Sender: "Debbugs-submit" X-Spam-Score: -3.3 (---) Hi, Ludovic Court=C3=A8s skribis: > Ludovic Court=C3=A8s skribis: > >> + (define (try-fetch choices) >> + (match choices >> + (((uri compression file-size) rest ...) >> + (guard (c ((and (pair? rest) (network-error? c)) >> + (warning (G_ "download from '~a' failed, trying next = URL~%") >> + (uri->string uri)) > > I realized we can change =E2=80=98network-error?=E2=80=99 to =E2=80=98htt= p-get-error?=E2=80=99 above. > Otherwise, we could find ourselves trying several nar URLs on the same > server when the error is ETIMEDOUT or ECONNREFUSED, which would be a > waste of time. Pushed as 8af9a2aa5fa2fa5b00234c1cbe12e9aff60888a0. Ludo=E2=80=99. ------------=_1685398802-8768-1 Content-Type: message/rfc822 Content-Disposition: inline Content-Transfer-Encoding: 7bit Received: (at submit) by debbugs.gnu.org; 22 May 2023 15:31:09 +0000 Received: from localhost ([127.0.0.1]:35934 helo=debbugs.gnu.org) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1q17V6-0004sV-Kw for submit@debbugs.gnu.org; Mon, 22 May 2023 11:31:09 -0400 Received: from lists.gnu.org ([209.51.188.17]:43202) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1q17V4-0004sN-F8 for submit@debbugs.gnu.org; Mon, 22 May 2023 11:31:07 -0400 Received: from eggs.gnu.org ([2001:470:142:3::10]) by lists.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1q17V3-0007JC-CE for guix-patches@gnu.org; Mon, 22 May 2023 11:31:06 -0400 Received: from fencepost.gnu.org ([2001:470:142:3::e]) by eggs.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1q17V1-0001E5-PC; Mon, 22 May 2023 11:31:04 -0400 DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=gnu.org; s=fencepost-gnu-org; h=MIME-Version:Date:Subject:To:From:in-reply-to: references; bh=W8FgW+p0K19l9XTbOwJpeMnNIILn3UGIhOSd+7VQ6f4=; b=WMKZnyqJO8dQpq ehNwetFFkQANvEyS2EOB5ibdDFU8hFVRlzuJECOyndywjI6tKiPFz38iMvXaYEM10bPSH9qk5MCAl NHx81MVHbamkLOuZeS8H31hZ0Kxal5VI78bt/mm+Utn8ixgCaGqDbYt/eXHPHiNsy0ylIXR+DwpAA cYV+ft79LqFlcjg8erkwN7psQG2tEDjE4vC/IOX2gre10KDf02wViXvIAPGIkw6VnZtjzI+7RNvgm I+Cbe/ec6RzNB7gEO4CFck3EbH5biPeBLF7EzjYdd3CS3i2i/h8v8iXg876GK45InULqX4aDHxE07 QcSMdPHJsAWhkcceLWOw==; Received: from [193.50.110.247] (helo=gnu.org) by fencepost.gnu.org with esmtpsa (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1q17V0-0003C9-65; Mon, 22 May 2023 11:31:02 -0400 From: =?UTF-8?q?Ludovic=20Court=C3=A8s?= To: guix-patches@gnu.org Subject: [PATCH] substitute: If a server's nar URL is 404, try the next one(s). Date: Mon, 22 May 2023 17:30:50 +0200 Message-Id: X-Mailer: git-send-email 2.40.1 MIME-Version: 1.0 X-Debbugs-Cc: Christopher Baines , Josselin Poiret , Ludovic Courtès , Mathieu Othacehe , Ricardo Wurmus , Simon Tournier , Tobias Geerinckx-Rice Content-Transfer-Encoding: 8bit X-Spam-Score: -2.3 (--) X-Debbugs-Envelope-To: submit Cc: =?UTF-8?q?Ludovic=20Court=C3=A8s?= X-BeenThere: debbugs-submit@debbugs.gnu.org X-Mailman-Version: 2.1.18 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: debbugs-submit-bounces@debbugs.gnu.org Sender: "Debbugs-submit" X-Spam-Score: -3.3 (---) If a substitute server advertises in its narinfo, for example, both a /zstd and a /lzip URL but the /zstd URL is unreachable, try the /lzip URL. Fixes . * guix/narinfo.scm (narinfo-preferred-uris): New procedure. (narinfo-best-uri): Rebase on top of it. * guix/scripts/substitute.scm (download-nar)[try-fetch]: New procedure. Use 'narinfo-preferred-uris' and 'try-fetch' to attempt all the URLs of NARINFO. * tests/substitute.scm (request-substitution): Remove 'parameterize'. Delete DESTINATION. ("substitute, preferred nar URL is 404, other is 200"): New test. --- guix/narinfo.scm | 21 +++++++++++++++++---- guix/scripts/substitute.scm | 33 ++++++++++++++++++++++----------- tests/substitute.scm | 36 +++++++++++++++++++++++++++++++----- 3 files changed, 70 insertions(+), 20 deletions(-) diff --git a/guix/narinfo.scm b/guix/narinfo.scm index 741c7ad406..a149d9a901 100644 --- a/guix/narinfo.scm +++ b/guix/narinfo.scm @@ -54,6 +54,7 @@ (define-module (guix narinfo) narinfo-hash-algorithm+value narinfo-hash->sha256 + narinfo-preferred-uris narinfo-best-uri valid-narinfo? @@ -309,9 +310,11 @@ (define (decompresses-faster? compression1 compression2) ("gzip" (string=? compression2 "lzip")) (_ #f))) -(define* (narinfo-best-uri narinfo #:key fast-decompression?) - "Select the \"best\" URI to download NARINFO's nar, and return three values: -the URI, its compression method (a string), and the compressed file size. +(define* (narinfo-preferred-uris narinfo #:key fast-decompression?) + "Return the sorted list of \"preferred\" nar URIs from NARINFO (preferred +comes first) where each entry is a tuple containing: the URI, its compression +method (a string), and the compressed file size. + When FAST-DECOMPRESSION? is true, prefer substitutes with faster decompression (typically zstd) rather than substitutes with a higher compression ratio (typically lzip)." @@ -343,6 +346,16 @@ (define* (narinfo-best-uri narinfo #:key fast-decompression?) ((uri2 compression2 . _) (decompresses-faster? compression2 compression1)))))) - (match (sort choices (if fast-decompression? (negate speedstring uri))))) - (let ((uri compression file-size - (narinfo-best-uri narinfo - #:fast-decompression? - %prefer-fast-decompression?))) - (unless print-build-trace? - (format (current-error-port) - (G_ "Downloading ~a...~%") (uri->string uri))) + (define (try-fetch choices) + (match choices + (((uri compression file-size) rest ...) + (guard (c ((and (pair? rest) (network-error? c)) + (warning (G_ "download from '~a' failed, trying next URL~%") + (uri->string uri)) + (try-fetch rest))) + (let ((port download-size (fetch uri))) + (unless print-build-trace? + (format (current-error-port) + (G_ "Downloading ~a...~%") (uri->string uri))) + (values port uri compression download-size)))) + (() + (leave (G_ "no valid nar URLs for ~a at ~a~%") + (narinfo-path narinfo) + (narinfo-uri-base narinfo))))) - (let* ((raw download-size - ;; 'guix publish' without '--cache' doesn't specify a - ;; Content-Length, so DOWNLOAD-SIZE is #f in this case. - (fetch uri)) + (let ((choices (narinfo-preferred-uris narinfo + #:fast-decompression? + %prefer-fast-decompression?))) + ;; 'guix publish' without '--cache' doesn't specify a Content-Length, so + ;; DOWNLOAD-SIZE is #f in this case. + (let* ((raw uri compression download-size (try-fetch choices)) (progress (let* ((dl-size (or download-size (and (equal? compression "none") diff --git a/tests/substitute.scm b/tests/substitute.scm index 9032a50268..8df3938b59 100644 --- a/tests/substitute.scm +++ b/tests/substitute.scm @@ -64,11 +64,11 @@ (define-syntax-rule (test-quit name error-rx exp) (define (request-substitution item destination) "Run 'guix substitute --substitute' to fetch ITEM to DESTINATION." - (parameterize ((guix-warning-port (current-error-port))) - (with-input-from-string (string-append "substitute " item " " - destination "\n") - (lambda () - (guix-substitute "--substitute"))))) + (false-if-exception (delete-file destination)) + (with-input-from-string (string-append "substitute " item " " + destination "\n") + (lambda () + (guix-substitute "--substitute")))) (define %public-key ;; This key is known to be in the ACL by default. @@ -613,6 +613,32 @@ (define-syntax-rule (with-narinfo* narinfo directory body ...) (lambda () (false-if-exception (delete-file "substitute-retrieved"))))))))))) +(test-equal "substitute, preferred nar URL is 404, other is 200" + "Substitutable data." + (with-narinfo* (string-append %narinfo "Signature: " (signature-field %narinfo)) + %main-substitute-directory + + (with-http-server `((200 ,(string-append %narinfo "Signature: " + (signature-field %narinfo) + "\n" + "URL: example.nar.lz\n" + "Compression: lzip\n")) + (404 "Sorry, nar.lz is missing!") + (200 ,(call-with-input-file + (string-append %main-substitute-directory + "/example.nar") + get-bytevector-all))) + (dynamic-wind + (const #t) + (lambda () + (parameterize ((substitute-urls (list (%local-url)))) + (request-substitution (string-append (%store-prefix) + "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo") + "substitute-retrieved")) + (call-with-input-file "substitute-retrieved" get-string-all)) + (lambda () + (false-if-exception (delete-file "substitute-retrieved"))))))) + (test-quit "substitute, narinfo is available but nar is missing" "failed to find alternative substitute" (with-narinfo* base-commit: dff1689bb37e5303868584d3f1d7a33cbcb7f51e -- 2.40.1 ------------=_1685398802-8768-1--