From unknown Sun Jun 15 08:57:45 2025 X-Loop: help-debbugs@gnu.org Subject: [bug#45104] pull: Add a "with-substitutes" option. Resent-From: Mathieu Othacehe Original-Sender: "Debbugs-submit" Resent-CC: guix-patches@gnu.org Resent-Date: Mon, 07 Dec 2020 15:41:02 +0000 Resent-Message-ID: Resent-Sender: help-debbugs@gnu.org X-GNU-PR-Message: report 45104 X-GNU-PR-Package: guix-patches X-GNU-PR-Keywords: To: 45104@debbugs.gnu.org X-Debbugs-Original-To: guix-patches@gnu.org Received: via spool by submit@debbugs.gnu.org id=B.160735560516320 (code B ref -1); Mon, 07 Dec 2020 15:41:02 +0000 Received: (at submit) by debbugs.gnu.org; 7 Dec 2020 15:40:05 +0000 Received: from localhost ([127.0.0.1]:55002 helo=debbugs.gnu.org) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1kmIcP-0004F9-23 for submit@debbugs.gnu.org; Mon, 07 Dec 2020 10:40:05 -0500 Received: from lists.gnu.org ([209.51.188.17]:59678) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1kmIcM-0004Ez-Md for submit@debbugs.gnu.org; Mon, 07 Dec 2020 10:40:03 -0500 Received: from eggs.gnu.org ([2001:470:142:3::10]:44856) by lists.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1kmIcM-0002Zp-Dc for guix-patches@gnu.org; Mon, 07 Dec 2020 10:40:02 -0500 Received: from fencepost.gnu.org ([2001:470:142:3::e]:57408) by eggs.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1kmIcM-0000wv-4V for guix-patches@gnu.org; Mon, 07 Dec 2020 10:40:02 -0500 Received: from [2a01:e0a:19b:d9a0:34f0:bcad:cad1:16d6] (port=34604 helo=cervin) by fencepost.gnu.org with esmtpsa (TLS1.2:RSA_AES_256_CBC_SHA1:256) (Exim 4.82) (envelope-from ) id 1kmIcL-0006bB-7Q for guix-patches@gnu.org; Mon, 07 Dec 2020 10:40:01 -0500 From: Mathieu Othacehe Date: Mon, 07 Dec 2020 16:39:59 +0100 Message-ID: <87eek1vd4g.fsf@gnu.org> User-Agent: Gnus/5.13 (Gnus v5.13) Emacs/27.1 (gnu/linux) MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="=-=-=" X-Spam-Score: -2.3 (--) X-BeenThere: debbugs-submit@debbugs.gnu.org X-Mailman-Version: 2.1.18 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: debbugs-submit-bounces@debbugs.gnu.org Sender: "Debbugs-submit" X-Spam-Score: -3.3 (---) --=-=-= Content-Type: text/plain Hello, When "guix pull" is run before the CI server is done baking substitutes, the user may need to build some derivations, which can be long on low-end hardware. This patch adds an option to "guix pull" so that it updates to the latest commit with available substitutes. This work is still in progress, but I'd like to gather some impressions on that before going further. Thanks, Mathieu --=-=-= Content-Type: text/x-diff; charset=utf-8 Content-Disposition: inline; filename=0001-scripts-pull-Add-with-substitutes-option.patch Content-Transfer-Encoding: quoted-printable >From d399f8dbb9e38a82241b9048b8b04758fae10005 Mon Sep 17 00:00:00 2001 From: Mathieu Othacehe Date: Mon, 7 Dec 2020 16:12:22 +0100 Subject: [PATCH] scripts: pull: Add "with-substitutes" option. --- guix/scripts/pull.scm | 38 +++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/guix/scripts/pull.scm b/guix/scripts/pull.scm index 83cdc1d1eb..4609f8614e 100644 --- a/guix/scripts/pull.scm +++ b/guix/scripts/pull.scm @@ -2,6 +2,7 @@ ;;; Copyright =C2=A9 2013, 2014, 2015, 2017, 2018, 2019, 2020 Ludovic Cour= t=C3=A8s ;;; Copyright =C2=A9 2017 Marius Bakke ;;; Copyright =C2=A9 2020 Tobias Geerinckx-Rice +;;; Copyright =C2=A9 2020 Mathieu Othacehe ;;; ;;; This file is part of GNU Guix. ;;; @@ -20,6 +21,7 @@ =20 (define-module (guix scripts pull) #:use-module ((guix ui) #:hide (display-profile-content)) + #:use-module (guix ci) #:use-module (guix colors) #:use-module (guix utils) #:use-module ((guix status) #:select (with-status-verbosity)) @@ -64,6 +66,7 @@ #:re-export (display-profile-content channel-commit-hyperlink) #:export (channel-list + find-lastest-commit-with-substitutes guix-pull)) =20 @@ -169,6 +172,9 @@ Download and deploy the latest version of Guix.\n")) (lambda (opt name arg result) (alist-cons 'validate-pull warn-about-backward-updates result))) + (option '("with-substitutes") #f #f + (lambda (opt name arg result) + (alist-cons 'with-substitutes? #t result))) (option '("disable-authentication") #f #f (lambda (opt name arg result) (alist-cons 'authenticate-channels? #f result))) @@ -526,6 +532,26 @@ true, display what would be built without actually bui= lding it." (leave (G_ "while creating symlink '~a': ~a~%") link (strerror (system-error-errno args)))))))) =20 + +;;; +;;; Substitutes. +;;; + +(define (find-lastest-commit-with-substitutes) + (let ((urls %default-substitute-urls)) + (any (lambda (url) + (let* ((build (match (latest-builds url 1 + #:job "guix.x86_64-linux" + #:status 0) ;success + ((build) build))) + (evaluation (evaluation url + (build-evaluation build))) + (commit (match (evaluation-checkouts evaluation) + ((checkout) + (checkout-commit checkout))))) + commit)) + urls))) + ;;; ;;; Queries. @@ -731,8 +757,9 @@ Use '~/.config/guix/channels.scm' instead.")) =20 (let ((ref (assoc-ref opts 'ref)) (url (or (assoc-ref opts 'repository-url) - (environment-variable)))) - (if (or ref url) + (environment-variable))) + (with-substitutes? (assoc-ref opts 'with-substitutes?))) + (if (or ref url with-substitutes?) (match (find guix-channel? channels) ((? channel? guix) ;; Apply '--url', '--commit', and '--branch' to the 'guix' chan= nel. @@ -745,7 +772,12 @@ Use '~/.config/guix/channels.scm' instead.")) (channel (inherit guix) (url url) (commit #f) (branch branch))) (#f - (channel (inherit guix) (url url)))) + (let ((commit + (and with-substitutes? + (find-lastest-commit-with-substitutes)))) + (channel (inherit guix) + (url url) + (commit commit))))) (remove guix-channel? channels)))) (#f ;no 'guix' channel, failure will e= nsue channels)) --=20 2.29.2 --=-=-=-- From unknown Sun Jun 15 08:57:45 2025 X-Loop: help-debbugs@gnu.org Subject: [bug#45104] pull: Add a "with-substitutes" option. Resent-From: zimoun Original-Sender: "Debbugs-submit" Resent-CC: guix-patches@gnu.org Resent-Date: Mon, 07 Dec 2020 17:12:02 +0000 Resent-Message-ID: Resent-Sender: help-debbugs@gnu.org X-GNU-PR-Message: followup 45104 X-GNU-PR-Package: guix-patches X-GNU-PR-Keywords: To: Mathieu Othacehe , 45104@debbugs.gnu.org Received: via spool by 45104-submit@debbugs.gnu.org id=B45104.16073611209911 (code B ref 45104); Mon, 07 Dec 2020 17:12:02 +0000 Received: (at 45104) by debbugs.gnu.org; 7 Dec 2020 17:12:00 +0000 Received: from localhost ([127.0.0.1]:55349 helo=debbugs.gnu.org) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1kmK3L-0002Zn-Ux for submit@debbugs.gnu.org; Mon, 07 Dec 2020 12:12:00 -0500 Received: from mail-wr1-f41.google.com ([209.85.221.41]:43260) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1kmK3K-0002ZU-Nj for 45104@debbugs.gnu.org; Mon, 07 Dec 2020 12:11:59 -0500 Received: by mail-wr1-f41.google.com with SMTP id y17so4080868wrr.10 for <45104@debbugs.gnu.org>; Mon, 07 Dec 2020 09:11:58 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20161025; h=from:to:subject:in-reply-to:references:date:message-id:mime-version :content-transfer-encoding; bh=om+lleiirzyMuPAXOh0kk5eg3+YBg5eeG6AyF2Y5TAc=; b=gDDpVTcq0H8vOrXgg+HjnxP2+Ml0K+oD7ExTkP1/H8Yxa903K4ehstJbVelVI+Kgxx KfINLA7cAoxTv608svK7hjry/hdVZPI3QkhLpIEJLT9sNFgv8MFroqCUlIaN3sbI50nC lNnZn4cxj3TZFGmqD82BW9V6Gwyi0wBW0KubSWE1+SAcu39PVBMRNxog6lqsp+qPQZDR /1P94RR6ETvV47HzjFyRju++WMYAb9r/FTmx/N8V9pPIvQ6qzAM3Jr9h2OrLNxv4rcZt jFHDTN72TTyG3eN19rvkfX6IZiEk2Tfq3T26PRAuGhPAVQqmFsYfxG50eNz9HDQjDHB9 ncqA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:subject:in-reply-to:references:date :message-id:mime-version:content-transfer-encoding; bh=om+lleiirzyMuPAXOh0kk5eg3+YBg5eeG6AyF2Y5TAc=; b=t9Ks8MSB/sxKsYcStwdwO4VajxegYB3fyCfN0iOlwj11y4BdKNOQCjQtpFgItk6Jd3 /FFyx8km8izL7MnBXugf3PkwIZZ7T6+PeuIrbRaiyeK4MkBu5vU6s2TQVxqBAwi8OQTt nr6S/jX+wmwWOj7TE9CZwg9SZGFXlCK5LnU423FSp8YwMf8TgvVUaRl3gFBLNxJQtsbU vESaf41AQVZnFA0x9SFx8Oeh9dFUqKKIM08ujajEzCGWW5fYkfMso+AqQlOi3C9MU9tK zM4g2lpOgAQW+U7XyzpFrSTOroLMV6VJuxj3yBiMOTwdeO6qJ6iLO0XCIRbFDCWgBRQx 0hVg== X-Gm-Message-State: AOAM532SvNKf6MTdPvYaatSPONA9nqyW35dAfqLsnST+zfPCGzlR1gs7 2c7T8B+C4w4ZYT2Vz3+d1y87JFB3ie4= X-Google-Smtp-Source: ABdhPJwsLZNH0KcZVj1WTEOOKjdpQB0ohSk6QrKkx1ga1JwgPqOuhu5mRU/7hocbiUd1zF/7bqwE0A== X-Received: by 2002:adf:9b95:: with SMTP id d21mr21024846wrc.335.1607361112562; Mon, 07 Dec 2020 09:11:52 -0800 (PST) Received: from lili ([2a01:e0a:59b:9120:65d2:2476:f637:db1e]) by smtp.gmail.com with ESMTPSA id r2sm14477813wrn.83.2020.12.07.09.11.51 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Mon, 07 Dec 2020 09:11:52 -0800 (PST) From: zimoun In-Reply-To: <87eek1vd4g.fsf@gnu.org> References: <87eek1vd4g.fsf@gnu.org> Date: Mon, 07 Dec 2020 18:05:16 +0100 Message-ID: <86k0tt5yyb.fsf@gmail.com> MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable X-Spam-Score: 0.0 (/) X-BeenThere: debbugs-submit@debbugs.gnu.org X-Mailman-Version: 2.1.18 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: debbugs-submit-bounces@debbugs.gnu.org Sender: "Debbugs-submit" X-Spam-Score: -1.0 (-) Hi Mathieu, On Mon, 07 Dec 2020 at 16:39, Mathieu Othacehe wrote: > When "guix pull" is run before the CI server is done baking substitutes, > the user may need to build some derivations, which can be long on > low-end hardware. Yeah=E2=80=A6 > This patch adds an option to "guix pull" so that it updates to the > latest commit with available substitutes. Cool! It improves my ugly bash to pull 2 weeks behind. :-) > This work is still in progress, but I'd like to gather some impressions > on that before going further. With your patch, it is possible to never pull something. Because the push are faster than the CI is able to build. Therefore, an improvement could to be to check the latest commit on the CI, if not available fallback to the previous, repeat until one is available or repeat X time and raise a warning. Cheers, simon From unknown Sun Jun 15 08:57:45 2025 X-Loop: help-debbugs@gnu.org Subject: [bug#45104] pull: Add a "with-substitutes" option. Resent-From: Christopher Baines Original-Sender: "Debbugs-submit" Resent-CC: guix-patches@gnu.org Resent-Date: Tue, 08 Dec 2020 19:18:01 +0000 Resent-Message-ID: Resent-Sender: help-debbugs@gnu.org X-GNU-PR-Message: followup 45104 X-GNU-PR-Package: guix-patches X-GNU-PR-Keywords: To: Mathieu Othacehe Cc: 45104@debbugs.gnu.org Received: via spool by 45104-submit@debbugs.gnu.org id=B45104.160745507517378 (code B ref 45104); Tue, 08 Dec 2020 19:18:01 +0000 Received: (at 45104) by debbugs.gnu.org; 8 Dec 2020 19:17:55 +0000 Received: from localhost ([127.0.0.1]:59812 helo=debbugs.gnu.org) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1kmiUk-0004WE-ND for submit@debbugs.gnu.org; Tue, 08 Dec 2020 14:17:54 -0500 Received: from mira.cbaines.net ([212.71.252.8]:35942) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1kmiUi-0004W6-Na for 45104@debbugs.gnu.org; Tue, 08 Dec 2020 14:17:53 -0500 Received: from localhost (188.29.103.89.threembb.co.uk [188.29.103.89]) by mira.cbaines.net (Postfix) with ESMTPSA id BA08D27BC01; Tue, 8 Dec 2020 19:17:51 +0000 (GMT) Received: from capella (localhost [127.0.0.1]) by localhost (OpenSMTPD) with ESMTP id 750974a9; Tue, 8 Dec 2020 19:17:49 +0000 (UTC) References: <87eek1vd4g.fsf@gnu.org> User-agent: mu4e 1.4.13; emacs 27.1 From: Christopher Baines In-reply-to: <87eek1vd4g.fsf@gnu.org> Date: Tue, 08 Dec 2020 19:17:47 +0000 Message-ID: <87a6uohztw.fsf@cbaines.net> MIME-Version: 1.0 Content-Type: multipart/signed; boundary="=-=-="; micalg=pgp-sha512; protocol="application/pgp-signature" X-Spam-Score: -0.0 (/) X-BeenThere: debbugs-submit@debbugs.gnu.org X-Mailman-Version: 2.1.18 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: debbugs-submit-bounces@debbugs.gnu.org Sender: "Debbugs-submit" X-Spam-Score: -1.0 (-) --=-=-= Content-Type: text/plain Content-Transfer-Encoding: quoted-printable Mathieu Othacehe writes: > When "guix pull" is run before the CI server is done baking substitutes, > the user may need to build some derivations, which can be long on > low-end hardware. > > This patch adds an option to "guix pull" so that it updates to the > latest commit with available substitutes. > > This work is still in progress, but I'd like to gather some impressions > on that before going further. Hey! I think it's definitely a nice idea, although I think there are some things in the naming. My first thought is that the option doesn't really do what it says it will do. The option is named --with-substitutes, but that actually means something like "with successful build on ci.guix.gnu.org". This could cause confusion if you're not fetching substitutes from ci.guix.gnu.org, and it starts building things, or perhaps if you're fetching substitutes from two servers, one of which is ci.guix.gnu.org which doesn't have a substitute, but the other server does, this option would fetch an older revision than what's truly available. Maybe part of what I've said above is incorrect if a users substitute URLs are used, but there also seems to be an assuption that something which provides substitutes provides a Cuirass compatible API, which isn't always the case. One more thought on this is that it's probably important to convey that this doesn't pick a revision where substitutes will be available. I think this is something some users would be eager to assume, and then be confused when it doesn't work. Thanks, Chris --=-=-= Content-Type: application/pgp-signature; name="signature.asc" -----BEGIN PGP SIGNATURE----- iQKkBAEBCgCPFiEEPonu50WOcg2XVOCyXiijOwuE9XcFAl/P0VtfFIAAAAAALgAo aXNzdWVyLWZwckBub3RhdGlvbnMub3BlbnBncC5maWZ0aGhvcnNlbWFuLm5ldDNF ODlFRUU3NDU4RTcyMEQ5NzU0RTBCMjVFMjhBMzNCMEI4NEY1NzcRHG1haWxAY2Jh aW5lcy5uZXQACgkQXiijOwuE9XcBQQ/42tYVXO2CBpNOELqx4BGTY5C8wDH64KNZ rlhxUAaovMdBFxZ4Fim8UzMZS85wGKBaW8aoyw1wYHlTBJNgr9z6fjOM8F/AWi8Y eyput0DcMZ36hhyfL1vgJ77+lnKxfWARrtEMxDrRP1j3jtQjnc3nAOm+5j4eFY7r aal527/pJKPzAOooslF4/VABgxonoq23V5qwytjYoCergQFGPva5Ktu2LK9EjG5Z n+maaC+SCCn0bUHZ+zkPirOY7FTM4g6rZiKBknnykygl9Nu9r0EEv4xKGz4wVgxt p4HFmgQnD+bHEzWstQnWr6U5pfX8WHUfBWpTg5UFyDsfcBniL8lbE6/rZpXuldxP 1bZqJqzDd56mJ728Ld7xXqv81JlXCFnyou7fW/waVRNwNKTZVlpw/DDvLmiQP0Ig Y7CT1kUnP3gwUtlZgL3ffNqE7oOEbEf/s4ibK7aV9dQ9axexDZjxcX9QHtM9WaDf Aakn/+hbFt6JTt1TUozFQVDeK9vRmDwD2g2aXXcihjUU5byyxLiyqj5MNjgzVsH1 SNNhKZPJL4w54wFjfkghUqeqe0+wYg+0PBWD8IV6+6MFODoUqIWUJHfhL9V/BaK5 T5XHjVAN9BJp36AZFlAeBlvwyXp9k0r+ZSjbbUKCDi8KB4vR23qNDY//XTmCB6lJ EV723X4YjQ== =3ZFW -----END PGP SIGNATURE----- --=-=-=-- From unknown Sun Jun 15 08:57:45 2025 X-Loop: help-debbugs@gnu.org Subject: [bug#45104] pull: Add a "with-substitutes" option. Resent-From: Ludovic =?UTF-8?Q?Court=C3=A8s?= Original-Sender: "Debbugs-submit" Resent-CC: guix-patches@gnu.org Resent-Date: Mon, 14 Dec 2020 11:07:02 +0000 Resent-Message-ID: Resent-Sender: help-debbugs@gnu.org X-GNU-PR-Message: followup 45104 X-GNU-PR-Package: guix-patches X-GNU-PR-Keywords: To: Christopher Baines Cc: Mathieu Othacehe , 45104@debbugs.gnu.org Received: via spool by 45104-submit@debbugs.gnu.org id=B45104.160794397232314 (code B ref 45104); Mon, 14 Dec 2020 11:07:02 +0000 Received: (at 45104) by debbugs.gnu.org; 14 Dec 2020 11:06:12 +0000 Received: from localhost ([127.0.0.1]:51381 helo=debbugs.gnu.org) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1kolg9-0008P5-Aw for submit@debbugs.gnu.org; Mon, 14 Dec 2020 06:06:12 -0500 Received: from eggs.gnu.org ([209.51.188.92]:58186) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1kolg4-0008OF-Hm for 45104@debbugs.gnu.org; Mon, 14 Dec 2020 06:06:07 -0500 Received: from fencepost.gnu.org ([2001:470:142:3::e]:50194) by eggs.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1kolfz-0006gm-8Y; Mon, 14 Dec 2020 06:05:59 -0500 Received: from [2a01:e0a:1d:7270:af76:b9b:ca24:c465] (port=40050 helo=ribbon) by fencepost.gnu.org with esmtpsa (TLS1.2:RSA_AES_256_CBC_SHA1:256) (Exim 4.82) (envelope-from ) id 1kolfv-0004Xh-Mt; Mon, 14 Dec 2020 06:05:57 -0500 From: Ludovic =?UTF-8?Q?Court=C3=A8s?= References: <87eek1vd4g.fsf@gnu.org> <87a6uohztw.fsf@cbaines.net> Date: Mon, 14 Dec 2020 12:05:54 +0100 In-Reply-To: <87a6uohztw.fsf@cbaines.net> (Christopher Baines's message of "Tue, 08 Dec 2020 19:17:47 +0000") Message-ID: <877dpktzot.fsf@gnu.org> User-Agent: Gnus/5.13 (Gnus v5.13) Emacs/27.1 (gnu/linux) MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable X-Spam-Score: -2.3 (--) X-BeenThere: debbugs-submit@debbugs.gnu.org X-Mailman-Version: 2.1.18 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: debbugs-submit-bounces@debbugs.gnu.org Sender: "Debbugs-submit" X-Spam-Score: -3.3 (---) Hi! Christopher Baines skribis: > My first thought is that the option doesn't really do what it says it > will do. The option is named --with-substitutes, but that actually means > something like "with successful build on ci.guix.gnu.org". This could > cause confusion if you're not fetching substitutes from ci.guix.gnu.org, > and it starts building things, or perhaps if you're fetching substitutes > from two servers, one of which is ci.guix.gnu.org which doesn't have a > substitute, but the other server does, this option would fetch an older > revision than what's truly available. > > Maybe part of what I've said above is incorrect if a users substitute > URLs are used, but there also seems to be an assuption that something > which provides substitutes provides a Cuirass compatible API, which > isn't always the case. > > One more thought on this is that it's probably important to convey that > this doesn't pick a revision where substitutes will be available. I > think this is something some users would be eager to assume, and then be > confused when it doesn't work. Agreed on these points. Another option would be to leave it up to users to customize their channel file to run pretty much the code you posted, like: --8<---------------cut here---------------start------------->8--- (use-modules (guix ci) (srfi srfi-1) (ice-9 match)) (define (latest-commit-successfully-built) "Return the latest commit for which substitutes are (potentially) available." (let* ((evaluations (filter (lambda (evaluation) (and (evaluation-complete? evaluation) (string=3D? "guix-modular-master" (evaluation-spec evaluation)))) (latest-evaluations "https://ci.guix.gnu.org"= )))) (any (lambda (evaluation) (match (evaluation-checkouts evaluation) ((checkout) (checkout-commit checkout)) (_ #f))) evaluations))) ;; Pull the latest commit fully built on berlin.guixsd.org. (list (channel (name 'guix) (url "https://git.savannah.gnu.org/git/guix.git") (commit (pk 'commit (latest-commit-successfully-built))))) --8<---------------cut here---------------end--------------->8--- We don=E2=80=99t want people to copy/paste all this, so we could instead pr= ovide and document a procedure that takes a channel and substitute URL and returns a channel, like: (channel-with-substitutes-available (channel (name 'guix) =E2=80=A6) "https://ci.guix.gnu.org") and optionally with a manifest or a list of packages that should be available as substitutes: (channel-with-substitutes-available (channel (name 'guix) =E2=80=A6) "https://ci.guix.gnu.org" (specifications->manifest '("emacs" "guile"))) WDYT? It does mean that we=E2=80=99re asking users to do extra work. Perhaps the= re could still be a command-line option that would call =E2=80=98channel-with-substitutes-available=E2=80=99 for you, but at least = it would take an explicit URL and clarify what Chris mentioned? BTW, doing all this is safer today because =E2=80=98guix pull=E2=80=99 will= detect and prevent downgrades. Though an attacker who manages to break into ci.guix.gnu.org could cause all the users of =E2=80=98channel-with-substitutes-available=E2=80=99 to no longer receive u= pdates or to receive them more slowly than they appear in Git simply by making CI even slower than it currently is. Thoughts? Ludo=E2=80=99. From unknown Sun Jun 15 08:57:45 2025 X-Loop: help-debbugs@gnu.org Subject: [bug#45104] pull: Add a "with-substitutes" option. Resent-From: zimoun Original-Sender: "Debbugs-submit" Resent-CC: guix-patches@gnu.org Resent-Date: Mon, 14 Dec 2020 12:34:02 +0000 Resent-Message-ID: Resent-Sender: help-debbugs@gnu.org X-GNU-PR-Message: followup 45104 X-GNU-PR-Package: guix-patches X-GNU-PR-Keywords: To: Ludovic =?UTF-8?Q?Court=C3=A8s?= , Christopher Baines Cc: Mathieu Othacehe , 45104@debbugs.gnu.org Received: via spool by 45104-submit@debbugs.gnu.org id=B45104.160794923116640 (code B ref 45104); Mon, 14 Dec 2020 12:34:02 +0000 Received: (at 45104) by debbugs.gnu.org; 14 Dec 2020 12:33:51 +0000 Received: from localhost ([127.0.0.1]:51486 helo=debbugs.gnu.org) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1kon30-0004KK-Lg for submit@debbugs.gnu.org; Mon, 14 Dec 2020 07:33:50 -0500 Received: from mail-wr1-f43.google.com ([209.85.221.43]:39351) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1kon2z-0004K6-2T for 45104@debbugs.gnu.org; Mon, 14 Dec 2020 07:33:49 -0500 Received: by mail-wr1-f43.google.com with SMTP id c5so12552598wrp.6 for <45104@debbugs.gnu.org>; Mon, 14 Dec 2020 04:33:48 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20161025; h=from:to:cc:subject:in-reply-to:references:date:message-id :mime-version:content-transfer-encoding; bh=Y2pU/JUIPK08J+4ouBO/GRjWSxj+do7SWKMJY/EYPps=; b=c2wEFYxowwlvPiWohRJZgIEszCxuNlLpr5XA7N0QIvNqw4dz9cwFFu2IP6co3eKZ2d gT1Y9hEc4KM3IvjOCMbWi3//JDrCGwKPKGdjrQYPz12R0uYmFsejy8HFW7ETO3vqV9VJ nzhKd0JDBk0dUEj6mrJAzEmfvcFJAxvSRNtJiTVO4PNJK47Y/WUd3hWpOh1SP/C+Nco6 GkIxmrCp+CvasdixdYk7EO8wmhPr7qPAaMlZyH+Pay3ZUCYzXFmm3YKHP6oM332EWruI pUfvIzhDOeDF3mdK7Izhxpu0wQoRc29wwXPjx4zRsfqwjYjFHDzjziyMvgFrT/SXCptJ /oPw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:in-reply-to:references:date :message-id:mime-version:content-transfer-encoding; bh=Y2pU/JUIPK08J+4ouBO/GRjWSxj+do7SWKMJY/EYPps=; b=bVkH5haA454dzfbBF4VIUJVGsjGpB8kHZFvjvuUlWuOHmVh9yE3K29RI212cwBLnl8 5eYFYp/OD9mwHg9Pjao3RHQiJGw4LUzS0OzRmXiMrT4+29Fg+BfWtm2JWh7m+Qu63uzr rdn/dJ/G2nUMGAoRHwRpywKyzhGdLJFNSH+PbKrBYTJ6SNKR/Kcb9tCHN4Jp9AeIxlCP /o2Jb3V5Jx5txgOndnFxuisrVrXFFxRRnIZf/sYs4l1qy3j0tYEiJL1TiKd3Ij134cyY BjZhCbpndAytWXkWhKAVJmSQwnI+aBqkL9fY7ZxZDUb0WiB48Pf/mILRDHNk6m120zOv 4l3A== X-Gm-Message-State: AOAM532pFB+xHcRyYRdZpg64BF7COh6driOI+c2AnUPUHvyhcjCSFB8K W1pXDir+6QoZhqosz+rMDMvDvjMva18= X-Google-Smtp-Source: ABdhPJyoSoQZhZX76Ti/XzN4Mns4M3olGdoooo6eiOT6JxTQl3RuSS8uV4zzi+b9zR6McfNSz4daGg== X-Received: by 2002:a5d:6789:: with SMTP id v9mr17501104wru.86.1607949222725; Mon, 14 Dec 2020 04:33:42 -0800 (PST) Received: from lili ([2a01:e0a:59b:9120:65d2:2476:f637:db1e]) by smtp.gmail.com with ESMTPSA id v125sm29867829wme.42.2020.12.14.04.33.40 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Mon, 14 Dec 2020 04:33:41 -0800 (PST) From: zimoun In-Reply-To: <877dpktzot.fsf@gnu.org> References: <87eek1vd4g.fsf@gnu.org> <87a6uohztw.fsf@cbaines.net> <877dpktzot.fsf@gnu.org> Date: Mon, 14 Dec 2020 12:39:21 +0100 Message-ID: <86o8iwobva.fsf@gmail.com> MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable X-Spam-Score: 0.0 (/) X-BeenThere: debbugs-submit@debbugs.gnu.org X-Mailman-Version: 2.1.18 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: debbugs-submit-bounces@debbugs.gnu.org Sender: "Debbugs-submit" X-Spam-Score: -1.0 (-) Hi, On Mon, 14 Dec 2020 at 12:05, Ludovic Court=C3=A8s wrote: > (channel-with-substitutes-available > (channel (name 'guix) =E2=80=A6) > "https://ci.guix.gnu.org") > > and optionally with a manifest or a list of packages that should be > available as substitutes: > > (channel-with-substitutes-available > (channel (name 'guix) =E2=80=A6) > "https://ci.guix.gnu.org" > (specifications->manifest '("emacs" "guile"))) Sounds good to me. > BTW, doing all this is safer today because =E2=80=98guix pull=E2=80=99 wi= ll detect and > prevent downgrades. Though an attacker who manages to break into > ci.guix.gnu.org could cause all the users of > =E2=80=98channel-with-substitutes-available=E2=80=99 to no longer receive= updates or to > receive them more slowly than they appear in Git simply by making CI > even slower than it currently is. As mentioned earlier, if =E2=80=9Cguix pull=E2=80=9D completes only when su= bstitutes is available, then depending on the CI, the user can never complete the =E2=80=9Cpull=E2=80=9D and so stay =E2=80=9Cblocked=E2=80=9C. For example, reusing your example, Emacs is not currently available and it has been since say 1 month. Therefore until Emacs becomes available for the fast moving HEAD, the user would be =E2=80=9Cblocked=E2=80=9D. Or = I miss something. All the best, simon From unknown Sun Jun 15 08:57:45 2025 X-Loop: help-debbugs@gnu.org Subject: [bug#45104] pull: Add a "with-substitutes" option. Resent-From: Mathieu Othacehe Original-Sender: "Debbugs-submit" Resent-CC: guix-patches@gnu.org Resent-Date: Tue, 15 Dec 2020 10:26:02 +0000 Resent-Message-ID: Resent-Sender: help-debbugs@gnu.org X-GNU-PR-Message: followup 45104 X-GNU-PR-Package: guix-patches X-GNU-PR-Keywords: To: Christopher Baines , Ludovic =?UTF-8?Q?Court=C3=A8s?= Cc: 45104@debbugs.gnu.org Received: via spool by 45104-submit@debbugs.gnu.org id=B45104.160802790830418 (code B ref 45104); Tue, 15 Dec 2020 10:26:02 +0000 Received: (at 45104) by debbugs.gnu.org; 15 Dec 2020 10:25:08 +0000 Received: from localhost ([127.0.0.1]:55715 helo=debbugs.gnu.org) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1kp7W0-0007uY-HQ for submit@debbugs.gnu.org; Tue, 15 Dec 2020 05:25:08 -0500 Received: from eggs.gnu.org ([209.51.188.92]:48844) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1kp7Vy-0007u4-AB for 45104@debbugs.gnu.org; Tue, 15 Dec 2020 05:25:06 -0500 Received: from fencepost.gnu.org ([2001:470:142:3::e]:48925) by eggs.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1kp7Vs-0005b1-6o; Tue, 15 Dec 2020 05:25:00 -0500 Received: from 225.71.114.78.rev.sfr.net ([78.114.71.225]:56324 helo=cervin) by fencepost.gnu.org with esmtpsa (TLS1.2:RSA_AES_256_CBC_SHA1:256) (Exim 4.82) (envelope-from ) id 1kp7Vr-0006qz-5p; Tue, 15 Dec 2020 05:24:59 -0500 From: Mathieu Othacehe References: <87eek1vd4g.fsf@gnu.org> <87a6uohztw.fsf@cbaines.net> <877dpktzot.fsf@gnu.org> Date: Tue, 15 Dec 2020 11:24:55 +0100 In-Reply-To: <877dpktzot.fsf@gnu.org> ("Ludovic =?UTF-8?Q?Court=C3=A8s?="'s message of "Mon, 14 Dec 2020 12:05:54 +0100") Message-ID: <878s9zfjt4.fsf@gnu.org> User-Agent: Gnus/5.13 (Gnus v5.13) Emacs/27.1 (gnu/linux) MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable X-Spam-Score: -2.3 (--) X-BeenThere: debbugs-submit@debbugs.gnu.org X-Mailman-Version: 2.1.18 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: debbugs-submit-bounces@debbugs.gnu.org Sender: "Debbugs-submit" X-Spam-Score: -3.3 (---) Hey Chris and Ludo, > Agreed on these points. Yes I think you are definitely right on that point. > (and (evaluation-complete? evaluation) > (string=3D? "guix-modular-master" > (evaluation-spec > evaluation)))) On Berlin, evaluations can be completed for days, but the associated builds never started. I think that searching directly for a completed build provides a stronger guarantee of available substitutes. > ;; Pull the latest commit fully built on berlin.guixsd.org. > (list (channel > (name 'guix) > (url "https://git.savannah.gnu.org/git/guix.git") > (commit (pk 'commit (latest-commit-successfully-built))))) Providing such a procedure definitely makes sense though. > (channel-with-substitutes-available > (channel (name 'guix) =E2=80=A6) > "https://ci.guix.gnu.org" > (specifications->manifest '("emacs" "guile"))) Yes it would be the ultimate thing! However, while finding the latest commit with an available substitute for a derivation is quite easy, finding a commit with available derivations for N derivations seems way more difficult. > It does mean that we=E2=80=99re asking users to do extra work. Perhaps t= here > could still be a command-line option that would call > =E2=80=98channel-with-substitutes-available=E2=80=99 for you, but at leas= t it would take > an explicit URL and clarify what Chris mentioned? Yes, the user would then have to provide the channels that need available substitutes, the URL to use for the substitution check and maybe a manifest that also needs available substitutes. The channels list could default to '("guix") and the URL to "https://ci.guix.gnu.org" as it would be a sensible default for most Guix users I think. > BTW, doing all this is safer today because =E2=80=98guix pull=E2=80=99 wi= ll detect and > prevent downgrades. Though an attacker who manages to break into > ci.guix.gnu.org could cause all the users of > =E2=80=98channel-with-substitutes-available=E2=80=99 to no longer receive= updates or to > receive them more slowly than they appear in Git simply by making CI > even slower than it currently is. Yes, the downgrade check definitely helps here, as it's often what will happen with our lagging CI. Regarding the security aspect, I think that breaking into ci.guix.gnu.org can have other way more impacting consequences. Thanks, Mathieu From unknown Sun Jun 15 08:57:45 2025 X-Loop: help-debbugs@gnu.org Subject: [bug#45104] pull: Add a "with-substitutes" option. Resent-From: Mathieu Othacehe Original-Sender: "Debbugs-submit" Resent-CC: guix-patches@gnu.org Resent-Date: Tue, 15 Dec 2020 10:31:01 +0000 Resent-Message-ID: Resent-Sender: help-debbugs@gnu.org X-GNU-PR-Message: followup 45104 X-GNU-PR-Package: guix-patches X-GNU-PR-Keywords: To: zimoun Cc: 45104@debbugs.gnu.org, Ludovic =?UTF-8?Q?Court=C3=A8s?= , Christopher Baines Received: via spool by 45104-submit@debbugs.gnu.org id=B45104.160802822531037 (code B ref 45104); Tue, 15 Dec 2020 10:31:01 +0000 Received: (at 45104) by debbugs.gnu.org; 15 Dec 2020 10:30:25 +0000 Received: from localhost ([127.0.0.1]:55720 helo=debbugs.gnu.org) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1kp7b7-00084X-8g for submit@debbugs.gnu.org; Tue, 15 Dec 2020 05:30:25 -0500 Received: from eggs.gnu.org ([209.51.188.92]:49992) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1kp7b6-00084L-Cc for 45104@debbugs.gnu.org; Tue, 15 Dec 2020 05:30:24 -0500 Received: from fencepost.gnu.org ([2001:470:142:3::e]:48973) by eggs.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1kp7b0-0007o7-Km; Tue, 15 Dec 2020 05:30:18 -0500 Received: from 225.71.114.78.rev.sfr.net ([78.114.71.225]:56374 helo=cervin) by fencepost.gnu.org with esmtpsa (TLS1.2:RSA_AES_256_CBC_SHA1:256) (Exim 4.82) (envelope-from ) id 1kp7ax-0005v9-TR; Tue, 15 Dec 2020 05:30:18 -0500 From: Mathieu Othacehe In-Reply-To: <86o8iwobva.fsf@gmail.com> (zimoun's message of "Mon, 14 Dec 2020 12:39:21 +0100") References: <87eek1vd4g.fsf@gnu.org> <87a6uohztw.fsf@cbaines.net> <877dpktzot.fsf@gnu.org> <86o8iwobva.fsf@gmail.com> User-Agent: Gnus/5.13 (Gnus v5.13) Emacs/27.1 (gnu/linux) Date: Tue, 15 Dec 2020 11:30:12 +0100 Message-ID: <873607fjkb.fsf@gnu.org> MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable X-Spam-Score: -2.3 (--) X-BeenThere: debbugs-submit@debbugs.gnu.org X-Mailman-Version: 2.1.18 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: debbugs-submit-bounces@debbugs.gnu.org Sender: "Debbugs-submit" X-Spam-Score: -3.3 (---) Hello zimoun, > As mentioned earlier, if =E2=80=9Cguix pull=E2=80=9D completes only when = substitutes is > available, then depending on the CI, the user can never complete the > =E2=80=9Cpull=E2=80=9D and so stay =E2=80=9Cblocked=E2=80=9C. There's nothing blocking in what I'm proposing. "guix pull" first asks to the CI what's the latest commit with available substitutes for "guix", and then tries to update to that commit. If it results in a downgrade then, "guix pull" detects it. The corner case where there are no commits with available substitutes is not handled, but I guess we could decide to update to the latest commit in that case. Thanks, Mathieu From unknown Sun Jun 15 08:57:45 2025 X-Loop: help-debbugs@gnu.org Subject: [bug#45104] pull: Add a "with-substitutes" option. Resent-From: zimoun Original-Sender: "Debbugs-submit" Resent-CC: guix-patches@gnu.org Resent-Date: Tue, 15 Dec 2020 12:57:01 +0000 Resent-Message-ID: Resent-Sender: help-debbugs@gnu.org X-GNU-PR-Message: followup 45104 X-GNU-PR-Package: guix-patches X-GNU-PR-Keywords: To: Mathieu Othacehe Cc: 45104@debbugs.gnu.org, Ludovic =?UTF-8?Q?Court=C3=A8s?= , Christopher Baines Received: via spool by 45104-submit@debbugs.gnu.org id=B45104.160803700921887 (code B ref 45104); Tue, 15 Dec 2020 12:57:01 +0000 Received: (at 45104) by debbugs.gnu.org; 15 Dec 2020 12:56:49 +0000 Received: from localhost ([127.0.0.1]:55863 helo=debbugs.gnu.org) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1kp9sm-0005gw-Mi for submit@debbugs.gnu.org; Tue, 15 Dec 2020 07:56:48 -0500 Received: from mail-wr1-f46.google.com ([209.85.221.46]:36223) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1kp9sl-0005gj-Bl for 45104@debbugs.gnu.org; Tue, 15 Dec 2020 07:56:47 -0500 Received: by mail-wr1-f46.google.com with SMTP id t16so19749234wra.3 for <45104@debbugs.gnu.org>; Tue, 15 Dec 2020 04:56:47 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20161025; h=from:to:cc:subject:in-reply-to:references:date:message-id :mime-version:content-transfer-encoding; bh=2O+yx9nj0I2g/sUtjypyZpV5Y07JVWXmUheZPEJIl2k=; b=q/QsCXAENdtudYi1KyOkHKLHUcn4u1fiqfQwwXsXXCKjzZBdkxoa/9cNkoKWfpF7C5 ycSuQBn8KnvojIqy/YGWFJTqcQN8w1A0mzxMWxmBXL4ejCeEgrI2z4qToJ4Vmjxi8joe APMDGtV25MGiJG3cFcDh7Iy14ui9CtLKzEQtUJkDn7zHq3ddfu29iIL3FHEVlqKPq/Yi 6Si7DoJUuU/dyeHbyFDyMtplJbTZh7f4nNWJ77c4b62p+4T+NwdSYN39sVeowdLS0aT1 K/YK9QC0kNgforXMW4X/kV5EV35dcwod7f2LHyVmL0e/MPleGkMYkhvSd2sXkTsUbTls XAcw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:in-reply-to:references:date :message-id:mime-version:content-transfer-encoding; bh=2O+yx9nj0I2g/sUtjypyZpV5Y07JVWXmUheZPEJIl2k=; b=POi7aeP3GFoeBvhiWPfTviWKI4d5nFUrAAJLJOCX62Gv2lcWv+tmHfz3Fx1Z/DrJqw XVLQpQ46kZAd5djfGGY0BtblbrkmD2UsPSN5v2Y4MtmIxQ4iIz3dcgU/+Gk92MAvXzsw Xhw0cr3vQmYI4AwB7IsbmWMXTwMpPFepneTETHcCfPp9lX8Jy3jJz50D/2BKJyF1hiEI +g2by8QFCgasuMti1ihYyVCPz8TLL88+sgVNqeVb9UMMEMvb8XCVO/OmTPk8z3/gPXWz 4N3qb1G87ZpoCGElwdchlkZLBSGSXVCCi8/we4V53OnRi2jW8OKAsU0tI4gaf55eql0s KQ+g== X-Gm-Message-State: AOAM532eDMDh4mcLjJPfi8oUKzzapyDmZ/Jn+AdZnMEz3Cfl93L9GxUY 6KqO2IvHjRRNqq6s5OyxhSBdf2417oY= X-Google-Smtp-Source: ABdhPJza3Lj5F7IKEx/oLTSy34Yb0w5MxtEAmnY+T1d1o/asfZvBoyFeefxtrrA1fwuhZRwE6lceeQ== X-Received: by 2002:adf:e8c9:: with SMTP id k9mr33625080wrn.107.1608037001232; Tue, 15 Dec 2020 04:56:41 -0800 (PST) Received: from lili ([2a01:e0a:59b:9120:65d2:2476:f637:db1e]) by smtp.gmail.com with ESMTPSA id q15sm36793137wrw.75.2020.12.15.04.56.40 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 15 Dec 2020 04:56:40 -0800 (PST) From: zimoun In-Reply-To: <873607fjkb.fsf@gnu.org> References: <87eek1vd4g.fsf@gnu.org> <87a6uohztw.fsf@cbaines.net> <877dpktzot.fsf@gnu.org> <86o8iwobva.fsf@gmail.com> <873607fjkb.fsf@gnu.org> Date: Tue, 15 Dec 2020 13:51:17 +0100 Message-ID: <861rfrnsfu.fsf@gmail.com> MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable X-Spam-Score: 0.0 (/) X-BeenThere: debbugs-submit@debbugs.gnu.org X-Mailman-Version: 2.1.18 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: debbugs-submit-bounces@debbugs.gnu.org Sender: "Debbugs-submit" X-Spam-Score: -1.0 (-) Hi Mathieu, On Tue, 15 Dec 2020 at 11:30, Mathieu Othacehe wrote: > There's nothing blocking in what I'm proposing. "guix pull" first asks > to the CI what's the latest commit with available substitutes for > "guix", and then tries to update to that commit. If it results in a > downgrade then, "guix pull" detects it. Thanks for the explanation. I have actually overlooked the code. Well, the proposal was what your patch is doing somehow. :-) Now, since I have played with =E2=80=9Cguix repl=E2=80=9D, all is clear. ;-) Aside, the option name =E2=80=99--with-substitutes=E2=80=99 could be mislea= ding. Or clearly state that it is substitutes for =E2=80=9Cguix=E2=80=9D. Cheers, simon From unknown Sun Jun 15 08:57:45 2025 X-Loop: help-debbugs@gnu.org Subject: [bug#45104] pull: Add a "with-substitutes" option. Resent-From: Ludovic =?UTF-8?Q?Court=C3=A8s?= Original-Sender: "Debbugs-submit" Resent-CC: guix-patches@gnu.org Resent-Date: Tue, 15 Dec 2020 22:04:01 +0000 Resent-Message-ID: Resent-Sender: help-debbugs@gnu.org X-GNU-PR-Message: followup 45104 X-GNU-PR-Package: guix-patches X-GNU-PR-Keywords: To: Mathieu Othacehe Cc: 45104@debbugs.gnu.org, Christopher Baines Received: via spool by 45104-submit@debbugs.gnu.org id=B45104.16080698407919 (code B ref 45104); Tue, 15 Dec 2020 22:04:01 +0000 Received: (at 45104) by debbugs.gnu.org; 15 Dec 2020 22:04:00 +0000 Received: from localhost ([127.0.0.1]:59399 helo=debbugs.gnu.org) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1kpIQG-00023b-Oc for submit@debbugs.gnu.org; Tue, 15 Dec 2020 17:04:00 -0500 Received: from eggs.gnu.org ([209.51.188.92]:53594) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1kpIQF-00023O-UZ for 45104@debbugs.gnu.org; Tue, 15 Dec 2020 17:03:56 -0500 Received: from fencepost.gnu.org ([2001:470:142:3::e]:36064) by eggs.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1kpIQA-0005mN-Nx; Tue, 15 Dec 2020 17:03:50 -0500 Received: from [2a01:e0a:1d:7270:af76:b9b:ca24:c465] (port=36818 helo=ribbon) by fencepost.gnu.org with esmtpsa (TLS1.2:RSA_AES_256_CBC_SHA1:256) (Exim 4.82) (envelope-from ) id 1kpIQ7-0005iK-DN; Tue, 15 Dec 2020 17:03:48 -0500 From: Ludovic =?UTF-8?Q?Court=C3=A8s?= References: <87eek1vd4g.fsf@gnu.org> <87a6uohztw.fsf@cbaines.net> <877dpktzot.fsf@gnu.org> <878s9zfjt4.fsf@gnu.org> X-URL: http://www.fdn.fr/~lcourtes/ X-Revolutionary-Date: 25 Frimaire an 229 de la =?UTF-8?Q?R=C3=A9volution?= X-PGP-Key-ID: 0x090B11993D9AEBB5 X-PGP-Key: http://www.fdn.fr/~lcourtes/ludovic.asc X-PGP-Fingerprint: 3CE4 6455 8A84 FDC6 9DB4 0CFB 090B 1199 3D9A EBB5 X-OS: x86_64-pc-linux-gnu Date: Tue, 15 Dec 2020 23:03:45 +0100 In-Reply-To: <878s9zfjt4.fsf@gnu.org> (Mathieu Othacehe's message of "Tue, 15 Dec 2020 11:24:55 +0100") Message-ID: <875z52loam.fsf@gnu.org> User-Agent: Gnus/5.13 (Gnus v5.13) Emacs/27.1 (gnu/linux) MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable X-Spam-Score: -2.3 (--) X-BeenThere: debbugs-submit@debbugs.gnu.org X-Mailman-Version: 2.1.18 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: debbugs-submit-bounces@debbugs.gnu.org Sender: "Debbugs-submit" X-Spam-Score: -3.3 (---) Hi, Mathieu Othacehe skribis: >> (and (evaluation-complete? evaluation) >> (string=3D? "guix-modular-master" >> (evaluation-spec >> evaluation)))) > > On Berlin, evaluations can be completed for days, but the associated > builds never started. I think that searching directly for a completed > build provides a stronger guarantee of available substitutes. Yes, something like you proposed probably makes more sense. My point is just that we could make the procedure available as part of the API and document it as something people can use in their channels file. >> ;; Pull the latest commit fully built on berlin.guixsd.org. >> (list (channel >> (name 'guix) >> (url "https://git.savannah.gnu.org/git/guix.git") >> (commit (pk 'commit (latest-commit-successfully-built))))) > > Providing such a procedure definitely makes sense though. > >> (channel-with-substitutes-available >> (channel (name 'guix) =E2=80=A6) >> "https://ci.guix.gnu.org" >> (specifications->manifest '("emacs" "guile"))) > > Yes it would be the ultimate thing! However, while finding the latest > commit with an available substitute for a derivation is quite easy, > finding a commit with available derivations for N derivations seems way > more difficult. Right! >> It does mean that we=E2=80=99re asking users to do extra work. Perhaps = there >> could still be a command-line option that would call >> =E2=80=98channel-with-substitutes-available=E2=80=99 for you, but at lea= st it would take >> an explicit URL and clarify what Chris mentioned? > > Yes, the user would then have to provide the channels that need > available substitutes, the URL to use for the substitution check and > maybe a manifest that also needs available substitutes. > > The channels list could default to '("guix") and the URL to > "https://ci.guix.gnu.org" as it would be a sensible default for most > Guix users I think. Yes, choosing good defaults can make it less intimidating. >> BTW, doing all this is safer today because =E2=80=98guix pull=E2=80=99 w= ill detect and >> prevent downgrades. Though an attacker who manages to break into >> ci.guix.gnu.org could cause all the users of >> =E2=80=98channel-with-substitutes-available=E2=80=99 to no longer receiv= e updates or to >> receive them more slowly than they appear in Git simply by making CI >> even slower than it currently is. > > Yes, the downgrade check definitely helps here, as it's often what will > happen with our lagging CI. Regarding the security aspect, I think that > breaking into ci.guix.gnu.org can have other way more impacting > consequences. Yeah, though here we=E2=80=99re opening a new vulnerability channel, indepe= ndent of substitutes. It changes the threat model. Thanks, Ludo=E2=80=99. From unknown Sun Jun 15 08:57:45 2025 X-Loop: help-debbugs@gnu.org Subject: [bug#45104] pull: Add a "with-substitutes" option. Resent-From: Mathieu Othacehe Original-Sender: "Debbugs-submit" Resent-CC: guix-patches@gnu.org Resent-Date: Fri, 29 Jan 2021 13:24:02 +0000 Resent-Message-ID: Resent-Sender: help-debbugs@gnu.org X-GNU-PR-Message: followup 45104 X-GNU-PR-Package: guix-patches X-GNU-PR-Keywords: To: Ludovic =?UTF-8?Q?Court=C3=A8s?= Cc: 45104@debbugs.gnu.org, Christopher Baines Received: via spool by 45104-submit@debbugs.gnu.org id=B45104.161192661112258 (code B ref 45104); Fri, 29 Jan 2021 13:24:02 +0000 Received: (at 45104) by debbugs.gnu.org; 29 Jan 2021 13:23:31 +0000 Received: from localhost ([127.0.0.1]:51366 helo=debbugs.gnu.org) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1l5TkI-0003Bd-Kc for submit@debbugs.gnu.org; Fri, 29 Jan 2021 08:23:31 -0500 Received: from eggs.gnu.org ([209.51.188.92]:41726) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1l5TkG-0003Ao-3c for 45104@debbugs.gnu.org; Fri, 29 Jan 2021 08:23:29 -0500 Received: from fencepost.gnu.org ([2001:470:142:3::e]:49773) by eggs.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1l5Tk9-0007du-Sh; Fri, 29 Jan 2021 08:23:21 -0500 Received: from [2a01:e0a:19b:d9a0:10b6:a2c3:9f4e:6dae] (port=56646 helo=cervin) by fencepost.gnu.org with esmtpsa (TLS1.2:RSA_AES_256_CBC_SHA1:256) (Exim 4.82) (envelope-from ) id 1l5Tk7-0007lf-NL; Fri, 29 Jan 2021 08:23:21 -0500 From: Mathieu Othacehe References: <87eek1vd4g.fsf@gnu.org> <87a6uohztw.fsf@cbaines.net> <877dpktzot.fsf@gnu.org> <878s9zfjt4.fsf@gnu.org> <875z52loam.fsf@gnu.org> Date: Fri, 29 Jan 2021 14:23:18 +0100 In-Reply-To: <875z52loam.fsf@gnu.org> ("Ludovic =?UTF-8?Q?Court=C3=A8s?="'s message of "Tue, 15 Dec 2020 23:03:45 +0100") Message-ID: <871re3khbt.fsf@gnu.org> User-Agent: Gnus/5.13 (Gnus v5.13) Emacs/27.1 (gnu/linux) MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="=-=-=" X-Spam-Score: -0.7 (/) X-BeenThere: debbugs-submit@debbugs.gnu.org X-Mailman-Version: 2.1.18 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: debbugs-submit-bounces@debbugs.gnu.org Sender: "Debbugs-submit" X-Spam-Score: -1.7 (-) --=-=-= Content-Type: text/plain Hello, Here's a patch implementing "channel-with-substitutes-available". The following step will be to add the matching option for the "guix pull" command, as discussed previously. Thanks, Mathieu --=-=-= Content-Type: text/x-diff; charset=utf-8 Content-Disposition: inline; filename=0001-guix-channels-Introduce-channel-with-substitutes-ava.patch Content-Transfer-Encoding: quoted-printable >From 8f93ced042f7f694c0c7a473686faa3c00497aa5 Mon Sep 17 00:00:00 2001 From: Mathieu Othacehe Date: Fri, 29 Jan 2021 13:48:44 +0100 Subject: [PATCH] guix: channels: Introduce "channel-with-substitutes-available". * guix/channels.scm (find-latest-commit-with-substitutes, channel-with-substitutes-available): New procedures. * guix/scripts/pull.scm (guix-pull): Move "channel-list" call inside the %current-system parameter scope. * doc/guix.texi (Channels with substitutes): New section. --- doc/guix.texi | 27 ++++++++++- guix/channels.scm | 29 +++++++++++- guix/scripts/pull.scm | 103 +++++++++++++++++++++--------------------- 3 files changed, 106 insertions(+), 53 deletions(-) diff --git a/doc/guix.texi b/doc/guix.texi index ff9e8da2e0..6587a49d0e 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -40,7 +40,7 @@ Copyright @copyright{} 2016, 2017, 2018, 2019, 2020 Julie= n Lepiller@* Copyright @copyright{} 2016 Alex ter Weele@* Copyright @copyright{} 2016, 2017, 2018, 2019 Christopher Baines@* Copyright @copyright{} 2017, 2018, 2019 Cl=C3=A9ment Lassieur@* -Copyright @copyright{} 2017, 2018, 2020 Mathieu Othacehe@* +Copyright @copyright{} 2017, 2018, 2020, 2021 Mathieu Othacehe@* Copyright @copyright{} 2017 Federico Beffa@* Copyright @copyright{} 2017, 2018 Carlo Zancanaro@* Copyright @copyright{} 2017 Thomas Danckaert@* @@ -245,6 +245,7 @@ Channels * Specifying Channel Authorizations:: Defining channel authors authorizat= ions. * Primary URL:: Distinguishing mirror to original. * Writing Channel News:: Communicating information to channel's use= rs. +* Channels with substitutes:: Using channels with available substitutes. =20 Development =20 @@ -4919,6 +4920,7 @@ updates. * Specifying Channel Authorizations:: Defining channel authors authorizat= ions. * Primary URL:: Distinguishing mirror to original. * Writing Channel News:: Communicating information to channel's use= rs. +* Channels with substitutes:: Using channels with available substitutes. @end menu =20 @node Specifying Additional Channels @@ -5390,6 +5392,29 @@ xgettext -o news.po -l scheme -ken etc/news.txt To sum up, yes, you could use your channel as a blog. But beware, this is @emph{not quite} what your users might expect. =20 +@node Channels with substitutes +@section Channels with substitutes + +When running @command{guix pull}, Guix will first compile the +definitions of every available package. This is an expensive operation +for which substitutes (@pxref{Substitutes}) may be available. The +following snippet in @file{channels.scm} will ensure that @command{guix +pull} uses the latest commit with available substitutes for the package +definitions. This is done by querying the continuous integration +server at @url{https://ci.guix.gnu.org}. + +Be careful, this does not mean that all the packages that you will +install after running @command{guix pull} will have available +substitutes. It only ensures that @command{guix pull} will not try to +compile package definitions. This is particularly useful when using +machines with limited resources. + +@lisp +(list + (channel-with-substitutes-available + %default-guix-channel + "https://ci.guix.gnu.org")) +@end lisp =20 @c ********************************************************************* @node Development diff --git a/guix/channels.scm b/guix/channels.scm index 0c84eed477..817e30d738 100644 --- a/guix/channels.scm +++ b/guix/channels.scm @@ -20,6 +20,7 @@ =20 (define-module (guix channels) #:use-module (git) + #:use-module (guix ci) #:use-module (guix git) #:use-module (guix git-authenticate) #:use-module ((guix openpgp) @@ -98,7 +99,9 @@ channel-news-entry-title channel-news-entry-body =20 - channel-news-for-commit)) + channel-news-for-commit + + channel-with-substitutes-available)) =20 ;;; Commentary: ;;; @@ -1044,6 +1047,30 @@ NEW. When OLD is omitted or is #f, return all the n= ews entries of CHANNEL." '() (apply throw key error rest))))) =20 +(define (find-latest-commit-with-substitutes url) + (let* ((job-name (string-append "guix." (%current-system))) + (build (match (latest-builds url 1 + #:job job-name + #:status 0) ;success + ((build) build) + (_ #f))) + (evaluation (and build + (evaluation url (build-evaluation build)))) + (commit (and evaluation + (match (evaluation-checkouts evaluation) + ((checkout) + (checkout-commit checkout)))))) + commit)) + +(define (channel-with-substitutes-available chan url) + (let ((commit (find-latest-commit-with-substitutes url))) + (unless commit + (warning (G_ "could not find available substitutes at ~a~%") + url)) + (channel + (inherit chan) + (commit commit)))) + ;;; Local Variables: ;;; eval: (put 'with-guile 'scheme-indent-function 1) ;;; End: diff --git a/guix/scripts/pull.scm b/guix/scripts/pull.scm index 83cdc1d1eb..4e0ab5d341 100644 --- a/guix/scripts/pull.scm +++ b/guix/scripts/pull.scm @@ -765,60 +765,61 @@ Use '~/.config/guix/channels.scm' instead.")) #:argument-handler no-argume= nts)) (substitutes? (assoc-ref opts 'substitutes?)) (dry-run? (assoc-ref opts 'dry-run?)) - (channels (channel-list opts)) (profile (or (assoc-ref opts 'profile) %current-profile)) (current-channels (profile-channels profile)) (validate-pull (assoc-ref opts 'validate-pull)) (authenticate? (assoc-ref opts 'authenticate-channels?))) - (cond ((assoc-ref opts 'query) - (process-query opts profile)) - ((assoc-ref opts 'generation) - (process-generation-change opts profile)) - (else - (with-store store - (with-status-verbosity (assoc-ref opts 'verbosity) - (parameterize ((%current-system (assoc-ref opts 'system)) - (%graft? (assoc-ref opts 'graft?))) - (with-build-handler (build-notifier #:use-substitutes? - substitutes? - #:verbosity - (assoc-ref opts 'v= erbosity) - #:dry-run? dry-run= ?) - (set-build-options-from-command-line store opts) - (ensure-default-profile) - (honor-x509-certificates store) - - (let ((instances - (latest-channel-instances store channels - #:current-channels - current-channels - #:validate-pull - validate-pull - #:authenticate? - authenticate?))) - (format (current-error-port) - (N_ "Building from this channel:~%" - "Building from these channels:~%" - (length instances))) - (for-each (lambda (instance) - (let ((channel - (channel-instance-channel insta= nce))) - (format (current-error-port) - " ~10a~a\t~a~%" - (channel-name channel) - (channel-url channel) - (string-take - (channel-instance-commit in= stance) - 7)))) - instances) - (parameterize ((%guile-for-build - (package-derivation - store - (if (assoc-ref opts 'bootstrap?) - %bootstrap-guile - (default-guile))))) - (with-profile-lock profile - (run-with-store store - (build-and-install instances profile))))))))= ))))))) + (cond + ((assoc-ref opts 'query) + (process-query opts profile)) + ((assoc-ref opts 'generation) + (process-generation-change opts profile)) + (else + (with-store store + (with-status-verbosity (assoc-ref opts 'verbosity) + (parameterize ((%current-system (assoc-ref opts 'system)) + (%graft? (assoc-ref opts 'graft?))) + (with-build-handler (build-notifier #:use-substitutes? + substitutes? + #:verbosity + (assoc-ref opts 'verbos= ity) + #:dry-run? dry-run?) + (set-build-options-from-command-line store opts) + (ensure-default-profile) + (honor-x509-certificates store) + + (let* ((channels (channel-list opts)) + (instances + (latest-channel-instances store channels + #:current-channels + current-channels + #:validate-pull + validate-pull + #:authenticate? + authenticate?))) + (format (current-error-port) + (N_ "Building from this channel:~%" + "Building from these channels:~%" + (length instances))) + (for-each (lambda (instance) + (let ((channel + (channel-instance-channel instance))) + (format (current-error-port) + " ~10a~a\t~a~%" + (channel-name channel) + (channel-url channel) + (string-take + (channel-instance-commit instanc= e) + 7)))) + instances) + (parameterize ((%guile-for-build + (package-derivation + store + (if (assoc-ref opts 'bootstrap?) + %bootstrap-guile + (default-guile))))) + (with-profile-lock profile + (run-with-store store + (build-and-install instances profile)))))))))))))= )) =20 ;;; pull.scm ends here --=20 2.29.2 --=-=-=-- From unknown Sun Jun 15 08:57:45 2025 X-Loop: help-debbugs@gnu.org Subject: [bug#45104] pull: Add a "with-substitutes" option. Resent-From: Mathieu Othacehe Original-Sender: "Debbugs-submit" Resent-CC: guix-patches@gnu.org Resent-Date: Fri, 29 Jan 2021 13:37:01 +0000 Resent-Message-ID: Resent-Sender: help-debbugs@gnu.org X-GNU-PR-Message: followup 45104 X-GNU-PR-Package: guix-patches X-GNU-PR-Keywords: To: Ludovic =?UTF-8?Q?Court=C3=A8s?= Cc: 45104@debbugs.gnu.org, Christopher Baines Received: via spool by 45104-submit@debbugs.gnu.org id=B45104.161192740213786 (code B ref 45104); Fri, 29 Jan 2021 13:37:01 +0000 Received: (at 45104) by debbugs.gnu.org; 29 Jan 2021 13:36:42 +0000 Received: from localhost ([127.0.0.1]:51382 helo=debbugs.gnu.org) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1l5Tx4-0003aH-5q for submit@debbugs.gnu.org; Fri, 29 Jan 2021 08:36:42 -0500 Received: from eggs.gnu.org ([209.51.188.92]:43922) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1l5Tx2-0003a5-7Q for 45104@debbugs.gnu.org; Fri, 29 Jan 2021 08:36:40 -0500 Received: from fencepost.gnu.org ([2001:470:142:3::e]:50140) by eggs.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1l5Tww-0004Wt-Lr; Fri, 29 Jan 2021 08:36:34 -0500 Received: from [2a01:e0a:19b:d9a0:10b6:a2c3:9f4e:6dae] (port=56768 helo=cervin) by fencepost.gnu.org with esmtpsa (TLS1.2:RSA_AES_256_CBC_SHA1:256) (Exim 4.82) (envelope-from ) id 1l5Twu-0002Am-Hp; Fri, 29 Jan 2021 08:36:34 -0500 From: Mathieu Othacehe References: <87eek1vd4g.fsf@gnu.org> <87a6uohztw.fsf@cbaines.net> <877dpktzot.fsf@gnu.org> <878s9zfjt4.fsf@gnu.org> <875z52loam.fsf@gnu.org> <871re3khbt.fsf@gnu.org> Date: Fri, 29 Jan 2021 14:36:30 +0100 In-Reply-To: <871re3khbt.fsf@gnu.org> (Mathieu Othacehe's message of "Fri, 29 Jan 2021 14:23:18 +0100") Message-ID: <87y2gbj25d.fsf@gnu.org> User-Agent: Gnus/5.13 (Gnus v5.13) Emacs/27.1 (gnu/linux) MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="=-=-=" X-Spam-Score: -0.7 (/) X-BeenThere: debbugs-submit@debbugs.gnu.org X-Mailman-Version: 2.1.18 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: debbugs-submit-bounces@debbugs.gnu.org Sender: "Debbugs-submit" X-Spam-Score: -1.7 (-) --=-=-= Content-Type: text/plain Here's a v2 with the missing docstrings. Thanks, Mathieu --=-=-= Content-Type: text/x-diff; charset=utf-8 Content-Disposition: inline; filename=0001-guix-channels-Introduce-channel-with-substitutes-ava.patch Content-Transfer-Encoding: quoted-printable >From 31dad6456825a329ba0f07c95e3e99258d186a8f Mon Sep 17 00:00:00 2001 From: Mathieu Othacehe Date: Fri, 29 Jan 2021 13:48:44 +0100 Subject: [PATCH v2] guix: channels: Introduce "channel-with-substitutes-available". * guix/channels.scm (find-latest-commit-with-substitutes, channel-with-substitutes-available): New procedures. * guix/scripts/pull.scm (guix-pull): Move "channel-list" call inside the %current-system parameter scope. * doc/guix.texi (Channels with substitutes): New section. --- doc/guix.texi | 27 ++++++++++- guix/channels.scm | 37 ++++++++++++++- guix/scripts/pull.scm | 103 +++++++++++++++++++++--------------------- 3 files changed, 114 insertions(+), 53 deletions(-) diff --git a/doc/guix.texi b/doc/guix.texi index ff9e8da2e0..6587a49d0e 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -40,7 +40,7 @@ Copyright @copyright{} 2016, 2017, 2018, 2019, 2020 Julie= n Lepiller@* Copyright @copyright{} 2016 Alex ter Weele@* Copyright @copyright{} 2016, 2017, 2018, 2019 Christopher Baines@* Copyright @copyright{} 2017, 2018, 2019 Cl=C3=A9ment Lassieur@* -Copyright @copyright{} 2017, 2018, 2020 Mathieu Othacehe@* +Copyright @copyright{} 2017, 2018, 2020, 2021 Mathieu Othacehe@* Copyright @copyright{} 2017 Federico Beffa@* Copyright @copyright{} 2017, 2018 Carlo Zancanaro@* Copyright @copyright{} 2017 Thomas Danckaert@* @@ -245,6 +245,7 @@ Channels * Specifying Channel Authorizations:: Defining channel authors authorizat= ions. * Primary URL:: Distinguishing mirror to original. * Writing Channel News:: Communicating information to channel's use= rs. +* Channels with substitutes:: Using channels with available substitutes. =20 Development =20 @@ -4919,6 +4920,7 @@ updates. * Specifying Channel Authorizations:: Defining channel authors authorizat= ions. * Primary URL:: Distinguishing mirror to original. * Writing Channel News:: Communicating information to channel's use= rs. +* Channels with substitutes:: Using channels with available substitutes. @end menu =20 @node Specifying Additional Channels @@ -5390,6 +5392,29 @@ xgettext -o news.po -l scheme -ken etc/news.txt To sum up, yes, you could use your channel as a blog. But beware, this is @emph{not quite} what your users might expect. =20 +@node Channels with substitutes +@section Channels with substitutes + +When running @command{guix pull}, Guix will first compile the +definitions of every available package. This is an expensive operation +for which substitutes (@pxref{Substitutes}) may be available. The +following snippet in @file{channels.scm} will ensure that @command{guix +pull} uses the latest commit with available substitutes for the package +definitions. This is done by querying the continuous integration +server at @url{https://ci.guix.gnu.org}. + +Be careful, this does not mean that all the packages that you will +install after running @command{guix pull} will have available +substitutes. It only ensures that @command{guix pull} will not try to +compile package definitions. This is particularly useful when using +machines with limited resources. + +@lisp +(list + (channel-with-substitutes-available + %default-guix-channel + "https://ci.guix.gnu.org")) +@end lisp =20 @c ********************************************************************* @node Development diff --git a/guix/channels.scm b/guix/channels.scm index 0c84eed477..a04eae1c10 100644 --- a/guix/channels.scm +++ b/guix/channels.scm @@ -20,6 +20,7 @@ =20 (define-module (guix channels) #:use-module (git) + #:use-module (guix ci) #:use-module (guix git) #:use-module (guix git-authenticate) #:use-module ((guix openpgp) @@ -98,7 +99,9 @@ channel-news-entry-title channel-news-entry-body =20 - channel-news-for-commit)) + channel-news-for-commit + + channel-with-substitutes-available)) =20 ;;; Commentary: ;;; @@ -1044,6 +1047,38 @@ NEW. When OLD is omitted or is #f, return all the n= ews entries of CHANNEL." '() (apply throw key error rest))))) =20 +(define (find-latest-commit-with-substitutes url) + "Return the latest commit with available substitutes for the Guix package +definitions at URL. Return false if no commit were found." + (let* ((job-name (string-append "guix." (%current-system))) + (build (match (latest-builds url 1 + #:job job-name + #:status 0) ;success + ((build) build) + (_ #f))) + (evaluation (and build + (evaluation url (build-evaluation build)))) + (commit (and evaluation + (match (evaluation-checkouts evaluation) + ((checkout) + (checkout-commit checkout)))))) + commit)) + +(define (channel-with-substitutes-available chan url) + "Return a channel inheriting from CHAN but which commit field is set to = the +latest commit with available substitutes for the Guix package definitions = at +URL. The current system is taken into account. + +If no commit with available substitutes were found, the commit field is se= t to +false and a warning message is printed." + (let ((commit (find-latest-commit-with-substitutes url))) + (unless commit + (warning (G_ "could not find available substitutes at ~a~%") + url)) + (channel + (inherit chan) + (commit commit)))) + ;;; Local Variables: ;;; eval: (put 'with-guile 'scheme-indent-function 1) ;;; End: diff --git a/guix/scripts/pull.scm b/guix/scripts/pull.scm index 83cdc1d1eb..4e0ab5d341 100644 --- a/guix/scripts/pull.scm +++ b/guix/scripts/pull.scm @@ -765,60 +765,61 @@ Use '~/.config/guix/channels.scm' instead.")) #:argument-handler no-argume= nts)) (substitutes? (assoc-ref opts 'substitutes?)) (dry-run? (assoc-ref opts 'dry-run?)) - (channels (channel-list opts)) (profile (or (assoc-ref opts 'profile) %current-profile)) (current-channels (profile-channels profile)) (validate-pull (assoc-ref opts 'validate-pull)) (authenticate? (assoc-ref opts 'authenticate-channels?))) - (cond ((assoc-ref opts 'query) - (process-query opts profile)) - ((assoc-ref opts 'generation) - (process-generation-change opts profile)) - (else - (with-store store - (with-status-verbosity (assoc-ref opts 'verbosity) - (parameterize ((%current-system (assoc-ref opts 'system)) - (%graft? (assoc-ref opts 'graft?))) - (with-build-handler (build-notifier #:use-substitutes? - substitutes? - #:verbosity - (assoc-ref opts 'v= erbosity) - #:dry-run? dry-run= ?) - (set-build-options-from-command-line store opts) - (ensure-default-profile) - (honor-x509-certificates store) - - (let ((instances - (latest-channel-instances store channels - #:current-channels - current-channels - #:validate-pull - validate-pull - #:authenticate? - authenticate?))) - (format (current-error-port) - (N_ "Building from this channel:~%" - "Building from these channels:~%" - (length instances))) - (for-each (lambda (instance) - (let ((channel - (channel-instance-channel insta= nce))) - (format (current-error-port) - " ~10a~a\t~a~%" - (channel-name channel) - (channel-url channel) - (string-take - (channel-instance-commit in= stance) - 7)))) - instances) - (parameterize ((%guile-for-build - (package-derivation - store - (if (assoc-ref opts 'bootstrap?) - %bootstrap-guile - (default-guile))))) - (with-profile-lock profile - (run-with-store store - (build-and-install instances profile))))))))= ))))))) + (cond + ((assoc-ref opts 'query) + (process-query opts profile)) + ((assoc-ref opts 'generation) + (process-generation-change opts profile)) + (else + (with-store store + (with-status-verbosity (assoc-ref opts 'verbosity) + (parameterize ((%current-system (assoc-ref opts 'system)) + (%graft? (assoc-ref opts 'graft?))) + (with-build-handler (build-notifier #:use-substitutes? + substitutes? + #:verbosity + (assoc-ref opts 'verbos= ity) + #:dry-run? dry-run?) + (set-build-options-from-command-line store opts) + (ensure-default-profile) + (honor-x509-certificates store) + + (let* ((channels (channel-list opts)) + (instances + (latest-channel-instances store channels + #:current-channels + current-channels + #:validate-pull + validate-pull + #:authenticate? + authenticate?))) + (format (current-error-port) + (N_ "Building from this channel:~%" + "Building from these channels:~%" + (length instances))) + (for-each (lambda (instance) + (let ((channel + (channel-instance-channel instance))) + (format (current-error-port) + " ~10a~a\t~a~%" + (channel-name channel) + (channel-url channel) + (string-take + (channel-instance-commit instanc= e) + 7)))) + instances) + (parameterize ((%guile-for-build + (package-derivation + store + (if (assoc-ref opts 'bootstrap?) + %bootstrap-guile + (default-guile))))) + (with-profile-lock profile + (run-with-store store + (build-and-install instances profile)))))))))))))= )) =20 ;;; pull.scm ends here --=20 2.29.2 --=-=-=-- From unknown Sun Jun 15 08:57:45 2025 X-Loop: help-debbugs@gnu.org Subject: [bug#45104] pull: Add a "with-substitutes" option. Resent-From: Ludovic =?UTF-8?Q?Court=C3=A8s?= Original-Sender: "Debbugs-submit" Resent-CC: guix-patches@gnu.org Resent-Date: Sun, 31 Jan 2021 16:20:02 +0000 Resent-Message-ID: Resent-Sender: help-debbugs@gnu.org X-GNU-PR-Message: followup 45104 X-GNU-PR-Package: guix-patches X-GNU-PR-Keywords: To: Mathieu Othacehe Cc: 45104@debbugs.gnu.org, Christopher Baines Received: via spool by 45104-submit@debbugs.gnu.org id=B45104.161210994829580 (code B ref 45104); Sun, 31 Jan 2021 16:20:02 +0000 Received: (at 45104) by debbugs.gnu.org; 31 Jan 2021 16:19:08 +0000 Received: from localhost ([127.0.0.1]:57802 helo=debbugs.gnu.org) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1l6FRK-0007h0-Mt for submit@debbugs.gnu.org; Sun, 31 Jan 2021 11:19:08 -0500 Received: from eggs.gnu.org ([209.51.188.92]:46996) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1l6FRH-0007gK-P8 for 45104@debbugs.gnu.org; Sun, 31 Jan 2021 11:19:05 -0500 Received: from fencepost.gnu.org ([2001:470:142:3::e]:35054) by eggs.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1l6FRC-0001Ia-BI; Sun, 31 Jan 2021 11:18:58 -0500 Received: from [2a01:e0a:1d:7270:af76:b9b:ca24:c465] (port=50406 helo=ribbon) by fencepost.gnu.org with esmtpsa (TLS1.2:RSA_AES_256_CBC_SHA1:256) (Exim 4.82) (envelope-from ) id 1l6FRA-0004N0-T8; Sun, 31 Jan 2021 11:18:57 -0500 From: Ludovic =?UTF-8?Q?Court=C3=A8s?= References: <87eek1vd4g.fsf@gnu.org> <87a6uohztw.fsf@cbaines.net> <877dpktzot.fsf@gnu.org> <878s9zfjt4.fsf@gnu.org> <875z52loam.fsf@gnu.org> <871re3khbt.fsf@gnu.org> <87y2gbj25d.fsf@gnu.org> X-URL: http://www.fdn.fr/~lcourtes/ X-Revolutionary-Date: 12 =?UTF-8?Q?Pluvi=C3=B4se?= an 229 de la =?UTF-8?Q?R=C3=A9volution?= X-PGP-Key-ID: 0x090B11993D9AEBB5 X-PGP-Key: http://www.fdn.fr/~lcourtes/ludovic.asc X-PGP-Fingerprint: 3CE4 6455 8A84 FDC6 9DB4 0CFB 090B 1199 3D9A EBB5 X-OS: x86_64-pc-linux-gnu Date: Sun, 31 Jan 2021 17:18:55 +0100 In-Reply-To: <87y2gbj25d.fsf@gnu.org> (Mathieu Othacehe's message of "Fri, 29 Jan 2021 14:36:30 +0100") Message-ID: <87eei1nkpc.fsf@gnu.org> User-Agent: Gnus/5.13 (Gnus v5.13) Emacs/27.1 (gnu/linux) MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable X-Spam-Score: -0.7 (/) X-BeenThere: debbugs-submit@debbugs.gnu.org X-Mailman-Version: 2.1.18 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: debbugs-submit-bounces@debbugs.gnu.org Sender: "Debbugs-submit" X-Spam-Score: -1.7 (-) Hi! Mathieu Othacehe skribis: > From 31dad6456825a329ba0f07c95e3e99258d186a8f Mon Sep 17 00:00:00 2001 > From: Mathieu Othacehe > Date: Fri, 29 Jan 2021 13:48:44 +0100 > Subject: [PATCH v2] guix: channels: Introduce > "channel-with-substitutes-available". > > * guix/channels.scm (find-latest-commit-with-substitutes, > channel-with-substitutes-available): New procedures. > * guix/scripts/pull.scm (guix-pull): Move "channel-list" call inside the > %current-system parameter scope. > * doc/guix.texi (Channels with substitutes): New section. Yay! > +@node Channels with substitutes > +@section Channels with substitutes Should be =E2=80=9CChannels With Substitutes=E2=80=9D > +When running @command{guix pull}, Guix will first compile the > +definitions of every available package. This is an expensive operation > +for which substitutes (@pxref{Substitutes}) may be available. The > +following snippet in @file{channels.scm} will ensure that @command{guix > +pull} uses the latest commit with available substitutes for the package > +definitions. This is done by querying the continuous integration ^ I=E2=80=99d replace the period with a colon and insert the example right he= re. > +server at @url{https://ci.guix.gnu.org}. > + > +Be careful, this does not mean that all the packages that you will s/Be careful,/Note that/ > +install after running @command{guix pull} will have available > +substitutes. It only ensures that @command{guix pull} will not try to > +compile package definitions. This is particularly useful when using > +machines with limited resources. > + > +@lisp > +(list > + (channel-with-substitutes-available ^ Nipick: maybe move on the previous line. > (define-module (guix channels) > #:use-module (git) > + #:use-module (guix ci) Could we instead move these two procedures to (guix ci), with an #:autoload for (guix channels), so that (guix channels) remains focused on core functionality? It does mean that in the example above we need to add an explicit (use-modules (guix ci)). Otherwise LGTM, thanks! Ludo=E2=80=99. From unknown Sun Jun 15 08:57:45 2025 MIME-Version: 1.0 X-Mailer: MIME-tools 5.505 (Entity 5.505) X-Loop: help-debbugs@gnu.org From: help-debbugs@gnu.org (GNU bug Tracking System) To: Mathieu Othacehe Subject: bug#45104: closed (Re: [bug#45104] pull: Add a "with-substitutes" option.) Message-ID: References: <87sg6hypl6.fsf@gnu.org> <87eek1vd4g.fsf@gnu.org> X-Gnu-PR-Message: they-closed 45104 X-Gnu-PR-Package: guix-patches Reply-To: 45104@debbugs.gnu.org Date: Sun, 31 Jan 2021 17:39:03 +0000 Content-Type: multipart/mixed; boundary="----------=_1612114743-4642-1" This is a multi-part message in MIME format... ------------=_1612114743-4642-1 Content-Disposition: inline Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Your bug report #45104: pull: Add a "with-substitutes" option. which was filed against the guix-patches package, has been closed. The explanation is attached below, along with your original report. If you require more details, please reply to 45104@debbugs.gnu.org. --=20 45104: http://debbugs.gnu.org/cgi/bugreport.cgi?bug=3D45104 GNU Bug Tracking System Contact help-debbugs@gnu.org with problems ------------=_1612114743-4642-1 Content-Type: message/rfc822 Content-Disposition: inline Content-Transfer-Encoding: 7bit Received: (at 45104-done) by debbugs.gnu.org; 31 Jan 2021 17:38:15 +0000 Received: from localhost ([127.0.0.1]:57899 helo=debbugs.gnu.org) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1l6Gfv-0001Bn-3l for submit@debbugs.gnu.org; Sun, 31 Jan 2021 12:38:15 -0500 Received: from eggs.gnu.org ([209.51.188.92]:57078) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1l6Gfq-0001BM-Vm for 45104-done@debbugs.gnu.org; Sun, 31 Jan 2021 12:38:11 -0500 Received: from fencepost.gnu.org ([2001:470:142:3::e]:36105) by eggs.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1l6Gfk-0003V8-9V; Sun, 31 Jan 2021 12:38:04 -0500 Received: from [2a01:e0a:19b:d9a0:10b6:a2c3:9f4e:6dae] (port=34644 helo=cervin) by fencepost.gnu.org with esmtpsa (TLS1.2:RSA_AES_256_CBC_SHA1:256) (Exim 4.82) (envelope-from ) id 1l6Gfi-0001I4-JA; Sun, 31 Jan 2021 12:38:03 -0500 From: Mathieu Othacehe To: Ludovic =?utf-8?Q?Court=C3=A8s?= Subject: Re: [bug#45104] pull: Add a "with-substitutes" option. References: <87eek1vd4g.fsf@gnu.org> <87a6uohztw.fsf@cbaines.net> <877dpktzot.fsf@gnu.org> <878s9zfjt4.fsf@gnu.org> <875z52loam.fsf@gnu.org> <871re3khbt.fsf@gnu.org> <87y2gbj25d.fsf@gnu.org> <87eei1nkpc.fsf@gnu.org> Date: Sun, 31 Jan 2021 18:37:57 +0100 In-Reply-To: <87eei1nkpc.fsf@gnu.org> ("Ludovic =?utf-8?Q?Court=C3=A8s=22'?= =?utf-8?Q?s?= message of "Sun, 31 Jan 2021 17:18:55 +0100") Message-ID: <87sg6hypl6.fsf@gnu.org> User-Agent: Gnus/5.13 (Gnus v5.13) Emacs/27.1 (gnu/linux) MIME-Version: 1.0 Content-Type: text/plain X-Spam-Score: -0.7 (/) X-Debbugs-Envelope-To: 45104-done Cc: 45104-done@debbugs.gnu.org, Christopher Baines X-BeenThere: debbugs-submit@debbugs.gnu.org X-Mailman-Version: 2.1.18 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: debbugs-submit-bounces@debbugs.gnu.org Sender: "Debbugs-submit" X-Spam-Score: -1.7 (-) Hey Ludo, > It does mean that in the example above we need to add an explicit > (use-modules (guix ci)). > > Otherwise LGTM, thanks! I took your remarks into account and pushed as 041a9466ea23d6ae811491bcf529bf9487317b48. Thanks for reviewing! Mathieu ------------=_1612114743-4642-1 Content-Type: message/rfc822 Content-Disposition: inline Content-Transfer-Encoding: 7bit Received: (at submit) by debbugs.gnu.org; 7 Dec 2020 15:40:05 +0000 Received: from localhost ([127.0.0.1]:55002 helo=debbugs.gnu.org) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1kmIcP-0004F9-23 for submit@debbugs.gnu.org; Mon, 07 Dec 2020 10:40:05 -0500 Received: from lists.gnu.org ([209.51.188.17]:59678) by debbugs.gnu.org with esmtp (Exim 4.84_2) (envelope-from ) id 1kmIcM-0004Ez-Md for submit@debbugs.gnu.org; Mon, 07 Dec 2020 10:40:03 -0500 Received: from eggs.gnu.org ([2001:470:142:3::10]:44856) by lists.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1kmIcM-0002Zp-Dc for guix-patches@gnu.org; Mon, 07 Dec 2020 10:40:02 -0500 Received: from fencepost.gnu.org ([2001:470:142:3::e]:57408) by eggs.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1kmIcM-0000wv-4V for guix-patches@gnu.org; Mon, 07 Dec 2020 10:40:02 -0500 Received: from [2a01:e0a:19b:d9a0:34f0:bcad:cad1:16d6] (port=34604 helo=cervin) by fencepost.gnu.org with esmtpsa (TLS1.2:RSA_AES_256_CBC_SHA1:256) (Exim 4.82) (envelope-from ) id 1kmIcL-0006bB-7Q for guix-patches@gnu.org; Mon, 07 Dec 2020 10:40:01 -0500 From: Mathieu Othacehe To: guix-patches@gnu.org Subject: pull: Add a "with-substitutes" option. Date: Mon, 07 Dec 2020 16:39:59 +0100 Message-ID: <87eek1vd4g.fsf@gnu.org> User-Agent: Gnus/5.13 (Gnus v5.13) Emacs/27.1 (gnu/linux) MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="=-=-=" X-Spam-Score: -2.3 (--) X-Debbugs-Envelope-To: submit X-BeenThere: debbugs-submit@debbugs.gnu.org X-Mailman-Version: 2.1.18 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: debbugs-submit-bounces@debbugs.gnu.org Sender: "Debbugs-submit" X-Spam-Score: -3.3 (---) --=-=-= Content-Type: text/plain Hello, When "guix pull" is run before the CI server is done baking substitutes, the user may need to build some derivations, which can be long on low-end hardware. This patch adds an option to "guix pull" so that it updates to the latest commit with available substitutes. This work is still in progress, but I'd like to gather some impressions on that before going further. Thanks, Mathieu --=-=-= Content-Type: text/x-diff; charset=utf-8 Content-Disposition: inline; filename=0001-scripts-pull-Add-with-substitutes-option.patch Content-Transfer-Encoding: quoted-printable >From d399f8dbb9e38a82241b9048b8b04758fae10005 Mon Sep 17 00:00:00 2001 From: Mathieu Othacehe Date: Mon, 7 Dec 2020 16:12:22 +0100 Subject: [PATCH] scripts: pull: Add "with-substitutes" option. --- guix/scripts/pull.scm | 38 +++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/guix/scripts/pull.scm b/guix/scripts/pull.scm index 83cdc1d1eb..4609f8614e 100644 --- a/guix/scripts/pull.scm +++ b/guix/scripts/pull.scm @@ -2,6 +2,7 @@ ;;; Copyright =C2=A9 2013, 2014, 2015, 2017, 2018, 2019, 2020 Ludovic Cour= t=C3=A8s ;;; Copyright =C2=A9 2017 Marius Bakke ;;; Copyright =C2=A9 2020 Tobias Geerinckx-Rice +;;; Copyright =C2=A9 2020 Mathieu Othacehe ;;; ;;; This file is part of GNU Guix. ;;; @@ -20,6 +21,7 @@ =20 (define-module (guix scripts pull) #:use-module ((guix ui) #:hide (display-profile-content)) + #:use-module (guix ci) #:use-module (guix colors) #:use-module (guix utils) #:use-module ((guix status) #:select (with-status-verbosity)) @@ -64,6 +66,7 @@ #:re-export (display-profile-content channel-commit-hyperlink) #:export (channel-list + find-lastest-commit-with-substitutes guix-pull)) =20 @@ -169,6 +172,9 @@ Download and deploy the latest version of Guix.\n")) (lambda (opt name arg result) (alist-cons 'validate-pull warn-about-backward-updates result))) + (option '("with-substitutes") #f #f + (lambda (opt name arg result) + (alist-cons 'with-substitutes? #t result))) (option '("disable-authentication") #f #f (lambda (opt name arg result) (alist-cons 'authenticate-channels? #f result))) @@ -526,6 +532,26 @@ true, display what would be built without actually bui= lding it." (leave (G_ "while creating symlink '~a': ~a~%") link (strerror (system-error-errno args)))))))) =20 + +;;; +;;; Substitutes. +;;; + +(define (find-lastest-commit-with-substitutes) + (let ((urls %default-substitute-urls)) + (any (lambda (url) + (let* ((build (match (latest-builds url 1 + #:job "guix.x86_64-linux" + #:status 0) ;success + ((build) build))) + (evaluation (evaluation url + (build-evaluation build))) + (commit (match (evaluation-checkouts evaluation) + ((checkout) + (checkout-commit checkout))))) + commit)) + urls))) + ;;; ;;; Queries. @@ -731,8 +757,9 @@ Use '~/.config/guix/channels.scm' instead.")) =20 (let ((ref (assoc-ref opts 'ref)) (url (or (assoc-ref opts 'repository-url) - (environment-variable)))) - (if (or ref url) + (environment-variable))) + (with-substitutes? (assoc-ref opts 'with-substitutes?))) + (if (or ref url with-substitutes?) (match (find guix-channel? channels) ((? channel? guix) ;; Apply '--url', '--commit', and '--branch' to the 'guix' chan= nel. @@ -745,7 +772,12 @@ Use '~/.config/guix/channels.scm' instead.")) (channel (inherit guix) (url url) (commit #f) (branch branch))) (#f - (channel (inherit guix) (url url)))) + (let ((commit + (and with-substitutes? + (find-lastest-commit-with-substitutes)))) + (channel (inherit guix) + (url url) + (commit commit))))) (remove guix-channel? channels)))) (#f ;no 'guix' channel, failure will e= nsue channels)) --=20 2.29.2 --=-=-=-- ------------=_1612114743-4642-1--