Package: guix-patches;
Reported by: pukkamustard <pukkamustard <at> posteo.net>
Date: Thu, 16 Dec 2021 16:18:02 UTC
Severity: important
Tags: patch
View this message in rfc822 format
From: pukkamustard <pukkamustard <at> posteo.net> To: 52555 <at> debbugs.gnu.org Cc: pukkamustard <pukkamustard <at> posteo.net>, ludo <at> gnu.org, maximedevos <at> telenet.be, Christopher Baines <guix <at> cbaines.net>, Josselin Poiret <dev <at> jpoiret.xyz>, Ludovic Courtès <ludo <at> gnu.org>, Mathieu Othacehe <othacehe <at> gnu.org>, Ricardo Wurmus <rekado <at> elephly.net>, Simon Tournier <zimon.toutoune <at> gmail.com>, Tobias Geerinckx-Rice <me <at> tobias.gr> Subject: [bug#52555] [PATCH v4 6/7] substitute: Decode substitutes using ERIS. Date: Thu, 28 Dec 2023 09:40:09 +0000
* guix/scripts/substitute.scm: Decode substitutes using ERIS.
* guix/eris.scm (eris-decode-store-item): New function.
* nix/nix-daemon/guix-daemon.cc (options): Add eris-store-url option.
---
guix/eris.scm | 45 +++++++++++++++++----
guix/scripts/substitute.scm | 76 +++++++++++++++++++++++++++--------
nix/nix-daemon/guix-daemon.cc | 5 +++
3 files changed, 102 insertions(+), 24 deletions(-)
diff --git a/guix/eris.scm b/guix/eris.scm
index 3fbedd0cb7..1aa52e69dd 100644
--- a/guix/eris.scm
+++ b/guix/eris.scm
@@ -21,6 +21,7 @@ (define-module (guix eris)
#:use-module (eris)
#:use-module (eris fs)
#:use-module (eris sqlite)
+ #:use-module (sqlite3)
#:use-module (eris coap)
#:use-module (eris read-capability)
@@ -31,7 +32,8 @@ (define-module (guix eris)
#:use-module (srfi srfi-171)
#:export (%eris-store-url
- eris-encode-store-item))
+ eris-encode-store-item
+ eris-decode-store-item))
(define %eris-store-url
(make-parameter
@@ -83,8 +85,7 @@ (define (guix-eris-block-reducer)
(() (ecbr))
((conn ref-block) (ecbr conn ref-block))
((conn)
- (ecbr conn)
- (close-port conn)))))
+ (ecbr conn)))))
;; TODO
;; ('coap+tcp #f)
@@ -97,10 +98,40 @@ (define (guix-eris-block-reducer)
;; SRFI-171 that counts the number of blocks.
rcount)))
+(define (call-with-guix-eris-block-ref proc)
+ (let ((store-url (%eris-store-url)))
+ (if (uri? store-url)
+ (match (uri-scheme store-url)
+
+ ('sqlite
+ (let ((db (eris-sqlite-open (uri-path store-url))))
+ (proc (lambda (ref) (eris-sqlite-ref db ref)))
+ (sqlite-close db)))
+
+ ('coap+unix
+ (let ((conn (open-coap-unix-socket (uri-path store-url)))
+ (req-uri (build-uri 'coap #:path ".well-known/eris")))
+ (proc
+ (lambda (ref)
+ (eris-coap-block-ref req-uri ref #:connection conn)))
+ (close-port conn)))
+
+ (_ (error "Don't know how to handle ERIS store URL "
+ (uri->string (%eris-store-url)))))
+
+ (error "No ERIS store to get blocks."))))
+
(define* (eris-encode-store-item item)
"Encodes the store item ITEM using ERIS and returns the read capability as
string."
- (eris-read-capability->string
- (eris-fs-encode item
- #:convergence-secret (%guix-eris-convergence-secret)
- #:block-reducer (guix-eris-block-reducer))))
+ (eris-fs-encode item
+ #:convergence-secret (%guix-eris-convergence-secret)
+ #:block-reducer (guix-eris-block-reducer)))
+
+(define* (eris-decode-store-item eris-urn destination)
+ "Decode a store item with read-capability ERIS-URN to DESTINATION."
+ (call-with-guix-eris-block-ref
+ (lambda (block-ref)
+ (eris-fs-decode eris-urn destination
+ #:block-ref block-ref)
+ #t)))
diff --git a/guix/scripts/substitute.scm b/guix/scripts/substitute.scm
index 37cd08e289..3c060f1c89 100755
--- a/guix/scripts/substitute.scm
+++ b/guix/scripts/substitute.scm
@@ -45,6 +45,8 @@ (define-module (guix scripts substitute)
. guix:open-connection-for-uri)))
#:autoload (gnutls) (error/invalid-session error/again error/interrupted)
#:use-module (guix progress)
+ #:use-module (guix eris)
+ #:use-module (fibers)
#:use-module ((guix build syscalls)
#:select (set-thread-name))
#:use-module (ice-9 rdelim)
@@ -656,9 +658,11 @@ (define* (process-substitution/fallback port narinfo destination
(()
(loop rest)))))))
+
(define* (process-substitution port store-item destination
#:key cache-urls acl
- deduplicate? print-build-trace?)
+ deduplicate? print-build-trace?
+ (eris? #f))
"Substitute STORE-ITEM (a store file name) from CACHE-URLS, and write it to
DESTINATION as a nar file. Verify the substitute against ACL, and verify its
hash against what appears in the narinfo. When DEDUPLICATE? is true, and if
@@ -674,20 +678,56 @@ (define* (process-substitution port store-item destination
(leave (G_ "no valid substitute for '~a'~%")
store-item))
- (guard (c ((network-error? c)
- (format (current-error-port)
- (G_ "retrying download of '~a' with other substitute URLs...~%")
- store-item)
- (process-substitution/fallback port narinfo destination
- #:cache-urls cache-urls
- #:acl acl
- #:deduplicate? deduplicate?
- #:print-build-trace?
- print-build-trace?)))
- (download-nar narinfo destination
- #:status-port port
- #:deduplicate? deduplicate?
- #:print-build-trace? print-build-trace?)))
+ (if (and eris?
+ (%eris-store-url)
+ (narinfo-eris-urn narinfo))
+
+ (unless
+
+ ;; Attempt to fetch substitute via ERIS
+ (let ((eris-urn (narinfo-eris-urn narinfo)))
+ (format (current-error-port)
+ (G_ "Downloading ~a...~%") (uri->string eris-urn))
+ (run-fibers
+ (lambda ()
+ (guard
+ (c (else
+ (format (current-error-port)
+ (G_ "failed to decode substitute from ERIS URN ~a: ~a"
+ (uri->string eris-urn)
+ c))
+ #f))
+ (eris-decode-store-item eris-urn destination)
+ ;; Tell the daemon that we're done.
+ (format port "success ~a ~a~%"
+ (narinfo-hash narinfo) (narinfo-size narinfo))))))
+
+ ;; Retry without ERIS on failure.
+ (process-substitution port store-item destination
+ #:cache-urls cache-urls
+ #:acl acl
+ #:deduplicate? deduplicate?
+ #:print-build-trace? print-build-trace?
+ #:eris? #f))
+
+ (guard (c ((network-error? c)
+ (format (current-error-port)
+ (G_ "retrying download of '~a' with other substitute URLs...~%")
+ store-item)
+ (process-substitution/fallback port narinfo destination
+ #:cache-urls cache-urls
+ #:acl acl
+ #:deduplicate? deduplicate?
+ #:print-build-trace?
+ print-build-trace?)))
+ (download-nar narinfo destination
+ #:status-port port
+ #:deduplicate? deduplicate?
+ #:print-build-trace? print-build-trace?))))
+
+
+
+
;;;
@@ -876,7 +916,8 @@ (define-command (guix-substitute . args)
;; Download STORE-PATH and store it as a Nar in file DESTINATION.
;; Specify the number of columns of the terminal so the progress
;; report displays nicely.
- (parameterize ((current-terminal-columns (client-terminal-columns)))
+ (parameterize ((current-terminal-columns (client-terminal-columns))
+ (%eris-store-url (find-daemon-option "eris-store-url")))
(let loop ()
(match (read-line)
((? eof-object?)
@@ -887,7 +928,8 @@ (define-command (guix-substitute . args)
#:acl (current-acl)
#:deduplicate? deduplicate?
#:print-build-trace?
- print-build-trace?)
+ print-build-trace?
+ #:eris? #t)
(loop))))))
(opts
(leave (G_ "~a: unrecognized options~%") opts))))))
diff --git a/nix/nix-daemon/guix-daemon.cc b/nix/nix-daemon/guix-daemon.cc
index d7ab9c5e64..d6b054bc6c 100644
--- a/nix/nix-daemon/guix-daemon.cc
+++ b/nix/nix-daemon/guix-daemon.cc
@@ -90,6 +90,7 @@ builds derivations on behalf of its clients.");
#define GUIX_OPT_MAX_SILENT_TIME 19
#define GUIX_OPT_LOG_COMPRESSION 20
#define GUIX_OPT_DISCOVER 21
+#define GUIX_OPT_ERIS_STORE_URL 22
static const struct argp_option options[] =
{
@@ -132,6 +133,8 @@ static const struct argp_option options[] =
n_("use the specified compression type for build logs") },
{ "discover", GUIX_OPT_DISCOVER, "yes/no", OPTION_ARG_OPTIONAL,
n_("use substitute servers discovered on the local network") },
+ { "eris-store-url", GUIX_OPT_ERIS_STORE_URL, n_("URL"), 0,
+ n_("use URL to retrieve blocks of ERIS encoded substitutes") },
/* '--disable-deduplication' was known as '--disable-store-optimization'
up to Guix 0.7 included, so keep the alias around. */
@@ -270,6 +273,8 @@ parse_opt (int key, char *arg, struct argp_state *state)
useDiscover = string_to_bool (arg);
settings.set ("discover", useDiscover ? "true" : "false");
break;
+ case GUIX_OPT_ERIS_STORE_URL:
+ settings.set ("eris-store-url", arg);
case GUIX_OPT_DEBUG:
verbosity = lvlDebug;
break;
--
2.41.0
GNU bug tracking system
Copyright (C) 1999 Darren O. Benham,
1997,2003 nCipher Corporation Ltd,
1994-97 Ian Jackson.