Package: guix-patches;
Reported by: Katherine Cox-Buday <cox.katherine.e <at> gmail.com>
Date: Fri, 23 Oct 2020 14:08:01 UTC
Severity: normal
Done: Ludovic Courtès <ludo <at> gnu.org>
Bug is archived. No further changes may be made.
Message #74 received at 44178 <at> debbugs.gnu.org (full text, mbox):
From: JOULAUD François <Francois.JOULAUD <at> radiofrance.com> To: "44178 <at> debbugs.gnu.org" <44178 <at> debbugs.gnu.org> Cc: Ludovic Courtès <ludo <at> gnu.org>, Katherine Cox-Buday <cox.katherine.e <at> gmail.com> Subject: [PATCHv3] Create importer for Go modules Date: Fri, 19 Feb 2021 16:21:03 +0000
This patch add a `guix import go` command.
It was tested with several big repositories and mostly works. Several
features are lacking (see TODO in source code) but we will do the
improvments step-by-step in future patches.
* doc/guix.texi: doc about go importer and guile-lib dependency
* gnu/packages/package-management.scm: added guile-lib dependency
* guix/self.scm: add guile-lib dependency
* guix/build-system/go.scm: go-version->git-ref function
* guix/import/go.scm: Created Go importer
* guix/scripts/import/go.scm: Subcommand for Go importer
* guix/scripts/import.scm: Declare subcommand guix import go
* tests/import-go.scm: Tests for parse-go.mod procedure
Signed-off-by: Francois Joulaud <francois.joulaud <at> radiofrance.com>
---
doc/guix.texi | 26 ++
gnu/packages/package-management.scm | 2 +
guix/build-system/go.scm | 35 ++-
guix/import/go.scm | 416 ++++++++++++++++++++++++++++
guix/scripts/import.scm | 2 +-
guix/scripts/import/go.scm | 118 ++++++++
guix/self.scm | 5 +-
tests/import-go.scm | 144 ++++++++++
8 files changed, 745 insertions(+), 3 deletions(-)
create mode 100644 guix/import/go.scm
create mode 100644 guix/scripts/import/go.scm
create mode 100644 tests/import-go.scm
diff --git a/doc/guix.texi b/doc/guix.texi
index 5d28fca837..89c8abd261 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -861,6 +861,10 @@ substitutes (@pxref{Invoking guix publish}).
@uref{https://ngyro.com/software/guile-semver.html, Guile-Semver} for
the @code{crate} importer (@pxref{Invoking guix import}).
+@item
+@uref{https://www.nongnu.org/guile-lib/doc/ref/htmlprag/, guile-lib} for
+the @code{crate} importer (@pxref{Invoking guix import}).
+
@item
When @url{http://www.bzip.org, libbz2} is available,
@command{guix-daemon} can use it to compress build logs.
@@ -11493,6 +11497,28 @@ Select the given repository (a repository name). Possible values include:
of coq packages.
@end itemize
@end table
+
+@item go
+@cindex go
+Import metadata for a Go module using
+@uref{https://proxy.golang.org, proxy.golang.org}.
+
+This importer is highly experimental. See the source code for more info
+about the current state.
+
+@example
+guix import go gopkg.in/yaml.v2
+@end example
+
+Additional options include:
+
+@table @code
+@item --recursive
+@itemx -r
+Traverse the dependency graph of the given upstream package recursively
+and generate package expressions for all those packages that are not yet
+in Guix.
+@end table
@end table
The structure of the @command{guix import} code is modular. It would be
diff --git a/gnu/packages/package-management.scm b/gnu/packages/package-management.scm
index 9fb8c40a31..06bb5bd2df 100644
--- a/gnu/packages/package-management.scm
+++ b/gnu/packages/package-management.scm
@@ -304,6 +304,7 @@ $(prefix)/etc/init.d\n")))
'((assoc-ref inputs "guile"))))
(avahi (assoc-ref inputs "guile-avahi"))
(gcrypt (assoc-ref inputs "guile-gcrypt"))
+ (guile-lib (assoc-ref inputs "guile-lib"))
(json (assoc-ref inputs "guile-json"))
(sqlite (assoc-ref inputs "guile-sqlite3"))
(zlib (assoc-ref inputs "guile-zlib"))
@@ -367,6 +368,7 @@ $(prefix)/etc/init.d\n")))
`(("guile-avahi" ,guile-avahi)))
("guile-gcrypt" ,guile-gcrypt)
("guile-json" ,guile-json-4)
+ ("guile-lib" ,guile-lib)
("guile-sqlite3" ,guile-sqlite3)
("guile-zlib" ,guile-zlib)
("guile-lzlib" ,guile-lzlib)
diff --git a/guix/build-system/go.scm b/guix/build-system/go.scm
index f8ebaefb27..594e0cb4f3 100644
--- a/guix/build-system/go.scm
+++ b/guix/build-system/go.scm
@@ -26,9 +26,42 @@
#:use-module (guix build-system gnu)
#:use-module (guix packages)
#:use-module (ice-9 match)
+ #:use-module (ice-9 regex)
#:export (%go-build-system-modules
go-build
- go-build-system))
+ go-build-system
+
+ go-version->git-ref))
+
+(define (go-version->git-ref version)
+ "GO-VERSION->GIT-REF parse pseudo-versions and extract the commit
+ hash from it, defaulting to full VERSION if we don't recognise a
+ pseudo-version pattern."
+ ;; A module version like v1.2.3 is introduced by tagging a revision in
+ ;; the underlying source repository. Untagged revisions can be referred
+ ;; to using a "pseudo-version" like v0.0.0-yyyymmddhhmmss-abcdefabcdef,
+ ;; where the time is the commit time in UTC and the final suffix is the
+ ;; prefix of the commit hash.
+ ;; cf. https://golang.org/cmd/go/#hdr-Pseudo_versions
+ (let* ((version
+ ;; if a source code repository has a v2.0.0 or later tag for
+ ;; a file tree with no go.mod, the version is considered to be
+ ;; part of the v1 module's available versions and is given an
+ ;; +incompatible suffix
+ ;; https://golang.org/cmd/go/#hdr-Module_compatibility_and_semantic_versioning
+ (if (string-suffix? "+incompatible" version)
+ (string-drop-right version 13)
+ version))
+ (re (string-concatenate
+ (list
+ "(v?[0-9]\\.[0-9]\\.[0-9])" ; "v" prefix can be omitted in version prefix
+ "(-|-pre\\.0\\.|-0\\.)" ; separator
+ "([0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9])-" ; timestamp
+ "([0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f])"))) ; commit hash
+ (match (string-match re version)))
+ (if match
+ (match:substring match 4)
+ version)))
;; Commentary:
;;
diff --git a/guix/import/go.scm b/guix/import/go.scm
new file mode 100644
index 0000000000..fead355bd2
--- /dev/null
+++ b/guix/import/go.scm
@@ -0,0 +1,416 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e <at> gmail.com>
+;;; Copyright © 2020 Helio Machado <0x2b3bfa0+guix <at> googlemail.com>
+;;; Copyright © 2021 François Joulaud <francois.joulaud <at> radiofrance.com>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+;;; (guix import golang) wants to make easier to create Guix package
+;;; declaration for Go modules.
+;;;
+;;; Modules in Go are "collection of related Go packages" which are
+;;; "the unit of source code interchange and versioning".
+;;; Modules are generally hosted in a repository.
+;;;
+;;; At this point it should handle correctly modules which
+;;; have only Go dependencies and are accessible from proxy.golang.org
+;;; (or configured GOPROXY).
+;;;
+;;; We want it to work more or less this way:
+;;; - get latest version for the module from GOPROXY
+;;; - infer VCS root repo from which we will check-out source by
+;;; + recognising known patterns (like github.com)
+;;; + or (TODO) recognising .vcs suffix
+;;; + or parsing meta tag in html served at the URL
+;;; + or (TODO) if nothing else works by using zip file served by GOPROXY
+;;; - get go.mod from GOPROXY (which is able to synthetize one if needed)
+;;; - extract list of dependencies from this go.mod
+;;;
+;;; We translate Go module paths to a Guix package name under the
+;;; assumption that there will be no collision.
+
+;;; TODO list
+;;; - get correct hash in vcs->origin
+;;; - print partial result during recursive imports (need to catch
+;;; exceptions)
+;;; - infer repo from module path with VCS qualifier
+;;; (e.g. site.example/my/path/to/repo.git/and/subdir/module)
+;;; - don't print fetch messages to stdout
+;;; - pre-fill synopsis, description and license
+
+(define-module (guix import go)
+ #:use-module (ice-9 match)
+ #:use-module (ice-9 rdelim)
+ #:use-module (ice-9 receive)
+ #:use-module (ice-9 regex)
+ #:use-module (guix build-system go)
+ #:use-module (htmlprag)
+ #:use-module (sxml xpath)
+ #:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-9)
+ #:use-module (srfi srfi-11)
+ #:use-module (json)
+ #:use-module ((guix download) #:prefix download:)
+ #:use-module (guix git)
+ #:use-module (guix import utils)
+ #:use-module (guix import json)
+ #:use-module (guix packages)
+ #:use-module (guix upstream)
+ #:use-module (guix utils)
+ #:use-module ((guix licenses) #:prefix license:)
+ #:use-module (guix base16)
+ #:use-module (guix base32)
+ #:use-module (guix memoization)
+ #:use-module ((guix build download) #:prefix build-download:)
+ #:use-module (web uri)
+
+ #:export (go-module->guix-package
+ go-module-recursive-import
+ infer-module-root-repo))
+
+
+(define (go-path-escape path)
+ "Escape a module path by replacing every uppercase letter with an exclamation
+mark followed with its lowercase equivalent, as per the module Escaped Paths
+specification. https://godoc.org/golang.org/x/mod/module#hdr-Escaped_Paths"
+ (define (escape occurrence)
+ (string-append "!" (string-downcase (match:substring occurrence))))
+ (regexp-substitute/global #f "[A-Z]" path 'pre escape 'post))
+
+
+(define (go-module-latest-version goproxy-url module-path)
+ "Fetches the version number of the latest version for MODULE-PATH from the
+given GOPROXY-URL server."
+ (assoc-ref
+ (json-fetch (format #f "~a/~a/@latest" goproxy-url
+ (go-path-escape module-path)))
+ "Version"))
+
+(define go-module-latest-version* (memoize go-module-latest-version))
+
+(define (fetch-go.mod goproxy-url module-path version file)
+ "Fetches go.mod from the given GOPROXY-URL server for the given MODULE-PATH
+and VERSION."
+ (let ((url (format #f "~a/~a/@v/~a.mod" goproxy-url
+ (go-path-escape module-path)
+ (go-path-escape version))))
+ (parameterize ((current-output-port (current-error-port)))
+ (build-download:url-fetch url
+ file
+ #:print-build-trace? #f))))
+
+(define (parse-go.mod go.mod-path)
+ (parse-go.mod-port (open-input-file go.mod-path)))
+
+(define (parse-go.mod-port go.mod-port)
+ "PARSE-GO.MOD takes a filename in GO.MOD-PATH and extract a list of
+requirements from it."
+ ;; We parse only a subset of https://golang.org/ref/mod#go-mod-file-grammar
+ ;; which we think necessary for our use case.
+ (define (toplevel results)
+ "Main parser, RESULTS is a pair of alist serving as accumulator for
+ all encountered requirements and replacements."
+ (let ((line (read-line)))
+ (cond
+ ((eof-object? line)
+ ;; parsing ended, give back the result
+ results)
+ ((string=? line "require (")
+ ;; a require block begins, delegate parsing to IN-REQUIRE
+ (in-require results))
+ ((string=? line "replace (")
+ ;; a replace block begins, delegate parsing to IN-REPLACE
+ (in-replace results))
+ ((string-prefix? "require " line)
+ ;; a require directive by itself
+ (let* ((stripped-line (string-drop line 8))
+ (new-results (require-directive results stripped-line)))
+ (toplevel new-results)))
+ ((string-prefix? "replace " line)
+ ;; a replace directive by itself
+ (let* ((stripped-line (string-drop line 8))
+ (new-results (replace-directive results stripped-line)))
+ (toplevel new-results)))
+ (#t
+ ;; unrecognised line, ignore silently
+ (toplevel results)))))
+ (define (in-require results)
+ (let ((line (read-line)))
+ (cond
+ ((eof-object? line)
+ ;; this should never happen here but we ignore silently
+ results)
+ ((string=? line ")")
+ ;; end of block, coming back to toplevel
+ (toplevel results))
+ (#t
+ (in-require (require-directive results line))))))
+ (define (in-replace results)
+ (let ((line (read-line)))
+ (cond
+ ((eof-object? line)
+ ;; this should never happen here but we ignore silently
+ results)
+ ((string=? line ")")
+ ;; end of block, coming back to toplevel
+ (toplevel results))
+ (#t
+ (in-replace (replace-directive results line))))))
+ (define (replace-directive results line)
+ "Extract replaced modules and new requirements from replace directive
+ in LINE and add to RESULTS."
+ ;; ReplaceSpec = ModulePath [ Version ] "=>" FilePath newline
+ ;; | ModulePath [ Version ] "=>" ModulePath Version newline .
+ (let* ((requirements (car results))
+ (replaced (cdr results))
+ (re (string-concatenate
+ '("([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?"
+ "[[:blank:]]+" "=>" "[[:blank:]]+"
+ "([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?")))
+ (match (string-match re line))
+ (module-path (match:substring match 1))
+ (version (match:substring match 3))
+ (new-module-path (match:substring match 4))
+ (new-version (match:substring match 6))
+ (new-replaced (acons module-path version replaced))
+ (new-requirements
+ (if (string-match "^\\.?\\./" new-module-path)
+ requirements
+ (acons new-module-path new-version requirements))))
+ (cons new-requirements new-replaced)))
+ (define (require-directive results line)
+ "Extract requirement from LINE and add it to RESULTS."
+ (let* ((requirements (car results))
+ (replaced (cdr results))
+ ;; A line in a require directive is composed of a module path and
+ ;; a version separated by whitespace and an optionnal '//' comment at
+ ;; the end.
+ (re (string-concatenate
+ '("^[[:blank:]]*"
+ "([^[:blank:]]+)[[:blank:]]+([^[:blank:]]+)"
+ "([[:blank:]]+//.*)?")))
+ (match (string-match re line))
+ (module-path (match:substring match 1))
+ ;; we saw double-quoted string in the wild without escape
+ ;; sequences so we just trim the quotes
+ (module-path (string-trim-both module-path #\"))
+ (version (match:substring match 2)))
+ (cons (acons module-path version requirements) replaced)))
+ (with-input-from-port go.mod-port
+ (lambda ()
+ (let* ((results (toplevel '(() . ())))
+ (requirements (car results))
+ (replaced (cdr results)))
+ ;; At last we remove replaced modules from the requirements list
+ (fold
+ (lambda (replacedelem requirements)
+ (alist-delete! (car replacedelem) requirements))
+ requirements
+ replaced)))))
+
+(define (infer-module-root-repo module-path)
+ "Go modules can be defined at any level of a repository's tree, but querying
+for the meta tag usually can only be done at the webpage at the root of the
+repository. Therefore, it is sometimes necessary to try and derive a module's
+root path from its path. For a set of well-known forges, the pattern of what
+consists of a module's root page is known before hand."
+ ;; See the following URL for the official Go equivalent:
+ ;; https://github.com/golang/go/blob/846dce9d05f19a1f53465e62a304dea21b99f910/src/cmd/go/internal/vcs/vcs.go#L1026-L1087
+ ;;
+ ;; TODO: handle module path with VCS qualifier as described in
+ ;; https://golang.org/ref/mod#vcs-find and
+ ;; https://golang.org/cmd/go/#hdr-Remote_import_paths
+ (define-record-type <vcs>
+ (make-vcs url-prefix root-regex type)
+ vcs?
+ (url-prefix vcs-url-prefix)
+ (root-regex vcs-root-regex)
+ (type vcs-type))
+ (let* ((known-vcs
+ (list
+ (make-vcs
+ "github.com"
+ "^(github\\.com/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)
+ (make-vcs
+ "bitbucket.org"
+ "^(bitbucket\\.org/([A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+))(/[A-Za-z0-9_.\\-]+)*$"
+ 'unknown)
+ (make-vcs
+ "hub.jazz.net/git/"
+ "^(hub\\.jazz\\.net/git/[a-z0-9]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)
+ (make-vcs
+ "git.apache.org"
+ "^(git\\.apache\\.org/[a-z0-9_.\\-]+\\.git)(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)
+ (make-vcs
+ "git.openstack.org"
+ "^(git\\.openstack\\.org/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(\\.git)?(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)))
+ (vcs (find (lambda (vcs) (string-prefix? (vcs-url-prefix vcs) module-path))
+ known-vcs)))
+ (if vcs
+ (match:substring (string-match (vcs-root-regex vcs) module-path) 1)
+ module-path)))
+
+(define (go-module->guix-package-name module-path)
+ "Converts a module's path to the canonical Guix format for Go packages."
+ (string-downcase
+ (string-append "go-"
+ (string-replace-substring
+ (string-replace-substring
+ module-path
+ "." "-")
+ "/" "-"))))
+
+(define-record-type <module-meta>
+ (make-module-meta import-prefix vcs repo-root)
+ module-meta?
+ (import-prefix module-meta-import-prefix)
+ ;; VCS field is a symbol
+ (vcs module-meta-vcs)
+ (repo-root module-meta-repo-root))
+
+(define (fetch-module-meta-data module-path)
+ "Fetches module meta-data from a module's landing page. This is
+ necessary because goproxy servers don't currently provide all the
+ information needed to build a package."
+ ;; <meta name="go-import" content="import-prefix vcs repo-root">
+ (define (meta-go-import->module-meta text)
+ "Takes the content of the go-import meta tag as TEXT and gives back
+ a MODULE-META record"
+ (define (get-component s start)
+ (let*
+ ((start (string-skip s char-set:whitespace start))
+ (end (string-index s char-set:whitespace start))
+ (end (if end end (string-length s)))
+ (result (substring s start end)))
+ (values result end)))
+ (let*-values (((import-prefix end) (get-component text 0))
+ ((vcs end) (get-component text end))
+ ((repo-root end) (get-component text end)))
+ (make-module-meta import-prefix (string->symbol vcs) repo-root)))
+ (define (html->meta-go-import port)
+ "Read PORT with HTML content. Find the go-import meta tag and gives
+ back its content as a string."
+ (let* ((parsedhtml (html->sxml port))
+ (extract-content (node-join
+ (select-kids (node-typeof? 'html))
+ (select-kids (node-typeof? 'head))
+ (select-kids (node-typeof? 'meta))
+ (select-kids (node-typeof? '@))
+ (node-self
+ (node-join
+ (select-kids (node-typeof? 'name))
+ (select-kids (node-equal? "go-import"))))
+ (select-kids (node-typeof? 'content))
+ (select-kids (lambda (_) #t))))
+ (content (car (extract-content parsedhtml))))
+ content))
+ (let* ((port (build-download:http-fetch (string->uri (format #f "https://~a?go-get=1" module-path))))
+ (meta-go-import (html->meta-go-import port))
+ (module-metadata (meta-go-import->module-meta meta-go-import)))
+ (close-port port)
+ module-metadata))
+
+(define (module-meta-data-repo-url meta-data goproxy-url)
+ "Return the URL where the fetcher which will be used can download the source
+control."
+ (if (member (module-meta-vcs meta-data)'(fossil mod))
+ goproxy-url
+ (module-meta-repo-root meta-data)))
+
+(define (vcs->origin vcs-type vcs-repo-url version file)
+ "Generate the `origin' block of a package depending on what type of source
+control system is being used."
+ (case vcs-type
+ ((git)
+ `(origin
+ (method git-fetch)
+ (uri (git-reference
+ (url ,vcs-repo-url)
+ (commit (go-version->git-ref version))))
+ (file-name (git-file-name name version))
+ (sha256
+ (base32
+ ;; FIXME: get hash for git repo checkout
+ "0000000000000000000000000000000000000000000000000000"))))
+ ((hg)
+ `(origin
+ (method hg-fetch)
+ (uri (hg-reference
+ (url ,vcs-repo-url)
+ (changeset ,version)))
+ (file-name (format #f "~a-~a-checkout" name version))))
+ ((svn)
+ `(origin
+ (method svn-fetch)
+ (uri (svn-reference
+ (url ,vcs-repo-url)
+ (revision (string->number version))
+ (recursive? #f)))
+ (file-name (format #f "~a-~a-checkout" name version))
+ (sha256
+ (base32
+ ,(guix-hash-url file)))))
+ (else
+ (raise-exception (format #f "unsupported vcs type: ~a" vcs-type)))))
+
+(define* (go-module->guix-package module-path #:key (goproxy-url "https://proxy.golang.org"))
+ (call-with-temporary-output-file
+ (lambda (temp port)
+ (let* ((latest-version (go-module-latest-version* goproxy-url module-path))
+ (go.mod-path (fetch-go.mod goproxy-url module-path latest-version
+ temp))
+ (dependencies (map car (parse-go.mod temp)))
+ (guix-name (go-module->guix-package-name module-path))
+ (root-module-path (infer-module-root-repo module-path))
+ ;; VCS type and URL are not included in goproxy information. For
+ ;; this we need to fetch it from the official module page.
+ (meta-data (fetch-module-meta-data root-module-path))
+ (vcs-type (module-meta-vcs meta-data))
+ (vcs-repo-url (module-meta-data-repo-url meta-data goproxy-url)))
+ (values
+ `(package
+ (name ,guix-name)
+ ;; Elide the "v" prefix Go uses
+ (version ,(string-trim latest-version #\v))
+ (source
+ ,(vcs->origin vcs-type vcs-repo-url latest-version temp))
+ (build-system go-build-system)
+ (arguments
+ '(#:import-path ,root-module-path))
+ ,@(maybe-inputs (map go-module->guix-package-name dependencies))
+ ;; TODO(katco): It would be nice to make an effort to fetch this
+ ;; from known forges, e.g. GitHub
+ (home-page ,(format #f "https://~a" root-module-path))
+ (synopsis "A Go package")
+ (description ,(format #f "~a is a Go package." guix-name))
+ (license #f))
+ dependencies)))))
+
+(define go-module->guix-package* (memoize go-module->guix-package))
+
+(define* (go-module-recursive-import package-name
+ #:key (goproxy-url "https://proxy.golang.org"))
+ (recursive-import
+ package-name
+ #:repo->guix-package (lambda* (name . _)
+ (go-module->guix-package*
+ name
+ #:goproxy-url goproxy-url))
+ #:guix-name go-module->guix-package-name))
diff --git a/guix/scripts/import.scm b/guix/scripts/import.scm
index 0a3863f965..1d2b45d942 100644
--- a/guix/scripts/import.scm
+++ b/guix/scripts/import.scm
@@ -77,7 +77,7 @@ rather than \\n."
;;;
(define importers '("gnu" "nix" "pypi" "cpan" "hackage" "stackage" "elpa" "gem"
- "cran" "crate" "texlive" "json" "opam"))
+ "go" "cran" "crate" "texlive" "json" "opam"))
(define (resolve-importer name)
(let ((module (resolve-interface
diff --git a/guix/scripts/import/go.scm b/guix/scripts/import/go.scm
new file mode 100644
index 0000000000..fde7555973
--- /dev/null
+++ b/guix/scripts/import/go.scm
@@ -0,0 +1,118 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e <at> gmail.com>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (guix scripts import go)
+ #:use-module (guix ui)
+ #:use-module (guix utils)
+ #:use-module (guix scripts)
+ #:use-module (guix import go)
+ #:use-module (guix scripts import)
+ #:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-11)
+ #:use-module (srfi srfi-37)
+ #:use-module (ice-9 match)
+ #:use-module (ice-9 format)
+ #:export (guix-import-go))
+
+
+;;;
+;;; Command-line options.
+;;;
+
+(define %default-options
+ '())
+
+(define (show-help)
+ (display (G_ "Usage: guix import go PACKAGE-PATH
+Import and convert the Go module for PACKAGE-PATH.\n"))
+ (display (G_ "
+ -h, --help display this help and exit"))
+ (display (G_ "
+ -V, --version display version information and exit"))
+ (display (G_ "
+ -r, --recursive generate package expressions for all Go modules\
+ that are not yet in Guix"))
+ (display (G_ "
+ -p, --goproxy=GOPROXY specify which goproxy server to use"))
+ (newline)
+ (show-bug-report-information))
+
+(define %options
+ ;; Specification of the command-line options.
+ (cons* (option '(#\h "help") #f #f
+ (lambda args
+ (show-help)
+ (exit 0)))
+ (option '(#\V "version") #f #f
+ (lambda args
+ (show-version-and-exit "guix import go")))
+ (option '(#\r "recursive") #f #f
+ (lambda (opt name arg result)
+ (alist-cons 'recursive #t result)))
+ (option '(#\p "goproxy") #t #f
+ (lambda (opt name arg result)
+ (alist-cons 'goproxy
+ (string->symbol arg)
+ (alist-delete 'goproxy result))))
+ %standard-import-options))
+
+
+;;;
+;;; Entry point.
+;;;
+
+(define (guix-import-go . args)
+ (define (parse-options)
+ ;; Return the alist of option values.
+ (args-fold* args %options
+ (lambda (opt name arg result)
+ (leave (G_ "~A: unrecognized option~%") name))
+ (lambda (arg result)
+ (alist-cons 'argument arg result))
+ %default-options))
+
+ (let* ((opts (parse-options))
+ (args (filter-map (match-lambda
+ (('argument . value)
+ value)
+ (_ #f))
+ (reverse opts))))
+ (match args
+ ((module-name)
+ (if (assoc-ref opts 'recursive)
+ (map (match-lambda
+ ((and ('package ('name name) . rest) pkg)
+ `(define-public ,(string->symbol name)
+ ,pkg))
+ (_ #f))
+ (go-module-recursive-import module-name
+ #:goproxy-url
+ (or (assoc-ref opts 'goproxy)
+ "https://proxy.golang.org")))
+ (let ((sexp (go-module->guix-package module-name
+ #:goproxy-url
+ (or (assoc-ref opts 'goproxy)
+ "https://proxy.golang.org"))))
+ (unless sexp
+ (leave (G_ "failed to download meta-data for module '~a'~%")
+ module-name))
+ sexp)))
+ (()
+ (leave (G_ "too few arguments~%")))
+ ((many ...)
+ (leave (G_ "too many arguments~%"))))))
diff --git a/guix/self.scm b/guix/self.scm
index 35fba1152d..ed5ee9ddea 100644
--- a/guix/self.scm
+++ b/guix/self.scm
@@ -814,6 +814,9 @@ itself."
(define guile-ssh
(specification->package "guile-ssh"))
+ (define guile-lib
+ (specification->package "guile-lib"))
+
(define guile-git
(specification->package "guile-git"))
@@ -842,7 +845,7 @@ itself."
(append-map transitive-package-dependencies
(list guile-gcrypt gnutls guile-git guile-avahi
guile-json guile-semver guile-ssh guile-sqlite3
- guile-zlib guile-lzlib guile-zstd)))
+ guile-lib guile-zlib guile-lzlib guile-zstd)))
(define *core-modules*
(scheme-node "guix-core"
diff --git a/tests/import-go.scm b/tests/import-go.scm
new file mode 100644
index 0000000000..ad4185684c
--- /dev/null
+++ b/tests/import-go.scm
@@ -0,0 +1,144 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2021 François Joulaud <francois.joulaud <at> radiofrance.com>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+;;; Summary
+;; Tests for guix/import/go.scm
+
+(define-module (test-import-go)
+ #:use-module (guix import go)
+ #:use-module (guix base32)
+ #:use-module (ice-9 iconv)
+ #:use-module (ice-9 match)
+ #:use-module (srfi srfi-64))
+
+(define fixture-go-mod-simple
+ "module my/thing
+go 1.12
+require other/thing v1.0.2
+require new/thing/v2 v2.3.4
+exclude old/thing v1.2.3
+replace bad/thing v1.4.5 => good/thing v1.4.5
+")
+
+(define fixture-go-mod-with-block
+ "module M
+
+require (
+ A v1
+ B v1.0.0
+ C v1.0.0
+ D v1.2.3
+ E dev
+)
+
+exclude D v1.2.3
+")
+
+
+(define fixture-go-mod-complete
+ "module M
+
+go 1.13
+
+replace github.com/myname/myproject/myapi => ./api
+
+replace github.com/mymname/myproject/thissdk => ../sdk
+
+replace launchpad.net/gocheck => github.com/go-check/check v0.0.0-20140225173054-eb6ee6f84d0a
+
+require (
+ github.com/user/project v1.1.11
+ github.com/user/project/sub/directory v1.1.12
+ bitbucket.org/user/project v1.11.20
+ bitbucket.org/user/project/sub/directory v1.11.21
+ launchpad.net/project v1.1.13
+ launchpad.net/project/series v1.1.14
+ launchpad.net/project/series/sub/directory v1.1.15
+ launchpad.net/~user/project/branch v1.1.16
+ launchpad.net/~user/project/branch/sub/directory v1.1.17
+ hub.jazz.net/git/user/project v1.1.18
+ hub.jazz.net/git/user/project/sub/directory v1.1.19
+ k8s.io/kubernetes/subproject v1.1.101
+ one.example.com/abitrary/repo v1.1.111
+ two.example.com/abitrary/repo v0.0.2
+ \"quoted.example.com/abitrary/repo\" v0.0.2
+)
+
+replace two.example.com/abitrary/repo => github.com/corp/arbitrary-repo v0.0.2
+
+replace (
+ golang.org/x/sys => golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a // pinned to release-branch.go1.13
+ golang.org/x/tools => golang.org/x/tools v0.0.0-20190821162956-65e3620a7ae7 // pinned to release-branch.go1.13
+)
+
+")
+
+(test-begin "import go")
+
+(test-equal "go-path-escape"
+ "github.com/!azure/!avere"
+ ((@@ (guix import go) go-path-escape) "github.com/Azure/Avere"))
+
+
+
+;; We define a function for all similar tests with different go.mod files
+(define (testing-parse-mod name expected input)
+ (define (inf? p1 p2)
+ (string<? (car p1) (car p2)))
+ (let ((input-port (open-input-string input)))
+ (test-equal name
+ (sort expected inf?)
+ (sort
+ ( (@@ (guix import go) parse-go.mod-port)
+ input-port)
+ inf?))))
+
+(testing-parse-mod "parse-go.mod-simple"
+ '(("good/thing" . "v1.4.5")
+ ("new/thing/v2" . "v2.3.4")
+ ("other/thing" . "v1.0.2"))
+ fixture-go-mod-simple)
+
+(testing-parse-mod "parse-go.mod-with-block"
+ '(("A" . "v1")
+ ("B" . "v1.0.0")
+ ("C" . "v1.0.0")
+ ("D" . "v1.2.3")
+ ("E" . "dev"))
+ fixture-go-mod-with-block)
+
+(testing-parse-mod "parse-go.mod-complete"
+ '(("github.com/corp/arbitrary-repo" . "v0.0.2")
+ ("quoted.example.com/abitrary/repo" . "v0.0.2")
+ ("one.example.com/abitrary/repo" . "v1.1.111")
+ ("hub.jazz.net/git/user/project/sub/directory" . "v1.1.19")
+ ("hub.jazz.net/git/user/project" . "v1.1.18")
+ ("launchpad.net/~user/project/branch/sub/directory" . "v1.1.17")
+ ("launchpad.net/~user/project/branch" . "v1.1.16")
+ ("launchpad.net/project/series/sub/directory" . "v1.1.15")
+ ("launchpad.net/project/series" . "v1.1.14")
+ ("launchpad.net/project" . "v1.1.13")
+ ("bitbucket.org/user/project/sub/directory" . "v1.11.21")
+ ("bitbucket.org/user/project" . "v1.11.20")
+ ("k8s.io/kubernetes/subproject" . "v1.1.101")
+ ("github.com/user/project/sub/directory" . "v1.1.12")
+ ("github.com/user/project" . "v1.1.11")
+ ("github.com/go-check/check" . "v0.0.0-20140225173054-eb6ee6f84d0a"))
+ fixture-go-mod-complete)
+
+(test-end "import go")
--
2.28.0
GNU bug tracking system
Copyright (C) 1999 Darren O. Benham,
1997,2003 nCipher Corporation Ltd,
1994-97 Ian Jackson.