Subject: [PATCH 2/2] gnu: Add debops.

From: Hartmut Goebel <[email-masked]>
Date: Sun, 8 Sep 2019 13:34:59 +0200

* gnu/packages/admin.scm (debops): New variable.
* gnu/packages/patches/debops-constants-for-external-program-names.patch,
  gnu/packages/patches/debops-debops-defaults-fall-back-to-less.patch:
  New files.
* gnu/local.mk: Add them.
---
 gnu/local.mk                                  |   2 +
 gnu/packages/admin.scm                        |  98 +++++++
 ...constants-for-external-program-names.patch | 273 ++++++++++++++++++
 ...ps-debops-defaults-fall-back-to-less.patch |  45 +++
 4 files changed, 418 insertions(+)
 create mode 100644 gnu/packages/patches/debops-constants-for-external-program-names.patch
 create mode 100644 gnu/packages/patches/debops-debops-defaults-fall-back-to-less.patch

diff --git a/gnu/local.mk b/gnu/local.mk
index b7a5ef825b..2651d57781 100644
--- a/gnu/local.mk
+++ b/gnu/local.mk
@@ -760,6 +760,8 @@ dist_patch_DATA =						\
   %D%/packages/patches/dbus-helper-search-path.patch		\
   %D%/packages/patches/dbus-CVE-2019-12749.patch		\
   %D%/packages/patches/dealii-mpi-deprecations.patch		\
+  %D%/packages/patches/debops-constants-for-external-program-names.patch \
+  %D%/packages/patches/debops-debops-defaults-fall-back-to-less.patch \
   %D%/packages/patches/deja-dup-use-ref-keyword-for-iter.patch	\
   %D%/packages/patches/dfu-programmer-fix-libusb.patch		\
   %D%/packages/patches/diffutils-gets-undeclared.patch		\
diff --git a/gnu/packages/admin.scm b/gnu/packages/admin.scm
index 3e75b73dcb..d21f4ab66a 100644
--- a/gnu/packages/admin.scm
+++ b/gnu/packages/admin.scm
@@ -25,6 +25,7 @@
 ;;; Copyright © 2018 Pierre Neidhardt <[email-masked]>
 ;;; Copyright © 2019 Brett Gilio <[email-masked]>
 ;;; Copyright © 2019 Björn Höfling <[email-masked]>
+;;; Copyright © 2019 Hartmut Goebel <[email-masked]>
 ;;;
 ;;; This file is part of GNU Guix.
 ;;;
@@ -58,6 +59,7 @@
   #:use-module (gnu packages base)
   #:use-module (gnu packages bash)
   #:use-module (gnu packages check)
+  #:use-module (gnu packages crypto)
   #:use-module (gnu packages cyrus-sasl)
   #:use-module (gnu packages dns)
   #:use-module (gnu packages file)
@@ -110,6 +112,7 @@
   #:use-module (gnu packages boost)
   #:use-module (gnu packages elf)
   #:use-module (gnu packages mpi)
+  #:use-module (gnu packages version-control)
   #:use-module (gnu packages web))
 
 (define-public aide
@@ -1901,6 +1904,101 @@ ad hoc task execution, and multinode orchestration---including trivializing
 things like zero-downtime rolling updates with load balancers.")
     (license license:gpl3+)))
 
+(define-public debops
+  (package
+    (name "debops")
+    (version "1.1.0")
+    (source
+     (origin
+       (method git-fetch)
+       (uri (git-reference
+             (url "https://github.com/debops/debops")
+             (commit (string-append "v" version))))
+       (file-name (git-file-name name version))
+       (sha256
+        (base32 "052b2dykdn35pdpn9s4prawl6nl6yzih8nyf54hpvhpisvjrm1v5"))
+       (patches
+        (search-patches "debops-constants-for-external-program-names.patch"
+                        "debops-debops-defaults-fall-back-to-less.patch"))))
+    (build-system python-build-system)
+    (native-inputs
+     `(("git" ,git)))
+    (inputs
+     `(("ansible" ,ansible)
+       ("encfs" ,encfs)
+       ("fuse" ,fuse)
+       ("util-linux" ,util-linux)  ;; for umount
+       ("findutils" ,findutils)
+       ("gnupg" ,gnupg)
+       ("which" ,which)))
+    (propagated-inputs
+     `(("python-future" ,python-future)
+       ("python-distro" ,python-distro)))
+    (arguments
+     `(#:tests? #f
+       #:phases
+       (modify-phases %standard-phases
+         (add-after 'unpack 'nuke-debops-update
+           (lambda _
+             (chmod "bin/debops-update" #o755) ; FIXME work-around git-fetch issue
+             (with-output-to-file "bin/debops-update"
+               (lambda ()
+                 (format #t "#!/bin/sh
+echo 'debops is installed via guix. guix-update is useless in this case.
+Please use `guix package -u debops` instead.'")))
+             #t))
+         ;; patch shebangs only in actuall scripts, not in files included in
+         ;; roles (which are to be delivered to the targte systems)
+         (delete `patch-generated-file-shebangs)
+         (replace 'patch-source-shebangs
+           (lambda _
+             (for-each patch-shebang
+                       (find-files "bin"
+                                   (lambda (file stat)
+                                     ;; Filter out symlinks.
+                                     (eq? 'regular (stat:type stat)))
+                                   #:stat lstat))))
+         (add-after 'unpack 'fix-paths
+           (lambda _
+             (define (substitute-program-names file)
+               ;; e.g. ANSIBLE_PLAYBOOK = '/gnu/store/…/bin/ansible-playbook'
+               (for-each
+                (lambda (name)
+                  (let ((varname (string-upcase
+                                  (string-map
+                                   (lambda (c) (if (char=? c #\-) #\_ c))
+                                   name))))
+                    (substitute* file
+                      (((string-append "^(" varname " = )'.*'") line prefix)
+                       (string-append prefix "'" (which name) "'")))))
+                '("ansible-playbook" "encfs" "find" "fusermount"
+                  "umount" "gpg" "ansible" "which")))
+             (for-each substitute-program-names
+                       '("bin/debops"
+                         "bin/debops-padlock"
+                         "bin/debops-task"
+                         "debops/__init__.py"
+                         "debops/cmds/__init__.py"))
+             #t)))))
+    (home-page "https://www.debops.org/")
+    (synopsis "Collection of general-purpose Ansible roles")
+    (description "The Ansible roles provided by that can be used to manage
+Debian or Ubuntu hosts.  In addition, a default set of Ansible playbooks can
+be used to apply the provided roles in a controlled way, using Ansible
+inventory groups.
+
+The roles are written with a high customization in mind, which can be done
+using Ansible inventory.  This way the role and playbook code can be shared
+between multiple environments, with different configuration in to each one.
+
+Services can be managed on a single host, or spread between multiple hosts.
+DebOps provides support for different SQL and NoSQL databases, web servers,
+programming languages and specialized applications useful in a data center
+environment or in a cluster.  The project can also be used to deploy
+virtualization environments using KVM/libvirt, Docker or LXC technologies to
+manage virtual machines and/or containers.")
+    (license license:gpl3+)))
+
 (define-public emacs-ansible-doc
   (let ((commit "86083a7bb2ed0468ca64e52076b06441a2f8e9e0"))
     (package
diff --git a/gnu/packages/patches/debops-constants-for-external-program-names.patch b/gnu/packages/patches/debops-constants-for-external-program-names.patch
new file mode 100644
index 0000000000..d599dc475f
--- /dev/null
+++ b/gnu/packages/patches/debops-constants-for-external-program-names.patch
@@ -0,0 +1,273 @@
+From fcb9e679c219c559e07a84b3035555a10886c570 Mon Sep 17 00:00:00 2001
+From: Hartmut Goebel <[email-masked]>
+Date: Thu, 8 Aug 2019 15:19:48 +0200
+Subject: [PATCH] Scripts: Use constants for external program names.
+
+This makes it much, much easier to replace the program
+with one using an absolute path. This is necessary for
+e.g. Guix to keep references to these external programs.
+---
+ bin/debops              |  8 +++++---
+ bin/debops-padlock      | 20 ++++++++++++++------
+ bin/debops-task         |  7 +++++--
+ bin/debops-update       | 17 ++++++++++-------
+ debops/__init__.py      | 16 +++++++++++-----
+ debops/cmds/__init__.py |  6 ++++--
+ 6 files changed, 49 insertions(+), 25 deletions(-)
+
+diff --git a/bin/debops b/bin/debops
+index 2b7ad3f88..1a8b0cae1 100755
+--- a/bin/debops
++++ b/bin/debops
+@@ -59,6 +59,9 @@ ConfigFileHeader = """\
+ # You can manipulate the contents of this file via `.debops.cfg`.
+ """
+ 
++# External programms used. List here for easy substitution for
++# hard-coded paths.
++ANSIBLE_PLAYBOOK = 'ansible-playbook'
+ 
+ def write_config(filename, config):
+     cfgparser = configparser.ConfigParser()
+@@ -131,7 +134,7 @@ def gen_ansible_cfg(filename, config, project_root, playbooks_path,
+             os.path.join(playbooks_path, "roles"),
+             "/etc/ansible/roles")))
+ 
+-    ansible_version_out = subprocess.check_output(["ansible-playbook",
++    ansible_version_out = subprocess.check_output([ANSIBLE_PLAYBOOK,
+                                                    "--version"]).decode()
+ 
+     # Get first line and split by spaces to get second 'word'.
+@@ -256,12 +259,11 @@ def main(cmd_args):
+         print("Running Ansible playbooks:")
+         for element in play_list:
+             print(element)
+-        return subprocess.call(['ansible-playbook'] + play_list + arg_list)
++        return subprocess.call([ANSIBLE_PLAYBOOK] + play_list + arg_list)
+     finally:
+         if revert_unlock:
+             padlock_lock(encfs_encrypted)
+ 
+-
+ try:
+     sys.exit(main(sys.argv[1:]))
+ except KeyboardInterrupt:
+diff --git a/bin/debops-padlock b/bin/debops-padlock
+index bfdfb8e06..706a26091 100755
+--- a/bin/debops-padlock
++++ b/bin/debops-padlock
+@@ -67,6 +67,14 @@ devrandom = os.environ.get('DEVRANDOM', "/dev/urandom")
+ 
+ SCRIPT_FILENAME = 'padlock-script'
+ 
++# External programms used. List here for easy substitution for
++# hard-coded paths.
++ENCFS = 'encfs'
++FIND = 'find'
++FUSERMOUNT = 'fusermount'
++UMOUNT = 'umount'
++GPG = 'gpg'
++
+ # ---- DebOps environment setup ----
+ 
+ 
+@@ -80,9 +88,9 @@ def main(subcommand_func, **kwargs):
+     # Make sure required commands are present
+     # OS X compatibility
+     if sys.platform == 'darwin':
+-        require_commands('encfs', 'find', 'umount', 'gpg')
++        require_commands(ENCFS, FIND, UMOUNT, GPG)
+     else:
+-        require_commands('encfs', 'find', 'fusermount', 'gpg')
++        require_commands(ENCFS, FIND, FUSERMOUNT, GPG)
+ 
+     inventory_path = find_inventorypath(project_root, required=False)
+     # If inventory hasn't been found automatically, assume it's the default
+@@ -121,7 +129,7 @@ def init(encfs_decrypted, encfs_encrypted, recipients):
+     # Generate a random password and encrypt it with GPG keys of recipients.
+     print("Generating a random", ENCFS_KEYFILE_LENGTH, "char password")
+     pwd = gen_pwd()
+-    gpg = subprocess.Popen(['gpg', '--encrypt', '--armor',
++    gpg = subprocess.Popen([GPG, '--encrypt', '--armor',
+                             '--output', encfs_keyfile] + recipients,
+                            stdin=subprocess.PIPE)
+     gpg.communicate(pwd.encode('utf-8'))
+@@ -133,9 +141,9 @@ def init(encfs_decrypted, encfs_encrypted, recipients):
+     # NB2: We can not use padlock_unlock here, because the config file
+     # does not yet exist.
+     encfs = subprocess.Popen([
+-        'encfs', encfs_encrypted, encfs_decrypted,
++        ENCFS, encfs_encrypted, encfs_decrypted,
+         '--extpass',
+-        'gpg --decrypt --no-mdc-warning --output - '+shquote(encfs_keyfile)],
++        GPG + ' --decrypt --no-mdc-warning --output - '+shquote(encfs_keyfile)],
+         stdin=subprocess.PIPE)
+     encfs.communicate(('p\n'+pwd).encode('utf-8'))
+ 
+@@ -154,7 +162,7 @@ def init(encfs_decrypted, encfs_encrypted, recipients):
+ 
+     # Protect the EncFS configuration file by also encrypting it with
+     # the GPG keys of recipients.
+-    subprocess.call(['gpg', '--encrypt', '--armor',
++    subprocess.call([GPG, '--encrypt', '--armor',
+                      '--output', encfs_configfile+'.asc']
+                     + recipients + [encfs_configfile])
+     os.remove(encfs_configfile)
+diff --git a/bin/debops-task b/bin/debops-task
+index 223e5f834..dc31ad4e6 100755
+--- a/bin/debops-task
++++ b/bin/debops-task
+@@ -49,11 +49,14 @@ project_root = find_debops_project(required=True)
+ # todo: need to decide on semantics!
+ # config = read_config(project_root)
+ 
++# External programms used. List here for easy substitution for
++# hard-coded paths.
++ANSIBLE = 'ansible'
+ 
+ # ---- Main script ----
+ 
+ # Make sure required commands are present
+-require_commands('ansible')
++require_commands(ANSIBLE)
+ 
+ ansible_inventory = find_inventorypath(project_root)
+ 
+@@ -71,5 +74,5 @@ if INSECURE:
+     os.environ['ANSIBLE_HOST_KEY_CHECKING'] = 'False'
+ 
+ # Run ansible with custom environment
+-cmd = ['ansible'] + module + sys.argv[1:]
++cmd = [ANSIBLE] + module + sys.argv[1:]
+ subprocess.call(cmd)
+diff --git a/bin/debops-update b/bin/debops-update
+index 88c5e2c82..e650ba029 100755
+--- a/bin/debops-update
++++ b/bin/debops-update
+@@ -90,6 +90,9 @@ GALAXY_REQUIREMENTS = "galaxy/requirements.txt"
+ # Default Ansible Galaxy user account name
+ GALAXY_ACCOUNT = "debops"
+ 
++# External programms used. List here for easy substitution for
++# hard-coded paths.
++GIT = 'git'
+ 
+ # ---- Functions ----
+ 
+@@ -137,7 +140,7 @@ def clone_git_repository(repo_uri, branch, destination, dry_run=False):
+     if dry_run:
+         print("Cloning '%s' to %s..." % (repo_uri, destination))
+     else:
+-        subprocess.call(['git', 'clone', '--quiet', '--branch', branch,
++        subprocess.call([GIT, 'clone', '--quiet', '--branch', branch,
+                          repo_uri, destination])
+ 
+ 
+@@ -152,22 +155,22 @@ def update_git_repository(path, dry_run=False, remote_uri=False):
+     os.chdir(path)
+ 
+     if dry_run:
+-        subprocess.call(['git', 'fetch'])
+-        subprocess.call(['git', 'diff', 'HEAD', 'origin', '--stat'])
++        subprocess.call([GIT, 'fetch'])
++        subprocess.call([GIT, 'diff', 'HEAD', 'origin', '--stat'])
+     else:
+         # Get the current sha of the head branch
+         current_sha = subprocess.check_output(
+-                ['git', 'rev-parse', 'HEAD']).strip()
++                [GIT, 'rev-parse', 'HEAD']).strip()
+ 
+         # Fetch it silently and store the new sha
+-        subprocess.call(['git', 'fetch', '--quiet'])
++        subprocess.call([GIT, 'fetch', '--quiet'])
+         fetch_sha = subprocess.check_output(
+-                ['git', 'rev-parse', 'FETCH_HEAD']).strip()
++                [GIT, 'rev-parse', 'FETCH_HEAD']).strip()
+ 
+         if current_sha != fetch_sha:
+             print()
+             print('--')
+-            subprocess.call(['git', 'merge', fetch_sha])
++            subprocess.call([GIT, 'merge', fetch_sha])
+ 
+             if remote_uri:
+                 compare_uri = (remote_uri + '/compare/' + current_sha[:7]
+diff --git a/debops/__init__.py b/debops/__init__.py
+index 1c2cedcb0..42a76fdcd 100644
+--- a/debops/__init__.py
++++ b/debops/__init__.py
+@@ -93,6 +93,12 @@ ENCFS_KEYFILE = ".encfs6.keyfile"
+ # Length of the random EncFS password stored in encrypted keyfile
+ ENCFS_KEYFILE_LENGTH = 256
+ 
++# External programms used. List here for easy substitution for
++# hard-coded paths.
++ENCFS = 'encfs'
++FUSERMOUNT = 'fusermount'
++UMOUNT = 'umount'
++GPG = 'gpg'
+ 
+ # ---- Functions ----
+ 
+@@ -180,9 +186,9 @@ def padlock_lock(encrypted_path):
+         return False
+     # OS X compatibility
+     if sys.platform == 'darwin':
+-        subprocess.call(['umount', decrypted_path])
++        subprocess.call([UMOUNT, decrypted_path])
+     else:
+-        subprocess.call(['fusermount', '-u', decrypted_path])
++        subprocess.call([FUSERMOUNT, '-u', decrypted_path])
+     return True
+ 
+ 
+@@ -237,14 +243,14 @@ def padlock_unlock(encrypted_path):
+     # Start encfs. It will wait for input on the `configfile` named
+     # pipe.
+     encfs = subprocess.Popen([
+-        'encfs', encrypted_path, decrypted_path,
++        ENCFS, encrypted_path, decrypted_path,
+         '--extpass',
+-        'gpg --decrypt --no-mdc-warning --output - %s' % shquote(keyfile)])
++        GPG + ' --decrypt --no-mdc-warning --output - %s' % shquote(keyfile)])
+     # now decrypt the config and write it into the named pipe
+     with open(configfile, 'w') as fh:
+         # NB: gpg must write to stdout to avoid it is asking whether
+         # the file should be overwritten
+-        subprocess.Popen(['gpg',
++        subprocess.Popen([GPG,
+                           '--decrypt', '--no-mdc-warning', '--output', '-',
+                           crypted_configfile], stdout=fh).wait()
+     encfs.wait()
+diff --git a/debops/cmds/__init__.py b/debops/cmds/__init__.py
+index b221fa191..d9477be80 100644
+--- a/debops/cmds/__init__.py
++++ b/debops/cmds/__init__.py
+@@ -55,6 +55,9 @@ SCRIPT_NAME = os.path.basename(sys.argv[0])
+ # command line)
+ INSECURE = bool(os.environ.get('INSECURE', False))
+ 
++# External programms used. List here for easy substitution for
++# hard-coded paths.
++WHICH = 'which'
+ 
+ def error_msg(message, severity="Error"):
+     """
+@@ -64,13 +67,12 @@ def error_msg(message, severity="Error"):
+     if severity == "Error":
+         raise SystemExit(1)
+ 
+-
+ def require_commands(*cmd_names):
+     """
+     Check if required commands exist.
+     """
+     def command_exists(cmd_name):
+-        which = "where" if platform.system() == "Windows" else "which"
++        which = "where" if platform.system() == "Windows" else WHICH
+         return not subprocess.call([which, cmd_name],
+                                    stdout=DEVNULL, stderr=subprocess.STDOUT)
+ 
+-- 
+2.21.0
+
diff --git a/gnu/packages/patches/debops-debops-defaults-fall-back-to-less.patch b/gnu/packages/patches/debops-debops-defaults-fall-back-to-less.patch
new file mode 100644
index 0000000000..bbb6b7c08e
--- /dev/null
+++ b/gnu/packages/patches/debops-debops-defaults-fall-back-to-less.patch
@@ -0,0 +1,45 @@
+From 5059daf8bd59a83f520c14731173ea76ce8b8661 Mon Sep 17 00:00:00 2001
+From: Hartmut Goebel <[email-masked]>
+Date: Sun, 8 Sep 2019 13:09:15 +0200
+Subject: [PATCH] [debops-defaults] If `view` is not available, try less, etc.
+
+---
+ bin/debops-defaults | 21 +++++++++++++++------
+ 1 file changed, 15 insertions(+), 6 deletions(-)
+
+diff --git a/bin/debops-defaults b/bin/debops-defaults
+index 9dd87fe0a..3e3db4c41 100755
+--- a/bin/debops-defaults
++++ b/bin/debops-defaults
+@@ -96,13 +96,22 @@ def main(role_list):
+     config = read_config(project_root)
+     playbooks_path = find_playbookpath(config, project_root, required=True)
+ 
+-    # Make sure required commands are present
+-    require_commands('view')
+-
+-    if sys.stdout.isatty():
++    # Check if one of the output commands is present
++    sys.stdout = io.BytesIO()  # suppress error message, if any
++    for cmd_args in (('view', '+set ft=yaml', '-'),
++                     ('less', '-'),
++                     ('more', '-')):
++        try:
++            require_commands(cmd_args[0])
++            break
++        except SystemExit:
++            # this command was not found
++            cmd_args = None
++    sys.stdout = sys.__stdout__
++
++    if cmd_args and sys.stdout.isatty():
+         # if script is run as standalone, redirect to view
+-        view = subprocess.Popen(['view', '+set ft=yaml', '-'],
+-                                stdin=subprocess.PIPE)
++        view = subprocess.Popen(cmd_args, stdin=subprocess.PIPE)
+         try:
+             aggregate_defaults(playbooks_path, role_list, view.stdin)
+         except IOError as e:
+-- 
+2.21.0
+
-- 
2.21.0