Subject: Re: bug#47260: Package GNU MediaGoblin as a Guix service

From: Ben Sturmfels <[email-masked]>
Date: Tue, 06 Apr 2021 22:01:56 +1000

On Thu, 01 Apr 2021, Ben Sturmfels wrote:

> 5. Get a basic Guix service working, with sqlite3 and without the
> offloaded media transcoding currently using Celery task queue with a
> Redis broker.

Woo! After a lot of trial and error, I finally have a basic MediaGoblin
running entirely under Guix with no virtualenv trickery!

After applying the attached patch to my guix repo, I grab a copy of the
basic config files and enable audio and video:

  curl https://git.savannah.gnu.org/cgit/mediagoblin.git/plain/mediagoblin.example.ini > mediagoblin.ini
  curl https://git.savannah.gnu.org/cgit/mediagoblin.git/plain/paste.ini > paste.ini
  echo "[[mediagoblin.media_types.audio]]" >> mediagoblin.ini
  echo "[[mediagoblin.media_types.video]]" >> mediagoblin.ini

Build MediaGoblin, which downloads from our master branch and runs the
full test suite successfully:

  ~/ws/guix/pre-inst-env guix build mediagoblin

Then install MediaGoblin in a container (not working in a non-container
guix environment or without explicit "python"):

  ~/ws/guix/pre-inst-env guix environment --container --network --share=$HOME/.bash_history --ad-hoc mediagoblin python

Create an sqlite3 database and add a user:

  gmg dbupdate
  gmg adduser --username admin --password a --email [email-masked]
  gmg changepw admin a

Upload an image, audio and video via CLI:

  gmg addmedia admin image.jpg
  gmg addmedia admin audio.wav
  gmg addmedia admin video.mp4

Start the web interface:

  CELERY_ALWAYS_EAGER=true paster serve paste.ini

The web interface is working. Looks like we're missing some CSS
(probably due to files not being included in the setuptools package),
but that's a minor issue.

Getting there!

Regards,
Ben
>From 704cd6aa56f17d4f705d33f1f58ff03e2581a2da Mon Sep 17 00:00:00 2001
From: Ben Sturmfels <[email-masked]>
Date: Tue, 6 Apr 2021 12:48:47 +1000
Subject: [PATCH] Add MediaGoblin package.

---
 gnu/local.mk                 |   1 +
 gnu/packages/mediagoblin.scm | 266 +++++++++++++++++++++++++++++++++++
 2 files changed, 267 insertions(+)
 create mode 100644 gnu/packages/mediagoblin.scm

diff --git a/gnu/local.mk b/gnu/local.mk
index f2d595f2cc..b2daa3cfe6 100644
--- a/gnu/local.mk
+++ b/gnu/local.mk
@@ -467,6 +467,7 @@ GNU_SYSTEM_MODULES =				\
   %D%/packages/python-science.scm		\
   %D%/packages/python-web.scm			\
   %D%/packages/python-xyz.scm			\
+  %D%/packages/mediagoblin.scm			\
   %D%/packages/toys.scm				\
   %D%/packages/tryton.scm			\
   %D%/packages/qt.scm				\
diff --git a/gnu/packages/mediagoblin.scm b/gnu/packages/mediagoblin.scm
new file mode 100644
index 0000000000..87649c965a
--- /dev/null
+++ b/gnu/packages/mediagoblin.scm
@@ -0,0 +1,266 @@
+;; Install with:
+;;
+;; ~/ws/guix/pre-inst-env guix environment --container --network --share=$HOME/.bash_history --ad-hoc mediagoblin
+;;
+;; Doesn't currently work when not in a container:
+;;
+;; ~/ws/guix/pre-inst-env guix environment --ad-hoc mediagoblin
+
+(define-module (gnu packages mediagoblin)
+  #:use-module (ice-9 match)
+  #:use-module (srfi srfi-1)
+  #:use-module (guix packages)
+  #:use-module (guix licenses)
+  #:use-module (guix download)
+  #:use-module (guix git-download)
+  #:use-module (guix build-system gnu)
+  #:use-module (guix build-system python)
+  #:use-module (gnu packages)
+  #:use-module (gnu packages xiph)  ; flac for embedded libsndfile
+  #:use-module (gnu packages autotools)
+  #:use-module (gnu packages base)
+  #:use-module (gnu packages certs)
+  #:use-module (gnu packages check)
+  #:use-module (gnu packages compression)  ; unzip for embedded python-wtforms
+  #:use-module (gnu packages databases)
+  #:use-module (gnu packages libffi)  ; cffi for embedded python-soundfile
+  #:use-module (gnu packages openldap)
+  #:use-module (gnu packages pdf)
+  #:use-module (gnu packages pkg-config)  ; embedded libsndfile
+  #:use-module (gnu packages python)
+  #:use-module (gnu packages python-crypto)
+  #:use-module (gnu packages python-web)
+  #:use-module (gnu packages python-xyz)
+  #:use-module (gnu packages sphinx)
+  #:use-module (gnu packages gstreamer)
+  #:use-module (gnu packages glib)
+  #:use-module (gnu packages pulseaudio)
+  #:use-module (gnu packages rsync)
+  #:use-module (gnu packages ssh)
+  #:use-module (gnu packages time)
+  #:use-module (gnu packages video)
+  #:use-module (gnu packages version-control)
+  #:use-module (gnu packages xml)
+  #:use-module ((guix licenses) #:select (bsd-3 gpl2+) #:prefix license:))
+
+(define python-wtforms
+  (package
+    (name "python-wtforms")
+    (version "2.3.3")
+    (source
+     (origin
+       (method url-fetch)
+       (uri (pypi-uri "WTForms" version))
+       (sha256
+        (base32
+         ;; Interesting, if this has is that of a lower version, it blindly
+         ;; ignores the version number above and you silently get the older
+         ;; version.
+         "17427m7p9nn9byzva697dkykykwcp2br3bxvi8vciywlmkh5s6c1"))))
+    (build-system python-build-system)
+    (arguments
+     `(#:tests? #f))  ; TODO: Fix tests for upgraded version.
+    (propagated-inputs
+    `(("python-markupsafe" ,python-markupsafe)))
+    (native-inputs
+     `(("unzip" ,unzip)))  ; CHECK WHETHER NEEDED - not in `guix import` but is in old package.
+    (home-page "http://wtforms.simplecodes.com/")
+    (synopsis
+     "Form validation and rendering library for Python web development")
+    (description
+     "WTForms is a flexible forms validation and rendering library
+for Python web development.  It is very similar to the web form API
+available in Django, but is a standalone package.")
+    (license license:bsd-3)))
+
+
+;; Copied from guix/gnu/packages/pulseaudio.scm in the core-updates branch which
+;; adds flac/ogg/vorbis/opus support. This is required for building
+;; python-soundfile (March 2021).
+(define libsndfile
+  (package
+    (name "libsndfile")
+    (version "1.0.30")
+    (source (origin
+             (method url-fetch)
+             (uri (string-append "https://github.com/erikd/libsndfile"
+                                 "/releases/download/v" version
+                                 "/libsndfile-" version ".tar.bz2"))
+             (sha256
+              (base32
+               "06k1wj3lwm7vf21s8yqy51k6nrkn9z610bj1gxb618ag5hq77wlx"))
+             (modules '((ice-9 textual-ports) (guix build utils)))
+             (snippet
+              '(begin
+                 ;; Remove carriage returns (CRLF) to prevent bogus
+                 ;; errors from bash like "$'\r': command not found".
+                 (let ((data (call-with-input-file
+                                 "tests/pedantic-header-test.sh.in"
+                               (lambda (port)
+                                 (string-join
+                                  (string-split (get-string-all port)
+                                                #\return))))))
+                   (call-with-output-file "tests/pedantic-header-test.sh.in"
+                     (lambda (port) (format port data))))
+
+                 ;; While at it, fix hard coded executable name.
+                 (substitute* "tests/test_wrapper.sh.in"
+                   (("^/usr/bin/env") "env"))
+                 #t))))
+    (build-system gnu-build-system)
+    (propagated-inputs
+     `(("flac" ,flac)
+       ("libogg" ,libogg)
+       ("libvorbis" ,libvorbis)
+       ("opus" ,opus)))
+    (native-inputs
+     `(("pkg-config" ,pkg-config)
+       ("python" ,python)))
+    (home-page "http://www.mega-nerd.com/libsndfile/")
+    (synopsis "Reading and writing files containing sampled sound")
+    (description
+     "Libsndfile is a C library for reading and writing files containing
+sampled sound (such as MS Windows WAV and the Apple/SGI AIFF format) through
+one standard library interface.
+
+It was designed to handle both little-endian (such as WAV) and
+big-endian (such as AIFF) data, and to compile and run correctly on
+little-endian (such as Intel and DEC/Compaq Alpha) processor systems as well
+as big-endian processor systems such as Motorola 68k, Power PC, MIPS and
+SPARC.  Hopefully the design of the library will also make it easy to extend
+for reading and writing new sound file formats.")
+    (license license:gpl2+)))
+
+;; Need soundfile for new Python 3 audio spectrograms. Can me merged into Guix
+;; once core-updates is merged.
+(define python-soundfile
+  (package
+    (name "python-soundfile")
+    (version "0.10.3.post1")
+    (source
+     (origin
+       (method url-fetch)
+       (uri (pypi-uri "SoundFile" version))
+       (sha256
+        (base32
+         "0yqhrfz7xkvqrwdxdx2ydy4h467sk7z3gf984y1x2cq7cm1gy329"))))
+    (build-system python-build-system)
+    (native-inputs
+     `(("python-pytest" ,python-pytest)))
+    (propagated-inputs
+     `(("python-cffi" ,python-cffi)
+       ("libsndfile" ,libsndfile)
+       ("python-numpy" ,python-numpy)))
+    (arguments
+     `(#:phases
+       (modify-phases %standard-phases
+         (add-before 'build 'set-library-file-name
+           (lambda* (#:key inputs #:allow-other-keys)
+             (let ((libsndfile (assoc-ref inputs "libsndfile")))
+               (substitute* "soundfile.py"
+                 (("_find_library\\('sndfile'\\)")
+                  (string-append "'" libsndfile "/lib/libsndfile.so.1'")))
+               #t))))))
+    (home-page "https://github.com/bastibe/python-soundfile")
+    (synopsis "An audio library based on libsndfile, CFFI and NumPy")
+    (description
+     "The soundfile module can read and write sound files, representing audio
+data as NumPy arrays.")
+    (license license:bsd-3)))
+
+;; =================================================================
+
+(define-public mediagoblin
+  (package
+    (name "mediagoblin")
+    (version "0.12.0.dev")
+    (source
+     (origin
+       (method git-fetch)
+       (uri (git-reference
+             (url "https://git.savannah.gnu.org/git/mediagoblin.git")
+             (commit "3d72ccf4dfb106d8b9dcc5ac4cb754d6efa80649")))
+       (file-name (git-file-name name version))
+       (sha256
+        (base32 "0i6h8781nlnnqrw8jf0n0264p2fvc3z9rcp8vzv4a7aghiqmn2b2"))))
+    (build-system python-build-system)
+    (arguments
+     `(
+       #:tests? #f
+       ;; #:phases (modify-phases %standard-phases
+       ;;            (replace 'check
+       ;;              (lambda _
+       ;;                (setenv "PYTHONPATH"
+       ;;                        (string-append (getcwd) ":"
+       ;;                                       (getenv "PYTHONPATH")))
+       ;;                ;; Translations needed for tests to pass. Probably
+       ;;                ;; should be done during build stage?
+       ;;                (invoke "./devtools/compile_translations.sh")
+       ;;                (invoke "pytest" "./mediagoblin/tests" "-rs" "--boxed")
+       ;;                #t)))
+                ))
+    (native-inputs
+     `(("nss-certs" ,nss-certs)
+       ("python-pytest" ,python-pytest)
+       ("python-pytest-forked" ,python-pytest-forked)
+       ("python-pytest-xdist" ,python-pytest-xdist)
+       ("python-webtest" ,python-webtest)))
+    ;; lle-bout suggests avoiding propagated-inputs and insteading creating
+    ;; wrappers scripts. See:
+    ;; https://debbugs.gnu.org/cgi/bugreport.cgi?bug=47260#44
+    (propagated-inputs
+     `(("python-alembic" ,python-alembic)
+       ("python-babel" ,python-babel)
+       ("python-celery" ,python-celery)
+       ("python-configobj" ,python-configobj)
+       ("python-dateutil" ,python-dateutil)
+       ("python-docutils" ,python-docutils)  ; What for?
+       ("python-email-validator" ,python-email-validator)
+       ("python-exif-read" ,python-exif-read)
+       ("python-feedgenerator" ,python-feedgenerator)
+       ("python-itsdangerous" ,python-itsdangerous)
+       ("python-jinja2" ,python-jinja2)
+       ("python-jsonschema" ,python-jsonschema)
+       ("python-ldap" ,python-ldap)  ; For LDAP plugin
+       ("python-lxml" ,python-lxml)
+       ("python-markdown" ,python-markdown)
+       ("python-oauthlib" ,python-oauthlib)
+       ("python-openid" ,python-openid)  ; For OpenID plugin
+       ("python-pastescript" ,python-pastescript)
+       ("python-pillow" ,python-pillow)
+       ("python-py-bcrypt" ,python-py-bcrypt)
+       ("python-pyld" ,python-pyld)
+       ("python-pytz" ,python-pytz)
+       ("python-requests" ,python-requests)  ; For Persona plugin, batchaddmedia
+       ("python-setuptools" ,python-setuptools)  ; What for?
+       ("python-soundfile" ,python-soundfile)
+       ("python-sphinx" ,python-sphinx)
+       ("python-sqlalchemy" ,python-sqlalchemy)
+       ("python-translitcodec" ,python-translitcodec)
+       ("python-unidecode" ,python-unidecode)
+       ("python-waitress" ,python-waitress)
+       ("python-werkzeug" ,python-werkzeug)
+       ("python-wtforms" ,python-wtforms)
+
+       ;; Audio/video media
+       ("gobject-introspection" ,gobject-introspection)
+       ("gst-libav" ,gst-libav)
+       ("gst-plugins-bad" ,gst-plugins-bad)
+       ("gst-plugins-base" ,gst-plugins-base)
+       ("gst-plugins-good" ,gst-plugins-good)
+       ("gst-plugins-ugly" ,gst-plugins-ugly)
+       ("gstreamer" ,gstreamer)
+       ("openh264" ,openh264)
+       ("python-gst" ,python-gst)  ; For tests to pass
+       ("python-numpy" ,python-numpy)  ; Audio spectrograms
+       ("python-pygobject" ,python-pygobject)
+
+       ;; PDF media.
+       ("poppler" ,poppler)
+
+       ))
+    (home-page "https://mediagoblin.org/")
+    (synopsis "Web application for media publishing")
+    (description "MediaGoblin is a web application for publishing all kinds of
+media.")
+    (license agpl3+)))
-- 
2.31.1