gnu: Add python-pytest-helpers-namespace.

[jackhill/guix/guix.git] / gnu / packages / parallel.scm

diff --git a/gnu/packages/parallel.scm b/gnu/packages/parallel.scm
index 8b64878..05964cd 100644 (file)
--- a/gnu/packages/parallel.scm
+++ b/gnu/packages/parallel.scm
@@ -1,14 +1,17 @@
 ;;; GNU Guix --- Functional package management for GNU
 ;;; GNU Guix --- Functional package management for GNU

-;;; Copyright © 2013, 2014 Eric Bavier <bavier@member.fsf.org>
+;;; Copyright © 2013, 2014, 2020 Eric Bavier <bavier@posteo.net>

 ;;; Copyright © 2015 Mark H Weaver <mhw@netris.org>
 ;;; Copyright © 2015 Mark H Weaver <mhw@netris.org>

-;;; Copyright © 2015, 2016, 2017, 2018 Efraim Flashner <efraim@flashner.co.il>
+;;; Copyright © 2015, 2016, 2017, 2018, 2020, 2021 Efraim Flashner <efraim@flashner.co.il>

 ;;; Copyright © 2016 Pjotr Prins <pjotr.guix@thebird.nl>
 ;;; Copyright © 2016 Andreas Enge <andreas@enge.fr>
 ;;; Copyright © 2016 Pjotr Prins <pjotr.guix@thebird.nl>
 ;;; Copyright © 2016 Andreas Enge <andreas@enge.fr>

-;;; Copyright © 2016 Ricardo Wurmus <rekado@elephly.net>
+;;; Copyright © 2016, 2020, 2021 Ricardo Wurmus <rekado@elephly.net>

 ;;; Copyright © 2016 Ben Woodcroft <donttrustben@gmail.com>
 ;;; Copyright © 2017, 2018 Rutger Helling <rhelling@mykolab.com>
 ;;; Copyright © 2016 Ben Woodcroft <donttrustben@gmail.com>
 ;;; Copyright © 2017, 2018 Rutger Helling <rhelling@mykolab.com>

-;;; Copyright © 2018, 2019 Tobias Geerinckx-Rice <me@tobias.gr>
+;;; Copyright © 2018–2021 Tobias Geerinckx-Rice <me@tobias.gr>

 ;;; Copyright © 2018 Clément Lassieur <clement@lassieur.org>
 ;;; Copyright © 2018 Clément Lassieur <clement@lassieur.org>

+;;; Copyright © 2019, 2020 Ludovic Courtès <ludo@gnu.org>
+;;; Copyright © 2020 Roel Janssen <roel@gnu.org>
+;;; Copyright © 2021 Stefan Reichör <stefan@xsteve.at>

 ;;;
 ;;; This file is part of GNU Guix.
 ;;;
 ;;;
 ;;; This file is part of GNU Guix.
 ;;;


@@ -26,10 +29,14 @@
 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
 
 (define-module (gnu packages parallel)
 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
 
 (define-module (gnu packages parallel)

-  #:use-module (guix build-system gnu)

   #:use-module (guix download)
   #:use-module (guix download)

+  #:use-module (guix git-download)
+  #:use-module (guix build-system gnu)
+  #:use-module (guix build-system python)

   #:use-module ((guix licenses) #:prefix license:)
   #:use-module ((guix licenses) #:prefix license:)

+  #:use-module ((guix utils) #:select (target-64bit?))

   #:use-module (guix packages)
   #:use-module (guix packages)

+  #:use-module (guix utils)

   #:use-module (gnu packages)
   #:use-module (gnu packages admin)
   #:use-module (gnu packages autotools)
   #:use-module (gnu packages)
   #:use-module (gnu packages admin)
   #:use-module (gnu packages autotools)


@@ -40,6 +47,8 @@
   #:use-module (gnu packages perl)
   #:use-module (gnu packages pkg-config)
   #:use-module (gnu packages python)
   #:use-module (gnu packages perl)
   #:use-module (gnu packages pkg-config)
   #:use-module (gnu packages python)

+  #:use-module (gnu packages python-science)
+  #:use-module (gnu packages python-xyz)

   #:use-module (gnu packages readline)
   #:use-module (gnu packages tcl)
   #:use-module (gnu packages tls)
   #:use-module (gnu packages readline)
   #:use-module (gnu packages tcl)
   #:use-module (gnu packages tls)


@@ -48,14 +57,14 @@
 (define-public parallel
   (package
     (name "parallel")
 (define-public parallel
   (package
     (name "parallel")

-    (version "20190422")
+    (version "20210322")

     (source
      (origin
       (method url-fetch)
       (uri (string-append "mirror://gnu/parallel/parallel-"
                           version ".tar.bz2"))
       (sha256
     (source
      (origin
       (method url-fetch)
       (uri (string-append "mirror://gnu/parallel/parallel-"
                           version ".tar.bz2"))
       (sha256

-       (base32 "0xdl5fnh1vpjp3zpmqsrbbgjixazlhl4d9awk42nz4snzynysjxl"))))
+       (base32 "152np0jg4n94sbl2p2fzxjfnssiyp5sg7r5wx6s8p893b921pxwq"))))

     (build-system gnu-build-system)
     (arguments
      `(#:phases
     (build-system gnu-build-system)
     (arguments
      `(#:phases


@@ -98,10 +107,38 @@ or more computers.  Jobs can consist of single commands or of scripts
 and they are executed on lists of files, hosts, users or other items.")
     (license license:gpl3+)))
 
 and they are executed on lists of files, hosts, users or other items.")
     (license license:gpl3+)))
 

+(define-public xe
+  (package
+    (name "xe")
+    (version "0.11")
+    (source
+     (origin
+       (method git-fetch)
+       (uri (git-reference
+             (url "https://github.com/leahneukirchen/xe")
+             (commit (string-append "v" version))))
+       (file-name (git-file-name name version))
+       (sha256
+        (base32 "04jr8f6jcijr0bsmn8ajm0aj35qh9my3xjsaq64h8lwg5bpyn29x"))))
+    (build-system gnu-build-system)
+    (arguments
+     `(#:tests? #f
+       #:make-flags (list (string-append "CC=" ,(cc-for-target))
+                          (string-append "PREFIX=" %output))
+       #:phases (modify-phases %standard-phases
+                  (delete 'configure))))
+    (synopsis "Execute a command for every argument")
+    (description
+     "The xe utility constructs command lines from specified arguments,
+combining some of the best features of xargs(1) and apply(1).  Parallel
+execution is also possible.")
+    (home-page "https://github.com/leahneukirchen/xe")
+    (license license:public-domain)))
+

 (define-public slurm
   (package
    (name "slurm")
 (define-public slurm
   (package
    (name "slurm")

-   (version "17.11.3")
+   (version "20.11.3")

    (source (origin
             (method url-fetch)
             (uri (string-append
    (source (origin
             (method url-fetch)
             (uri (string-append


@@ -109,43 +146,64 @@ and they are executed on lists of files, hosts, users or other items.")
                   version ".tar.bz2"))
             (sha256
              (base32
                   version ".tar.bz2"))
             (sha256
              (base32

-              "1x3i6z03d9m46fvj1cslrapm1drvgyqch9pn4xf23kvbz4gkhaps"))
+              "1s70x6yh60sx63dgmp5rlhq8jcz7kxv9pk8gbs9v1jg8zps5h5bk"))

             (modules '((guix build utils)))
             (snippet
              '(begin
             (modules '((guix build utils)))
             (snippet
              '(begin

+                ;; According to
+                ;; <https://lists.gnu.org/archive/html/guix-devel/2016-02/msg00534.html>
+                ;; there are non-free bits under contribs/, though it's not
+                ;; clear which ones.  libpmi is clearly free (it used to be
+                ;; under src/api/), so remove all of contribs/ except
+                ;; contribs/pmi/.

                 (substitute* "configure.ac"
                 (substitute* "configure.ac"

-                  (("^[[:space:]]+contribs/.*$") ""))
+                  (("^[[:space:]]+contribs/(.*)$" all directory)
+                   (if (and (string-prefix? "pmi" directory)
+                            (not (string-prefix? "pmi2" directory)))
+                       all
+                       "")))
+
+                (rename-file "contribs/pmi" "tmp-pmi")

                 (delete-file-recursively "contribs")
                 (delete-file-recursively "contribs")

+                (mkdir "contribs")
+                (rename-file "tmp-pmi" "contribs/pmi")

                 #t))))
    ;; FIXME: More optional inputs could be added,
    ;; in particular mysql and gtk+.
                 #t))))
    ;; FIXME: More optional inputs could be added,
    ;; in particular mysql and gtk+.

-   (inputs `(("expect" ,expect)
-             ("freeipmi" ,freeipmi)
-             ("hwloc" ,hwloc "lib")
+   (inputs `(("freeipmi" ,freeipmi)
+             ("hwloc" ,hwloc-2 "lib")

              ("json-c" ,json-c)
              ("linux-pam" , linux-pam)
              ("munge" ,munge)
              ("numactl" ,numactl)
              ("json-c" ,json-c)
              ("linux-pam" , linux-pam)
              ("munge" ,munge)
              ("numactl" ,numactl)

-             ("openssl" ,openssl)
-             ("perl" ,perl)
-             ("python" ,python-wrapper)

              ("readline" ,readline)))
    (native-inputs
     `(("autoconf" ,autoconf)
              ("readline" ,readline)))
    (native-inputs
     `(("autoconf" ,autoconf)

-      ("pkg-config" ,pkg-config)))
+      ("expect" ,expect)
+      ("perl" ,perl)
+      ("pkg-config" ,pkg-config)
+      ("python" ,python-wrapper)))

    (build-system gnu-build-system)
    (arguments
     `(#:configure-flags
       (list "--enable-pam" "--sysconfdir=/etc/slurm"
    (build-system gnu-build-system)
    (arguments
     `(#:configure-flags
       (list "--enable-pam" "--sysconfdir=/etc/slurm"

+            "--disable-static"

             (string-append "--with-freeipmi=" (assoc-ref %build-inputs "freeipmi"))
             (string-append "--with-hwloc=" (assoc-ref %build-inputs "hwloc"))
             (string-append "--with-json=" (assoc-ref %build-inputs "json-c"))
             (string-append "--with-munge=" (assoc-ref %build-inputs "munge"))
             (string-append "--with-freeipmi=" (assoc-ref %build-inputs "freeipmi"))
             (string-append "--with-hwloc=" (assoc-ref %build-inputs "hwloc"))
             (string-append "--with-json=" (assoc-ref %build-inputs "json-c"))
             (string-append "--with-munge=" (assoc-ref %build-inputs "munge"))

-            (string-append "--with-ssl=" (assoc-ref %build-inputs "openssl")))
+
+            ;; 32-bit support is marked as deprecated and needs to be
+            ;; explicitly enabled.
+            ,@(if (target-64bit?) '() '("--enable-deprecated")))

       #:phases
       (modify-phases %standard-phases
         (add-after 'unpack 'autoconf
       #:phases
       (modify-phases %standard-phases
         (add-after 'unpack 'autoconf

-          (lambda _ (invoke "autoconf")))))) ; configure.ac was patched
+          (lambda _ (invoke "autoconf")))         ;configure.ac was patched
+        (add-after 'install 'install-libpmi
+          (lambda _
+            ;; Open MPI expects libpmi to be provided by Slurm so install it.
+            (invoke "make" "install" "-C" "contribs/pmi"))))))

    (home-page "https://slurm.schedmd.com/")
    (synopsis "Workload manager for cluster computing")
    (description
    (home-page "https://slurm.schedmd.com/")
    (synopsis "Workload manager for cluster computing")
    (description


@@ -162,23 +220,75 @@ by managing a queue of pending work.")
                   license:zlib        ; src/common/strnatcmp.c
                   license:gpl2+))))   ; the rest, often with OpenSSL exception
 
                   license:zlib        ; src/common/strnatcmp.c
                   license:gpl2+))))   ; the rest, often with OpenSSL exception
 

+;; The SLURM client/daemon protocol and file format changes from time to time
+;; in incompatible ways, as noted in
+;; <https://slurm.schedmd.com/troubleshoot.html#network>.  Thus, keep older
+;; releases here.  See also <https://issues.guix.gnu.org/44387>.
+;; As noted in the link, YY.MM is the release scheme, and the 'maintenance'
+;; digit does not introduce incompatibilities.
+
+(define-public slurm-20.02
+  (package
+    (inherit slurm)
+    (version "20.02.6-1")
+    (source (origin
+              (inherit (package-source slurm))
+              (method url-fetch)
+              (uri (string-append
+                    "https://download.schedmd.com/slurm/slurm-"
+                    version ".tar.bz2"))
+              (sha256
+               (base32
+                "0qj4blfymrd2ry2qmb58l3jbr4jwygc3adcfw7my27rippcijlyc"))))))
+
+(define-public slurm-19.05
+  (package
+    (inherit slurm)
+    (version "19.05.8")
+    (source (origin
+              (inherit (package-source slurm))
+              (method url-fetch)
+              (uri (string-append
+                    "https://download.schedmd.com/slurm/slurm-"
+                    version ".tar.bz2"))
+              (sha256
+               (base32
+                "10c9j4a9a6d4ibpf75006mn03p8xgpaprc247x2idakysjf2fw43"))))))
+
+;; Same as Debian 10
+(define-public slurm-18.08
+  (package
+    (inherit slurm)
+    (version "18.08.9")
+    (source
+      (origin
+        (inherit (package-source slurm))
+        (uri (string-append
+               "https://download.schedmd.com/slurm/slurm-"
+               version ".tar.bz2"))
+        (sha256
+         (base32
+          "1bgrpz75m7l4xhirsd0fvnkzlkrl8v2qpmjcz60barc5qm2kn457"))))))
+

 (define-public slurm-drmaa
   (package
     (name "slurm-drmaa")
 (define-public slurm-drmaa
   (package
     (name "slurm-drmaa")

-    (version "1.0.7")
+    (version "1.1.2")

     (source (origin
               (method url-fetch)
     (source (origin
               (method url-fetch)

-              (uri "http://apps.man.poznan.pl/trac/slurm-drmaa/downloads/9")
-              (file-name (string-append name "-" version ".tar.gz"))
+              (uri (string-append
+                    "https://github.com/natefoo/slurm-drmaa/releases/download/"
+                    version "/slurm-drmaa-" version ".tar.gz"))

               (sha256
                (base32
               (sha256
                (base32

-                "0grw55hmny2mc4nc0y1arnvxd2k0dcdfn476kzs180fibjxgfw14"))))
+                "0dn8ypqxdaq3k4jqwwx7msckxnmr6n2z5j68yffp50yy07ajbzjv"))))

     (build-system gnu-build-system)
     (build-system gnu-build-system)

+    (arguments `(#:tests? #f)) ; The tests require "bats".

     (inputs
      `(("slurm" ,slurm)))
     (native-inputs
      `(("which" ,which)))
     (inputs
      `(("slurm" ,slurm)))
     (native-inputs
      `(("which" ,which)))

-    (home-page "http://apps.man.poznan.pl/trac/slurm-drmaa")
+    (home-page "https://github.com/natefoo/slurm-drmaa")

     (synopsis "Distributed resource management application API for SLURM")
     (description
      "PSNC DRMAA for Simple Linux Utility for Resource Management (SLURM) is
     (synopsis "Distributed resource management application API for SLURM")
     (description
      "PSNC DRMAA for Simple Linux Utility for Resource Management (SLURM) is


@@ -188,3 +298,50 @@ to SLURM.  Using DRMAA, grid applications builders, portal developers and ISVs
 can use the same high-level API to link their software with different
 cluster/resource management systems.")
     (license license:gpl3+)))
 can use the same high-level API to link their software with different
 cluster/resource management systems.")
     (license license:gpl3+)))

+
+(define-public python-slurm-magic
+  (let ((commit "73dd1a2b85799f7dae4b3f1cd9027536eff0c4d7")
+        (revision "0"))
+    (package
+      (name "python-slurm-magic")
+      (version (git-version "0.0" revision commit))
+      (home-page "https://github.com/NERSC/slurm-magic")
+      (source (origin
+                (method git-fetch)
+                (uri (git-reference (url home-page)
+                                    (commit commit)))
+                (sha256
+                 (base32
+                  "19pp2vs0wm8mx0arz9n6lw9wgyv70w9wyi4y6b91qc5j3bz5igfs"))
+                (file-name (git-file-name name version))))
+      (build-system python-build-system)
+      (arguments
+       '(#:phases (modify-phases %standard-phases
+                    (add-before 'build 'set-slurm-path
+                      (lambda* (#:key inputs #:allow-other-keys)
+                        ;; The '_execute' method tries to exec 'salloc'
+                        ;; etc. from $PATH.  Record the absolute file name
+                        ;; instead.
+                        (let ((slurm (assoc-ref inputs "slurm")))
+                          (substitute* "slurm_magic.py"
+                            (("name = (.*)$" _ value)
+                             (string-append "name = \""
+                                            slurm "/bin/\" + "
+                                            value "\n")))
+                          #t))))))
+      (inputs
+       `(("slurm" ,slurm)))
+      (propagated-inputs
+       `(("python-ipython" ,python-ipython)
+         ("python-pandas" ,python-pandas)))
+      (synopsis "Control the SLURM batch scheduler from Jupyter Notebook")
+      (description
+       "This package implements Jupyter/IPython
+@uref{http://ipython.readthedocs.io/en/stable/interactive/magics.html, magic
+commands} for interacting with the SLURM workload manager.  SLURM magic simply
+wraps command-line executables and the commands themselves should look like
+their command-line counterparts.  Commands are spawned via @code{subprocess}
+and output captured in the notebook.  Whatever arguments are accepted by a
+SLURM command line executable are also accepted by the corresponding magic
+command---e.g., @code{%salloc}, @code{%sbatch}, etc.")
+      (license license:bsd-3))))