gnu/packages/parallel.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2013, 2014, 2020 Eric Bavier <bavier@posteo.net>
   3 ;;; Copyright © 2015 Mark H Weaver <mhw@netris.org>
   4 ;;; Copyright © 2015, 2016, 2017, 2018, 2020, 2021 Efraim Flashner <efraim@flashner.co.il>
   5 ;;; Copyright © 2016 Pjotr Prins <pjotr.guix@thebird.nl>
   6 ;;; Copyright © 2016 Andreas Enge <andreas@enge.fr>
   7 ;;; Copyright © 2016, 2020, 2021 Ricardo Wurmus <rekado@elephly.net>
   8 ;;; Copyright © 2016 Ben Woodcroft <donttrustben@gmail.com>
   9 ;;; Copyright © 2017, 2018 Rutger Helling <rhelling@mykolab.com>
  10 ;;; Copyright © 2018–2021 Tobias Geerinckx-Rice <me@tobias.gr>
  11 ;;; Copyright © 2018 Clément Lassieur <clement@lassieur.org>
  12 ;;; Copyright © 2019, 2020 Ludovic Courtès <ludo@gnu.org>
  13 ;;; Copyright © 2020 Roel Janssen <roel@gnu.org>
  14 ;;; Copyright © 2021 Stefan Reichör <stefan@xsteve.at>
  15 ;;;
  16 ;;; This file is part of GNU Guix.
  17 ;;;
  18 ;;; GNU Guix is free software; you can redistribute it and/or modify it
  19 ;;; under the terms of the GNU General Public License as published by
  20 ;;; the Free Software Foundation; either version 3 of the License, or (at
  21 ;;; your option) any later version.
  22 ;;;
  23 ;;; GNU Guix is distributed in the hope that it will be useful, but
  24 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  25 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  26 ;;; GNU General Public License for more details.
  27 ;;;
  28 ;;; You should have received a copy of the GNU General Public License
  29 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  30
  31 (define-module (gnu packages parallel)
  32   #:use-module (guix download)
  33   #:use-module (guix git-download)
  34   #:use-module (guix build-system gnu)
  35   #:use-module (guix build-system python)
  36   #:use-module ((guix licenses) #:prefix license:)
  37   #:use-module ((guix utils) #:select (target-64bit?))
  38   #:use-module (guix packages)
  39   #:use-module (guix utils)
  40   #:use-module (gnu packages)
  41   #:use-module (gnu packages admin)
  42   #:use-module (gnu packages autotools)
  43   #:use-module (gnu packages base)
  44   #:use-module (gnu packages freeipmi)
  45   #:use-module (gnu packages linux)
  46   #:use-module (gnu packages mpi)
  47   #:use-module (gnu packages perl)
  48   #:use-module (gnu packages pkg-config)
  49   #:use-module (gnu packages python)
  50   #:use-module (gnu packages python-science)
  51   #:use-module (gnu packages python-xyz)
  52   #:use-module (gnu packages readline)
  53   #:use-module (gnu packages tcl)
  54   #:use-module (gnu packages tls)
  55   #:use-module (gnu packages web))
  56
  57 (define-public parallel
  58   (package
  59     (name "parallel")
  60     (version "20210322")
  61     (source
  62      (origin
  63       (method url-fetch)
  64       (uri (string-append "mirror://gnu/parallel/parallel-"
  65                           version ".tar.bz2"))
  66       (sha256
  67        (base32 "152np0jg4n94sbl2p2fzxjfnssiyp5sg7r5wx6s8p893b921pxwq"))))
  68     (build-system gnu-build-system)
  69     (arguments
  70      `(#:phases
  71        (modify-phases %standard-phases
  72          (add-after 'unpack 'patch-bin-sh
  73            (lambda _
  74              (for-each
  75               (lambda (file)
  76                 (substitute* file
  77                   ;; Patch hard coded '/bin/sh' in the line ending in:
  78                   ;; $Global::shell = $ENV{'PARALLEL_SHELL'} ||
  79                   ;;  parent_shell($$) || $ENV{'SHELL'} || "/bin/sh";
  80                   (("/bin/sh\\\";\n$") (string-append (which "sh") "\";\n"))))
  81               (list "src/parallel" "src/sem"))
  82              #t))
  83          (add-after 'install 'wrap-program
  84            (lambda* (#:key inputs outputs #:allow-other-keys)
  85              (let ((out (assoc-ref outputs "out")))
  86                (wrap-program (string-append out "/bin/parallel")
  87                  `("PATH" ":" prefix
  88                    ,(map (lambda (input)
  89                            (string-append (assoc-ref inputs input) "/bin"))
  90                          '("perl"
  91                            "procps"))))
  92                #t)))
  93          (add-after 'wrap-program 'post-install-test
  94            (lambda* (#:key outputs #:allow-other-keys)
  95              (invoke (string-append
  96                       (assoc-ref outputs "out") "/bin/parallel")
  97                      "echo"
  98                      ":::" "1" "2" "3"))))))
  99     (inputs
 100      `(("perl" ,perl)
 101        ("procps" ,procps)))
 102     (home-page "https://www.gnu.org/software/parallel/")
 103     (synopsis "Build and execute command lines in parallel")
 104     (description
 105      "GNU Parallel is a tool for executing shell jobs in parallel using one
 106 or more computers.  Jobs can consist of single commands or of scripts
 107 and they are executed on lists of files, hosts, users or other items.")
 108     (license license:gpl3+)))
 109
 110 (define-public xe
 111   (package
 112     (name "xe")
 113     (version "0.11")
 114     (source
 115      (origin
 116        (method git-fetch)
 117        (uri (git-reference
 118              (url "https://github.com/leahneukirchen/xe")
 119              (commit (string-append "v" version))))
 120        (file-name (git-file-name name version))
 121        (sha256
 122         (base32 "04jr8f6jcijr0bsmn8ajm0aj35qh9my3xjsaq64h8lwg5bpyn29x"))))
 123     (build-system gnu-build-system)
 124     (arguments
 125      `(#:tests? #f
 126        #:make-flags (list (string-append "CC=" ,(cc-for-target))
 127                           (string-append "PREFIX=" %output))
 128        #:phases (modify-phases %standard-phases
 129                   (delete 'configure))))
 130     (synopsis "Execute a command for every argument")
 131     (description
 132      "The xe utility constructs command lines from specified arguments,
 133 combining some of the best features of xargs(1) and apply(1).  Parallel
 134 execution is also possible.")
 135     (home-page "https://github.com/leahneukirchen/xe")
 136     (license license:public-domain)))
 137
 138 (define-public slurm
 139   (package
 140    (name "slurm")
 141    (version "20.11.3")
 142    (source (origin
 143             (method url-fetch)
 144             (uri (string-append
 145                   "https://download.schedmd.com/slurm/slurm-"
 146                   version ".tar.bz2"))
 147             (sha256
 148              (base32
 149               "1s70x6yh60sx63dgmp5rlhq8jcz7kxv9pk8gbs9v1jg8zps5h5bk"))
 150             (modules '((guix build utils)))
 151             (snippet
 152              '(begin
 153                 ;; According to
 154                 ;; <https://lists.gnu.org/archive/html/guix-devel/2016-02/msg00534.html>
 155                 ;; there are non-free bits under contribs/, though it's not
 156                 ;; clear which ones.  libpmi is clearly free (it used to be
 157                 ;; under src/api/), so remove all of contribs/ except
 158                 ;; contribs/pmi/.
 159                 (substitute* "configure.ac"
 160                   (("^[[:space:]]+contribs/(.*)$" all directory)
 161                    (if (and (string-prefix? "pmi" directory)
 162                             (not (string-prefix? "pmi2" directory)))
 163                        all
 164                        "")))
 165
 166                 (rename-file "contribs/pmi" "tmp-pmi")
 167                 (delete-file-recursively "contribs")
 168                 (mkdir "contribs")
 169                 (rename-file "tmp-pmi" "contribs/pmi")
 170                 #t))))
 171    ;; FIXME: More optional inputs could be added,
 172    ;; in particular mysql and gtk+.
 173    (inputs `(("freeipmi" ,freeipmi)
 174              ("hwloc" ,hwloc-2 "lib")
 175              ("json-c" ,json-c)
 176              ("linux-pam" , linux-pam)
 177              ("munge" ,munge)
 178              ("numactl" ,numactl)
 179              ("readline" ,readline)))
 180    (native-inputs
 181     `(("autoconf" ,autoconf)
 182       ("expect" ,expect)
 183       ("perl" ,perl)
 184       ("pkg-config" ,pkg-config)
 185       ("python" ,python-wrapper)))
 186    (build-system gnu-build-system)
 187    (arguments
 188     `(#:configure-flags
 189       (list "--enable-pam" "--sysconfdir=/etc/slurm"
 190             "--disable-static"
 191             (string-append "--with-freeipmi=" (assoc-ref %build-inputs "freeipmi"))
 192             (string-append "--with-hwloc=" (assoc-ref %build-inputs "hwloc"))
 193             (string-append "--with-json=" (assoc-ref %build-inputs "json-c"))
 194             (string-append "--with-munge=" (assoc-ref %build-inputs "munge"))
 195
 196             ;; 32-bit support is marked as deprecated and needs to be
 197             ;; explicitly enabled.
 198             ,@(if (target-64bit?) '() '("--enable-deprecated")))
 199       #:phases
 200       (modify-phases %standard-phases
 201         (add-after 'unpack 'autoconf
 202           (lambda _ (invoke "autoconf")))         ;configure.ac was patched
 203         (add-after 'install 'install-libpmi
 204           (lambda _
 205             ;; Open MPI expects libpmi to be provided by Slurm so install it.
 206             (invoke "make" "install" "-C" "contribs/pmi"))))))
 207    (home-page "https://slurm.schedmd.com/")
 208    (synopsis "Workload manager for cluster computing")
 209    (description
 210     "SLURM is a fault-tolerant and highly scalable cluster management and job
 211 scheduling system for large and small clusters.  It allocates access to
 212 resources (computer nodes) to users for some duration of time, provides a
 213 framework for starting, executing, and monitoring work (typically a parallel
 214 job) on a set of allocated nodes, and arbitrates contention for resources
 215 by managing a queue of pending work.")
 216    (license (list license:bsd-2       ; src/common/log.[ch], src/common/uthash
 217                   license:expat       ; slurm/pmi.h
 218                   license:isc         ; src/common/strlcpy.c
 219                   license:lgpl2.1+    ; hilbert.[ch], src/common/slurm_time.h
 220                   license:zlib        ; src/common/strnatcmp.c
 221                   license:gpl2+))))   ; the rest, often with OpenSSL exception
 222
 223 ;; The SLURM client/daemon protocol and file format changes from time to time
 224 ;; in incompatible ways, as noted in
 225 ;; <https://slurm.schedmd.com/troubleshoot.html#network>.  Thus, keep older
 226 ;; releases here.  See also <https://issues.guix.gnu.org/44387>.
 227 ;; As noted in the link, YY.MM is the release scheme, and the 'maintenance'
 228 ;; digit does not introduce incompatibilities.
 229
 230 (define-public slurm-20.02
 231   (package
 232     (inherit slurm)
 233     (version "20.02.6-1")
 234     (source (origin
 235               (inherit (package-source slurm))
 236               (method url-fetch)
 237               (uri (string-append
 238                     "https://download.schedmd.com/slurm/slurm-"
 239                     version ".tar.bz2"))
 240               (sha256
 241                (base32
 242                 "0qj4blfymrd2ry2qmb58l3jbr4jwygc3adcfw7my27rippcijlyc"))))))
 243
 244 (define-public slurm-19.05
 245   (package
 246     (inherit slurm)
 247     (version "19.05.8")
 248     (source (origin
 249               (inherit (package-source slurm))
 250               (method url-fetch)
 251               (uri (string-append
 252                     "https://download.schedmd.com/slurm/slurm-"
 253                     version ".tar.bz2"))
 254               (sha256
 255                (base32
 256                 "10c9j4a9a6d4ibpf75006mn03p8xgpaprc247x2idakysjf2fw43"))))))
 257
 258 ;; Same as Debian 10
 259 (define-public slurm-18.08
 260   (package
 261     (inherit slurm)
 262     (version "18.08.9")
 263     (source
 264       (origin
 265         (inherit (package-source slurm))
 266         (uri (string-append
 267                "https://download.schedmd.com/slurm/slurm-"
 268                version ".tar.bz2"))
 269         (sha256
 270          (base32
 271           "1bgrpz75m7l4xhirsd0fvnkzlkrl8v2qpmjcz60barc5qm2kn457"))))))
 272
 273 (define-public slurm-drmaa
 274   (package
 275     (name "slurm-drmaa")
 276     (version "1.1.2")
 277     (source (origin
 278               (method url-fetch)
 279               (uri (string-append
 280                     "https://github.com/natefoo/slurm-drmaa/releases/download/"
 281                     version "/slurm-drmaa-" version ".tar.gz"))
 282               (sha256
 283                (base32
 284                 "0dn8ypqxdaq3k4jqwwx7msckxnmr6n2z5j68yffp50yy07ajbzjv"))))
 285     (build-system gnu-build-system)
 286     (arguments `(#:tests? #f)) ; The tests require "bats".
 287     (inputs
 288      `(("slurm" ,slurm)))
 289     (native-inputs
 290      `(("which" ,which)))
 291     (home-page "https://github.com/natefoo/slurm-drmaa")
 292     (synopsis "Distributed resource management application API for SLURM")
 293     (description
 294      "PSNC DRMAA for Simple Linux Utility for Resource Management (SLURM) is
 295 an implementation of Open Grid Forum DRMAA 1.0 (Distributed Resource
 296 Management Application API) specification for submission and control of jobs
 297 to SLURM.  Using DRMAA, grid applications builders, portal developers and ISVs
 298 can use the same high-level API to link their software with different
 299 cluster/resource management systems.")
 300     (license license:gpl3+)))
 301
 302 (define-public python-slurm-magic
 303   (let ((commit "73dd1a2b85799f7dae4b3f1cd9027536eff0c4d7")
 304         (revision "0"))
 305     (package
 306       (name "python-slurm-magic")
 307       (version (git-version "0.0" revision commit))
 308       (home-page "https://github.com/NERSC/slurm-magic")
 309       (source (origin
 310                 (method git-fetch)
 311                 (uri (git-reference (url home-page)
 312                                     (commit commit)))
 313                 (sha256
 314                  (base32
 315                   "19pp2vs0wm8mx0arz9n6lw9wgyv70w9wyi4y6b91qc5j3bz5igfs"))
 316                 (file-name (git-file-name name version))))
 317       (build-system python-build-system)
 318       (arguments
 319        '(#:phases (modify-phases %standard-phases
 320                     (add-before 'build 'set-slurm-path
 321                       (lambda* (#:key inputs #:allow-other-keys)
 322                         ;; The '_execute' method tries to exec 'salloc'
 323                         ;; etc. from $PATH.  Record the absolute file name
 324                         ;; instead.
 325                         (let ((slurm (assoc-ref inputs "slurm")))
 326                           (substitute* "slurm_magic.py"
 327                             (("name = (.*)$" _ value)
 328                              (string-append "name = \""
 329                                             slurm "/bin/\" + "
 330                                             value "\n")))
 331                           #t))))))
 332       (inputs
 333        `(("slurm" ,slurm)))
 334       (propagated-inputs
 335        `(("python-ipython" ,python-ipython)
 336          ("python-pandas" ,python-pandas)))
 337       (synopsis "Control the SLURM batch scheduler from Jupyter Notebook")
 338       (description
 339        "This package implements Jupyter/IPython
 340 @uref{http://ipython.readthedocs.io/en/stable/interactive/magics.html, magic
 341 commands} for interacting with the SLURM workload manager.  SLURM magic simply
 342 wraps command-line executables and the commands themselves should look like
 343 their command-line counterparts.  Commands are spawned via @code{subprocess}
 344 and output captured in the notebook.  Whatever arguments are accepted by a
 345 SLURM command line executable are also accepted by the corresponding magic
 346 command---e.g., @code{%salloc}, @code{%sbatch}, etc.")
 347       (license license:bsd-3))))