1 ;;; GNU Guix --- Functional package management for GNU
2 ;;; Copyright © 2017, 2018, 2019 Marius Bakke <mbakke@fastmail.com>
3 ;;; Copyright © 2017 Rutger Helling <rhelling@mykolab.com>
5 ;;; This file is part of GNU Guix.
7 ;;; GNU Guix is free software; you can redistribute it and/or modify it
8 ;;; under the terms of the GNU General Public License as published by
9 ;;; the Free Software Foundation; either version 3 of the License, or (at
10 ;;; your option) any later version.
12 ;;; GNU Guix is distributed in the hope that it will be useful, but
13 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 ;;; GNU General Public License for more details.
17 ;;; You should have received a copy of the GNU General Public License
18 ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
20 (define-module (gnu packages storage)
21 #:use-module (guix download)
22 #:use-module ((guix licenses) #:prefix license:)
23 #:use-module (guix packages)
24 #:use-module (guix utils)
25 #:use-module (guix build-system cmake)
26 #:use-module (gnu packages)
27 #:use-module (gnu packages admin)
28 #:use-module (gnu packages assembly)
29 #:use-module (gnu packages authentication)
30 #:use-module (gnu packages bdw-gc)
31 #:use-module (gnu packages boost)
32 #:use-module (gnu packages check)
33 #:use-module (gnu packages compression)
34 #:use-module (gnu packages crypto)
35 #:use-module (gnu packages cryptsetup)
36 #:use-module (gnu packages curl)
37 #:use-module (gnu packages databases)
38 #:use-module (gnu packages disk)
39 #:use-module (gnu packages gcc)
40 #:use-module (gnu packages gperf)
41 #:use-module (gnu packages jemalloc)
42 #:use-module (gnu packages linux)
43 #:use-module (gnu packages lua)
44 #:use-module (gnu packages nss)
45 #:use-module (gnu packages openldap)
46 #:use-module (gnu packages perl)
47 #:use-module (gnu packages pkg-config)
48 #:use-module (gnu packages python)
49 #:use-module (gnu packages python-xyz)
50 #:use-module (gnu packages sphinx)
51 #:use-module (gnu packages tls)
52 #:use-module (gnu packages web)
53 #:use-module (gnu packages xml))
61 (uri (string-append "https://download.ceph.com/tarballs/ceph-"
65 "13f5qs7jpxprplk4irwlx90mc9gvm48fvd3q471xcqc3n6z1qywz"))
67 (search-patches "ceph-skip-unittest_blockdev.patch"
68 "ceph-skip-collect-sys-info-test.patch"
69 "ceph-detect-rocksdb.patch"
70 "ceph-volume-respect-PATH.patch"
71 "ceph-disable-cpu-optimizations.patch"))
72 (modules '((guix build utils)))
75 (for-each delete-file-recursively
76 '(;; TODO: Unbundle these:
89 (build-system cmake-build-system)
92 (let* ((out (assoc-ref %outputs "out"))
93 (lib (assoc-ref %outputs "lib"))
94 (libdir (string-append lib "/lib")))
95 (list (string-append "-DCMAKE_INSTALL_PREFIX=" out)
96 (string-append "-DCMAKE_INSTALL_LIBDIR=" libdir)
97 (string-append "-DCMAKE_INSTALL_INCLUDEDIR="
99 ;; We need both libdir and libdir/ceph in RUNPATH.
100 (string-append "-DCMAKE_INSTALL_RPATH="
101 libdir ";" libdir "/ceph")
102 (string-append "-DCMAKE_INSTALL_SYSCONFDIR=" out "/etc")
103 (string-append "-DCMAKE_INSTALL_MANDIR=" out "/share/man")
104 (string-append "-DCMAKE_INSTALL_DOCDIR=" out "/share/ceph/doc")
105 (string-append "-DCMAKE_INSTALL_LIBEXECDIR=" out "/libexec")
106 (string-append "-DKEYUTILS_INCLUDE_DIR="
107 (assoc-ref %build-inputs "keyutils") "/include")
108 (string-append "-DXFS_INCLUDE_DIR="
109 (assoc-ref %build-inputs "xfsprogs") "/include")
110 "-DCMAKE_INSTALL_LOCALSTATEDIR=/var"
111 "-DBUILD_SHARED_LIBS=ON"
112 "-DWITH_SYSTEM_ROCKSDB=ON"
113 "-DWITH_SYSTEM_BOOST=ON"
115 ;; TODO: Enable these when available in Guix.
116 "-DWITH_MGR_DASHBOARD_FRONTEND=OFF" ;requires node + nodeenv
117 "-DWITH_BABELTRACE=OFF"
121 ;; Use jemalloc instead of tcmalloc.
122 "-DALLOCATOR=jemalloc"))
123 ;; FIXME: Some of the tests leak Btrfs subvolumes on Btrfs. See
124 ;; <https://bugs.gnu.org/29674> for details. Disable tests until
128 (modify-phases %standard-phases
129 (add-after 'unpack 'patch-source
130 (lambda* (#:key outputs #:allow-other-keys)
131 (let ((out (assoc-ref outputs "out"))
132 (lib (assoc-ref outputs "lib")))
134 (substitute* "cmake/modules/Distutils.cmake"
135 ;; Prevent creation of Python eggs.
136 (("setup.py install")
137 "setup.py install --single-version-externally-managed --root=/"))
139 (substitute* (find-files "src/pybind" "^setup\\.py$")
140 ;; Here we inject an extra line to the `setup.py' of the
141 ;; Python C libraries so RUNPATH gets set up correctly.
142 (("^([[:blank:]]+)extra_compile_args=(.*)$" _ indent args)
143 (string-append indent "extra_compile_args=" args
144 indent "extra_link_args=['-Wl,-rpath="
147 (substitute* "src/ceph-disk/tox.ini"
148 ;; Disable flake8 test since it complains about too long lines.
149 (("envlist = flake8,py27") "envlist = py27"))
151 (substitute* "src/ceph-detect-init/tox.ini"
152 ;; Disable python3 tests until we at least get py2 working.
153 (("envlist = pep8,py27,py3") "envlist = pep8,py27"))
155 (substitute* "src/key_value_store/kv_flat_btree_async.cc"
156 (("/usr/include/") ""))
158 (substitute* "src/test/test_subprocess.cc"
159 (("/bin/sh") (which "sh")))
160 (substitute* "qa/standalone/special/ceph_objectstore_tool.py"
161 (("/bin/rm") (which "rm")))
162 (substitute* "src/ceph-disk/ceph_disk/main.py"
163 (("/bin/mount") "mount")
164 (("/bin/umount") "umount")
165 (("/sbin/blkid") (which "blkid"))
166 (("'cryptsetup'") (string-append "'" (which "cryptsetup") "'"))
167 (("'sgdisk'") (string-append "'" (which "sgdisk") "'"))
168 (("'parted'") (string-append "'" (which "parted") "'"))
169 (("'udevadm'") (string-append "'" (which "udevadm") "'")))
171 (substitute* "udev/50-rbd.rules"
172 (("/usr/bin/ceph-rbdnamer")
173 (string-append out "/bin/ceph-rbdnamer")))
174 (substitute* "udev/60-ceph-by-parttypeuuid.rules"
175 (("/sbin/blkid") (which "blkid")))
176 (substitute* "udev/95-ceph-osd.rules"
177 (("/usr/sbin/ceph-disk")
178 (string-append out "/bin/ceph-disk")))
180 (substitute* "src/test/run-cli-tests"
181 ;; Use our python-cram instead of the (un)bundled one.
183 (string-append "CRAM_BIN=" (which "cram") "\n")))
185 ;; Disable tests that are known to fail.
186 ;; TODO: The majority of these fail because
187 ;; 'qa/workunits/ceph-helpers.sh' expects to find
188 ;; /tmp/ceph-disk-virtualenv/bin/ceph-disk, but somehow
189 ;; src/ceph-disk/CMakeLists.txt fails to create it.
190 (substitute* "src/test/CMakeLists.txt"
191 ;; FIXME: These tests fails because `ceph-disk'
193 (("^add_ceph_test\\(test-ceph-helpers\\.sh.*$") "\n")
194 (("^add_ceph_test\\(test_pidfile\\.sh.*$") "\n")
195 ;; XXX Why does this fail.
196 (("^add_ceph_test\\(cephtool-test-mon\\.sh.*$") "\n")
197 ;; This fails due to missing '/etc/fstab'.
198 (("^add_ceph_test\\(cephtool-test-rados\\.sh.*$") "\n")
199 ;; `Bad messages to stderr: OSD has the store locked'
200 (("^add_ceph_test\\(ceph_objectstore_tool\\.py.*$") "\n")
201 ;; The bundled python-cram fork needs patching to work on
202 ;; guix, and the system version does not support --error-dir.
203 ;; https://bitbucket.org/brodie/cram/issues/9
204 (("^add_ceph_test\\(run-cli-tests.*$") "\n")
205 ;; FIXME: tox/virtualenv/pip does not discover the
206 ;; required packages and tries to go online.
207 (("^add_test\\(NAME run-tox-ceph-disk.*$") "\n")
208 (("^add_test\\(NAME run-tox-ceph-detect-init.*$") "\n")
209 ;; Also remove from the set_property block.
210 (("run-tox-ceph-disk") "")
211 (("run-tox-ceph-detect-init") ""))
212 ;; TODO: This also seems to fail because of /etc/os-release.
213 ;; How to make src/common/util.cc behave without it.
214 (substitute* "src/test/crush/CMakeLists.txt"
215 (("^add_ceph_test\\(crush-classes\\.sh.*$") "\n"))
216 ;; More 'ceph-disk' issues here.. :-(
217 (substitute* "src/test/erasure-code/CMakeLists.txt"
218 (("^add_ceph_test\\(test-erasure-code-plugins\\.sh.*$") "\n")
219 (("^add_ceph_test\\(test-erasure-code\\.sh.*$") "\n")
220 (("^add_ceph_test\\(test-erasure-eio\\.sh.*$") "\n"))
221 (substitute* "src/test/libradosstriper/CMakeLists.txt"
222 (("^add_ceph_test\\(rados-striper\\.sh.*$") "\n"))
223 (substitute* "src/test/mon/CMakeLists.txt"
224 (("^add_ceph_test\\(osd-crush\\.sh.*$") "\n")
225 (("^add_ceph_test\\(test_pool_quota\\.sh.*$") "\n")
226 (("^add_ceph_test\\(osd-pool-create\\.sh.*$") "\n"))
227 (substitute* "src/test/osd/CMakeLists.txt"
228 (("^add_ceph_test\\(osd-bench\\.sh.*$") "\n")
229 (("^add_ceph_test\\(osd-config\\.sh.*$") "\n")
230 (("add_ceph_test\\(osd-dup\\.sh.*$") "\n")
231 (("^add_ceph_test\\(osd-markdown\\.sh.*$") "\n")
232 (("^add_ceph_test\\(osd-reactivate\\.sh.*$") "\n")
233 (("^add_ceph_test\\(osd-reuse-id\\.sh.*$") "\n")
234 (("^add_ceph_test\\(osd-scrub-repair\\.sh.*$") "\n")
235 (("^add_ceph_test\\(osd-scrub-snaps\\.sh.*$") "\n")
236 (("^add_ceph_test\\(osd-copy-from\\.sh.*$") "\n")
237 (("^add_ceph_test\\(osd-fast-mark-down\\.sh.*$") "\n"))
239 (add-before 'check 'set-check-environment
241 ;; Run tests in parallel.
242 (setenv "CTEST_PARALLEL_LEVEL"
243 (number->string (parallel-job-count)))
244 ;; `pip' requires write access in $HOME.
245 (setenv "HOME" "/tmp")
247 (add-before 'install 'set-install-environment
248 (lambda* (#:key outputs #:allow-other-keys)
249 (let* ((out (assoc-ref outputs "out"))
251 (string-append out "/lib/python2.7/site-packages"))
253 (string-append out "/lib/python"
254 ,(version-major+minor
255 (package-version python))
257 ;; The Python install scripts refuses to function if
258 ;; the install directory is not on PYTHONPATH.
260 (string-append py2sitedir ":" py3sitedir ":"
261 (getenv "PYTHONPATH")))
263 (add-after 'install 'wrap-python-scripts
264 (lambda* (#:key inputs outputs #:allow-other-keys)
265 (let* ((out (assoc-ref outputs "out"))
266 (scripts '("ceph" "ceph-mgr" "ceph-volume"
268 "ceph-disk")) ;deprecated
269 (prettytable (assoc-ref inputs "python2-prettytable"))
270 (six (assoc-ref inputs "python2-six"))
271 (sitedir (lambda (package)
272 (string-append package
273 "/lib/python2.7/site-packages")))
274 (PYTHONPATH (string-append
277 (sitedir prettytable))))
278 (for-each (lambda (executable)
279 (wrap-program (string-append out "/bin/" executable)
280 `("PYTHONPATH" ":" prefix (,PYTHONPATH))))
283 (add-before 'validate-runpath 'remove-test-executables
284 (lambda* (#:key outputs #:allow-other-keys)
285 (let ((out (assoc-ref outputs "out")))
286 ;; FIXME: The BUILD_SHARED_LIBS CMake flag causes the test executables
287 ;; to link the bundled googletest dynamically, which in turn causes
288 ;; RUNPATH validation failures because 'libgtest.so' and friends do
289 ;; not get absolute RUNPATH entries. The next version of Ceph can use
290 ;; an external googletest; for now just remove the test executables.
291 (for-each delete-file (find-files (string-append out "/bin")
298 ("pkg-config" ,pkg-config)
299 ("python-cython" ,python-cython)
300 ("python-sphinx" ,python-sphinx)
304 ("inetutils" ,inetutils)
307 ("xmlstarlet" ,xmlstarlet)
308 ("python2-cram" ,python2-cram)
309 ("python2-virtualenv" ,python2-virtualenv)
311 ;; These dependencies are taken from test-requirements.txt
312 ;; of ceph-disk and ceph-detect-init. The latter can also
313 ;; test against python3, but let's try to get python2 tests
314 ;; working first since that is the default.
315 ("python2-configobj" ,python2-configobj)
316 ("python2-coverage" ,python2-coverage)
317 ("python2-discover" ,python2-discover)
318 ("python2-fixtures" ,python2-fixtures)
319 ("python2-flake8" ,python2-flake8)
320 ("python2-mock" ,python2-mock)
321 ("python2-nose" ,python2-nose)
322 ("python2-pip" ,python2-pip)
323 ("python2-pytest" ,python2-pytest)
324 ("python2-subunit" ,python2-subunit)
325 ("python2-testrepository" ,python2-testrepository)
326 ("python2-testtools" ,python2-testtools)
327 ("python2-tox" ,python2-tox)))
331 ("cryptsetup" ,cryptsetup)
335 ("gptfdisk" ,gptfdisk)
336 ("jemalloc" ,jemalloc)
337 ("keyutils" ,keyutils)
340 ("libatomic-ops" ,libatomic-ops)
343 ("oath-toolkit" ,oath-toolkit)
344 ("openldap" ,openldap)
348 ("python@2" ,python-2)
349 ("python2-prettytable" ,python2-prettytable) ;used by ceph_daemon.py
350 ("python2-six" ,python2-six) ;for ceph-mgr + plugins
351 ("python@3" ,python-3)
352 ("rapidjson" ,rapidjson)
356 ("util-linux" ,util-linux)
357 ("xfsprogs" ,xfsprogs)
359 (home-page "https://ceph.com/")
360 (synopsis "Distributed object store and file system")
362 "Ceph is a distributed storage system designed for reliability and
363 performance. It provides network-based block devices (RBD), a POSIX
364 compliant file system (CephFS), and offers compatibility with various
365 storage protocols (S3, NFS, and others) through the RADOS gateway.")
366 ;; The Ceph libraries are LGPL2.1 and most of the utilities fall under
367 ;; GPL2. The installed erasure code plugins are BSD-3 licensed and do
368 ;; not use the GPL code. The source archive includes a number of files
369 ;; carrying other licenses; consult COPYING for more information. Note
370 ;; that COPYING does not cover third-party bundled software.
371 (license (list license:lgpl2.1 license:gpl2 ;some files are 'or later'
372 license:cc-by-sa3.0 ;documentation
373 license:bsd-3 ;isa-l,jerasure,++
374 license:expat)))) ;civetweb,java bindings