X-Git-Url: http://git.hcoop.net/clinton/scripts.git/blobdiff_plain/24b2faa62c8b400bfee86ca6b27339aed65fc119..74bfc8b7d79cbf6025e8389d00a3f96d0d33c454:/hcoop-backup diff --git a/hcoop-backup b/hcoop-backup index 65d0bec..d771434 100755 --- a/hcoop-backup +++ b/hcoop-backup @@ -1,36 +1,45 @@ #!/bin/bash -e # -# run this script as root, on deleuze, in a directory that WILL NOT be -# part of the backup (/tmp is a good choice). +# it is dangerous to remove the "-e" above; please don't do that. # -# FIXME: -# postgres dumps -# mysql dumps - +# +# run this script as root, on deleuze +# -#KEYFILE=$1 +PATH=$PATH:/bin:/usr/bin:/sbin:/usr/sbin +#COMPRESS_EXT=.bz2 +#COMPRESS_PROG=bzip2 +COMPRESS_EXT=.gz +COMPRESS_PROG=gzip KEYFILE=/etc/backup-encryption-key -BACKUPDIR=/afs/megacz.com/hcoop-backup/ +BACKUPTMP=/var/backups/hcoop-backup +CURDATE=$(date -u +%Y.%m.%d) + +MOVE_OVER=$(dirname $0)/rsync.net-move-over -cd $BACKUPDIR -find * -prune -ctime +5 -exec rm -rf {} \; +IFS=$'\n' -SUBDIR=`date +%Y.%m.%d` -rm -rf $SUBDIR -mkdir -p $SUBDIR -cd $SUBDIR +# Initialize storage area +RSYNCDIR=/vicepa/hcoop-backups/files +rm -fr $RSYNCDIR +mkdir -p $RSYNCDIR/$CURDATE -echo 'I am in:' -pwd +# Initialize backup staging area +mkdir -p $BACKUPTMP +cd $BACKUPTMP + +groups +echo "I am in: $(pwd)" echo -echo building package lists... +echo "Building package lists..." dpkg-query -W -f='${Package}\n' > packages -(cd /; sudo find / /usr/ /usr/local/ /var/ -xdev) | sort | uniq > allfiles +(cd /; find / /usr/ /usr/local/ /var/ -xdev) | sort | uniq > allfiles dpkg-query -W -f='${Package}\n' | xargs dpkg -L | sort | uniq > debfiles -dpkg-query -W -f='${Conffiles}\n' | grep / | cut -b2- | sed 's_ .*__' | sort | uniq > conffiles +dpkg-query -W -f='${Conffiles}\n' | grep / | cut -b2- | \ + sed 's_ .*__' | sort | uniq > conffiles diff allfiles debfiles | grep '^<' | cut -b 3- | \ grep -v ^/var/cache | \ @@ -38,6 +47,7 @@ diff allfiles debfiles | grep '^<' | cut -b 3- | \ grep -v ^/var/lib/dpkg | \ grep -v ^/var/backups | \ grep -v ^/var/lib/changetrack | \ + grep -v ^/var/local/lib/spamd | \ grep -v ^/var/run | \ grep -v ^/var/lock | \ grep -v ^/var/lib/ucf | \ @@ -48,13 +58,21 @@ diff allfiles debfiles | grep '^<' | cut -b 3- | \ grep -v '^/$' | \ grep -v '^/usr/$' | \ grep -v ^/usr/src | \ - grep -v '^/usr/.*pyc' | \ - grep -v '^/usr/.*elc' | \ + grep -v '^/usr/.*\.pyc' | \ + grep -v '^/usr/.*\.elc' | \ + grep -v '^/usr/bin/perldoc\.stub$' | \ + grep -v '^/usr/bin/.*\.notslocate$' | \ + grep -v '^/usr/lib/courier/.*\.rand$' | \ + grep -v '^/usr/lib/gconv/gconv-modules\.cache$' | \ + grep -v '^/usr/lib/graphviz/config$' | \ + grep -v '^/usr/lib/locale/locale-archive$' | \ grep -v '^/usr/share/info/dir$' | \ - grep -v '^/usr/share/info/dir.old$' | \ + grep -v '^/usr/share/info/dir\.old$' | \ grep -v '^/usr/share/emacs21/site-lisp/' | \ grep -v '^/usr/share/emacs22/site-lisp/' | \ - cat > backupfiles + grep -v '^/usr/share/snmp/mibs/\.index$' | \ + grep -v '^/usr/share/vim/addons/doc/tags$' \ + > backupfiles cat conffiles >> backupfiles @@ -65,21 +83,14 @@ cat backupfiles | \ grep -v ^/var/log | \ grep -v ^/usr/lib/python2.4/ | \ grep -v ^/var/lib/python-support | \ + grep -v ^/usr/share/jed/lib | \ grep -v ^/usr/share/man | \ + grep -v ^/usr/share/perl5/IkiWiki/Plugin | \ grep -v ^/media | \ - grep -v ^/usr/lib/locale/locale-archive | \ grep -v ^/vmlinuz | \ grep -v ^/vmlinuz.old | \ - grep -v '^/sbin/[a-z\-]*.modutils$' | \ - cat > annoyingfiles- - -for A in `cat annoyingfiles-` -do \ - test -L "$A" || echo "$A" -done > annoyingfiles - -cat annoyingfiles | \ - grep -v ^/opt/dell/srvadmin/ | \ + grep -v '^/sbin/[a-z\-]*\.modutils$' | \ + grep -v ^/opt | \ grep -v ^/boot/ | \ grep -v ^/dev/ | \ grep -v ^/etc/ | \ @@ -90,39 +101,75 @@ cat annoyingfiles | \ grep -v ^/var/lib/mysql/ | \ grep -v ^/var/lib/postgres/ | \ grep -v ^/var/lib/postgresql/ | \ - cat > complain - -grep '[a-z/]' complain && \ - mail -a 'From: Adam Megacz ' \ - -s "automated message: annoying files found on deleuze (please do something about them)" admins@hcoop.net \ - < complain + xargs -I{} -d\\n -- bash -c "test -L '{}' || echo '{}'" > complain + +F=hcoop.backup.tar$COMPRESS_EXT.aescrypt +tar clpf - --ignore-failed-read --no-recursion -C / -T backupfiles | \ + $COMPRESS_PROG | \ + ccrypt -k $KEYFILE -e | \ + $MOVE_OVER $CURDATE $F + +# Acquire lock before messing with spamd +COUNT=0 +LOCK=/var/local/lib/spamd/.lock +while test -f $LOCK; do + sleep 2m + COUNT=$(expr $COUNT + 1) + if test $COUNT -eq 10; then + # Enough waiting. Kill the process. + P=$(cat $LOCK) || : + test -n "$P" && kill $P || : + rm -f $LOCK + break + fi +done +touch $LOCK -cp backupfiles /tmp/backupfiles -(cd /; sudo tar cvlpjf - --ignore-failed-read --no-recursion -C / -T /tmp/backupfiles) | \ - gzip | \ - ccrypt -k $KEYFILE -e \ - > hcoop.backup.tar.gz.aescrypt -du -chs hcoop.backup.tar.gz.aescrypt +F=common.spamd.tar$COMPRESS_EXT.aescrypt +tar clpf - --ignore-failed-read -C / /var/local/lib/spamd | \ + $COMPRESS_PROG | \ + ccrypt -k $KEYFILE -e > $F +rm -f $LOCK +< $F $MOVE_OVER $CURDATE $F +rm -f $F vos listvol deleuze | \ - grep -v common.debian | \ tail -n +2 | \ head -n -3 | \ cut -b1-34 | \ grep -v "\.backup .*$" | \ - grep -v "\.readonly .*$" \ + grep -v "\.readonly .*$" | \ + sed 's_^ .*__' | \ + sed 's_ .*$__' | \ + grep '[A-Za-z]' \ > volumes -for A in mysql postgres; do \ -echo backing up $A -tar -C /var/backups/databases/$A -cf - | \ - gzip | \ - ccrypt -k $KEYFILE -e \ - > database-$A.backup.tar.gz.aescrypt -done +cat volumes | \ + grep -v not-backed-up | \ + xargs -I{} -d\\n -- \ + bash -e -c \ + "F={}.dump$COMPRESS_EXT.aescrypt ; + vos dump -id {} -localauth -clone | + $COMPRESS_PROG | ccrypt -k $KEYFILE -e | + $MOVE_OVER $CURDATE \$F" || : + +echo "Backing up databases ..." +F=databases.tar$COMPRESS_EXT.aescrypt +tar -C /var/backups/databases/ -cf - . | \ + $COMPRESS_PROG | \ + ccrypt -k $KEYFILE -e | \ + $MOVE_OVER $CURDATE $F + +# Update file permissions so that rsync.net can access the backups +chmod -R go=,u-w $RSYNCDIR +chmod u+w $RSYNCDIR +chown -R rsync $RSYNCDIR + +# Complain to admins if there are unknown files +grep '[a-z/]' complain && \ + mail -a 'From: The Backup Program ' \ + -s "automated message: annoying files found on deleuze (please do something about them)" admins@hcoop.net \ + < complain \ + || : -for A in `cat volumes` -do \ - echo "dumping afs volume $A..." - time (cd /; sudo vos dump -id $A -localauth -clone) | gzip | ccrypt -k $KEYFILE -e > $A.dump.gz.aescrypt -done +echo "Done."