X-Git-Url: http://git.hcoop.net/clinton/scripts.git/blobdiff_plain/92a7af9747deeabe6c5eae4937fd2054a9f613a4..436baa0cc6238d3713f2067b4011431a9385f8cc:/hcoop-backup diff --git a/hcoop-backup b/hcoop-backup index dc67a9e..0d1d520 100755 --- a/hcoop-backup +++ b/hcoop-backup @@ -1,40 +1,67 @@ #!/bin/bash -e # -# run this script as root, on deleuze, in a directory that WILL NOT be -# part of the backup (/tmp is a good choice). +# it is dangerous to remove the "-e" above; please don't do that. # -# FIXME: -# postgres dumps -# mysql dumps +# +# run this script as root, on deleuze +# -# for catsync -PATH=$PATH:/home/megacz_admin/ -COMPRESS_EXT=.bz2 -COMPRESS_PROG=bzip2 +exec >& /var/log/backup-to-megacz.com-log -#KEYFILE=$1 -KEYFILE=/etc/backup-encryption-key -BACKUPDIR=/afs/megacz.com/hcoop-backup/ +PATH=$PATH:/bin:/usr/bin:/sbin:/usr/sbin +#COMPRESS_EXT=.bz2 +#COMPRESS_PROG=bzip2 +COMPRESS_EXT=.gz +COMPRESS_PROG=gzip +# units for BWLIMIT are KB/s +BWLIMIT=325 -cd $BACKUPDIR -find * -prune -ctime +5 -exec rm -rf {} \; +IFS=$'\n' +KEYFILE=/etc/backup-encryption-key +BACKUPDIR=/afs/megacz.com/private/hcoop-backup +BACKUPTMP=/var/backups/hcoop-backup SUBDIR=`date +%Y.%m.%d` + +#SYNC_CMD="rsync --bwlimit=$BWLIMIT --remove-source-files" + +function copy_over () { + # Move file to its offsite destination + # $1: file, $2: relative directory (optional) + if test -z "$1" || test -n "$3"; then + echo "Bad programming" + exit 1 + fi + local FILE=$1 + local DEST=$BACKUPDIR/$SUBDIR + if test -n "$2"; then + DEST=$DEST/$2 + fi + < $FILE catsync -b $BWLIMIT $DEST/$FILE + rm -f $FILE +} + +cd $BACKUPDIR +find . -mindepth 1 -maxdepth 1 -type d -ctime +7 \ + -execdir rm -fr '{}' \; || true rm -rf $SUBDIR mkdir -p $SUBDIR -cd $SUBDIR +mkdir -p $BACKUPTMP +cd $BACKUPTMP +groups echo 'I am in:' pwd echo echo building package lists... dpkg-query -W -f='${Package}\n' > packages -(cd /; sudo find / /usr/ /usr/local/ /var/ -xdev) | sort | uniq > allfiles +(cd /; find / /usr/ /usr/local/ /var/ -xdev) | sort | uniq > allfiles dpkg-query -W -f='${Package}\n' | xargs dpkg -L | sort | uniq > debfiles -dpkg-query -W -f='${Conffiles}\n' | grep / | cut -b2- | sed 's_ .*__' | sort | uniq > conffiles +dpkg-query -W -f='${Conffiles}\n' | grep / | cut -b2- | \ + sed 's_ .*__' | sort | uniq > conffiles diff allfiles debfiles | grep '^<' | cut -b 3- | \ grep -v ^/var/cache | \ @@ -42,6 +69,7 @@ diff allfiles debfiles | grep '^<' | cut -b 3- | \ grep -v ^/var/lib/dpkg | \ grep -v ^/var/backups | \ grep -v ^/var/lib/changetrack | \ + grep -v ^/var/local/lib/spamd | \ grep -v ^/var/run | \ grep -v ^/var/lock | \ grep -v ^/var/lib/ucf | \ @@ -55,6 +83,8 @@ diff allfiles debfiles | grep '^<' | cut -b 3- | \ grep -v '^/usr/.*\.pyc' | \ grep -v '^/usr/.*\.elc' | \ grep -v '^/usr/bin/perldoc\.stub$' | \ + grep -v '^/usr/bin/.*\.notslocate$' | \ + grep -v '^/usr/lib/courier/.*\.rand$' | \ grep -v '^/usr/lib/gconv/gconv-modules\.cache$' | \ grep -v '^/usr/lib/graphviz/config$' | \ grep -v '^/usr/lib/locale/locale-archive$' | \ @@ -62,8 +92,9 @@ diff allfiles debfiles | grep '^<' | cut -b 3- | \ grep -v '^/usr/share/info/dir\.old$' | \ grep -v '^/usr/share/emacs21/site-lisp/' | \ grep -v '^/usr/share/emacs22/site-lisp/' | \ - grep -v '^/usr/share/vim/addons/doc/tags$' | \ - cat > backupfiles + grep -v '^/usr/share/snmp/mibs/\.index$' | \ + grep -v '^/usr/share/vim/addons/doc/tags$' \ + > backupfiles cat conffiles >> backupfiles @@ -74,20 +105,13 @@ cat backupfiles | \ grep -v ^/var/log | \ grep -v ^/usr/lib/python2.4/ | \ grep -v ^/var/lib/python-support | \ + grep -v ^/usr/share/jed/lib | \ grep -v ^/usr/share/man | \ grep -v ^/usr/share/perl5/IkiWiki/Plugin | \ grep -v ^/media | \ grep -v ^/vmlinuz | \ grep -v ^/vmlinuz.old | \ grep -v '^/sbin/[a-z\-]*\.modutils$' | \ - cat > annoyingfiles- - -for A in `cat annoyingfiles-` -do \ - test -L "$A" || echo "$A" -done > annoyingfiles - -cat annoyingfiles | \ grep -v ^/opt/dell/srvadmin/ | \ grep -v ^/boot/ | \ grep -v ^/dev/ | \ @@ -99,19 +123,39 @@ cat annoyingfiles | \ grep -v ^/var/lib/mysql/ | \ grep -v ^/var/lib/postgres/ | \ grep -v ^/var/lib/postgresql/ | \ - cat > complain + xargs -I{} -d\\n -- bash -c "test -L '{}' || echo '{}'" > complain -grep '[a-z/]' complain && \ - mail -a 'From: Adam Megacz ' \ - -s "automated message: annoying files found on deleuze (please do something about them)" admins@hcoop.net \ - < complain +F=hcoop.backup.tar$COMPRESS_EXT.aescrypt +tar clpf - --ignore-failed-read --no-recursion -C / -T backupfiles | \ + $COMPRESS_PROG | \ + ccrypt -k $KEYFILE -e > $F +copy_over $F + +# Acquire lock before messing with spamd +COUNT=0 +LOCK=/var/local/lib/spamd/.lock +while test -f $LOCK; do + sleep 2m + COUNT=$(expr $COUNT + 1) + if test $COUNT -eq 10; then + # Enough waiting. Kill the process. + P=$(cat $LOCK) || : + test -n "$P" && kill $P || : + rm -f $LOCK + break + fi +done +touch $LOCK -cp backupfiles /tmp/backupfiles -(cd /; sudo tar cvlpjf - --ignore-failed-read --no-recursion -C / -T /tmp/backupfiles) | \ +F=common.spamd.tar$COMPRESS_EXT.aescrypt +tar clpf - --ignore-failed-read -C / /var/local/lib/spamd | \ $COMPRESS_PROG | \ - ccrypt -k $KEYFILE -e | \ - catsync hcoop.backup.tar$COMPRESS_EXT.aescrypt -du -chs hcoop.backup.tar$COMPRESS_EXT.aescrypt + ccrypt -k $KEYFILE -e > $F.new +rm -f $LOCK +copy_over $F.new .. + +test -s $BACKUPDIR/$F.new && \ + mv $BACKUPDIR/$F.new $BACKUPDIR/$F vos listvol deleuze | \ tail -n +2 | \ @@ -119,19 +163,33 @@ vos listvol deleuze | \ cut -b1-34 | \ grep -v "\.backup .*$" | \ grep -v "\.readonly .*$" | \ - cat > volumes + sed 's_^ .*__' | \ + sed 's_ .*$__' | \ + grep '[A-Za-z]' \ + > volumes + +cat volumes | \ + grep -v not-backed-up | \ + xargs -I{} -d\\n -- \ + bash -c \ + "F={}.dump$COMPRESS_EXT.aescrypt ; + vos dump -id {} -localauth -clone | + $COMPRESS_PROG | ccrypt -k $KEYFILE -e > \$F ; + < \$F catsync -b $BWLIMIT $BACKUPDIR/$SUBDIR/\$F ; + rm -f \$F" echo backing up databases +F=databases.tar$COMPRESS_EXT.aescrypt tar -C /var/backups/databases/ -cf - . | \ $COMPRESS_PROG | \ - ccrypt -k $KEYFILE -e \ - | catsync databases.tar$COMPRESS_EXT.aescrypt - -for A in `cat volumes | grep -v not-backed-up` -do \ - echo "dumping afs volume $A..." - time (cd /; sudo vos dump -id $A -localauth -clone) | \ - $COMPRESS_PROG | \ - ccrypt -k $KEYFILE -e | \ - catsync $A.dump$COMPRESS_EXT.aescrypt -done + ccrypt -k $KEYFILE -e > $F +copy_over $F + +grep '[a-z/]' complain && \ + mail -a 'From: The Backup Program ' \ + -s "automated message: annoying files found on deleuze (please do something about them)" admins@hcoop.net \ + < complain \ + || true + +echo done +