| 1 | #!/bin/bash -e |
| 2 | |
| 3 | # |
| 4 | # it is dangerous to remove the "-e" above; please don't do that. |
| 5 | # |
| 6 | |
| 7 | # |
| 8 | # run this script as root, on deleuze |
| 9 | # |
| 10 | |
| 11 | exec >& /var/log/backup-to-megacz.com-log |
| 12 | |
| 13 | PATH=$PATH:/bin:/usr/bin:/sbin:/usr/sbin |
| 14 | #COMPRESS_EXT=.bz2 |
| 15 | #COMPRESS_PROG=bzip2 |
| 16 | COMPRESS_EXT=.gz |
| 17 | COMPRESS_PROG=gzip |
| 18 | # units for BWLIMIT are KB/s |
| 19 | BWLIMIT=325 |
| 20 | |
| 21 | IFS=$'\n' |
| 22 | |
| 23 | KEYFILE=/etc/backup-encryption-key |
| 24 | BACKUPDIR=/afs/megacz.com/private/hcoop-backup |
| 25 | BACKUPTMP=/var/backups/hcoop-backup |
| 26 | SUBDIR=`date +%Y.%m.%d` |
| 27 | |
| 28 | #SYNC_CMD="rsync --bwlimit=$BWLIMIT --remove-source-files" |
| 29 | |
| 30 | function copy_over () { |
| 31 | # Move file to its offsite destination |
| 32 | # $1: file, $2: relative directory (optional) |
| 33 | if test -z "$1" || test -n "$3"; then |
| 34 | echo "Bad programming" |
| 35 | exit 1 |
| 36 | fi |
| 37 | local FILE=$1 |
| 38 | local DEST=$BACKUPDIR/$SUBDIR |
| 39 | if test -n "$2"; then |
| 40 | DEST=$DEST/$2 |
| 41 | fi |
| 42 | < $FILE catsync -b $BWLIMIT $DEST/$FILE |
| 43 | rm -f $FILE |
| 44 | } |
| 45 | |
| 46 | cd $BACKUPDIR |
| 47 | find . -mindepth 1 -maxdepth 1 -type d -ctime +7 \ |
| 48 | -execdir rm -fr '{}' \; || true |
| 49 | rm -rf $SUBDIR |
| 50 | mkdir -p $SUBDIR |
| 51 | mkdir -p $BACKUPTMP |
| 52 | cd $BACKUPTMP |
| 53 | |
| 54 | groups |
| 55 | echo 'I am in:' |
| 56 | pwd |
| 57 | echo |
| 58 | |
| 59 | echo building package lists... |
| 60 | dpkg-query -W -f='${Package}\n' > packages |
| 61 | (cd /; find / /usr/ /usr/local/ /var/ -xdev) | sort | uniq > allfiles |
| 62 | dpkg-query -W -f='${Package}\n' | xargs dpkg -L | sort | uniq > debfiles |
| 63 | dpkg-query -W -f='${Conffiles}\n' | grep / | cut -b2- | \ |
| 64 | sed 's_ .*__' | sort | uniq > conffiles |
| 65 | |
| 66 | diff allfiles debfiles | grep '^<' | cut -b 3- | \ |
| 67 | grep -v ^/var/cache | \ |
| 68 | grep -v ^/var/tmp | \ |
| 69 | grep -v ^/var/lib/dpkg | \ |
| 70 | grep -v ^/var/backups | \ |
| 71 | grep -v ^/var/lib/changetrack | \ |
| 72 | grep -v ^/var/local/lib/spamd | \ |
| 73 | grep -v ^/var/run | \ |
| 74 | grep -v ^/var/lock | \ |
| 75 | grep -v ^/var/lib/ucf | \ |
| 76 | grep -v ^/vicepa | \ |
| 77 | grep -v ^/home | \ |
| 78 | grep -v ^/tmp | \ |
| 79 | grep -v '^/afs$' | \ |
| 80 | grep -v '^/$' | \ |
| 81 | grep -v '^/usr/$' | \ |
| 82 | grep -v ^/usr/src | \ |
| 83 | grep -v '^/usr/.*\.pyc' | \ |
| 84 | grep -v '^/usr/.*\.elc' | \ |
| 85 | grep -v '^/usr/bin/perldoc\.stub$' | \ |
| 86 | grep -v '^/usr/bin/.*\.notslocate$' | \ |
| 87 | grep -v '^/usr/lib/courier/.*\.rand$' | \ |
| 88 | grep -v '^/usr/lib/gconv/gconv-modules\.cache$' | \ |
| 89 | grep -v '^/usr/lib/graphviz/config$' | \ |
| 90 | grep -v '^/usr/lib/locale/locale-archive$' | \ |
| 91 | grep -v '^/usr/share/info/dir$' | \ |
| 92 | grep -v '^/usr/share/info/dir\.old$' | \ |
| 93 | grep -v '^/usr/share/emacs21/site-lisp/' | \ |
| 94 | grep -v '^/usr/share/emacs22/site-lisp/' | \ |
| 95 | grep -v '^/usr/share/snmp/mibs/\.index$' | \ |
| 96 | grep -v '^/usr/share/vim/addons/doc/tags$' \ |
| 97 | > backupfiles |
| 98 | |
| 99 | cat conffiles >> backupfiles |
| 100 | |
| 101 | cat backupfiles | \ |
| 102 | grep -v ^/home | \ |
| 103 | grep -v ^/usr/local | \ |
| 104 | grep -v ^/var/spool | \ |
| 105 | grep -v ^/var/log | \ |
| 106 | grep -v ^/usr/lib/python2.4/ | \ |
| 107 | grep -v ^/var/lib/python-support | \ |
| 108 | grep -v ^/usr/share/jed/lib | \ |
| 109 | grep -v ^/usr/share/man | \ |
| 110 | grep -v ^/usr/share/perl5/IkiWiki/Plugin | \ |
| 111 | grep -v ^/media | \ |
| 112 | grep -v ^/vmlinuz | \ |
| 113 | grep -v ^/vmlinuz.old | \ |
| 114 | grep -v '^/sbin/[a-z\-]*\.modutils$' | \ |
| 115 | grep -v ^/opt/dell/srvadmin/ | \ |
| 116 | grep -v ^/boot/ | \ |
| 117 | grep -v ^/dev/ | \ |
| 118 | grep -v ^/etc/ | \ |
| 119 | grep -v ^/root/ | \ |
| 120 | grep -v ^/var/ | \ |
| 121 | grep -v ^/lib/modules/ | \ |
| 122 | grep -v ^/var/domtool/ | \ |
| 123 | grep -v ^/var/lib/mysql/ | \ |
| 124 | grep -v ^/var/lib/postgres/ | \ |
| 125 | grep -v ^/var/lib/postgresql/ | \ |
| 126 | xargs -I{} -d\\n -- bash -c "test -L '{}' || echo '{}'" > complain |
| 127 | |
| 128 | F=hcoop.backup.tar$COMPRESS_EXT.aescrypt |
| 129 | tar clpf - --ignore-failed-read --no-recursion -C / -T backupfiles | \ |
| 130 | $COMPRESS_PROG | \ |
| 131 | ccrypt -k $KEYFILE -e > $F |
| 132 | copy_over $F |
| 133 | |
| 134 | # Acquire lock before messing with spamd |
| 135 | COUNT=0 |
| 136 | LOCK=/var/local/lib/spamd/.lock |
| 137 | while test -f $LOCK; do |
| 138 | sleep 2m |
| 139 | COUNT=$(expr $COUNT + 1) |
| 140 | if test $COUNT -eq 10; then |
| 141 | # Enough waiting. Kill the process. |
| 142 | P=$(cat $LOCK) || : |
| 143 | test -n "$P" && kill $P || : |
| 144 | rm -f $LOCK |
| 145 | break |
| 146 | fi |
| 147 | done |
| 148 | touch $LOCK |
| 149 | |
| 150 | F=common.spamd.tar$COMPRESS_EXT.aescrypt |
| 151 | tar clpf - --ignore-failed-read -C / /var/local/lib/spamd | \ |
| 152 | $COMPRESS_PROG | \ |
| 153 | ccrypt -k $KEYFILE -e > $F.new |
| 154 | rm -f $LOCK |
| 155 | copy_over $F.new .. |
| 156 | |
| 157 | test -s $BACKUPDIR/$F.new && \ |
| 158 | mv $BACKUPDIR/$F.new $BACKUPDIR/$F |
| 159 | |
| 160 | vos listvol deleuze | \ |
| 161 | tail -n +2 | \ |
| 162 | head -n -3 | \ |
| 163 | cut -b1-34 | \ |
| 164 | grep -v "\.backup .*$" | \ |
| 165 | grep -v "\.readonly .*$" | \ |
| 166 | sed 's_^ .*__' | \ |
| 167 | sed 's_ .*$__' | \ |
| 168 | grep '[A-Za-z]' \ |
| 169 | > volumes |
| 170 | |
| 171 | cat volumes | \ |
| 172 | grep -v not-backed-up | \ |
| 173 | xargs -I{} -d\\n -- \ |
| 174 | bash -c \ |
| 175 | "F={}.dump$COMPRESS_EXT.aescrypt ; |
| 176 | vos dump -id {} -localauth -clone | |
| 177 | $COMPRESS_PROG | ccrypt -k $KEYFILE -e > \$F ; |
| 178 | < \$F catsync -b $BWLIMIT $BACKUPDIR/$SUBDIR/\$F ; |
| 179 | rm -f \$F" |
| 180 | |
| 181 | echo backing up databases |
| 182 | F=databases.tar$COMPRESS_EXT.aescrypt |
| 183 | tar -C /var/backups/databases/ -cf - . | \ |
| 184 | $COMPRESS_PROG | \ |
| 185 | ccrypt -k $KEYFILE -e > $F |
| 186 | copy_over $F |
| 187 | |
| 188 | grep '[a-z/]' complain && \ |
| 189 | mail -a 'From: The Backup Program <backups@deleuze.hcoop.net>' \ |
| 190 | -s "automated message: annoying files found on deleuze (please do something about them)" admins@hcoop.net \ |
| 191 | < complain \ |
| 192 | || true |
| 193 | |
| 194 | echo done |
| 195 | |