6d52e269 |
1 | #!/bin/bash -e |
2 | |
3 | # |
4b645870 |
4 | # it is dangerous to remove the "-e" above; please don't do that. |
6d52e269 |
5 | # |
6 | |
4b645870 |
7 | # |
8 | # run this script as root, on deleuze |
9 | # |
6d52e269 |
10 | |
be9bd94d |
11 | exec >& /var/log/backup-to-s3-log |
44b7f284 |
12 | |
4b645870 |
13 | PATH=$PATH:/bin:/usr/bin:/sbin:/usr/sbin |
44b7f284 |
14 | #COMPRESS_EXT=.bz2 |
15 | #COMPRESS_PROG=bzip2 |
16 | COMPRESS_EXT=.gz |
17 | COMPRESS_PROG=gzip |
bee5bcbc |
18 | # units for BWLIMIT are KB/s |
451f65fd |
19 | BWLIMIT=325 |
24b2faa6 |
20 | |
4b645870 |
21 | IFS=$'\n' |
22 | |
be9bd94d |
23 | SCRIPTDIR=$(dirname $0) |
6d52e269 |
24 | KEYFILE=/etc/backup-encryption-key |
c13b2355 |
25 | BUCKET=hcoop.net-backups |
26 | BACKUPDIR=full |
be9bd94d |
27 | BACKUPTMP=/var/backups/hcoop-backup-testing |
c13b2355 |
28 | SUBDIR=$(date +%Y.%m.%d) |
bee5bcbc |
29 | |
c13b2355 |
30 | export S3_ACCESS_KEY_ID=$(cat ~mwolson_admin/.amazon/access.key) |
31 | export S3_SECRET_ACCESS_KEY=~mwolson_admin/.amazon/secret.key |
be9bd94d |
32 | |
33 | function s3_cmd () { |
c13b2355 |
34 | # $1: command (get|put|ls|rm) |
35 | # $2: bucket |
36 | # $3: destination |
37 | # $4: source file |
38 | $SCRIPTDIR/s3 $1 $2 $3 $4 ${BWLIMIT}K |
be9bd94d |
39 | } |
bee5bcbc |
40 | |
41 | function copy_over () { |
42 | # Move file to its offsite destination |
43 | # $1: file, $2: relative directory (optional) |
44 | if test -z "$1" || test -n "$3"; then |
45 | echo "Bad programming" |
46 | exit 1 |
47 | fi |
48 | local FILE=$1 |
49 | local DEST=$BACKUPDIR/$SUBDIR |
50 | if test -n "$2"; then |
51 | DEST=$DEST/$2 |
52 | fi |
c13b2355 |
53 | s3_cmd put $BUCKET $DEST/$FILE $FILE |
bee5bcbc |
54 | rm -f $FILE |
55 | } |
24b2faa6 |
56 | |
be9bd94d |
57 | function prune_old_backups () { |
58 | local oldpwd=$PWD |
59 | cd $BACKUPDIR |
60 | find . -mindepth 1 -maxdepth 1 -type d -ctime +7 \ |
61 | -execdir rm -fr '{}' \; || true |
62 | rm -rf $SUBDIR |
63 | mkdir -p $SUBDIR |
64 | cd $oldpwd |
65 | } |
66 | |
67 | prune_old_backups |
68 | |
bee5bcbc |
69 | mkdir -p $BACKUPTMP |
70 | cd $BACKUPTMP |
24b2faa6 |
71 | |
4b645870 |
72 | groups |
24b2faa6 |
73 | echo 'I am in:' |
74 | pwd |
75 | echo |
6d52e269 |
76 | |
77 | echo building package lists... |
24b2faa6 |
78 | dpkg-query -W -f='${Package}\n' > packages |
4b645870 |
79 | (cd /; find / /usr/ /usr/local/ /var/ -xdev) | sort | uniq > allfiles |
24b2faa6 |
80 | dpkg-query -W -f='${Package}\n' | xargs dpkg -L | sort | uniq > debfiles |
bee5bcbc |
81 | dpkg-query -W -f='${Conffiles}\n' | grep / | cut -b2- | \ |
82 | sed 's_ .*__' | sort | uniq > conffiles |
6d52e269 |
83 | |
84 | diff allfiles debfiles | grep '^<' | cut -b 3- | \ |
85 | grep -v ^/var/cache | \ |
86 | grep -v ^/var/tmp | \ |
87 | grep -v ^/var/lib/dpkg | \ |
88 | grep -v ^/var/backups | \ |
89 | grep -v ^/var/lib/changetrack | \ |
12e40abc |
90 | grep -v ^/var/local/lib/spamd | \ |
6d52e269 |
91 | grep -v ^/var/run | \ |
92 | grep -v ^/var/lock | \ |
93 | grep -v ^/var/lib/ucf | \ |
94 | grep -v ^/vicepa | \ |
95 | grep -v ^/home | \ |
96 | grep -v ^/tmp | \ |
97 | grep -v '^/afs$' | \ |
98 | grep -v '^/$' | \ |
99 | grep -v '^/usr/$' | \ |
100 | grep -v ^/usr/src | \ |
92a7af97 |
101 | grep -v '^/usr/.*\.pyc' | \ |
102 | grep -v '^/usr/.*\.elc' | \ |
103 | grep -v '^/usr/bin/perldoc\.stub$' | \ |
5b84f395 |
104 | grep -v '^/usr/bin/.*\.notslocate$' | \ |
d327aed8 |
105 | grep -v '^/usr/lib/courier/.*\.rand$' | \ |
50f51a78 |
106 | grep -v '^/usr/lib/gconv/gconv-modules\.cache$' | \ |
4df0bc18 |
107 | grep -v '^/usr/lib/graphviz/config$' | \ |
50f51a78 |
108 | grep -v '^/usr/lib/locale/locale-archive$' | \ |
24b2faa6 |
109 | grep -v '^/usr/share/info/dir$' | \ |
50f51a78 |
110 | grep -v '^/usr/share/info/dir\.old$' | \ |
24b2faa6 |
111 | grep -v '^/usr/share/emacs21/site-lisp/' | \ |
112 | grep -v '^/usr/share/emacs22/site-lisp/' | \ |
2c5daf49 |
113 | grep -v '^/usr/share/snmp/mibs/\.index$' | \ |
ae0e82f0 |
114 | grep -v '^/usr/share/vim/addons/doc/tags$' \ |
115 | > backupfiles |
6d52e269 |
116 | |
24b2faa6 |
117 | cat conffiles >> backupfiles |
118 | |
6d52e269 |
119 | cat backupfiles | \ |
120 | grep -v ^/home | \ |
121 | grep -v ^/usr/local | \ |
122 | grep -v ^/var/spool | \ |
123 | grep -v ^/var/log | \ |
124 | grep -v ^/usr/lib/python2.4/ | \ |
125 | grep -v ^/var/lib/python-support | \ |
ae0e82f0 |
126 | grep -v ^/usr/share/jed/lib | \ |
6d52e269 |
127 | grep -v ^/usr/share/man | \ |
4df0bc18 |
128 | grep -v ^/usr/share/perl5/IkiWiki/Plugin | \ |
6d52e269 |
129 | grep -v ^/media | \ |
130 | grep -v ^/vmlinuz | \ |
131 | grep -v ^/vmlinuz.old | \ |
92a7af97 |
132 | grep -v '^/sbin/[a-z\-]*\.modutils$' | \ |
6d52e269 |
133 | grep -v ^/opt/dell/srvadmin/ | \ |
134 | grep -v ^/boot/ | \ |
24b2faa6 |
135 | grep -v ^/dev/ | \ |
6d52e269 |
136 | grep -v ^/etc/ | \ |
137 | grep -v ^/root/ | \ |
138 | grep -v ^/var/ | \ |
139 | grep -v ^/lib/modules/ | \ |
140 | grep -v ^/var/domtool/ | \ |
141 | grep -v ^/var/lib/mysql/ | \ |
142 | grep -v ^/var/lib/postgres/ | \ |
143 | grep -v ^/var/lib/postgresql/ | \ |
bee5bcbc |
144 | xargs -I{} -d\\n -- bash -c "test -L '{}' || echo '{}'" > complain |
6d52e269 |
145 | |
bee5bcbc |
146 | F=hcoop.backup.tar$COMPRESS_EXT.aescrypt |
4b645870 |
147 | tar clpf - --ignore-failed-read --no-recursion -C / -T backupfiles | \ |
eede979f |
148 | $COMPRESS_PROG | \ |
bee5bcbc |
149 | ccrypt -k $KEYFILE -e > $F |
150 | copy_over $F |
481c2d5f |
151 | |
152 | # Acquire lock before messing with spamd |
153 | COUNT=0 |
154 | LOCK=/var/local/lib/spamd/.lock |
155 | while test -f $LOCK; do |
156 | sleep 2m |
157 | COUNT=$(expr $COUNT + 1) |
158 | if test $COUNT -eq 10; then |
159 | # Enough waiting. Kill the process. |
160 | P=$(cat $LOCK) || : |
161 | test -n "$P" && kill $P || : |
162 | rm -f $LOCK |
163 | break |
164 | fi |
165 | done |
166 | touch $LOCK |
167 | |
bee5bcbc |
168 | F=common.spamd.tar$COMPRESS_EXT.aescrypt |
481c2d5f |
169 | tar clpf - --ignore-failed-read -C / /var/local/lib/spamd | \ |
170 | $COMPRESS_PROG | \ |
bee5bcbc |
171 | ccrypt -k $KEYFILE -e > $F.new |
481c2d5f |
172 | rm -f $LOCK |
bee5bcbc |
173 | copy_over $F.new .. |
481c2d5f |
174 | |
bee5bcbc |
175 | test -s $BACKUPDIR/$F.new && \ |
176 | mv $BACKUPDIR/$F.new $BACKUPDIR/$F |
481c2d5f |
177 | |
24b2faa6 |
178 | vos listvol deleuze | \ |
24b2faa6 |
179 | tail -n +2 | \ |
180 | head -n -3 | \ |
181 | cut -b1-34 | \ |
182 | grep -v "\.backup .*$" | \ |
eede979f |
183 | grep -v "\.readonly .*$" | \ |
4b645870 |
184 | sed 's_^ .*__' | \ |
185 | sed 's_ .*$__' | \ |
ae0e82f0 |
186 | grep '[A-Za-z]' \ |
187 | > volumes |
24b2faa6 |
188 | |
4b645870 |
189 | cat volumes | \ |
190 | grep -v not-backed-up | \ |
191 | xargs -I{} -d\\n -- \ |
192 | bash -c \ |
bee5bcbc |
193 | "F={}.dump$COMPRESS_EXT.aescrypt ; |
194 | vos dump -id {} -localauth -clone | |
195 | $COMPRESS_PROG | ccrypt -k $KEYFILE -e > \$F ; |
ae0e82f0 |
196 | < \$F catsync -b $BWLIMIT $BACKUPDIR/$SUBDIR/\$F ; |
bee5bcbc |
197 | rm -f \$F" |
4b645870 |
198 | |
199 | echo backing up databases |
bee5bcbc |
200 | F=databases.tar$COMPRESS_EXT.aescrypt |
4b645870 |
201 | tar -C /var/backups/databases/ -cf - . | \ |
202 | $COMPRESS_PROG | \ |
bee5bcbc |
203 | ccrypt -k $KEYFILE -e > $F |
204 | copy_over $F |
4b645870 |
205 | |
206 | grep '[a-z/]' complain && \ |
bee5bcbc |
207 | mail -a 'From: The Backup Program <backups@deleuze.hcoop.net>' \ |
4b645870 |
208 | -s "automated message: annoying files found on deleuze (please do something about them)" admins@hcoop.net \ |
209 | < complain \ |
210 | || true |
44b7f284 |
211 | |
212 | echo done |
213 | |