HCoop
/
clinton
/
scripts.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
hcoop-backup: More work.
[clinton/scripts.git]
/
hcoop-backup
diff --git
a/hcoop-backup
b/hcoop-backup
index
6308105
..
6200a65
100755
(executable)
--- a/
hcoop-backup
+++ b/
hcoop-backup
@@
-17,6
+17,8
@@
COMPRESS_EXT=.gz
COMPRESS_PROG=gzip
# units for BWLIMIT are KB/s
BWLIMIT=325
COMPRESS_PROG=gzip
# units for BWLIMIT are KB/s
BWLIMIT=325
+# units for CHUNKSIZE are MB/s
+CHUNKSIZE=5000
IFS=$'\n'
IFS=$'\n'
@@
-32,10
+34,17
@@
export S3_SECRET_ACCESS_KEY=~mwolson_admin/.amazon/secret.key
function s3_cmd () {
# $1: command (get|put|ls|rm)
function s3_cmd () {
# $1: command (get|put|ls|rm)
- # $2: bucket
- # $3: destination
- # $4: source file
- $SCRIPTDIR/s3 $1 $2 $3 $4 ${BWLIMIT}K
+ # $2: remote file
+ # $3: local file
+ local cmd=$1
+ shift
+ local bwarg
+ if test "$cmd" = "put"; then
+ bwarg="${BWLIMIT}K";
+ else
+ bwarg=
+ fi
+ $SCRIPTDIR/s3 $cmd $BUCKET "$1" "$2" $bwarg
}
function copy_over () {
}
function copy_over () {
@@
-50,7
+59,11
@@
function copy_over () {
if test -n "$2"; then
DEST=$DEST/$2
fi
if test -n "$2"; then
DEST=$DEST/$2
fi
- s3_cmd put $BUCKET $DEST/$FILE $FILE
+ split -d -b ${CHUNKSIZE}m $FILE ${FILE}.
+ for i in ${FILE}.*; do
+ s3_cmd put $DEST/$i $i
+ rm -f $i
+ done
rm -f $FILE
}
rm -f $FILE
}
@@
-64,7
+77,7
@@
function prune_old_backups () {
cd $oldpwd
}
cd $oldpwd
}
-prune_old_backups
+
#
prune_old_backups
mkdir -p $BACKUPTMP
cd $BACKUPTMP
mkdir -p $BACKUPTMP
cd $BACKUPTMP