hcoop-backup: More work.

[clinton/scripts.git] / hcoop-backup

diff --git a/hcoop-backup b/hcoop-backup
index 9ecad5d..6200a65 100755 (executable)
--- a/hcoop-backup
+++ b/hcoop-backup
@@ -17,22 +17,34 @@ COMPRESS_EXT=.gz
 COMPRESS_PROG=gzip
 # units for BWLIMIT are KB/s
 BWLIMIT=325
+# units for CHUNKSIZE are MB/s
+CHUNKSIZE=5000
 
 IFS=$'\n'
 
 SCRIPTDIR=$(dirname $0)
 KEYFILE=/etc/backup-encryption-key
-BACKUPDIR=/hcoop.net-backups
+BUCKET=hcoop.net-backups
+BACKUPDIR=full
 BACKUPTMP=/var/backups/hcoop-backup-testing
-SUBDIR=`date +%Y.%m.%d`
+SUBDIR=$(date +%Y.%m.%d)
 
-S3_ACCESS_KEY=$(cat ~mwolson_admin/.amazon/access.key)
-S3_SECRET_KEY_FILE=~mwolson_admin/.amazon/secret.key
+export S3_ACCESS_KEY_ID=$(cat ~mwolson_admin/.amazon/access.key)
+export S3_SECRET_ACCESS_KEY=~mwolson_admin/.amazon/secret.key
 
 function s3_cmd () {
+    # $1: command (get|put|ls|rm)
+    # $2: remote file
+    # $3: local file
     local cmd=$1
     shift
-    $SCRIPTDIR/s3-$cmd -k $S3_ACCESS_KEY -s $S3_SECRET_KEY_FILE $@
+    local bwarg
+    if test "$cmd" = "put"; then
+        bwarg="${BWLIMIT}K";
+    else
+        bwarg=
+    fi
+    $SCRIPTDIR/s3 $cmd $BUCKET "$1" "$2" $bwarg
 }
 
 function copy_over () {
@@ -47,7 +59,11 @@ function copy_over () {
     if test -n "$2"; then
         DEST=$DEST/$2
     fi
-    s3_cmd put -b ${BWLIMIT}K -T $FILE $DEST/$FILE
+    split -d -b ${CHUNKSIZE}m $FILE ${FILE}.
+    for i in ${FILE}.*; do
+        s3_cmd put $DEST/$i $i
+        rm -f $i
+    done
     rm -f $FILE
 }
 
@@ -61,7 +77,7 @@ function prune_old_backups () {
     cd $oldpwd
 }
 
-prune_old_backups
+#prune_old_backups
 
 mkdir -p $BACKUPTMP
 cd $BACKUPTMP