398ce32e0181a02026cca93ac07a864dceb4626f
[clinton/scripts.git] / s3
1 #!/bin/bash
2 # basic amazon s3 operations
3 # Licensed under the terms of the GNU GPL v2
4 # Copyright 2007 Victor Lowther <victor.lowther@gmail.com>
5
6 CURL=/home/mwolson_admin/bin/curl
7 HMAC=$(dirname $0)/s3-hmac
8 ATTEMPTS=7
9 ATTEMPT_WAIT=10
10
11 # print a message and bail
12 die() {
13 echo $*
14 exit 1
15 }
16
17 # check to see if the variable name passed exists and holds a value.
18 # Die if it does not.
19 check_or_die() {
20 [[ ${!1} ]] || die "Environment variable ${1} is not set."
21 }
22
23 # check to see if we have all the needed S3 variables defined.
24 # Bail if we do not.
25 check_s3() {
26 local sak x
27 for x in S3_ACCESS_KEY_ID S3_SECRET_ACCESS_KEY; do
28 check_or_die ${x};
29 done
30 [[ -f ${S3_SECRET_ACCESS_KEY} ]] || die "S3_SECRET_ACCESS_KEY must point to a file!"
31 sak="$(wc -c "${S3_SECRET_ACCESS_KEY}")"
32 (( ${sak%%[!0-9 ]*} == 40 )) || \
33 die "S3 Secret Access Key is not exactly 40 bytes long. Please fix it."
34 }
35 # check to see if our external dependencies exist
36 check_dep() {
37 local res=0
38 while [[ $# -ne 0 ]]; do
39 which "${1}" >& /dev/null || { res=1; echo "${1} not found."; }
40 shift
41 done
42 (( res == 0 )) || die "aborting."
43 }
44
45 check_hmac() {
46 if test ! -f $HMAC || test ! -x $HMAC; then
47 die "hmac script not found or not executable."
48 fi
49 }
50
51 check_deps() {
52 check_dep openssl date cat grep
53 check_hmac
54 check_s3
55 }
56
57 urlenc() {
58 # $1 = string to url encode
59 # output is on stdout
60 # we don't urlencode everything, just enough stuff.
61 echo -n "${1}" |
62 sed 's/%/%25/g
63 s/ /%20/g
64 s/#/%23/g
65 s/\$/%24/g
66 s/\&/%26/g
67 s/+/%2b/g
68 s/,/%2c/g
69 s/:/%3a/g
70 s/;/%3b/g
71 s/?/%3f/g
72 s/@/%40/g
73 s/ /%09/g'
74 }
75
76 xmldec() {
77 # no parameters.
78 # accept input on stdin, put it on stdout.
79 # patches accepted to get more stuff
80 sed 's/\&quot;/\"/g
81 s/\&amp;/\&/g
82 s/\&lt;/</g
83 s/\&gt;/>/g'
84 }
85
86 ## basic S3 functionality. x-amz-header functionality is not implemented.
87 # make an S3 signature string, which will be output on stdout.
88 s3_signature_string() {
89 # $1 = HTTP verb
90 # $2 = date string, must be in UTC
91 # $3 = bucket name, if any
92 # $4 = resource path, if any
93 # $5 = content md5, if any
94 # $6 = content MIME type, if any
95 # $7 = canonicalized headers, if any
96 # signature string will be output on stdout
97 local verr="Must pass a verb to s3_signature_string!"
98 local verb="${1:?verr}"
99 local bucket="${3}"
100 local resource="${4}"
101 local derr="Must pass a date to s3_signature_string!"
102 local date="${2:?derr}"
103 local mime="${6}"
104 local md5="${5}"
105 local headers="${7}"
106 printf "%s\n%s\n%s\n%s\n%s%s%s" \
107 "${verb}" "${md5}" "${mime}" "${date}" \
108 "${headers}" "${bucket}" "${resource}" | \
109 $HMAC sha1 "${S3_SECRET_ACCESS_KEY}" | openssl base64 -e -a
110 }
111
112 # cheesy, but it is the best way to have multiple headers.
113 curl_headers() {
114 # each arg passed will be output on its own line
115 local parms=$#
116 for ((;$#;)); do
117 echo "header = \"${1}\""
118 shift
119 done
120 }
121
122 s3_curl() {
123 # invoke curl to do all the heavy HTTP lifting
124 # $1 = method (one of GET, PUT, or DELETE. HEAD is not handled yet.)
125 # $2 = remote bucket.
126 # $3 = remote name
127 # $4 = local name.
128 # $5 = bandwidth limit.
129 local bucket remote date sig md5 arg inout headers tries ret
130 # header handling is kinda fugly, but it works.
131 bucket="${2:+/${2}}/" # slashify the bucket
132 remote="$(urlenc "${3}")" # if you don't, strange things may happen.
133 stdopts="--connect-timeout 10 --fail --silent"
134 [[ $CURL_S3_DEBUG == true ]] && stdopts="${stdopts} --show-error --fail"
135 test -n "${5}" && stdopts="${stdopts} --limit-rate ${5}"
136 case "${1}" in
137 GET) arg="-o" inout="${4:--}" # stdout if no $4
138 ;;
139 PUT) [[ ${2} ]] || die "PUT can has bucket?"
140 if [[ ! ${3} ]]; then
141 arg="-X PUT"
142 headers[${#headers[@]}]="Content-Length: 0"
143 elif [[ -f ${4} ]]; then
144 md5="$(openssl dgst -md5 -binary "${4}"|openssl base64 -e -a)"
145 arg="-T" inout="${4}"
146 headers[${#headers[@]}]="Expect: 100-continue"
147 else
148 die "Cannot write non-existing file ${4}"
149 fi
150 ;;
151 DELETE) arg="-X DELETE"
152 ;;
153 HEAD) arg="-I" ;;
154 *) die "Unknown verb ${1}. It probably would not have worked anyways." ;;
155 esac
156 date="$(TZ=UTC date '+%a, %e %b %Y %H:%M:%S %z')"
157 sig=$(s3_signature_string ${1} "${date}" "${bucket}" "${remote}" "${md5}")
158
159 headers[${#headers[@]}]="Authorization: AWS ${S3_ACCESS_KEY_ID}:${sig}"
160 headers[${#headers[@]}]="Date: ${date}"
161 [[ ${md5} ]] && headers[${#headers[@]}]="Content-MD5: ${md5}"
162 tries=0
163 while true; do
164 $CURL ${arg} "${inout}" ${stdopts} -K <(curl_headers "${headers[@]}") \
165 "http://s3.amazonaws.com${bucket}${remote}"
166 ret=$?
167 test $ret -eq 0 && break;
168 if test $tries -lt $ATTEMPTS; then
169 tries=$(expr $tries + 1)
170 echo "Retrying ..."
171 sleep $ATTEMPT_WAIT
172 else
173 break
174 fi
175 done
176 return $ret
177 }
178
179 s3_put() {
180 # $1 = remote bucket to put it into
181 # $2 = remote name to put
182 # $3 = file to put. This must be present if $2 is.
183 # $4 = bandwidth limit.
184 s3_curl PUT "${1}" "${2}" "${3:-${2}}" "${4}"
185 return $?
186 }
187
188 s3_get() {
189 # $1 = bucket to get file from
190 # $2 = remote file to get
191 # $3 = local file to get into. Will be overwritten if it exists.
192 # If this contains a path, that path must exist before calling this.
193 # $4 = bandwidth limit.
194 s3_curl GET "${1}" "${2}" "${3:-${2}}" "${4}"
195 return $?
196 }
197
198 s3_test() {
199 # same args as s3_get, but uses the HEAD verb instead of the GET verb.
200 s3_curl HEAD "${1}" "${2}" >/dev/null
201 return $?
202 }
203
204 # Hideously ugly, but it works well enough.
205 s3_buckets() {
206 s3_get |grep -o '<Name>[^>]*</Name>' |sed 's/<[^>]*>//g' |xmldec
207 return $?
208 }
209
210 # this will only return the first thousand entries, alas
211 # Mabye some kind soul can fix this without writing an XML parser in bash?
212 # Also need to add xml entity handling.
213 s3_list() {
214 # $1 = bucket to list
215 [ "x${1}" == "x" ] && return 1
216 s3_get "${1}" |grep -o '<Key>[^>]*</Key>' |sed 's/<[^>]*>//g'| xmldec
217 return $?
218 }
219
220 s3_delete() {
221 # $1 = bucket to delete from
222 # $2 = item to delete
223 s3_curl DELETE "${1}" "${2}"
224 return $?
225 }
226
227 # because this uses s3_list, it suffers from the same flaws.
228 s3_rmrf() {
229 # $1 = bucket to delete everything from
230 s3_list "${1}" | while read f; do
231 s3_delete "${1}" "${f}";
232 done
233 }
234
235 check_deps
236 case $1 in
237 put) shift; s3_put "$@" ;;
238 get) shift; s3_get "$@" ;;
239 rm) shift; s3_delete "$@" ;;
240 ls) shift; s3_list "$@" ;;
241 test) shift; s3_test "$@" ;;
242 buckets) s3_buckets ;;
243 rmrf) shift; s3_rmrf "$@" ;;
244 *) die "Unknown command ${1}."
245 ;;
246 esac
247