s3: Make location of hmac script a constant.
[clinton/scripts.git] / s3
CommitLineData
a0d92e63 1#!/bin/bash
2# basic amazon s3 operations
3# Licensed under the terms of the GNU GPL v2
4# Copyright 2007 Victor Lowther <victor.lowther@gmail.com>
5
c347f520 6HMAC=$(dirname $0)/s3-hmac
a0d92e63 7
8# print a message and bail
9die() {
10 echo $*
11 exit 1
12}
13
14# check to see if the variable name passed exists and holds a value.
15# Die if it does not.
16check_or_die() {
17 [[ ${!1} ]] || die "Environment variable ${1} is not set."
18}
19
20# check to see if we have all the needed S3 variables defined.
21# Bail if we do not.
22check_s3() {
23 local sak x
24 for x in S3_ACCESS_KEY_ID S3_SECRET_ACCESS_KEY; do
25 check_or_die ${x};
26 done
27 [[ -f ${S3_SECRET_ACCESS_KEY} ]] || die "S3_SECRET_ACCESS_KEY must point to a file!"
28 sak="$(wc -c "${S3_SECRET_ACCESS_KEY}")"
29 (( ${sak%%[!0-9 ]*} == 40 )) || \
30 die "S3 Secret Access Key is not exactly 40 bytes long. Please fix it."
31}
32# check to see if our external dependencies exist
33check_dep() {
34 local res=0
35 while [[ $# -ne 0 ]]; do
36 which "${1}" >& /dev/null || { res=1; echo "${1} not found."; }
37 shift
38 done
39 (( res == 0 )) || die "aborting."
40}
41
c347f520 42check_hmac() {
43 if test ! -f $HMAC || test ! -x $HMAC; then
44 die "hmac script not found or not executable."
45 fi
46}
47
a0d92e63 48check_deps() {
c347f520 49 check_dep openssl date cat grep curl
50 check_hmac
a0d92e63 51 check_s3
52}
53
54urlenc() {
55 # $1 = string to url encode
56 # output is on stdout
57 # we don't urlencode everything, just enough stuff.
58 echo -n "${1}" |
59 sed 's/%/%25/g
60 s/ /%20/g
61 s/#/%23/g
62 s/\$/%24/g
63 s/\&/%26/g
64 s/+/%2b/g
65 s/,/%2c/g
66 s/:/%3a/g
67 s/;/%3b/g
68 s/?/%3f/g
69 s/@/%40/g
70 s/ /%09/g'
71}
72
73xmldec() {
74 # no parameters.
75 # accept input on stdin, put it on stdout.
76 # patches accepted to get more stuff
77 sed 's/\&quot;/\"/g
78 s/\&amp;/\&/g
79 s/\&lt;/</g
80 s/\&gt;/>/g'
81}
82
83## basic S3 functionality. x-amz-header functionality is not implemented.
84# make an S3 signature string, which will be output on stdout.
85s3_signature_string() {
86 # $1 = HTTP verb
87 # $2 = date string, must be in UTC
88 # $3 = bucket name, if any
89 # $4 = resource path, if any
90 # $5 = content md5, if any
91 # $6 = content MIME type, if any
92 # $7 = canonicalized headers, if any
93 # signature string will be output on stdout
94 local verr="Must pass a verb to s3_signature_string!"
95 local verb="${1:?verr}"
96 local bucket="${3}"
97 local resource="${4}"
98 local derr="Must pass a date to s3_signature_string!"
99 local date="${2:?derr}"
100 local mime="${6}"
101 local md5="${5}"
102 local headers="${7}"
103 printf "%s\n%s\n%s\n%s\n%s%s%s" \
104 "${verb}" "${md5}" "${mime}" "${date}" \
105 "${headers}" "${bucket}" "${resource}" | \
c347f520 106 $HMAC sha1 "${S3_SECRET_ACCESS_KEY}" | openssl base64 -e -a
a0d92e63 107}
108
109# cheesy, but it is the best way to have multiple headers.
110curl_headers() {
111 # each arg passed will be output on its own line
112 local parms=$#
113 for ((;$#;)); do
114 echo "header = \"${1}\""
115 shift
116 done
117}
118
119s3_curl() {
120 # invoke curl to do all the heavy HTTP lifting
121 # $1 = method (one of GET, PUT, or DELETE. HEAD is not handled yet.)
122 # $2 = remote bucket.
123 # $3 = remote name
124 # $4 = local name.
125 local bucket remote date sig md5 arg inout headers
126 # header handling is kinda fugly, but it works.
127 bucket="${2:+/${2}}/" # slashify the bucket
128 remote="$(urlenc "${3}")" # if you don't, strange things may happen.
129 stdopts="--connect-timeout 10 --fail --silent"
130 [[ $CURL_S3_DEBUG == true ]] && stdopts="${stdopts} --show-error --fail"
131 case "${1}" in
132 GET) arg="-o" inout="${4:--}" # stdout if no $4
133 ;;
134 PUT) [[ ${2} ]] || die "PUT can has bucket?"
135 if [[ ! ${3} ]]; then
136 arg="-X PUT"
137 headers[${#headers[@]}]="Content-Length: 0"
138 elif [[ -f ${4} ]]; then
139 md5="$(openssl dgst -md5 -binary "${4}"|openssl base64 -e -a)"
140 arg="-T" inout="${4}"
141 headers[${#headers[@]}]="Expect: 100-continue"
142 else
143 die "Cannot write non-existing file ${4}"
144 fi
145 ;;
146 DELETE) arg="-X DELETE"
147 ;;
148 HEAD) arg="-I" ;;
149 *) die "Unknown verb ${1}. It probably would not have worked anyways." ;;
150 esac
151 date="$(TZ=UTC date '+%a, %e %b %Y %H:%M:%S %z')"
152 sig=$(s3_signature_string ${1} "${date}" "${bucket}" "${remote}" "${md5}")
153
154 headers[${#headers[@]}]="Authorization: AWS ${S3_ACCESS_KEY_ID}:${sig}"
155 headers[${#headers[@]}]="Date: ${date}"
156 [[ ${md5} ]] && headers[${#headers[@]}]="Content-MD5: ${md5}"
157 curl ${arg} "${inout}" ${stdopts} -K <(curl_headers "${headers[@]}") \
158 "http://s3.amazonaws.com${bucket}${remote}"
159 return $?
160}
161
162s3_put() {
163 # $1 = remote bucket to put it into
164 # $2 = remote name to put
165 # $3 = file to put. This must be present if $2 is.
166 s3_curl PUT "${1}" "${2}" "${3:-${2}}"
167 return $?
168}
169
170s3_get() {
171 # $1 = bucket to get file from
172 # $2 = remote file to get
173 # $3 = local file to get into. Will be overwritten if it exists.
174 # If this contains a path, that path must exist before calling this.
175 s3_curl GET "${1}" "${2}" "${3:-${2}}"
176 return $?
177}
178
179s3_test() {
180 # same args as s3_get, but uses the HEAD verb instead of the GET verb.
181 s3_curl HEAD "${1}" "${2}" >/dev/null
182 return $?
183}
184
185# Hideously ugly, but it works well enough.
186s3_buckets() {
187 s3_get |grep -o '<Name>[^>]*</Name>' |sed 's/<[^>]*>//g' |xmldec
188 return $?
189}
190
191# this will only return the first thousand entries, alas
192# Mabye some kind soul can fix this without writing an XML parser in bash?
193# Also need to add xml entity handling.
194s3_list() {
195 # $1 = bucket to list
196 [ "x${1}" == "x" ] && return 1
197 s3_get "${1}" |grep -o '<Key>[^>]*</Key>' |sed 's/<[^>]*>//g'| xmldec
198 return $?
199}
200
201s3_delete() {
202 # $1 = bucket to delete from
203 # $2 = item to delete
204 s3_curl DELETE "${1}" "${2}"
205 return $?
206}
207
208# because this uses s3_list, it suffers from the same flaws.
209s3_rmrf() {
210 # $1 = bucket to delete everything from
211 s3_list "${1}" | while read f; do
212 s3_delete "${1}" "${f}";
213 done
214}
215
216check_deps
217case $1 in
218 put) shift; s3_put "$@" ;;
219 get) shift; s3_get "$@" ;;
220 rm) shift; s3_delete "$@" ;;
221 ls) shift; s3_list "$@" ;;
222 test) shift; s3_test "$@" ;;
223 buckets) s3_buckets ;;
224 rmrf) shift; s3_rmrf "$@" ;;
225 *) die "Unknown command ${1}."
226 ;;
227esac
228