s3-put: Implement bandwidth limit via -b argument.
[clinton/scripts.git] / s3-common-functions
CommitLineData
7f2282a7 1#! /usr/bin/env bash
2cat > /dev/null << EndOfLicence
3s3-bash
4Copyright 2007 Raphael James Cohn
5
6Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
7in compliance with the License.
8You may obtain a copy of the License at
9
10 http://www.apache.org/licenses/LICENSE-2.0
11
12Unless required by applicable law or agreed to in writing, software distributed under the License
13is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
14or implied. See the License for the specific language governing permissions and limitations under
15the License.
16EndOfLicence
17
18# Pragmas
19set -u
20set -e
21
22# Constants
23readonly version="0.02"
24readonly userSpecifiedDataErrorExitCode=1
25readonly invalidCommandLineOption=2
26readonly internalErrorExitCode=3
27readonly invalidEnvironmentExitCode=4
28readonly ipadXorByte=0x36
29readonly opadXorByte=0x5c
30
31# Command-like aliases
32readonly sha1="openssl dgst -sha1 -binary"
33readonly base64encode="openssl enc -base64 -e -in"
34readonly base64decode="openssl enc -base64 -d -in"
35
36# Globals
37declare -a temporaryFiles
38
39function base64EncodedMD5
40{
41 openssl dgst -md5 -binary "$1" | openssl enc -e -base64
42}
43
44function printErrorMessage
45{
46 printf "%s: %s\n" "$1" "$2" 1>&2
47}
48
49function printErrorHelpAndExit
50{
51 printErrorMessage "$weAreKnownAs" "$1"
52 printHelpAndExit $2
53}
54
55function checkProgramIsInEnvironment
56{
57 if [ ! -x "$(which $1)" ]; then
58 printErrorHelpAndExit "Environment Error: $1 not found on the path or not executable" $invalidEnvironmentExitCode
59 fi
60}
61
62# Do not use this from directly. Due to a bug in bash, array assignments do not work when the function is used with command substitution
63function createTemporaryFile
64{
65 local temporaryFile="$(mktemp "$temporaryDirectory/$$.$1.XXXXXXXX")" || printErrorHelpAndExit "Environment Error: Could not create a temporary file. Please check you /tmp folder permissions allow files and folders to be created and disc space." $invalidEnvironmentExitCode
66 local length="${#temporaryFiles[@]}"
67 temporaryFiles[$length]="$temporaryFile"
68}
69
70function mostRecentTemporaryFile
71{
72 local length="${#temporaryFiles[@]}"
73 local lastIndex
74 ((lastIndex = --length))
75 echo "${temporaryFiles[$lastIndex]}"
76}
77
78function deleteTemporaryFile
79{
80 rm -f "$1" || printErrorHelpAndExit "Environment Error: Could not delete a temporary file ($1)." $invalidEnvironmentExitCode
81}
82
83function removeTemporaryFiles
84{
85 length="${#temporaryFiles[@]}"
86 if [ $length -eq 0 ]; then
87 return
88 fi
89 for temporaryFile in ${temporaryFiles[@]}; do
90 deleteTemporaryFile "$temporaryFile"
91 done
92 temporaryFiles=()
93 length="${#temporaryFiles[@]}"
94}
95
96function checkEnvironment
97{
98 programs=(openssl curl od dd printf sed awk sort mktemp rm grep cp ls env bash)
99 for program in "${programs[@]}"; do
100 checkProgramIsInEnvironment "$program"
101 done
102
103 local temporaryFolder="${TMPDIR:-/tmp}"
104 if [ ! -x "$temporaryFolder" ]; then
105 printErrorHelpAndExit "Environment Error: The temporary directory ($temporaryFolder) does not exist. Please set the TMPDIR environment variable to your temporary directory" $invalidEnvironmentExitCode
106 fi
107 readonly temporaryDirectory="$temporaryFolder/s3-bash/$weAreKnownAs"
108 mkdir -p "$temporaryDirectory" || printErrorHelpAndExit "Environment Error: Could not create a temporary directory ($temporaryDiectory). Please check you /tmp folder permissions allow files and folders to be created and you have sufficient disc space" $invalidEnvironmentExitCode
109
110 #Check we can create and delete temporary files
111 createTemporaryFile "check"
112 temporaryFileCheck="$(mostRecentTemporaryFile)"
113 echo "Checking we can write to temporary files. If this is still here then we could not delete temporary files." > "$temporaryFileCheck"
114 removeTemporaryFiles
115}
116
117function setErrorTraps
118{
119 trap "removeTemporaryFiles; exit $internalErrorExitCode" INT TERM EXIT
120}
121
122function unsetErrorTraps
123{
124 trap - INT TERM EXIT
125}
126
127function verifyUrl
128{
129 if [ -z "$url" ]; then
130 printErrorHelpAndExit "URL not specified" $userSpecifiedDataErrorExitCode
131 elif echo $url | grep -q http://; then
132 printErrorHelpAndExit "URL starts with http://" $userSpecifiedDataErrorExitCode
133 elif echo $url | grep -q https://; then
134 printErrorHelpAndExit "URL starts with https://" $userSpecifiedDataErrorExitCode
135 elif echo $url | grep -v ^/; then
136 printErrorHelpAndExit "URL does not start with /" $userSpecifiedDataErrorExitCode
137 fi
138}
139
140function appendHash
141{
142 local fileToHash="$1"
143 local fileToWriteTo="$2"
144 $sha1 "$fileToHash" >> "$fileToWriteTo"
145}
146
147function writeHash
148{
149 local fileToHash="$1"
150 local fileToWriteTo="$2"
151 $sha1 -out "$fileToWriteTo" "$fileToHash"
152}
153
154function checkAwsKey
155{
156 local originalKeyFile="$1"
157 local keySize="$(ls -l "$originalKeyFile" | awk '{ print $5 }')"
158 if [ ! $keySize -eq 40 ]; then
159 printErrorHelpAndExit "We do not understand Amazon AWS secret keys which are not 40 bytes long. Have you included a carriage return or line feed by mistake at the end of the secret key file?" $userSpecifiedDataErrorExitCode
160 fi
161}
162
163function padDecodedKeyTo
164{
165 local originalKeyFile="$1"
166 local keyFile="$2"
167 cp "$originalKeyFile" "$keyFile"
168
169 local keySize=$(ls -l "$keyFile" | awk '{ print $5 }')
170 if [ $keySize -lt 64 ]; then
171 local zerosToWrite=$((64 - $keySize))
172 dd if=/dev/zero of=$keyFile bs=1 count=$zerosToWrite seek=$keySize 2> /dev/null
173 elif [ $keySize -gt 64 ]; then
174 echo "Warning: Support for hashing keys bigger than the SHA1 block size of 64 bytes is untested" 1>&2
175 writeHash "$originalKeyFile" "$keyFile"
176 local keySize=$(ls -l "$keyFile" | awk '{ print $5 }')
177 if [ $keySize -lt 64 ]; then
178 local zerosToWrite=$((64 - $keySize))
179 dd if=/dev/zero of=$keyFile bs=1 count=$zerosToWrite seek=$keySize 2> /dev/null
180 fi
181 exit 1
182 else
183 :
184 fi
185}
186
187function writeLongAsByte
188{
189 local byte="$1"
190 local file="$2"
191 printf "\\$(printf "%o" $byte)" >> "$file"
192}
193
194function readBytesAndXorAndWriteAsBytesTo
195{
196 local inputFile="$1"
197 local xorByte=$2
198 local outputFile="$3"
199
200 od -v -A n -t uC "$inputFile" | awk '{ OFS="\n"; for (i = 1; i <= NF; i++) print $i }' |
201 while read byte; do
202 ((xord = byte ^ xorByte))
203 writeLongAsByte $xord "$outputFile"
204 done
205}
206
207function writeHexByte
208{
209 local byte="$1"
210 local file="$2"
211 printf "\\$(printf "%o" 0x$byte)" >> "$file"
212}
213
214function writeHexString
215{
216 local hexString="$1"
217 for byte in $(echo $hexString | sed 's/../& /g'); do
218 writeHexByte "$byte" "$2"
219 done
220}
221
222function writeStringToSign
223{
224 local outputFile="$1"
225 echo $verb >> "$outputFile"
226 echo "$contentMD5" >> "$outputFile"
227 echo "$contentType" >> "$outputFile"
228 echo "$currentDateTime" >> "$outputFile"
229
230 writeStringToSignAmazonHeaders "$outputFile"
231
232 urlPath="$(echo "$url" | awk 'BEGIN { FS="[?]"} { print $1 }')"
233 urlQueryString="$(echo "$url" | awk 'BEGIN { FS="[?]"} { print $2 }')"
234 printf "$urlPath" >> "$outputFile"
235 if [ "$urlQueryString" = "acl" ] || [ "$urlQueryString" = "torrent" ]; then
236 printf "?" >> "$outputFile"
237 printf "$urlQueryString" >> "$outputFile"
238 fi
239}
240
241function writeStringToSignAmazonHeaders()
242{
243 local outputFile="$1"
244
245 #Convert all headers to lower case
246 #sort
247 #Strip ": " to ":"
248 #Add LF to each header
249 awk 'BEGIN { FS=": " } NF == 2 { print tolower($1) ":" $2 }' "$amazonHeaderFile" | sort >> "$outputFile"
250 #TODO: RFC 2616, section 4.2 (combine repeated headers' values)
251 #TODO: Unfold long lines (not supported elsewhere)
252}
253
254function computeAwsAuthorizationHeader
255{
256 checkAwsKey "$awsAccessSecretKeyIdFile"
257
258 createTemporaryFile "key"
259 local tempKeyFile="$(mostRecentTemporaryFile)"
260
261 createTemporaryFile "ipad"
262 local ipadHashingFile="$(mostRecentTemporaryFile)"
263
264 createTemporaryFile "opad"
265 local opadHashingFile="$(mostRecentTemporaryFile)"
266
267 createTemporaryFile "HMAC-SHA1"
268 local hmacSha1File="$(mostRecentTemporaryFile)"
269
270 padDecodedKeyTo "$awsAccessSecretKeyIdFile" "$tempKeyFile"
271 readBytesAndXorAndWriteAsBytesTo "$tempKeyFile" ipadXorByte "$ipadHashingFile"
272
273 writeStringToSign "$ipadHashingFile"
274
275 readBytesAndXorAndWriteAsBytesTo "$tempKeyFile" opadXorByte "$opadHashingFile"
276 appendHash "$ipadHashingFile" "$opadHashingFile"
277 writeHash "$opadHashingFile" "$hmacSha1File"
278
279 local signature="$($base64encode "$hmacSha1File")"
280
281 echo "Authorization: AWS $awsAccessKeyId:$signature"
282}
283
284function writeAmazonHeadersForCurl
285{
286 if [ ! -e "$amazonHeaderFile" ]; then
287 printErrorHelpAndExit "Amazon Header file does not exist" $userSpecifiedDataErrorExitCode
288 elif grep -q ^X-Amz-Date: "$amazonHeaderFile"; then
289 printErrorHelpAndExit "X-Amz-Date header not allowed" $userSpecifiedDataErrorExitCode
290 fi
291 # Consider using sed...
292 awk 'BEGIN { ORS=" "; FS="\0" } { print "--header \"" $1 "\""}' "$amazonHeaderFile" >> "$1"
293}
294
295function runCurl
296{
297 local verbAndAnyData="$1"
298 local fullUrl="$protocol://s3.amazonaws.com$url"
299 createTemporaryFile "curl"
300 local tempCurlCommand="$(mostRecentTemporaryFile)"
301 local cleanUpCommand="rm -f "$tempCurlCommand""
302
303 echo "#! /usr/bin/env bash" >> "$tempCurlCommand"
304 printf "curl %s %s --dump-header \"%s\" " "$verbose" "$verbAndAnyData" "$dumpHeaderFile" >> "$tempCurlCommand"
305 writeAmazonHeadersForCurl "$tempCurlCommand"
306 printf " --header \"%s\"" "Date: $currentDateTime" >> "$tempCurlCommand"
307 printf " --header \"%s\"" "$authorizationHeader" >> "$tempCurlCommand"
308 if [ ! -z "$contentType" ]; then
309 printf " --header \"Content-Type: %s\"" "$contentType" >> "$tempCurlCommand"
310 fi
311 if [ ! -z "$contentMD5" ]; then
312 printf " --header \"Content-MD5: %s\"" "$contentMD5" >> "$tempCurlCommand"
313 fi
314 printf " \"%s\"\n" "$fullUrl" >> "$tempCurlCommand"
315
316 unsetErrorTraps
317 exec env bash "$tempCurlCommand"
318}
319
320function initialise
321{
322 setErrorTraps
323 checkEnvironment
324}
325
326function main
327{
328 initialise
329 parseOptions "$@"
330 readonly currentDateTime="$(LC_TIME=C date "+%a, %d %h %Y %T %z")"
331 prepareToRunCurl
332 readonly authorizationHeader="$(computeAwsAuthorizationHeader)"
333 runCurl "$verbToPass"
334}