| 1 | #! /usr/bin/env bash |
| 2 | cat > /dev/null << EndOfLicence |
| 3 | s3-bash |
| 4 | Copyright 2007 Raphael James Cohn |
| 5 | |
| 6 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except |
| 7 | in compliance with the License. |
| 8 | You may obtain a copy of the License at |
| 9 | |
| 10 | http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | |
| 12 | Unless required by applicable law or agreed to in writing, software distributed under the License |
| 13 | is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express |
| 14 | or implied. See the License for the specific language governing permissions and limitations under |
| 15 | the License. |
| 16 | EndOfLicence |
| 17 | |
| 18 | # Pragmas |
| 19 | set -u |
| 20 | set -e |
| 21 | |
| 22 | # Constants |
| 23 | readonly version="0.02" |
| 24 | readonly userSpecifiedDataErrorExitCode=1 |
| 25 | readonly invalidCommandLineOption=2 |
| 26 | readonly internalErrorExitCode=3 |
| 27 | readonly invalidEnvironmentExitCode=4 |
| 28 | readonly ipadXorByte=0x36 |
| 29 | readonly opadXorByte=0x5c |
| 30 | |
| 31 | # Command-like aliases |
| 32 | readonly sha1="openssl dgst -sha1 -binary" |
| 33 | readonly base64encode="openssl enc -base64 -e -in" |
| 34 | readonly base64decode="openssl enc -base64 -d -in" |
| 35 | |
| 36 | # Globals |
| 37 | declare -a temporaryFiles |
| 38 | |
| 39 | function base64EncodedMD5 |
| 40 | { |
| 41 | openssl dgst -md5 -binary "$1" | openssl enc -e -base64 |
| 42 | } |
| 43 | |
| 44 | function printErrorMessage |
| 45 | { |
| 46 | printf "%s: %s\n" "$1" "$2" 1>&2 |
| 47 | } |
| 48 | |
| 49 | function printErrorHelpAndExit |
| 50 | { |
| 51 | printErrorMessage "$weAreKnownAs" "$1" |
| 52 | printHelpAndExit $2 |
| 53 | } |
| 54 | |
| 55 | function checkProgramIsInEnvironment |
| 56 | { |
| 57 | if [ ! -x "$(which $1)" ]; then |
| 58 | printErrorHelpAndExit "Environment Error: $1 not found on the path or not executable" $invalidEnvironmentExitCode |
| 59 | fi |
| 60 | } |
| 61 | |
| 62 | # Do not use this from directly. Due to a bug in bash, array assignments do not work when the function is used with command substitution |
| 63 | function createTemporaryFile |
| 64 | { |
| 65 | local temporaryFile="$(mktemp "$temporaryDirectory/$$.$1.XXXXXXXX")" || printErrorHelpAndExit "Environment Error: Could not create a temporary file. Please check you /tmp folder permissions allow files and folders to be created and disc space." $invalidEnvironmentExitCode |
| 66 | local length="${#temporaryFiles[@]}" |
| 67 | temporaryFiles[$length]="$temporaryFile" |
| 68 | } |
| 69 | |
| 70 | function mostRecentTemporaryFile |
| 71 | { |
| 72 | local length="${#temporaryFiles[@]}" |
| 73 | local lastIndex |
| 74 | ((lastIndex = --length)) |
| 75 | echo "${temporaryFiles[$lastIndex]}" |
| 76 | } |
| 77 | |
| 78 | function deleteTemporaryFile |
| 79 | { |
| 80 | rm -f "$1" || printErrorHelpAndExit "Environment Error: Could not delete a temporary file ($1)." $invalidEnvironmentExitCode |
| 81 | } |
| 82 | |
| 83 | function removeTemporaryFiles |
| 84 | { |
| 85 | length="${#temporaryFiles[@]}" |
| 86 | if [ $length -eq 0 ]; then |
| 87 | return |
| 88 | fi |
| 89 | for temporaryFile in ${temporaryFiles[@]}; do |
| 90 | deleteTemporaryFile "$temporaryFile" |
| 91 | done |
| 92 | temporaryFiles=() |
| 93 | length="${#temporaryFiles[@]}" |
| 94 | } |
| 95 | |
| 96 | function checkEnvironment |
| 97 | { |
| 98 | programs=(openssl curl od dd printf sed awk sort mktemp rm grep cp ls env bash) |
| 99 | for program in "${programs[@]}"; do |
| 100 | checkProgramIsInEnvironment "$program" |
| 101 | done |
| 102 | |
| 103 | local temporaryFolder="${TMPDIR:-/tmp}" |
| 104 | if [ ! -x "$temporaryFolder" ]; then |
| 105 | printErrorHelpAndExit "Environment Error: The temporary directory ($temporaryFolder) does not exist. Please set the TMPDIR environment variable to your temporary directory" $invalidEnvironmentExitCode |
| 106 | fi |
| 107 | readonly temporaryDirectory="$temporaryFolder/s3-bash/$weAreKnownAs" |
| 108 | mkdir -p "$temporaryDirectory" || printErrorHelpAndExit "Environment Error: Could not create a temporary directory ($temporaryDiectory). Please check you /tmp folder permissions allow files and folders to be created and you have sufficient disc space" $invalidEnvironmentExitCode |
| 109 | |
| 110 | #Check we can create and delete temporary files |
| 111 | createTemporaryFile "check" |
| 112 | temporaryFileCheck="$(mostRecentTemporaryFile)" |
| 113 | echo "Checking we can write to temporary files. If this is still here then we could not delete temporary files." > "$temporaryFileCheck" |
| 114 | removeTemporaryFiles |
| 115 | } |
| 116 | |
| 117 | function setErrorTraps |
| 118 | { |
| 119 | trap "removeTemporaryFiles; exit $internalErrorExitCode" INT TERM EXIT |
| 120 | } |
| 121 | |
| 122 | function unsetErrorTraps |
| 123 | { |
| 124 | trap - INT TERM EXIT |
| 125 | } |
| 126 | |
| 127 | function verifyUrl |
| 128 | { |
| 129 | if [ -z "$url" ]; then |
| 130 | printErrorHelpAndExit "URL not specified" $userSpecifiedDataErrorExitCode |
| 131 | elif echo $url | grep -q http://; then |
| 132 | printErrorHelpAndExit "URL starts with http://" $userSpecifiedDataErrorExitCode |
| 133 | elif echo $url | grep -q https://; then |
| 134 | printErrorHelpAndExit "URL starts with https://" $userSpecifiedDataErrorExitCode |
| 135 | elif echo $url | grep -v ^/; then |
| 136 | printErrorHelpAndExit "URL does not start with /" $userSpecifiedDataErrorExitCode |
| 137 | fi |
| 138 | } |
| 139 | |
| 140 | function appendHash |
| 141 | { |
| 142 | local fileToHash="$1" |
| 143 | local fileToWriteTo="$2" |
| 144 | $sha1 "$fileToHash" >> "$fileToWriteTo" |
| 145 | } |
| 146 | |
| 147 | function writeHash |
| 148 | { |
| 149 | local fileToHash="$1" |
| 150 | local fileToWriteTo="$2" |
| 151 | $sha1 -out "$fileToWriteTo" "$fileToHash" |
| 152 | } |
| 153 | |
| 154 | function checkAwsKey |
| 155 | { |
| 156 | local originalKeyFile="$1" |
| 157 | local keySize="$(ls -l "$originalKeyFile" | awk '{ print $5 }')" |
| 158 | if [ ! $keySize -eq 40 ]; then |
| 159 | printErrorHelpAndExit "We do not understand Amazon AWS secret keys which are not 40 bytes long. Have you included a carriage return or line feed by mistake at the end of the secret key file?" $userSpecifiedDataErrorExitCode |
| 160 | fi |
| 161 | } |
| 162 | |
| 163 | function padDecodedKeyTo |
| 164 | { |
| 165 | local originalKeyFile="$1" |
| 166 | local keyFile="$2" |
| 167 | cp "$originalKeyFile" "$keyFile" |
| 168 | |
| 169 | local keySize=$(ls -l "$keyFile" | awk '{ print $5 }') |
| 170 | if [ $keySize -lt 64 ]; then |
| 171 | local zerosToWrite=$((64 - $keySize)) |
| 172 | dd if=/dev/zero of=$keyFile bs=1 count=$zerosToWrite seek=$keySize 2> /dev/null |
| 173 | elif [ $keySize -gt 64 ]; then |
| 174 | echo "Warning: Support for hashing keys bigger than the SHA1 block size of 64 bytes is untested" 1>&2 |
| 175 | writeHash "$originalKeyFile" "$keyFile" |
| 176 | local keySize=$(ls -l "$keyFile" | awk '{ print $5 }') |
| 177 | if [ $keySize -lt 64 ]; then |
| 178 | local zerosToWrite=$((64 - $keySize)) |
| 179 | dd if=/dev/zero of=$keyFile bs=1 count=$zerosToWrite seek=$keySize 2> /dev/null |
| 180 | fi |
| 181 | exit 1 |
| 182 | else |
| 183 | : |
| 184 | fi |
| 185 | } |
| 186 | |
| 187 | function writeLongAsByte |
| 188 | { |
| 189 | local byte="$1" |
| 190 | local file="$2" |
| 191 | printf "\\$(printf "%o" $byte)" >> "$file" |
| 192 | } |
| 193 | |
| 194 | function readBytesAndXorAndWriteAsBytesTo |
| 195 | { |
| 196 | local inputFile="$1" |
| 197 | local xorByte=$2 |
| 198 | local outputFile="$3" |
| 199 | |
| 200 | od -v -A n -t uC "$inputFile" | awk '{ OFS="\n"; for (i = 1; i <= NF; i++) print $i }' | |
| 201 | while read byte; do |
| 202 | ((xord = byte ^ xorByte)) |
| 203 | writeLongAsByte $xord "$outputFile" |
| 204 | done |
| 205 | } |
| 206 | |
| 207 | function writeHexByte |
| 208 | { |
| 209 | local byte="$1" |
| 210 | local file="$2" |
| 211 | printf "\\$(printf "%o" 0x$byte)" >> "$file" |
| 212 | } |
| 213 | |
| 214 | function writeHexString |
| 215 | { |
| 216 | local hexString="$1" |
| 217 | for byte in $(echo $hexString | sed 's/../& /g'); do |
| 218 | writeHexByte "$byte" "$2" |
| 219 | done |
| 220 | } |
| 221 | |
| 222 | function writeStringToSign |
| 223 | { |
| 224 | local outputFile="$1" |
| 225 | echo $verb >> "$outputFile" |
| 226 | echo "$contentMD5" >> "$outputFile" |
| 227 | echo "$contentType" >> "$outputFile" |
| 228 | echo "$currentDateTime" >> "$outputFile" |
| 229 | |
| 230 | writeStringToSignAmazonHeaders "$outputFile" |
| 231 | |
| 232 | urlPath="$(echo "$url" | awk 'BEGIN { FS="[?]"} { print $1 }')" |
| 233 | urlQueryString="$(echo "$url" | awk 'BEGIN { FS="[?]"} { print $2 }')" |
| 234 | printf "$urlPath" >> "$outputFile" |
| 235 | if [ "$urlQueryString" = "acl" ] || [ "$urlQueryString" = "torrent" ]; then |
| 236 | printf "?" >> "$outputFile" |
| 237 | printf "$urlQueryString" >> "$outputFile" |
| 238 | fi |
| 239 | } |
| 240 | |
| 241 | function writeStringToSignAmazonHeaders() |
| 242 | { |
| 243 | local outputFile="$1" |
| 244 | |
| 245 | #Convert all headers to lower case |
| 246 | #sort |
| 247 | #Strip ": " to ":" |
| 248 | #Add LF to each header |
| 249 | awk 'BEGIN { FS=": " } NF == 2 { print tolower($1) ":" $2 }' "$amazonHeaderFile" | sort >> "$outputFile" |
| 250 | #TODO: RFC 2616, section 4.2 (combine repeated headers' values) |
| 251 | #TODO: Unfold long lines (not supported elsewhere) |
| 252 | } |
| 253 | |
| 254 | function computeAwsAuthorizationHeader |
| 255 | { |
| 256 | checkAwsKey "$awsAccessSecretKeyIdFile" |
| 257 | |
| 258 | createTemporaryFile "key" |
| 259 | local tempKeyFile="$(mostRecentTemporaryFile)" |
| 260 | |
| 261 | createTemporaryFile "ipad" |
| 262 | local ipadHashingFile="$(mostRecentTemporaryFile)" |
| 263 | |
| 264 | createTemporaryFile "opad" |
| 265 | local opadHashingFile="$(mostRecentTemporaryFile)" |
| 266 | |
| 267 | createTemporaryFile "HMAC-SHA1" |
| 268 | local hmacSha1File="$(mostRecentTemporaryFile)" |
| 269 | |
| 270 | padDecodedKeyTo "$awsAccessSecretKeyIdFile" "$tempKeyFile" |
| 271 | readBytesAndXorAndWriteAsBytesTo "$tempKeyFile" ipadXorByte "$ipadHashingFile" |
| 272 | |
| 273 | writeStringToSign "$ipadHashingFile" |
| 274 | |
| 275 | readBytesAndXorAndWriteAsBytesTo "$tempKeyFile" opadXorByte "$opadHashingFile" |
| 276 | appendHash "$ipadHashingFile" "$opadHashingFile" |
| 277 | writeHash "$opadHashingFile" "$hmacSha1File" |
| 278 | |
| 279 | local signature="$($base64encode "$hmacSha1File")" |
| 280 | |
| 281 | echo "Authorization: AWS $awsAccessKeyId:$signature" |
| 282 | } |
| 283 | |
| 284 | function writeAmazonHeadersForCurl |
| 285 | { |
| 286 | if [ ! -e "$amazonHeaderFile" ]; then |
| 287 | printErrorHelpAndExit "Amazon Header file does not exist" $userSpecifiedDataErrorExitCode |
| 288 | elif grep -q ^X-Amz-Date: "$amazonHeaderFile"; then |
| 289 | printErrorHelpAndExit "X-Amz-Date header not allowed" $userSpecifiedDataErrorExitCode |
| 290 | fi |
| 291 | # Consider using sed... |
| 292 | awk 'BEGIN { ORS=" "; FS="\0" } { print "--header \"" $1 "\""}' "$amazonHeaderFile" >> "$1" |
| 293 | } |
| 294 | |
| 295 | function runCurl |
| 296 | { |
| 297 | local verbAndAnyData="$1" |
| 298 | local fullUrl="$protocol://s3.amazonaws.com$url" |
| 299 | createTemporaryFile "curl" |
| 300 | local tempCurlCommand="$(mostRecentTemporaryFile)" |
| 301 | local cleanUpCommand="rm -f "$tempCurlCommand"" |
| 302 | |
| 303 | echo "#! /usr/bin/env bash" >> "$tempCurlCommand" |
| 304 | printf "curl %s %s --dump-header \"%s\" " "$verbose" "$verbAndAnyData" "$dumpHeaderFile" >> "$tempCurlCommand" |
| 305 | writeAmazonHeadersForCurl "$tempCurlCommand" |
| 306 | printf " --header \"%s\"" "Date: $currentDateTime" >> "$tempCurlCommand" |
| 307 | printf " --header \"%s\"" "$authorizationHeader" >> "$tempCurlCommand" |
| 308 | if [ ! -z "$contentType" ]; then |
| 309 | printf " --header \"Content-Type: %s\"" "$contentType" >> "$tempCurlCommand" |
| 310 | fi |
| 311 | if [ ! -z "$contentMD5" ]; then |
| 312 | printf " --header \"Content-MD5: %s\"" "$contentMD5" >> "$tempCurlCommand" |
| 313 | fi |
| 314 | printf " \"%s\"\n" "$fullUrl" >> "$tempCurlCommand" |
| 315 | |
| 316 | unsetErrorTraps |
| 317 | exec env bash "$tempCurlCommand" |
| 318 | } |
| 319 | |
| 320 | function initialise |
| 321 | { |
| 322 | setErrorTraps |
| 323 | checkEnvironment |
| 324 | } |
| 325 | |
| 326 | function main |
| 327 | { |
| 328 | initialise |
| 329 | parseOptions "$@" |
| 330 | readonly currentDateTime="$(LC_TIME=C date "+%a, %d %h %Y %T %z")" |
| 331 | prepareToRunCurl |
| 332 | readonly authorizationHeader="$(computeAwsAuthorizationHeader)" |
| 333 | runCurl "$verbToPass" |
| 334 | } |