drop aws-sdk and hand-roll an S3 client

better than including the huge aws-sdk with a whole bunch of hacks
This commit is contained in:
Peter Cai 2020-02-17 12:42:29 +08:00
parent d976e1838c
commit f32320c27f
No known key found for this signature in database
GPG Key ID: 71F5FB4E4F3FD54F
11 changed files with 245 additions and 254 deletions

View File

@ -1 +0,0 @@
module.exports = require("blob-polyfill").Blob

90
package-lock.json generated
View File

@ -329,58 +329,6 @@
"integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==",
"dev": true
},
"aws-sdk": {
"version": "2.619.0",
"resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.619.0.tgz",
"integrity": "sha512-qujQf27p3mrGZGCL+C+vXCEB08AMm6gS1572fgHrVLBXfy6SjhV4dzEYtt+ZptQm+8z0zuHsDqghJuBCcdpxqQ==",
"dev": true,
"requires": {
"buffer": "4.9.1",
"events": "1.1.1",
"ieee754": "1.1.13",
"jmespath": "0.15.0",
"querystring": "0.2.0",
"sax": "1.2.1",
"url": "0.10.3",
"uuid": "3.3.2",
"xml2js": "0.4.19"
},
"dependencies": {
"buffer": {
"version": "4.9.1",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz",
"integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=",
"dev": true,
"requires": {
"base64-js": "^1.0.2",
"ieee754": "^1.1.4",
"isarray": "^1.0.0"
}
},
"events": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz",
"integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=",
"dev": true
},
"punycode": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz",
"integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=",
"dev": true
},
"url": {
"version": "0.10.3",
"resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz",
"integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=",
"dev": true,
"requires": {
"punycode": "1.3.2",
"querystring": "0.2.0"
}
}
}
},
"balanced-match": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
@ -470,12 +418,6 @@
"file-uri-to-path": "1.0.0"
}
},
"blob-polyfill": {
"version": "4.0.20190430",
"resolved": "https://registry.npmjs.org/blob-polyfill/-/blob-polyfill-4.0.20190430.tgz",
"integrity": "sha512-REie9DM5XvHcqa9tuVT1QverzNpPbuRGffFGfwtHhZLHoToiAFP2YxHqVTQvET+UYnjRttGnlWag0+i3YgPKtw==",
"dev": true
},
"bluebird": {
"version": "3.7.2",
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
@ -2204,12 +2146,6 @@
"integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=",
"dev": true
},
"jmespath": {
"version": "0.15.0",
"resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz",
"integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=",
"dev": true
},
"json-loader": {
"version": "0.5.7",
"resolved": "https://registry.npmjs.org/json-loader/-/json-loader-0.5.7.tgz",
@ -2949,9 +2885,9 @@
}
},
"sax": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz",
"integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=",
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
"integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==",
"dev": true
},
"schema-utils": {
@ -3532,12 +3468,6 @@
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=",
"dev": true
},
"uuid": {
"version": "3.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz",
"integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==",
"dev": true
},
"vm-browserify": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-1.1.2.tgz",
@ -3620,19 +3550,19 @@
"dev": true
},
"xml2js": {
"version": "0.4.19",
"resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz",
"integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==",
"version": "0.4.23",
"resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz",
"integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==",
"dev": true,
"requires": {
"sax": ">=0.6.0",
"xmlbuilder": "~9.0.1"
"xmlbuilder": "~11.0.0"
}
},
"xmlbuilder": {
"version": "9.0.7",
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz",
"integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=",
"version": "11.0.1",
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz",
"integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==",
"dev": true
},
"xtend": {

View File

@ -11,11 +11,10 @@
"author": "Peter Cai <peter@typeblog.net>",
"license": "MIT",
"devDependencies": {
"aws-sdk": "^2.619.0",
"blob-polyfill": "^4.0.20190430",
"coffee-loader": "^0.9.0",
"coffeescript": "^2.5.1",
"json-loader": "^0.5.7",
"webpack": "^4.41.6"
"webpack": "^4.41.6",
"xml2js": "^0.4.23"
}
}

118
src/aws/auth.coffee Normal file
View File

@ -0,0 +1,118 @@
import * as crypto from '../crypto'
# Implement AWS signature authentication
class AwsAuth
constructor: (url, conf) ->
@url = new URL url
@date = new Date()
@conf = conf
@region = conf.region
# Setters
setMethod: (method) ->
@method = method.toUpperCase()
@
setQueryStringMap: (qsMap) ->
@qsMap = qsMap
@
setHeaderMap: (headerMap) ->
# headers MUST contain `x-amz-content-sha256: UNSIGNED-PAYLOAD`
@headerMap =
'host': @url.host
for [key, val] in Object.entries headerMap
key = key.toLowerCase()
val = val.trim()
if key == "content-type" || key.startsWith "x-amz-"
@headerMap[key] = val
if !@headerMap['x-amz-content-sha256']
throw "Must contain sha256 header"
@
setRegion: (region) ->
@region = region
@
setService: (service) ->
@service = service # "s3"
@
# Signature calculation: <https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html>
authorizedHeader: (origHeader) ->
origHeader['x-amz-date'] = @timeStampISO8601()
origHeader['authorization'] = await @authorizationHeader()
authorizationHeader: ->
"AWS4-HMAC-SHA256 " +
"Credential=" + @credential() + "," +
"SignedHeaders=" + @signedHeaders() + "," +
"Signature=" + await @calculateSignature()
credential: ->
@conf.accessKeyId + "/" + @scope()
calculateSignature: ->
crypto.hex await crypto.HMAC_SHA256 await @signingKey(), await @stringToSign()
signingKey: ->
accessKey = crypto.utf8Bytes "AWS4" + @conf.secretAccessKey
dateKey = await crypto.HMAC_SHA256 accessKey, @timeStampYYYYMMDD()
dateRegionKey = await crypto.HMAC_SHA256 dateKey, @region
dateRegionServiceKey = await crypto.HMAC_SHA256 dateRegionKey, @service
crypto.HMAC_SHA256 dateRegionServiceKey, "aws4_request"
stringToSign: ->
"AWS4-HMAC-SHA256" + "\n" +
@timeStampISO8601() + "\n" +
@scope() + "\n" +
await crypto.hex await crypto.SHA256 @canonicalRequest()
scope: ->
@timeStampYYYYMMDD() + "/" +
@region + "/" + @service + "/aws4_request"
canonicalRequest: ->
@method + "\n" +
@canonicalURI() + "\n" +
@canonicalQueryString() + "\n" +
@canonicalHeaders() + "\n" +
@signedHeaders() + "\n" +
@headerMap['x-amz-content-sha256']
canonicalURI: ->
encodeURI @url.pathname
canonicalQueryString: ->
return "" if not @qsMap
[...Object.entries @qsMap].sort()
.map (pair) ->
pair.map (x) -> encodeURIComponent x
.join "="
.join "&"
canonicalHeaders: ->
([...Object.entries @headerMap].sort()
.map (pair) ->
pair.join ':'
.join "\n") + "\n" # There's a trailing "\n"
signedHeaders: ->
[...Object.entries @headerMap].sort()
.map (pair) -> pair[0]
.join ";"
pad: (num) ->
if num < 10
"0" + num
else
num
timeStampISO8601: ->
@timeStampYYYYMMDD() + "T" +
@pad(@date.getUTCHours()) +
@pad(@date.getUTCMinutes()) +
@pad(@date.getUTCSeconds()) + "Z"
timeStampYYYYMMDD: ->
@date.getUTCFullYear() +
@pad(@date.getUTCMonth() + 1) +
@pad(@date.getUTCDate())
export default AwsAuth

79
src/aws/s3.coffee Normal file
View File

@ -0,0 +1,79 @@
import AwsAuth from "./auth"
import { parseStringPromise as parseXML } from "xml2js"
class S3
constructor: (@conf) ->
@baseURL = @conf.s3.endpoint + "/" + @conf.s3.bucket + "/"
# Wrapper of Fetch API with automatic AWS signature
# Note that query string in url is not supported
# options.method must be specified
request: (url, qsMap, options) ->
# We only support unsigned payload for now
options.headers['x-amz-content-sha256'] = 'UNSIGNED-PAYLOAD'
# Sign the request
auth = new AwsAuth url, @conf.aws
.setMethod options.method
.setQueryStringMap qsMap
.setHeaderMap options.headers
.setService "s3"
# Write needed authorization headers to header object
await auth.authorizedHeader options.headers
fetch url + "?" + auth.canonicalQueryString(), options
# Used for some requests to convert params to snake-cased x-amz-*
camelToSnake: (str) ->
str.replace /([A-Z])/g, "-$1"
.toLowerCase()[1..]
makeHeaders: (params) -> Object.fromEntries do =>
Object.entries params
.map ([key, val]) =>
key = @camelToSnake key
if not (key == "expires" or key.startsWith "content-" or key.startsWith "cache-")
key = 'x-amz-' + key
[key, val]
# See AWS docs for params
# params are passed in query string
# Paging not implemented yet; Need to
listObjects: (params) ->
# Send request using params as query string
resp = await @request @baseURL, params,
method: "GET"
headers:
'x-amz-request-payer': 'requester'
txt = await resp.text()
console.log txt
if not resp.ok
# no error handling yet
throw txt
result = await parseXML txt
if not result.ListBucketResult
return null
result.ListBucketResult
# params are in CamelCase, but converted to x-amz-* headers automatically
# Content* headers are exempt from the x-amz-* prefix, as well as Expires
# data can be a buffer or a readable stream
putObject: (key, data, params) ->
# Convert camel-cased params to snake-cased headers
headers = @makeHeaders params
# Send request
resp = await @request @baseURL + key, null,
method: 'PUT'
headers: headers
body: data
txt = await resp.text()
console.log txt
if not resp.ok
# no error handling yet
throw resp.status
else
txt
export default S3

25
src/crypto.coffee Normal file
View File

@ -0,0 +1,25 @@
utf8Bytes = (str) ->
new TextEncoder 'utf-8'
.encode str
hex = (buf) -> (Array.prototype.map.call new Uint8Array(buf),
(x) => ('00' + x.toString 16).slice(-2)).join ''
HMAC_SHA256_KEY = (buf) ->
crypto.subtle.importKey 'raw', buf,
{ name: 'HMAC', hash: 'SHA-256' }, true, [ 'sign' ]
HMAC_SHA256 = (key, str) ->
cryptoKey = await HMAC_SHA256_KEY key
buf = utf8Bytes str
await crypto.subtle.sign "HMAC", cryptoKey, buf
SHA256 = (str) ->
crypto.subtle.digest "SHA-256", utf8Bytes str
export {
utf8Bytes,
hex,
HMAC_SHA256,
SHA256
}

View File

@ -1,10 +1,11 @@
import * as util from './util'
import * as s3 from './s3'
import _ from './prelude'
import S3 from './aws/s3'
import config from '../config.json'
s3 = new S3 config
main = ->
s3.loadAWSConfig _
addEventListener 'fetch', (event) =>
event.respondWith handleRequest event
@ -38,19 +39,18 @@ handlePUT = (req, file) ->
loop
id = util.randomID _
path = util.idToPath id
files = await s3.listFiles path
break if !files or files.length == 0
files = await s3.listObjects
prefix: path
break if !files.Contents or files.Contents.length == 0
path = path + "/" + file
len = req.headers.get "content-length"
# Upload the file to S3
try
await s3.uploadFile # TODO: expiry date
Key: path
await s3.putObject path, req.body, # Expiration should be configured on S3 side
ContentType: req.headers.get "content-type"
ContentLength: len
Body: await util.readToBlob req.body
catch err
console.log err
return buildInvalidResponse err

View File

@ -1,26 +0,0 @@
import config from '../config.json'
import AWS from 'aws-sdk'
import _ from './prelude'
loadAWSConfig = ->
AWS.config.update config.aws
getS3 = ->
new AWS.S3
endpoint: new AWS.Endpoint config.s3.endpoint
uploadFile = (params) ->
params['Bucket'] = config.s3.bucket
getS3 _
.putObject params
.promise _
listFiles = (path) ->
(await getS3 _
.listObjects
Bucket: config.s3.bucket
Prefix: path
.promise _)
.Contents
export { loadAWSConfig, uploadFile, listFiles }

View File

@ -32,22 +32,10 @@ idToPath = (id) ->
id.split ''
.join '/'
# Convert a ReadableStream into Blob
# AWS-SDK does not support ReadableStream, unfortunately
readToBlob = (stream) ->
reader = stream.getReader()
ret = []
loop
{ done, value } = await reader.read()
break if done
ret.push value
new Blob ret
export {
getFileName,
validateLength,
MAX_UPLOAD_SIZE,
randomID,
idToPath,
readToBlob
idToPath
}

View File

@ -1,6 +1,3 @@
const webpack = require('webpack')
const path = require('path')
module.exports = {
target: "webworker",
entry: "./index.js",
@ -10,29 +7,8 @@ module.exports = {
minimize: false
},
resolve: {
extensions: ['.js', '.coffee'],
alias: {
'blob-shim': path.resolve(__dirname, './blob-shim.js'),
}
extensions: ['.js', '.coffee']
},
plugins: [
new webpack.NormalModuleReplacementPlugin(
// Rewritten xhr.js to use Fetch API
// Mostly from <https://github.com/aws/aws-sdk-js/issues/2807>
// Modified to fix a few bugs
/node_modules\/aws-sdk\/lib\/http\/xhr.js/,
'../../../../xhr-shim.js'
),
new webpack.NormalModuleReplacementPlugin(
// Force it to use node_parser
// Because we are not actually in browser
/node_modules\/aws-sdk\/lib\/xml\/browser_parser.js/,
'./node_parser.js'
),
new webpack.ProvidePlugin({
'Blob': 'blob-shim'
})
],
module: {
rules: [
{
@ -40,7 +16,7 @@ module.exports = {
use: [ 'coffee-loader' ]
},
{
type: 'javascript/auto', // Needed for aws-sdk
type: 'javascript/auto',
test: /\.json$/,
use: [ 'json-loader' ]
}

View File

@ -1,97 +0,0 @@
var AWS = require('./node_modules/aws-sdk/lib/core');
var EventEmitter = require('events').EventEmitter;
require('./node_modules/aws-sdk/lib/http');
/**
* @api private
*/
AWS.XHRClient = AWS.util.inherit({
handleRequest: function handleRequest(httpRequest, httpOptions, callback, errCallback) {
var self = this;
var endpoint = httpRequest.endpoint;
var emitter = new EventEmitter();
var href = endpoint.protocol + '//' + endpoint.hostname;
if (endpoint.port !== 80 && endpoint.port !== 443) {
href += ':' + endpoint.port;
}
href += httpRequest.path;
callback(emitter);
var headers = new Headers();
AWS.util.each(httpRequest.headers, function (key, value) {
if (key !== 'Content-Length' && key !== 'User-Agent' && key !== 'Host') {
headers.set(key, value);
}
});
var credentials = 'omit';
if (httpOptions.xhrWithCredentials) {
credentials = 'include';
}
var request = new Request(href, {
method: httpRequest.method,
headers: headers,
body: httpRequest.method == "GET" ? null : httpRequest.body
});
fetch(request).then(function(response) {
if (!response.ok) {
throw Error(response.statusText);
}
return response;
}).then(function(response) {
emitter.statusCode = response.status;
new Error(response.headers)
emitter.headers = self.parseHeaders(response.headers);
emitter.emit('headers', emitter.statusCode, emitter.headers);
response.text().then(function(res){
console.log(res);
self.finishRequest(res, emitter);
}).catch(function(err){
console.log(err);
});
}).catch(function(err) {
errCallback(AWS.util.error(new Error('Network Failure' + err), {
code: 'NetworkingError'
}));
});
return emitter;
},
parseHeaders: function parseHeaders(rawHeaders) {
var headers = {};
if (!rawHeaders) return headers;
for (var pair of rawHeaders.entries()) {
headers[pair[0]] = pair[1];
}
return headers;
},
finishRequest: function finishRequest(res, emitter) {
var buffer;
try {
buffer = new AWS.util.Buffer(res);
} catch (e) {}
if (buffer) emitter.emit('data', buffer);
emitter.emit('end');
}
});
/**
* @api private
*/
AWS.HttpClient.prototype = AWS.XHRClient.prototype;
/**
* @api private
*/
AWS.HttpClient.streamsApiVersion = 1;