diff --git a/blob-shim.js b/blob-shim.js deleted file mode 100644 index ec511e9..0000000 --- a/blob-shim.js +++ /dev/null @@ -1 +0,0 @@ -module.exports = require("blob-polyfill").Blob \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 244945e..514075c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -329,58 +329,6 @@ "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", "dev": true }, - "aws-sdk": { - "version": "2.619.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.619.0.tgz", - "integrity": "sha512-qujQf27p3mrGZGCL+C+vXCEB08AMm6gS1572fgHrVLBXfy6SjhV4dzEYtt+ZptQm+8z0zuHsDqghJuBCcdpxqQ==", - "dev": true, - "requires": { - "buffer": "4.9.1", - "events": "1.1.1", - "ieee754": "1.1.13", - "jmespath": "0.15.0", - "querystring": "0.2.0", - "sax": "1.2.1", - "url": "0.10.3", - "uuid": "3.3.2", - "xml2js": "0.4.19" - }, - "dependencies": { - "buffer": { - "version": "4.9.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", - "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", - "dev": true, - "requires": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4", - "isarray": "^1.0.0" - } - }, - "events": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", - "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=", - "dev": true - }, - "punycode": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=", - "dev": true - }, - "url": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", - "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", - "dev": true, - "requires": { - "punycode": "1.3.2", - "querystring": "0.2.0" - } - } - } - }, "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", @@ -470,12 +418,6 @@ "file-uri-to-path": "1.0.0" } }, - "blob-polyfill": { - "version": "4.0.20190430", - "resolved": "https://registry.npmjs.org/blob-polyfill/-/blob-polyfill-4.0.20190430.tgz", - "integrity": "sha512-REie9DM5XvHcqa9tuVT1QverzNpPbuRGffFGfwtHhZLHoToiAFP2YxHqVTQvET+UYnjRttGnlWag0+i3YgPKtw==", - "dev": true - }, "bluebird": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", @@ -2204,12 +2146,6 @@ "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=", "dev": true }, - "jmespath": { - "version": "0.15.0", - "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", - "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=", - "dev": true - }, "json-loader": { "version": "0.5.7", "resolved": "https://registry.npmjs.org/json-loader/-/json-loader-0.5.7.tgz", @@ -2949,9 +2885,9 @@ } }, "sax": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", - "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==", "dev": true }, "schema-utils": { @@ -3532,12 +3468,6 @@ "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", "dev": true }, - "uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==", - "dev": true - }, "vm-browserify": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-1.1.2.tgz", @@ -3620,19 +3550,19 @@ "dev": true }, "xml2js": { - "version": "0.4.19", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", - "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "version": "0.4.23", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", + "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==", "dev": true, "requires": { "sax": ">=0.6.0", - "xmlbuilder": "~9.0.1" + "xmlbuilder": "~11.0.0" } }, "xmlbuilder": { - "version": "9.0.7", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", - "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=", + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", "dev": true }, "xtend": { diff --git a/package.json b/package.json index 22e7c72..192b8ff 100644 --- a/package.json +++ b/package.json @@ -11,11 +11,10 @@ "author": "Peter Cai ", "license": "MIT", "devDependencies": { - "aws-sdk": "^2.619.0", - "blob-polyfill": "^4.0.20190430", "coffee-loader": "^0.9.0", "coffeescript": "^2.5.1", "json-loader": "^0.5.7", - "webpack": "^4.41.6" + "webpack": "^4.41.6", + "xml2js": "^0.4.23" } } diff --git a/src/aws/auth.coffee b/src/aws/auth.coffee new file mode 100644 index 0000000..6d70ace --- /dev/null +++ b/src/aws/auth.coffee @@ -0,0 +1,118 @@ +import * as crypto from '../crypto' + +# Implement AWS signature authentication +class AwsAuth + constructor: (url, conf) -> + @url = new URL url + @date = new Date() + @conf = conf + @region = conf.region + + # Setters + setMethod: (method) -> + @method = method.toUpperCase() + @ + setQueryStringMap: (qsMap) -> + @qsMap = qsMap + @ + setHeaderMap: (headerMap) -> + # headers MUST contain `x-amz-content-sha256: UNSIGNED-PAYLOAD` + @headerMap = + 'host': @url.host + for [key, val] in Object.entries headerMap + key = key.toLowerCase() + val = val.trim() + if key == "content-type" || key.startsWith "x-amz-" + @headerMap[key] = val + if !@headerMap['x-amz-content-sha256'] + throw "Must contain sha256 header" + @ + setRegion: (region) -> + @region = region + @ + setService: (service) -> + @service = service # "s3" + @ + + # Signature calculation: + authorizedHeader: (origHeader) -> + origHeader['x-amz-date'] = @timeStampISO8601() + origHeader['authorization'] = await @authorizationHeader() + + authorizationHeader: -> + "AWS4-HMAC-SHA256 " + + "Credential=" + @credential() + "," + + "SignedHeaders=" + @signedHeaders() + "," + + "Signature=" + await @calculateSignature() + + credential: -> + @conf.accessKeyId + "/" + @scope() + + calculateSignature: -> + crypto.hex await crypto.HMAC_SHA256 await @signingKey(), await @stringToSign() + + signingKey: -> + accessKey = crypto.utf8Bytes "AWS4" + @conf.secretAccessKey + dateKey = await crypto.HMAC_SHA256 accessKey, @timeStampYYYYMMDD() + dateRegionKey = await crypto.HMAC_SHA256 dateKey, @region + dateRegionServiceKey = await crypto.HMAC_SHA256 dateRegionKey, @service + crypto.HMAC_SHA256 dateRegionServiceKey, "aws4_request" + + stringToSign: -> + "AWS4-HMAC-SHA256" + "\n" + + @timeStampISO8601() + "\n" + + @scope() + "\n" + + await crypto.hex await crypto.SHA256 @canonicalRequest() + + scope: -> + @timeStampYYYYMMDD() + "/" + + @region + "/" + @service + "/aws4_request" + + canonicalRequest: -> + @method + "\n" + + @canonicalURI() + "\n" + + @canonicalQueryString() + "\n" + + @canonicalHeaders() + "\n" + + @signedHeaders() + "\n" + + @headerMap['x-amz-content-sha256'] + + canonicalURI: -> + encodeURI @url.pathname + + canonicalQueryString: -> + return "" if not @qsMap + [...Object.entries @qsMap].sort() + .map (pair) -> + pair.map (x) -> encodeURIComponent x + .join "=" + .join "&" + + canonicalHeaders: -> + ([...Object.entries @headerMap].sort() + .map (pair) -> + pair.join ':' + .join "\n") + "\n" # There's a trailing "\n" + + signedHeaders: -> + [...Object.entries @headerMap].sort() + .map (pair) -> pair[0] + .join ";" + + pad: (num) -> + if num < 10 + "0" + num + else + num + + timeStampISO8601: -> + @timeStampYYYYMMDD() + "T" + + @pad(@date.getUTCHours()) + + @pad(@date.getUTCMinutes()) + + @pad(@date.getUTCSeconds()) + "Z" + + timeStampYYYYMMDD: -> + @date.getUTCFullYear() + + @pad(@date.getUTCMonth() + 1) + + @pad(@date.getUTCDate()) + +export default AwsAuth \ No newline at end of file diff --git a/src/aws/s3.coffee b/src/aws/s3.coffee new file mode 100644 index 0000000..6adb506 --- /dev/null +++ b/src/aws/s3.coffee @@ -0,0 +1,79 @@ +import AwsAuth from "./auth" +import { parseStringPromise as parseXML } from "xml2js" + +class S3 + constructor: (@conf) -> + @baseURL = @conf.s3.endpoint + "/" + @conf.s3.bucket + "/" + + # Wrapper of Fetch API with automatic AWS signature + # Note that query string in url is not supported + # options.method must be specified + request: (url, qsMap, options) -> + # We only support unsigned payload for now + options.headers['x-amz-content-sha256'] = 'UNSIGNED-PAYLOAD' + + # Sign the request + auth = new AwsAuth url, @conf.aws + .setMethod options.method + .setQueryStringMap qsMap + .setHeaderMap options.headers + .setService "s3" + + # Write needed authorization headers to header object + await auth.authorizedHeader options.headers + fetch url + "?" + auth.canonicalQueryString(), options + + # Used for some requests to convert params to snake-cased x-amz-* + camelToSnake: (str) -> + str.replace /([A-Z])/g, "-$1" + .toLowerCase()[1..] + + makeHeaders: (params) -> Object.fromEntries do => + Object.entries params + .map ([key, val]) => + key = @camelToSnake key + if not (key == "expires" or key.startsWith "content-" or key.startsWith "cache-") + key = 'x-amz-' + key + [key, val] + + # See AWS docs for params + # params are passed in query string + # Paging not implemented yet; Need to + listObjects: (params) -> + # Send request using params as query string + resp = await @request @baseURL, params, + method: "GET" + headers: + 'x-amz-request-payer': 'requester' + txt = await resp.text() + console.log txt + if not resp.ok + # no error handling yet + throw txt + result = await parseXML txt + if not result.ListBucketResult + return null + result.ListBucketResult + + # params are in CamelCase, but converted to x-amz-* headers automatically + # Content* headers are exempt from the x-amz-* prefix, as well as Expires + # data can be a buffer or a readable stream + putObject: (key, data, params) -> + # Convert camel-cased params to snake-cased headers + headers = @makeHeaders params + + # Send request + resp = await @request @baseURL + key, null, + method: 'PUT' + headers: headers + body: data + + txt = await resp.text() + console.log txt + if not resp.ok + # no error handling yet + throw resp.status + else + txt + +export default S3 \ No newline at end of file diff --git a/src/crypto.coffee b/src/crypto.coffee new file mode 100644 index 0000000..605a3ae --- /dev/null +++ b/src/crypto.coffee @@ -0,0 +1,25 @@ +utf8Bytes = (str) -> + new TextEncoder 'utf-8' + .encode str + +hex = (buf) -> (Array.prototype.map.call new Uint8Array(buf), + (x) => ('00' + x.toString 16).slice(-2)).join '' + +HMAC_SHA256_KEY = (buf) -> + crypto.subtle.importKey 'raw', buf, + { name: 'HMAC', hash: 'SHA-256' }, true, [ 'sign' ] + +HMAC_SHA256 = (key, str) -> + cryptoKey = await HMAC_SHA256_KEY key + buf = utf8Bytes str + await crypto.subtle.sign "HMAC", cryptoKey, buf + +SHA256 = (str) -> + crypto.subtle.digest "SHA-256", utf8Bytes str + +export { + utf8Bytes, + hex, + HMAC_SHA256, + SHA256 +} \ No newline at end of file diff --git a/src/index.coffee b/src/index.coffee index 8ae48e3..956083a 100644 --- a/src/index.coffee +++ b/src/index.coffee @@ -1,10 +1,11 @@ import * as util from './util' -import * as s3 from './s3' import _ from './prelude' +import S3 from './aws/s3' +import config from '../config.json' + +s3 = new S3 config main = -> - s3.loadAWSConfig _ - addEventListener 'fetch', (event) => event.respondWith handleRequest event @@ -38,19 +39,18 @@ handlePUT = (req, file) -> loop id = util.randomID _ path = util.idToPath id - files = await s3.listFiles path - break if !files or files.length == 0 + files = await s3.listObjects + prefix: path + break if !files.Contents or files.Contents.length == 0 path = path + "/" + file len = req.headers.get "content-length" # Upload the file to S3 try - await s3.uploadFile # TODO: expiry date - Key: path + await s3.putObject path, req.body, # Expiration should be configured on S3 side ContentType: req.headers.get "content-type" ContentLength: len - Body: await util.readToBlob req.body catch err console.log err return buildInvalidResponse err diff --git a/src/s3.coffee b/src/s3.coffee deleted file mode 100644 index 03bb309..0000000 --- a/src/s3.coffee +++ /dev/null @@ -1,26 +0,0 @@ -import config from '../config.json' -import AWS from 'aws-sdk' -import _ from './prelude' - -loadAWSConfig = -> - AWS.config.update config.aws - -getS3 = -> - new AWS.S3 - endpoint: new AWS.Endpoint config.s3.endpoint - -uploadFile = (params) -> - params['Bucket'] = config.s3.bucket - getS3 _ - .putObject params - .promise _ - -listFiles = (path) -> - (await getS3 _ - .listObjects - Bucket: config.s3.bucket - Prefix: path - .promise _) - .Contents - -export { loadAWSConfig, uploadFile, listFiles } \ No newline at end of file diff --git a/src/util.coffee b/src/util.coffee index fa52687..7a7bab8 100644 --- a/src/util.coffee +++ b/src/util.coffee @@ -32,22 +32,10 @@ idToPath = (id) -> id.split '' .join '/' -# Convert a ReadableStream into Blob -# AWS-SDK does not support ReadableStream, unfortunately -readToBlob = (stream) -> - reader = stream.getReader() - ret = [] - loop - { done, value } = await reader.read() - break if done - ret.push value - new Blob ret - export { getFileName, validateLength, MAX_UPLOAD_SIZE, randomID, - idToPath, - readToBlob + idToPath } \ No newline at end of file diff --git a/webpack.config.js b/webpack.config.js index 77cfe97..9fd2829 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -1,6 +1,3 @@ -const webpack = require('webpack') -const path = require('path') - module.exports = { target: "webworker", entry: "./index.js", @@ -10,29 +7,8 @@ module.exports = { minimize: false }, resolve: { - extensions: ['.js', '.coffee'], - alias: { - 'blob-shim': path.resolve(__dirname, './blob-shim.js'), - } + extensions: ['.js', '.coffee'] }, - plugins: [ - new webpack.NormalModuleReplacementPlugin( - // Rewritten xhr.js to use Fetch API - // Mostly from - // Modified to fix a few bugs - /node_modules\/aws-sdk\/lib\/http\/xhr.js/, - '../../../../xhr-shim.js' - ), - new webpack.NormalModuleReplacementPlugin( - // Force it to use node_parser - // Because we are not actually in browser - /node_modules\/aws-sdk\/lib\/xml\/browser_parser.js/, - './node_parser.js' - ), - new webpack.ProvidePlugin({ - 'Blob': 'blob-shim' - }) - ], module: { rules: [ { @@ -40,7 +16,7 @@ module.exports = { use: [ 'coffee-loader' ] }, { - type: 'javascript/auto', // Needed for aws-sdk + type: 'javascript/auto', test: /\.json$/, use: [ 'json-loader' ] } diff --git a/xhr-shim.js b/xhr-shim.js deleted file mode 100644 index cf1bcf5..0000000 --- a/xhr-shim.js +++ /dev/null @@ -1,97 +0,0 @@ -var AWS = require('./node_modules/aws-sdk/lib/core'); -var EventEmitter = require('events').EventEmitter; -require('./node_modules/aws-sdk/lib/http'); - -/** - * @api private - */ - -AWS.XHRClient = AWS.util.inherit({ - handleRequest: function handleRequest(httpRequest, httpOptions, callback, errCallback) { - var self = this; - var endpoint = httpRequest.endpoint; - var emitter = new EventEmitter(); - var href = endpoint.protocol + '//' + endpoint.hostname; - if (endpoint.port !== 80 && endpoint.port !== 443) { - href += ':' + endpoint.port; - } - href += httpRequest.path; - - callback(emitter); - var headers = new Headers(); - - AWS.util.each(httpRequest.headers, function (key, value) { - if (key !== 'Content-Length' && key !== 'User-Agent' && key !== 'Host') { - headers.set(key, value); - } - }); - - var credentials = 'omit'; - - if (httpOptions.xhrWithCredentials) { - credentials = 'include'; - } - - var request = new Request(href, { - method: httpRequest.method, - headers: headers, - body: httpRequest.method == "GET" ? null : httpRequest.body - }); - - fetch(request).then(function(response) { - if (!response.ok) { - throw Error(response.statusText); - } - return response; - }).then(function(response) { - emitter.statusCode = response.status; - new Error(response.headers) - emitter.headers = self.parseHeaders(response.headers); - emitter.emit('headers', emitter.statusCode, emitter.headers); - response.text().then(function(res){ - console.log(res); - self.finishRequest(res, emitter); - }).catch(function(err){ - console.log(err); - }); - - }).catch(function(err) { - errCallback(AWS.util.error(new Error('Network Failure' + err), { - code: 'NetworkingError' - })); - - }); - - return emitter; - }, - - parseHeaders: function parseHeaders(rawHeaders) { - var headers = {}; - if (!rawHeaders) return headers; - for (var pair of rawHeaders.entries()) { - headers[pair[0]] = pair[1]; - } - - return headers; - }, - - finishRequest: function finishRequest(res, emitter) { - var buffer; - try { - buffer = new AWS.util.Buffer(res); - } catch (e) {} - - if (buffer) emitter.emit('data', buffer); - emitter.emit('end'); - } -}); - -/** - * @api private - */ -AWS.HttpClient.prototype = AWS.XHRClient.prototype; - -/** - * @api private - */ -AWS.HttpClient.streamsApiVersion = 1; \ No newline at end of file