bump aws-sdk to v3 for s3 (#10363)

* indent

* aws-sdk v3移行
This commit is contained in:
YS 2023-03-23 13:48:14 +09:00 committed by GitHub
parent 050787238b
commit 658901a47f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 1267 additions and 164 deletions

View file

@ -37,6 +37,9 @@
"@tensorflow/tfjs-node": "4.2.0"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.294.0",
"@aws-sdk/lib-storage": "^3.294.0",
"@aws-sdk/node-http-handler": "^3.292.0",
"@bull-board/api": "5.0.0",
"@bull-board/fastify": "5.0.0",
"@bull-board/ui": "5.0.0",
@ -59,7 +62,6 @@
"ajv": "8.12.0",
"archiver": "5.3.1",
"autwh": "0.1.0",
"aws-sdk": "2.1318.0",
"bcryptjs": "2.4.3",
"blurhash": "2.0.5",
"bull": "4.10.4",
@ -190,6 +192,7 @@
"@types/ws": "8.5.4",
"@typescript-eslint/eslint-plugin": "5.54.1",
"@typescript-eslint/parser": "5.54.1",
"aws-sdk-client-mock": "^2.1.1",
"cross-env": "7.0.3",
"eslint": "8.35.0",
"eslint-plugin-import": "2.27.5",

View file

@ -4,6 +4,7 @@ import { v4 as uuid } from 'uuid';
import sharp from 'sharp';
import { sharpBmp } from 'sharp-read-bmp';
import { IsNull } from 'typeorm';
import { DeleteObjectCommandInput, PutObjectCommandInput, NoSuchKey } from '@aws-sdk/client-s3';
import { DI } from '@/di-symbols.js';
import type { DriveFilesRepository, UsersRepository, DriveFoldersRepository, UserProfilesRepository } from '@/models/index.js';
import type { Config } from '@/config.js';
@ -36,7 +37,6 @@ import { bindThis } from '@/decorators.js';
import { RoleService } from '@/core/RoleService.js';
import { correctFilename } from '@/misc/correct-filename.js';
import { isMimeImage } from '@/misc/is-mime-image.js';
import type S3 from 'aws-sdk/clients/s3.js';
type AddFileArgs = {
/** User who wish to add file */
@ -81,6 +81,7 @@ type UploadFromUrlArgs = {
export class DriveService {
private registerLogger: Logger;
private downloaderLogger: Logger;
private deleteLogger: Logger;
constructor(
@Inject(DI.config)
@ -118,6 +119,7 @@ export class DriveService {
const logger = new Logger('drive', 'blue');
this.registerLogger = logger.createSubLogger('register', 'yellow');
this.downloaderLogger = logger.createSubLogger('downloader');
this.deleteLogger = logger.createSubLogger('delete');
}
/***
@ -368,7 +370,7 @@ export class DriveService {
Body: stream,
ContentType: type,
CacheControl: 'max-age=31536000, immutable',
} as S3.PutObjectRequest;
} as PutObjectCommandInput;
if (filename) params.ContentDisposition = contentDisposition(
'inline',
@ -378,21 +380,16 @@ export class DriveService {
);
if (meta.objectStorageSetPublicRead) params.ACL = 'public-read';
const s3 = this.s3Service.getS3(meta);
const upload = s3.upload(params, {
partSize: s3.endpoint.hostname === 'storage.googleapis.com' ? 500 * 1024 * 1024 : 8 * 1024 * 1024,
});
await upload.promise()
await this.s3Service.upload(meta, params)
.then(
result => {
if (result) {
if ('Bucket' in result) { // CompleteMultipartUploadCommandOutput
this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`);
} else {
this.registerLogger.error(`Upload Result Empty: key = ${key}, filename = ${filename}`);
} else { // AbortMultipartUploadCommandOutput
this.registerLogger.error(`Upload Result Aborted: key = ${key}, filename = ${filename}`);
}
},
})
.catch(
err => {
this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}`, err);
},
@ -720,24 +717,24 @@ export class DriveService {
@bindThis
public async deleteObjectStorageFile(key: string) {
const meta = await this.metaService.fetch();
const s3 = this.s3Service.getS3(meta);
try {
await s3.deleteObject({
Bucket: meta.objectStorageBucket!,
const param = {
Bucket: meta.objectStorageBucket,
Key: key,
}).promise();
} as DeleteObjectCommandInput;
await this.s3Service.delete(meta, param);
} catch (err: any) {
if (err.code === 'NoSuchKey') {
console.warn(`The object storage had no such key to delete: ${key}. Skipping this.`, err);
if (err.name === 'NoSuchKey') {
this.deleteLogger.warn(`The object storage had no such key to delete: ${key}. Skipping this.`, err as Error);
return;
}
} else {
throw new Error(`Failed to delete the file from the object storage with the given key: ${key}`, {
cause: err,
});
}
}
}
@bindThis
public async uploadFromUrl({

View file

@ -1,11 +1,16 @@
import { URL } from 'node:url';
import * as http from 'node:http';
import * as https from 'node:https';
import { Inject, Injectable } from '@nestjs/common';
import S3 from 'aws-sdk/clients/s3.js';
import { DeleteObjectCommand, S3Client } from '@aws-sdk/client-s3';
import { Upload } from '@aws-sdk/lib-storage';
import { NodeHttpHandler, NodeHttpHandlerOptions } from '@aws-sdk/node-http-handler';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import type { Meta } from '@/models/entities/Meta.js';
import { HttpRequestService } from '@/core/HttpRequestService.js';
import { bindThis } from '@/decorators.js';
import type { DeleteObjectCommandInput, PutObjectCommandInput } from '@aws-sdk/client-s3';
@Injectable()
export class S3Service {
@ -18,25 +23,47 @@ export class S3Service {
}
@bindThis
public getS3(meta: Meta) {
public getS3Client(meta: Meta): S3Client {
const u = meta.objectStorageEndpoint
? `${meta.objectStorageUseSSL ? 'https://' : 'http://'}${meta.objectStorageEndpoint}`
: `${meta.objectStorageUseSSL ? 'https://' : 'http://'}example.net`;
? `${meta.objectStorageUseSSL ? 'https' : 'http'}://${meta.objectStorageEndpoint}`
: `${meta.objectStorageUseSSL ? 'https' : 'http'}://example.net`; // dummy url to select http(s) agent
return new S3({
endpoint: meta.objectStorageEndpoint && meta.objectStorageEndpoint.length > 0
? meta.objectStorageEndpoint
: undefined,
accessKeyId: meta.objectStorageAccessKey!,
secretAccessKey: meta.objectStorageSecretKey!,
const agent = this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy);
const handlerOption: NodeHttpHandlerOptions = {};
if (meta.objectStorageUseSSL) {
handlerOption.httpsAgent = agent as https.Agent;
} else {
handlerOption.httpAgent = agent as http.Agent;
}
return new S3Client({
endpoint: meta.objectStorageEndpoint ? u : undefined,
credentials: (meta.objectStorageAccessKey !== null && meta.objectStorageSecretKey !== null) ? {
accessKeyId: meta.objectStorageAccessKey,
secretAccessKey: meta.objectStorageSecretKey,
} : undefined,
region: meta.objectStorageRegion ?? undefined,
sslEnabled: meta.objectStorageUseSSL,
s3ForcePathStyle: !meta.objectStorageEndpoint // AWS with endPoint omitted
? false
: meta.objectStorageS3ForcePathStyle,
httpOptions: {
agent: this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy),
},
tls: meta.objectStorageUseSSL,
forcePathStyle: meta.objectStorageEndpoint ? meta.objectStorageS3ForcePathStyle : false, // AWS with endPoint omitted
requestHandler: new NodeHttpHandler(handlerOption),
});
}
@bindThis
public async upload(meta: Meta, input: PutObjectCommandInput) {
const client = this.getS3Client(meta);
return new Upload({
client,
params: input,
partSize: (client.config.endpoint && (await client.config.endpoint()).hostname === 'storage.googleapis.com')
? 500 * 1024 * 1024
: 8 * 1024 * 1024,
}).done();
}
@bindThis
public delete(meta: Meta, input: DeleteObjectCommandInput) {
const client = this.getS3Client(meta);
return client.send(new DeleteObjectCommand(input));
}
}

View file

@ -1,55 +1,56 @@
process.env.NODE_ENV = 'test';
import { jest } from '@jest/globals';
import { Test } from '@nestjs/testing';
import { DeleteObjectCommandOutput, DeleteObjectCommand, NoSuchKey, InvalidObjectState, S3Client } from '@aws-sdk/client-s3';
import { mockClient } from 'aws-sdk-client-mock';
import { GlobalModule } from '@/GlobalModule.js';
import { DriveService } from '@/core/DriveService.js';
import { CoreModule } from '@/core/CoreModule.js';
import { S3Service } from '@/core/S3Service';
import type { Meta } from '@/models';
import type { DeleteObjectOutput } from 'aws-sdk/clients/s3';
import type { AWSError } from 'aws-sdk/lib/error';
import type { PromiseResult, Request } from 'aws-sdk/lib/request';
import type { TestingModule } from '@nestjs/testing';
describe('DriveService', () => {
let app: TestingModule;
let driveService: DriveService;
const s3Mock = mockClient(S3Client);
beforeEach(async () => {
beforeAll(async () => {
app = await Test.createTestingModule({
imports: [GlobalModule, CoreModule],
providers: [DriveService, S3Service],
providers: [DriveService],
}).compile();
app.enableShutdownHooks();
driveService = app.get<DriveService>(DriveService);
const s3Service = app.get<S3Service>(S3Service);
const s3 = s3Service.getS3({} as Meta);
// new S3() surprisingly does not return an instance of class S3.
// Let's use getPrototypeOf here to get a real prototype, since spying on S3.prototype doesn't work.
// TODO: Use `aws-sdk-client-mock` package when upgrading to AWS SDK v3.
jest.spyOn(Object.getPrototypeOf(s3), 'deleteObject').mockImplementation(() => {
// Roughly mock AWS request object
return {
async promise(): Promise<PromiseResult<DeleteObjectOutput, AWSError>> {
const err = new Error('mock') as AWSError;
err.code = 'NoSuchKey';
throw err;
},
} as Request<DeleteObjectOutput, AWSError>;
});
beforeEach(async () => {
s3Mock.reset();
});
afterAll(async () => {
await app.close();
});
describe('Object storage', () => {
test('delete a file', async () => {
s3Mock.on(DeleteObjectCommand)
.resolves({} as DeleteObjectCommandOutput);
await driveService.deleteObjectStorageFile('peace of the world');
});
test('delete a file then unexpected error', async () => {
s3Mock.on(DeleteObjectCommand)
.rejects(new InvalidObjectState({ $metadata: {}, message: '' }));
await expect(driveService.deleteObjectStorageFile('unexpected')).rejects.toThrowError(Error);
});
test('delete a file with no valid key', async () => {
try {
// Some S3 implementations returns 404 Not Found on deleting with a non-existent key
s3Mock.on(DeleteObjectCommand)
.rejects(new NoSuchKey({ $metadata: {}, message: 'allowed error.' }));
await driveService.deleteObjectStorageFile('lol no way');
} catch (err: any) {
console.log(err.cause);
throw err;
}
});
});
});

View file

@ -0,0 +1,77 @@
process.env.NODE_ENV = 'test';
import { Test } from '@nestjs/testing';
import { UploadPartCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
import { mockClient } from 'aws-sdk-client-mock';
import { GlobalModule } from '@/GlobalModule.js';
import { CoreModule } from '@/core/CoreModule.js';
import { S3Service } from '@/core/S3Service';
import { Meta } from '@/models';
import type { TestingModule } from '@nestjs/testing';
describe('S3Service', () => {
let app: TestingModule;
let s3Service: S3Service;
const s3Mock = mockClient(S3Client);
beforeAll(async () => {
app = await Test.createTestingModule({
imports: [GlobalModule, CoreModule],
providers: [S3Service],
}).compile();
app.enableShutdownHooks();
s3Service = app.get<S3Service>(S3Service);
});
beforeEach(async () => {
s3Mock.reset();
});
afterAll(async () => {
await app.close();
});
describe('upload', () => {
test('upload a file', async () => {
s3Mock.on(PutObjectCommand).resolves({});
await s3Service.upload({ objectStorageRegion: 'us-east-1' } as Meta, {
Bucket: 'fake',
Key: 'fake',
Body: 'x',
});
});
test('upload a large file', async () => {
s3Mock.on(CreateMultipartUploadCommand).resolves({ UploadId: '1' });
s3Mock.on(UploadPartCommand).resolves({ ETag: '1' });
s3Mock.on(CompleteMultipartUploadCommand).resolves({ Bucket: 'fake', Key: 'fake' });
await s3Service.upload({} as Meta, {
Bucket: 'fake',
Key: 'fake',
Body: 'x'.repeat(8 * 1024 * 1024 + 1), // デフォルトpartSizeにしている 8 * 1024 * 1024 を越えるサイズ
});
});
test('upload a file error', async () => {
s3Mock.on(PutObjectCommand).rejects({ name: 'Fake Error' });
await expect(s3Service.upload({ objectStorageRegion: 'us-east-1' } as Meta, {
Bucket: 'fake',
Key: 'fake',
Body: 'x',
})).rejects.toThrowError(Error);
});
test('upload a large file error', async () => {
s3Mock.on(UploadPartCommand).rejects();
await expect(s3Service.upload({} as Meta, {
Bucket: 'fake',
Key: 'fake',
Body: 'x'.repeat(8 * 1024 * 1024 + 1), // デフォルトpartSizeにしている 8 * 1024 * 1024 を越えるサイズ
})).rejects.toThrowError(Error);
});
});
});

File diff suppressed because it is too large Load diff