* wip

* update pnpm-lock

* use our own DevNull

* fix

* deliverJobConcurrencyをmacSocketsで割ってソケット数にする
This commit is contained in:
tamaina 2023-01-24 15:54:14 +09:00 committed by GitHub
parent b62894ff56
commit 6589e8a390
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 49 additions and 19 deletions

View file

@ -134,9 +134,34 @@ export class UndiciFetcher {
return res; return res;
} }
@bindThis
public async request(
url: string | URL,
options: { dispatcher?: undici.Dispatcher } & Omit<undici.Dispatcher.RequestOptions, 'origin' | 'path' | 'method'> & Partial<Pick<undici.Dispatcher.RequestOptions, 'method'>> = {},
privateOptions: { noOkError?: boolean; bypassProxy?: boolean; } = { noOkError: false, bypassProxy: false },
): Promise<undici.Dispatcher.ResponseData> {
const res = await undici.request(url, {
dispatcher: this.getAgentByUrl(new URL(url), privateOptions.bypassProxy),
...options,
headers: {
'user-agent': this.userAgent ?? '',
...(options.headers ?? {}),
},
}).catch((err) => {
this.logger?.error(`fetch error to ${typeof url === 'string' ? url : url.href}`, err);
throw new StatusError('Resource Unreachable', 500, 'Resource Unreachable');
});
if (res.statusCode >= 400) {
throw new StatusError(`${res.statusCode}`, res.statusCode, '');
}
return res;
}
@bindThis @bindThis
public async getJson<T extends unknown>(url: string, accept = 'application/json, */*', headers?: Record<string, string>): Promise<T> { public async getJson<T extends unknown>(url: string, accept = 'application/json, */*', headers?: Record<string, string>): Promise<T> {
const res = await this.fetch( const { body } = await this.request(
url, url,
{ {
headers: Object.assign({ headers: Object.assign({
@ -145,12 +170,12 @@ export class UndiciFetcher {
}, },
); );
return await res.json() as T; return await body.json() as T;
} }
@bindThis @bindThis
public async getHtml(url: string, accept = 'text/html, */*', headers?: Record<string, string>): Promise<string> { public async getHtml(url: string, accept = 'text/html, */*', headers?: Record<string, string>): Promise<string> {
const res = await this.fetch( const { body } = await this.request(
url, url,
{ {
headers: Object.assign({ headers: Object.assign({
@ -159,7 +184,7 @@ export class UndiciFetcher {
}, },
); );
return await res.text(); return await body.text();
} }
} }
@ -167,6 +192,7 @@ export class UndiciFetcher {
export class HttpRequestService { export class HttpRequestService {
public defaultFetcher: UndiciFetcher; public defaultFetcher: UndiciFetcher;
public fetch: UndiciFetcher['fetch']; public fetch: UndiciFetcher['fetch'];
public request: UndiciFetcher['request'];
public getHtml: UndiciFetcher['getHtml']; public getHtml: UndiciFetcher['getHtml'];
public defaultJsonFetcher: UndiciFetcher; public defaultJsonFetcher: UndiciFetcher;
public getJson: UndiciFetcher['getJson']; public getJson: UndiciFetcher['getJson'];
@ -221,11 +247,12 @@ export class HttpRequestService {
}, },
}; };
this.maxSockets = Math.max(64, this.config.deliverJobConcurrency ?? 128); this.maxSockets = Math.max(64, ((this.config.deliverJobConcurrency ?? 128) / (this.config.clusterLimit ?? 1)));
this.defaultFetcher = this.createFetcher({}, {}, this.logger); this.defaultFetcher = this.createFetcher({}, {}, this.logger);
this.fetch = this.defaultFetcher.fetch; this.fetch = this.defaultFetcher.fetch;
this.request = this.defaultFetcher.request;
this.getHtml = this.defaultFetcher.getHtml; this.getHtml = this.defaultFetcher.getHtml;
this.defaultJsonFetcher = this.createFetcher({ this.defaultJsonFetcher = this.createFetcher({

View file

@ -9,10 +9,12 @@ import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js'
import { LoggerService } from '@/core/LoggerService.js'; import { LoggerService } from '@/core/LoggerService.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import type Logger from '@/logger.js'; import type Logger from '@/logger.js';
import type { Dispatcher } from 'undici';
import { DevNull } from '@/misc/dev-null.js';
type Request = { type Request = {
url: string; url: string;
method: string; method: Dispatcher.HttpMethod;
headers: Record<string, string>; headers: Record<string, string>;
}; };
@ -163,7 +165,7 @@ export class ApRequestService {
}, },
}); });
await this.undiciFetcher.fetch( const response = await this.undiciFetcher.request(
url, url,
{ {
method: req.request.method, method: req.request.method,
@ -171,6 +173,7 @@ export class ApRequestService {
body, body,
}, },
); );
response.body.pipe(new DevNull());
} }
/** /**

View file

@ -0,0 +1,11 @@
import { Writable, WritableOptions } from "node:stream";
export class DevNull extends Writable implements NodeJS.WritableStream {
constructor(opts?: WritableOptions) {
super(opts);
}
_write (chunk: any, encoding: BufferEncoding, cb: (err?: Error | null) => void) {
setImmediate(cb);
}
}

View file

@ -3801,7 +3801,7 @@ packages:
/axios/0.24.0: /axios/0.24.0:
resolution: {integrity: sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==} resolution: {integrity: sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==}
dependencies: dependencies:
follow-redirects: 1.15.2 follow-redirects: 1.15.2_debug@4.3.4
transitivePeerDependencies: transitivePeerDependencies:
- debug - debug
dev: false dev: false
@ -6973,16 +6973,6 @@ packages:
readable-stream: 2.3.7 readable-stream: 2.3.7
dev: false dev: false
/follow-redirects/1.15.2:
resolution: {integrity: sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==}
engines: {node: '>=4.0'}
peerDependencies:
debug: '*'
peerDependenciesMeta:
debug:
optional: true
dev: false
/follow-redirects/1.15.2_debug@4.3.4: /follow-redirects/1.15.2_debug@4.3.4:
resolution: {integrity: sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==} resolution: {integrity: sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==}
engines: {node: '>=4.0'} engines: {node: '>=4.0'}
@ -6993,7 +6983,6 @@ packages:
optional: true optional: true
dependencies: dependencies:
debug: 4.3.4 debug: 4.3.4
dev: true
/for-each/0.3.3: /for-each/0.3.3:
resolution: {integrity: sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==} resolution: {integrity: sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==}