diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ad785d194..55df21e8aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ - ### Server -- +- 最初照会したユーザーの最新ノートを受け取るように --> diff --git a/chart/files/default.yml b/chart/files/default.yml index 90b574b99f..db3356d1a8 100644 --- a/chart/files/default.yml +++ b/chart/files/default.yml @@ -187,6 +187,10 @@ id: "aidx" # Sign to ActivityPub GET request (default: true) signToActivityPubGet: true +# Limit of notes to fetch from outbox with remote user first fetched (default: 5) +# https://github.com/misskey-dev/misskey/pull/11130 +outboxNotesFetchLimit: 5 + #allowedPrivateNetworks: [ # '127.0.0.1/32' #] diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index abbfdfed8f..f50bb98b61 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -85,6 +85,7 @@ type Source = { videoThumbnailGenerator?: string; signToActivityPubGet?: boolean; + outboxNotesFetchLimit?: number; perChannelMaxNoteCacheCount?: number; perUserNotificationsMaxCount?: number; diff --git a/packages/backend/src/core/activitypub/ApInboxService.ts b/packages/backend/src/core/activitypub/ApInboxService.ts index c47c72fda2..8590a104c9 100644 --- a/packages/backend/src/core/activitypub/ApInboxService.ts +++ b/packages/backend/src/core/activitypub/ApInboxService.ts @@ -27,7 +27,7 @@ import { QueueService } from '@/core/QueueService.js'; import type { UsersRepository, NotesRepository, FollowingsRepository, AbuseUserReportsRepository, FollowRequestsRepository } from '@/models/_.js'; import { bindThis } from '@/decorators.js'; import type { MiRemoteUser } from '@/models/entities/User.js'; -import { getApHrefNullable, getApId, getApIds, getApType, isAccept, isActor, isAdd, isAnnounce, isBlock, isCollection, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost } from './type.js'; +import { getApHrefNullable, getApId, getApIds, getApType, isAccept, isActor, isAdd, isAnnounce, isBlock, isCollection, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isMove, isOrderedCollectionPage, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost } from './type.js'; import { ApNoteService } from './models/ApNoteService.js'; import { ApLoggerService } from './ApLoggerService.js'; import { ApDbResolverService } from './ApDbResolverService.js'; @@ -87,11 +87,19 @@ export class ApInboxService { } @bindThis - public async performActivity(actor: MiRemoteUser, activity: IObject): Promise { - if (isCollectionOrOrderedCollection(activity)) { + public async performActivity(actor: MiRemoteUser, activity: IObject, { + limit = Infinity, + allow = null as (string[] | null) } = {}, + ): Promise { + if (isCollectionOrOrderedCollection(activity) || isOrderedCollectionPage(activity)) { const resolver = this.apResolverService.createResolver(); - for (const item of toArray(isCollection(activity) ? activity.items : activity.orderedItems)) { + for (const item of toArray(isCollection(activity) ? activity.items : activity.orderedItems).slice(0, limit)) { const act = await resolver.resolve(item); + const type = getApType(act); + if (allow && !allow.includes(type)) { + this.logger.info(`skipping activity type: ${type}`); + continue; + } try { await this.performOneActivity(actor, act); } catch (err) { @@ -367,7 +375,7 @@ export class ApInboxService { }); if (isPost(object)) { - this.createNote(resolver, actor, object, false, activity); + await this.createNote(resolver, actor, object, false, activity); } else { this.logger.warn(`Unknown type: ${getApType(object)}`); } diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index 0d70807335..f8a3b6a170 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -15,11 +15,11 @@ import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; import { LoggerService } from '@/core/LoggerService.js'; import type Logger from '@/logger.js'; -import { isCollectionOrOrderedCollection } from './type.js'; +import { isCollectionOrOrderedCollection, isOrderedCollectionPage } from './type.js'; import { ApDbResolverService } from './ApDbResolverService.js'; import { ApRendererService } from './ApRendererService.js'; import { ApRequestService } from './ApRequestService.js'; -import type { IObject, ICollection, IOrderedCollection } from './type.js'; +import type { IObject, ICollection, IOrderedCollection, IOrderedCollectionPage } from './type.js'; export class Resolver { private history: Set; @@ -64,6 +64,18 @@ export class Resolver { } } + public async resolveOrderedCollectionPage(value: string | IObject): Promise { + const collection = typeof value === 'string' + ? await this.resolve(value) + : value; + + if (isOrderedCollectionPage(collection)) { + return collection; + } else { + throw new Error(`unrecognized collection type: ${collection.type}`); + } + } + @bindThis public async resolve(value: string | IObject): Promise { if (typeof value !== 'string') { diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index d233bcc8b8..56cd9e44d0 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -38,7 +38,8 @@ import { MetaService } from '@/core/MetaService.js'; import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; import type { AccountMoveService } from '@/core/AccountMoveService.js'; import { checkHttps } from '@/misc/check-https.js'; -import { getApId, getApType, getOneApHrefNullable, isActor, isCollection, isCollectionOrOrderedCollection, isPropertyValue } from '../type.js'; +import { getApId, getApType, getOneApHrefNullable, isActor, isCollection, isCollectionOrOrderedCollection, isOrderedCollection, isOrderedCollectionPage, isPropertyValue } from '../type.js'; +import { ApInboxService } from '../ApInboxService.js'; import { extractApHashtags } from './tag.js'; import type { OnModuleInit } from '@nestjs/common'; import type { ApNoteService } from './ApNoteService.js'; @@ -68,6 +69,7 @@ export class ApPersonService implements OnModuleInit { private apResolverService: ApResolverService; private apNoteService: ApNoteService; private apImageService: ApImageService; + private apInboxService: ApInboxService; private apMfmService: ApMfmService; private mfmService: MfmService; private hashtagService: HashtagService; @@ -116,6 +118,7 @@ export class ApPersonService implements OnModuleInit { this.apResolverService = this.moduleRef.get('ApResolverService'); this.apNoteService = this.moduleRef.get('ApNoteService'); this.apImageService = this.moduleRef.get('ApImageService'); + this.apInboxService = this.moduleRef.get('ApInboxService'); this.apMfmService = this.moduleRef.get('ApMfmService'); this.mfmService = this.moduleRef.get('MfmService'); this.hashtagService = this.moduleRef.get('HashtagService'); @@ -384,7 +387,10 @@ export class ApPersonService implements OnModuleInit { } //#endregion - await this.updateFeatured(user.id, resolver).catch(err => this.logger.error(err)); + await Promise.allSettled([ + this.updateFeatured(user.id, resolver).catch(err => this.logger.error(err)), + this.updateOutboxFirstPage(user, person.outbox, resolver).catch(err => this.logger.error(err)), + ]); return user; } @@ -589,7 +595,7 @@ export class ApPersonService implements OnModuleInit { const unresolvedItems = isCollection(collection) ? collection.items : collection.orderedItems; const items = await Promise.all(toArray(unresolvedItems).map(x => _resolver.resolve(x))); - // Resolve and regist Notes + // Resolve and register Notes const limit = promiseLimit(2); const featuredNotes = await Promise.all(items .filter(item => getApType(item) === 'Note') // TODO: Noteでなくてもいいかも @@ -616,6 +622,35 @@ export class ApPersonService implements OnModuleInit { }); } + /** + * Retrieve outbox from an actor object. + * + * This only retrieves the first page for now. + */ + public async updateOutboxFirstPage(user: RemoteUser, outbox: IActor['outbox'], resolver: Resolver): Promise { + if (!this.config.outboxNotesFetchLimit) return; + + // https://www.w3.org/TR/activitypub/#actor-objects + // Outbox is a required property for all actors + if (!outbox) { + throw new Error('No outbox property'); + } + + this.logger.info(`Fetching the outbox for ${user.uri}: ${outbox}`); + + const collection = await resolver.resolveCollection(outbox); + if (!isOrderedCollection(collection)) { + throw new Error('Outbox must be an ordered collection'); + } + + const firstPage = collection.first ? + await resolver.resolveOrderedCollectionPage(collection.first) : + collection; + + // Perform activity but only the first outboxNotesFetchLimit ones with `type: Create` + await this.apInboxService.performActivity(user, firstPage, { limit: this.config.outboxNotesFetchLimit, allow: ['Create'] }); + } + /** * リモート由来のアカウント移行処理を行います * @param src 移行元アカウント（リモートかつupdatePerson後である必要がある、というかこれ自体がupdatePersonで呼ばれる前提） diff --git a/packages/backend/src/core/activitypub/type.ts b/packages/backend/src/core/activitypub/type.ts index 16ff86e894..719e6bfc0a 100644 --- a/packages/backend/src/core/activitypub/type.ts +++ b/packages/backend/src/core/activitypub/type.ts @@ -92,16 +92,37 @@ export interface IActivity extends IObject { }; } +// https://www.w3.org/TR/activitystreams-vocabulary/#dfn-collection export interface ICollection extends IObject { type: 'Collection'; totalItems: number; + current?: ICollectionPage | string; + first?: ICollectionPage | string; + last?: ICollectionPage | string; items: ApObject; } -export interface IOrderedCollection extends IObject { +// https://www.w3.org/TR/activitystreams-vocabulary/#dfn-orderedcollection +export interface IOrderedCollection extends Omit { type: 'OrderedCollection'; - totalItems: number; - orderedItems: ApObject; + + // orderedItems is not defined well + // https://github.com/w3c/activitystreams/issues/494 + orderedItems?: ApObject; +} + +// https://www.w3.org/TR/activitystreams-vocabulary/#dfn-collectionpage +export interface ICollectionPage extends Omit { + type: 'CollectionPage'; + partOf?: ICollection | string; + next?: ICollectionPage | string; + prev?: ICollectionPage | string; +} + +// https://www.w3.org/TR/activitystreams-vocabulary/#dfn-orderedcollectionpage +export interface IOrderedCollectionPage extends Omit, Omit { + type: 'OrderedCollectionPage'; + startIndex?: number, } export const validPost = ['Note', 'Question', 'Article', 'Audio', 'Document', 'Image', 'Page', 'Video', 'Event']; @@ -188,6 +209,9 @@ export const isCollection = (object: IObject): object is ICollection => export const isOrderedCollection = (object: IObject): object is IOrderedCollection => getApType(object) === 'OrderedCollection'; +export const isOrderedCollectionPage = (object: IObject): object is IOrderedCollectionPage => + getApType(object) === 'OrderedCollectionPage'; + export const isCollectionOrOrderedCollection = (object: IObject): object is ICollection | IOrderedCollection => isCollection(object) || isOrderedCollection(object); diff --git a/packages/backend/test/misc/mock-resolver.ts b/packages/backend/test/misc/mock-resolver.ts index 07abe515c3..9530ae01ab 100644 --- a/packages/backend/test/misc/mock-resolver.ts +++ b/packages/backend/test/misc/mock-resolver.ts @@ -68,7 +68,7 @@ export class MockResolver extends Resolver { const r = this.#responseMap.get(value); if (!r) { - throw new Error('Not registed for mock'); + throw new Error('Not registered for mock'); } const object = JSON.parse(r.content); diff --git a/packages/backend/test/unit/activitypub.ts b/packages/backend/test/unit/activitypub.ts index 75235b7948..20757905ea 100644 --- a/packages/backend/test/unit/activitypub.ts +++ b/packages/backend/test/unit/activitypub.ts @@ -17,7 +17,7 @@ import { GlobalModule } from '@/GlobalModule.js'; import { CoreModule } from '@/core/CoreModule.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { LoggerService } from '@/core/LoggerService.js'; -import type { IActor, IApDocument, ICollection, IPost } from '@/core/activitypub/type.js'; +import type { IActivity, IApDocument, IActor, ICollection, IObject, IOrderedCollection, IOrderedCollectionPage, IPost } from '@/core/activitypub/type.js'; import { MiMeta, MiNote } from '@/models/_.js'; import { secureRndstr } from '@/misc/secure-rndstr.js'; import { DownloadService } from '@/core/DownloadService.js'; @@ -29,6 +29,16 @@ const host = 'https://host1.test'; type NonTransientIActor = IActor & { id: string }; type NonTransientIPost = IPost & { id: string }; +type NonTransientICollection = ICollection & { id: string }; +type NonTransientIOrderedCollection = IOrderedCollection & { id: string }; +type NonTransientIOrderedCollectionPage = IOrderedCollectionPage & { id: string }; + +/** + * Use when the order of the array is not definitive + */ +function deepSortedEqual(array1: unknown[], array2: T): asserts array1 is T { + return assert.deepStrictEqual(array1.sort(), array2.sort()); +} function createRandomActor({ actorHost = host } = {}): NonTransientIActor { const preferredUsername = secureRndstr(8); @@ -60,7 +70,7 @@ function createRandomNotes(actor: NonTransientIActor, length: number): NonTransi return new Array(length).fill(null).map(() => createRandomNote(actor)); } -function createRandomFeaturedCollection(actor: NonTransientIActor, length: number): ICollection { +function createRandomFeaturedCollection(actor: NonTransientIActor, length: number): NonTransientICollection { const items = createRandomNotes(actor, length); return { @@ -72,6 +82,53 @@ function createRandomFeaturedCollection(actor: NonTransientIActor, length: numbe }; } +function createRandomActivities(actor: NonTransientIActor, type: string, length: number): IActivity[] { + return new Array(length).fill(null).map((): IActivity => { + const note = createRandomNote(actor); + + return { + type, + id: `${note.id}/activity`, + actor, + object: note, + }; + }); +} + +function createRandomNonPagedOutbox(actor: NonTransientIActor, length: number): NonTransientIOrderedCollection { + const orderedItems = createRandomActivities(actor, 'Create', length); + + return { + '@context': 'https://www.w3.org/ns/activitystreams', + type: 'OrderedCollection', + id: actor.outbox as string, + totalItems: orderedItems.length, + orderedItems, + }; +} + +function createRandomOutboxPage(actor: NonTransientIActor, id: string, length: number): NonTransientIOrderedCollectionPage { + const orderedItems = createRandomActivities(actor, 'Create', length); + + return { + '@context': 'https://www.w3.org/ns/activitystreams', + type: 'OrderedCollectionPage', + id, + totalItems: orderedItems.length, + orderedItems, + }; +} + +function createRandomPagedOutbox(actor: NonTransientIActor): NonTransientIOrderedCollection { + return { + '@context': 'https://www.w3.org/ns/activitystreams', + type: 'OrderedCollection', + id: actor.outbox as string, + totalItems: 10, + first: `${actor.outbox}?first`, + }; +} + async function createRandomRemoteUser( resolver: MockResolver, personService: ApPersonService, @@ -196,7 +253,7 @@ describe('ActivityPub', () => { describe('Renderer', () => { test('Render an announce with visibility: followers', () => { - rendererService.renderAnnounce(null, { + rendererService.renderAnnounce('hoge', { createdAt: new Date(0), visibility: 'followers', } as MiNote); @@ -216,7 +273,7 @@ describe('ActivityPub', () => { await personService.createPerson(actor.id, resolver); // All notes in `featured` are same-origin, no need to fetch notes again - assert.deepStrictEqual(resolver.remoteGetTrials(), [actor.id, actor.featured]); + deepSortedEqual(resolver.remoteGetTrials(), [actor.id, actor.featured, actor.outbox]); // Created notes without resolving anything for (const item of featured.items as IPost[]) { @@ -247,9 +304,9 @@ describe('ActivityPub', () => { await personService.createPerson(actor1.id, resolver); // actor2Note is from a different server and needs to be fetched again - assert.deepStrictEqual( + deepSortedEqual( resolver.remoteGetTrials(), - [actor1.id, actor1.featured, actor2Note.id, actor2.id], + [actor1.id, actor1.featured, actor1.outbox, actor2Note.id, actor2.id, actor2.outbox], ); const note = await noteService.fetchNote(actor2Note.id); @@ -276,6 +333,95 @@ describe('ActivityPub', () => { }); }); + describe('Outbox', () => { + test('Fetch non-paged outbox from IActor', async () => { + const actor = createRandomActor(); + const outbox = createRandomNonPagedOutbox(actor, 10); + + resolver.register(actor.id, actor); + resolver.register(actor.outbox as string, outbox); + + await personService.createPerson(actor.id, resolver); + + deepSortedEqual( + resolver.remoteGetTrials(), + [actor.id, actor.outbox], + ); + + for (const item of outbox.orderedItems as IActivity[]) { + const note = await noteService.fetchNote(item.object); + assert.ok(note); + assert.strictEqual(note.text, 'test test foo'); + assert.strictEqual(note.uri, (item.object as IObject).id); + } + }); + + test('Fetch paged outbox from IActor', async () => { + const actor = createRandomActor(); + const outbox = createRandomPagedOutbox(actor); + const page = createRandomOutboxPage(actor, outbox.id, 10); + + resolver.register(actor.id, actor); + resolver.register(actor.outbox as string, outbox); + resolver.register(outbox.first as string, page); + + await personService.createPerson(actor.id, resolver); + + deepSortedEqual( + resolver.remoteGetTrials(), + [actor.id, actor.outbox, outbox.first], + ); + + for (const item of page.orderedItems as IActivity[]) { + const note = await noteService.fetchNote(item.object); + assert.ok(note); + assert.strictEqual(note.text, 'test test foo'); + assert.strictEqual(note.uri, (item.object as IObject).id); + } + }); + + test('Fetch only the first 20 items', async () => { + const actor = createRandomActor(); + const outbox = createRandomNonPagedOutbox(actor, 200); + + resolver.register(actor.id, actor); + resolver.register(actor.outbox as string, outbox); + + await personService.createPerson(actor.id, resolver); + + const items = outbox.orderedItems as IActivity[]; + + deepSortedEqual( + resolver.remoteGetTrials(), + [actor.id, actor.outbox], + ); + + assert.ok(await noteService.fetchNote(items[19].object)); + assert.ok(!await noteService.fetchNote(items[20].object)); + }); + + test('Perform only Create activities', async () => { + const actor = createRandomActor(); + const outbox = createRandomNonPagedOutbox(actor, 0); + outbox.orderedItems = createRandomActivities(actor, 'Announce', 10); + + resolver.register(actor.id, actor); + resolver.register(actor.outbox as string, outbox); + + await personService.createPerson(actor.id, resolver); + + deepSortedEqual( + resolver.remoteGetTrials(), + [actor.id, actor.outbox], + ); + + for (const item of outbox.orderedItems as IActivity[]) { + const note = await noteService.fetchNote(item.object); + assert.ok(!note); + } + }); + }); + describe('Images', () => { test('Create images', async () => { const imageObject: IApDocument = {