Compare commits

...

8 commits

Author SHA1 Message Date
github-actions[bot]
9ea7340da6 Bump version to 2025.8.0-alpha.12 2025-08-15 13:44:57 +00:00
anatawa12
60f7278aff
fix: Remote Note Cleaning will delete notes embedded in a page (#16408)
* feat: preserve number of pages referencing the note

* chore: delete pages on account delete

* fix: notes on the pages are removed by CleanRemoteNotes

* test: add the simplest test for page embedded notes

* fix: section block is not considered

* fix: section block is not considered in migration

* chore: remove comments from columns

* revert unnecessary change

* add pageCount to webhook test

* fix type error on backend
2025-08-15 22:39:55 +09:00
syuilo
bae92a944d enhance(frontend): improve enableInfiniteScroll stability
Close #16318
2025-08-15 12:40:37 +09:00
syuilo
7d30768769 fix(frontend): Botプロテクションの設定の変更検知が正しくない問題を修正 2025-08-15 12:10:14 +09:00
github-actions[bot]
e444942c4e Bump version to 2025.8.0-alpha.11 2025-08-14 08:02:14 +00:00
饺子w (Yumechi)
90b9609341
enhance: performance for CleanRemoteNotesProcessorService (#16404)
* enhance: performance for CleanRemoteNotesProcessorService

Signed-off-by: eternal-flame-AD <yume@yumechi.jp>

* suggestions

Signed-off-by: eternal-flame-AD <yume@yumechi.jp>

* docs

Signed-off-by: eternal-flame-AD <yume@yumechi.jp>

* change initial limit to 100

Signed-off-by: eternal-flame-AD <yume@yumechi.jp>

* robustness for transient race conditions

Signed-off-by: eternal-flame-AD <yume@yumechi.jp>

* handle cursors in postgres

Signed-off-by: eternal-flame-AD <yume@yumechi.jp>

* robustness: transient errors and timeout handling

Signed-off-by: eternal-flame-AD <yume@yumechi.jp>

* use '0' as initial cursor

Signed-off-by: eternal-flame-AD <yume@yumechi.jp>

---------

Signed-off-by: eternal-flame-AD <yume@yumechi.jp>
2025-08-14 16:54:28 +09:00
syuilo
c25a922928 Merge branch 'develop' of https://github.com/misskey-dev/misskey into develop 2025-08-14 16:50:59 +09:00
syuilo
d26169ea32 Update about-misskey.vue 2025-08-14 16:50:56 +09:00
22 changed files with 600 additions and 206 deletions

View file

@ -33,6 +33,7 @@
- URLに`?safemode=true`を付ける
- PWAのショートカットで Safemode を選択して起動する
- Feat: ページのタブバーを下部に表示できるように
- Enhance: 「自動でもっと見る」オプションが有効になり、安定性が向上しました
- Enhance: コントロールパネルを検索できるように
- Enhance: トルコ語 (tr-TR) に対応
- Enhance: 不必要な翻訳データを読み込まなくなり、パフォーマンスが向上しました

View file

@ -1,6 +1,6 @@
{
"name": "misskey",
"version": "2025.8.0-alpha.10",
"version": "2025.8.0-alpha.12",
"codename": "nasubi",
"repository": {
"type": "git",

View file

@ -0,0 +1,58 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class PageCountInNote1755168347001 {
name = 'PageCountInNote1755168347001'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "note" ADD "pageCount" smallint NOT NULL DEFAULT '0'`);
// Update existing notes
// block_list CTE collects all page blocks on the pages including child blocks in the section blocks.
// The clipped_notes CTE counts how many distinct pages each note block is referenced in.
// Finally, we update the note table with the count of pages for each referenced note.
await queryRunner.query(`
WITH RECURSIVE block_list AS (
(
SELECT
page.id as page_id,
block as block
FROM page
CROSS JOIN LATERAL jsonb_array_elements(page.content) block
WHERE block->>'type' = 'note' OR block->>'type' = 'section'
)
UNION ALL
(
SELECT
block_list.page_id,
child_block AS block
FROM LATERAL (
SELECT page_id, block
FROM block_list
WHERE block_list.block->>'type' = 'section'
) block_list
CROSS JOIN LATERAL jsonb_array_elements(block_list.block->'children') child_block
WHERE child_block->>'type' = 'note' OR child_block->>'type' = 'section'
)
),
clipped_notes AS (
SELECT
(block->>'note') AS note_id,
COUNT(distinct block_list.page_id) AS count
FROM block_list
WHERE block_list.block->>'type' = 'note'
GROUP BY block->>'note'
)
UPDATE note
SET "pageCount" = clipped_notes.count
FROM clipped_notes
WHERE note.id = clipped_notes.note_id;
`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "note" DROP COLUMN "pageCount"`);
}
}

View file

@ -78,6 +78,7 @@ import { ChannelFollowingService } from './ChannelFollowingService.js';
import { ChatService } from './ChatService.js';
import { RegistryApiService } from './RegistryApiService.js';
import { ReversiService } from './ReversiService.js';
import { PageService } from './PageService.js';
import { ChartLoggerService } from './chart/ChartLoggerService.js';
import FederationChart from './chart/charts/federation.js';
@ -227,6 +228,7 @@ const $ChannelFollowingService: Provider = { provide: 'ChannelFollowingService',
const $ChatService: Provider = { provide: 'ChatService', useExisting: ChatService };
const $RegistryApiService: Provider = { provide: 'RegistryApiService', useExisting: RegistryApiService };
const $ReversiService: Provider = { provide: 'ReversiService', useExisting: ReversiService };
const $PageService: Provider = { provide: 'PageService', useExisting: PageService };
const $ChartLoggerService: Provider = { provide: 'ChartLoggerService', useExisting: ChartLoggerService };
const $FederationChart: Provider = { provide: 'FederationChart', useExisting: FederationChart };
@ -379,6 +381,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
ChatService,
RegistryApiService,
ReversiService,
PageService,
ChartLoggerService,
FederationChart,
@ -527,6 +530,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
$ChatService,
$RegistryApiService,
$ReversiService,
$PageService,
$ChartLoggerService,
$FederationChart,
@ -676,6 +680,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
ChatService,
RegistryApiService,
ReversiService,
PageService,
FederationChart,
NotesChart,
@ -822,6 +827,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
$ChatService,
$RegistryApiService,
$ReversiService,
$PageService,
$FederationChart,
$NotesChart,

View file

@ -0,0 +1,223 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Inject, Injectable } from '@nestjs/common';
import { DataSource, In, Not } from 'typeorm';
import { DI } from '@/di-symbols.js';
import {
type NotesRepository,
MiPage,
type PagesRepository,
MiDriveFile,
type UsersRepository,
MiNote,
} from '@/models/_.js';
import { bindThis } from '@/decorators.js';
import { RoleService } from '@/core/RoleService.js';
import { IdService } from '@/core/IdService.js';
import type { MiUser } from '@/models/User.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { ModerationLogService } from '@/core/ModerationLogService.js';
export interface PageBody {
title: string;
name: string;
summary: string | null;
content: Array<Record<string, any>>;
variables: Array<Record<string, any>>;
script: string;
eyeCatchingImage?: MiDriveFile | null;
font: string;
alignCenter: boolean;
hideTitleWhenPinned: boolean;
}
@Injectable()
export class PageService {
constructor(
@Inject(DI.db)
private db: DataSource,
@Inject(DI.pagesRepository)
private pagesRepository: PagesRepository,
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
private roleService: RoleService,
private moderationLogService: ModerationLogService,
private idService: IdService,
) {
}
@bindThis
public async create(
me: MiUser,
body: PageBody,
): Promise<MiPage> {
await this.pagesRepository.findBy({
userId: me.id,
name: body.name,
}).then(result => {
if (result.length > 0) {
throw new IdentifiableError('1a79e38e-3d83-4423-845b-a9d83ff93b61');
}
});
const page = await this.pagesRepository.insertOne(new MiPage({
id: this.idService.gen(),
updatedAt: new Date(),
title: body.title,
name: body.name,
summary: body.summary,
content: body.content,
variables: body.variables,
script: body.script,
eyeCatchingImageId: body.eyeCatchingImage ? body.eyeCatchingImage.id : null,
userId: me.id,
visibility: 'public',
alignCenter: body.alignCenter,
hideTitleWhenPinned: body.hideTitleWhenPinned,
font: body.font,
}));
const referencedNotes = this.collectReferencedNotes(page.content);
if (referencedNotes.length > 0) {
await this.notesRepository.increment({ id: In(referencedNotes) }, 'pageCount', 1);
}
return page;
}
@bindThis
public async update(
me: MiUser,
pageId: MiPage['id'],
body: Partial<PageBody>,
): Promise<void> {
await this.db.transaction(async (transaction) => {
const page = await transaction.findOne(MiPage, {
where: {
id: pageId,
},
lock: { mode: 'for_no_key_update' },
});
if (page == null) {
throw new IdentifiableError('66aefd3c-fdb2-4a71-85ae-cc18bea85d3f');
}
if (page.userId !== me.id) {
throw new IdentifiableError('d0017699-8256-46f1-aed4-bc03bed73616');
}
if (body.name != null) {
await transaction.findBy(MiPage, {
id: Not(pageId),
userId: me.id,
name: body.name,
}).then(result => {
if (result.length > 0) {
throw new IdentifiableError('d05bfe24-24b6-4ea2-a3ec-87cc9bf4daa4');
}
});
}
await transaction.update(MiPage, page.id, {
updatedAt: new Date(),
title: body.title,
name: body.name,
summary: body.summary === undefined ? page.summary : body.summary,
content: body.content,
variables: body.variables,
script: body.script,
alignCenter: body.alignCenter,
hideTitleWhenPinned: body.hideTitleWhenPinned,
font: body.font,
eyeCatchingImageId: body.eyeCatchingImage === undefined ? undefined : (body.eyeCatchingImage?.id ?? null),
});
console.log("page.content", page.content);
if (body.content != null) {
const beforeReferencedNotes = this.collectReferencedNotes(page.content);
const afterReferencedNotes = this.collectReferencedNotes(body.content);
const removedNotes = beforeReferencedNotes.filter(noteId => !afterReferencedNotes.includes(noteId));
const addedNotes = afterReferencedNotes.filter(noteId => !beforeReferencedNotes.includes(noteId));
if (removedNotes.length > 0) {
await transaction.decrement(MiNote, { id: In(removedNotes) }, 'pageCount', 1);
}
if (addedNotes.length > 0) {
await transaction.increment(MiNote, { id: In(addedNotes) }, 'pageCount', 1);
}
}
});
}
@bindThis
public async delete(me: MiUser, pageId: MiPage['id']): Promise<void> {
await this.db.transaction(async (transaction) => {
const page = await transaction.findOne(MiPage, {
where: {
id: pageId,
},
lock: { mode: 'pessimistic_write' }, // same lock level as DELETE
});
if (page == null) {
throw new IdentifiableError('66aefd3c-fdb2-4a71-85ae-cc18bea85d3f');
}
if (!await this.roleService.isModerator(me) && page.userId !== me.id) {
throw new IdentifiableError('d0017699-8256-46f1-aed4-bc03bed73616');
}
await transaction.delete(MiPage, page.id);
if (page.userId !== me.id) {
const user = await this.usersRepository.findOneByOrFail({ id: page.userId });
this.moderationLogService.log(me, 'deletePage', {
pageId: page.id,
pageUserId: page.userId,
pageUserUsername: user.username,
page,
});
}
const referencedNotes = this.collectReferencedNotes(page.content);
if (referencedNotes.length > 0) {
await transaction.decrement(MiNote, { id: In(referencedNotes) }, 'pageCount', 1);
}
});
}
collectReferencedNotes(content: MiPage['content']): string[] {
const referencingNotes = new Set<string>();
const recursiveCollect = (content: unknown[]) => {
for (const contentElement of content) {
if (typeof contentElement === 'object'
&& contentElement !== null
&& 'type' in contentElement) {
if (contentElement.type === 'note'
&& 'note' in contentElement
&& typeof contentElement.note === 'string') {
referencingNotes.add(contentElement.note);
}
if (contentElement.type === 'section'
&& 'children' in contentElement
&& Array.isArray(contentElement.children)) {
recursiveCollect(contentElement.children);
}
}
}
};
recursiveCollect(content);
return [...referencingNotes];
}
}

View file

@ -85,6 +85,7 @@ function generateDummyNote(override?: Partial<MiNote>): MiNote {
renoteCount: 10,
repliesCount: 5,
clippedCount: 0,
pageCount: 0,
reactions: {},
visibility: 'public',
uri: null,

View file

@ -114,6 +114,13 @@ export class MiNote {
})
public clippedCount: number;
// The number of note page blocks referencing this note.
// This column is used by Remote Note Cleaning and manually updated rather than automatically with triggers.
@Column('smallint', {
default: 0,
})
public pageCount: number;
@Column('jsonb', {
default: {},
})

View file

@ -5,6 +5,7 @@
import { setTimeout } from 'node:timers/promises';
import { Inject, Injectable } from '@nestjs/common';
import { DataSource, IsNull, LessThan, QueryFailedError, Not } from 'typeorm';
import { DI } from '@/di-symbols.js';
import type { MiMeta, MiNote, NotesRepository } from '@/models/_.js';
import type Logger from '@/logger.js';
@ -24,18 +25,31 @@ export class CleanRemoteNotesProcessorService {
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
@Inject(DI.db)
private db: DataSource,
private idService: IdService,
private queueLoggerService: QueueLoggerService,
) {
this.logger = this.queueLoggerService.logger.createSubLogger('clean-remote-notes');
}
@bindThis
private computeProgress(minId: string, maxId: string, cursorLeft: string) {
const minTs = this.idService.parse(minId).date.getTime();
const maxTs = this.idService.parse(maxId).date.getTime();
const cursorTs = this.idService.parse(cursorLeft).date.getTime();
return ((cursorTs - minTs) / (maxTs - minTs)) * 100;
}
@bindThis
public async process(job: Bull.Job<Record<string, unknown>>): Promise<{
deletedCount: number;
oldest: number | null;
newest: number | null;
skipped?: boolean;
skipped: boolean;
transientErrors: number;
}> {
if (!this.meta.enableRemoteNotesCleaning) {
this.logger.info('Remote notes cleaning is disabled, skipping...');
@ -44,6 +58,7 @@ export class CleanRemoteNotesProcessorService {
oldest: null,
newest: null,
skipped: true,
transientErrors: 0,
};
}
@ -52,12 +67,10 @@ export class CleanRemoteNotesProcessorService {
const maxDuration = this.meta.remoteNotesCleaningMaxProcessingDurationInMinutes * 60 * 1000; // Convert minutes to milliseconds
const startAt = Date.now();
const MAX_NOTE_COUNT_PER_QUERY = 50;
//#retion queries
// We use string literals instead of query builder for several reasons:
// - for removeCondition, we need to use it in having clause, which is not supported by Brackets.
// - for recursive part, we need to preserve the order of columns, but typeorm query builder does not guarantee the order of columns in the result query
//#region queries
// The date limit for the newest note to be considered for deletion.
// All notes newer than this limit will always be retained.
const newestLimit = this.idService.gen(Date.now() - (1000 * 60 * 60 * 24 * this.meta.remoteNotesCleaningExpiryDaysForEachNotes));
// The condition for removing the notes.
// The note must be:
@ -66,56 +79,94 @@ export class CleanRemoteNotesProcessorService {
// - not have clipped
// - not have pinned on the user profile
// - not has been favorite by any user
const removeCondition = 'note.id < :newestLimit'
+ ' AND note."clippedCount" = 0'
+ ' AND note."userHost" IS NOT NULL'
// using both userId and noteId instead of just noteId to use index on user_note_pining table.
// This is safe because notes are only pinned by the user who created them.
+ ' AND NOT EXISTS(SELECT 1 FROM "user_note_pining" WHERE "noteId" = note."id" AND "userId" = note."userId")'
// We cannot use userId trick because users can favorite notes from other users.
+ ' AND NOT EXISTS(SELECT 1 FROM "note_favorite" WHERE "noteId" = note."id")'
;
const removalCriteria = [
'note."id" < :newestLimit',
'note."clippedCount" = 0',
'note."pageCount" = 0',
'note."userHost" IS NOT NULL',
'NOT EXISTS (SELECT 1 FROM user_note_pining WHERE "noteId" = note."id")',
'NOT EXISTS (SELECT 1 FROM note_favorite WHERE "noteId" = note."id")',
].join(' AND ');
// The initiator query contains the oldest ${MAX_NOTE_COUNT_PER_QUERY} remote non-clipped notes
const initiatorQuery = this.notesRepository.createQueryBuilder('note')
const minId = (await this.notesRepository.createQueryBuilder('note')
.select('MIN(note.id)', 'minId')
.where({
id: LessThan(newestLimit),
userHost: Not(IsNull()),
replyId: IsNull(),
renoteId: IsNull(),
})
.getRawOne<{ minId?: MiNote['id'] }>())?.minId;
if (!minId) {
this.logger.info('No notes can possibly be deleted, skipping...');
return {
deletedCount: 0,
oldest: null,
newest: null,
skipped: false,
transientErrors: 0,
};
}
// start with a conservative limit and adjust it based on the query duration
const minimumLimit = 10;
let currentLimit = 100;
let cursorLeft = '0';
const candidateNotesCteName = 'candidate_notes';
// tree walk down all root notes, short-circuit when the first unremovable note is found
const candidateNotesQueryBase = this.notesRepository.createQueryBuilder('note')
.select('note."id"', 'id')
.addSelect('note."replyId"', 'replyId')
.addSelect('note."renoteId"', 'renoteId')
.addSelect('note."id"', 'rootId')
.addSelect('TRUE', 'isRemovable')
.addSelect('TRUE', 'isBase')
.where('note."id" > :cursorLeft')
.andWhere(removalCriteria)
.andWhere({ replyId: IsNull(), renoteId: IsNull() });
const candidateNotesQueryInductive = this.notesRepository.createQueryBuilder('note')
.select('note.id', 'id')
.where(removeCondition)
.andWhere('note.id > :cursor')
.orderBy('note.id', 'ASC')
.limit(MAX_NOTE_COUNT_PER_QUERY);
.addSelect('note."replyId"', 'replyId')
.addSelect('note."renoteId"', 'renoteId')
.addSelect('parent."rootId"', 'rootId')
.addSelect(removalCriteria, 'isRemovable')
.addSelect('FALSE', 'isBase')
.innerJoin(candidateNotesCteName, 'parent', 'parent."id" = note."replyId" OR parent."id" = note."renoteId"')
.where('parent."isRemovable" = TRUE');
// The union query queries the related notes and replies related to the initiator query
const unionQuery = `
SELECT "note"."id", "note"."replyId", "note"."renoteId", rn."initiatorId"
FROM "note" "note"
INNER JOIN "related_notes" "rn"
ON "note"."replyId" = rn.id
OR "note"."renoteId" = rn.id
OR "note"."id" = rn."replyId"
OR "note"."id" = rn."renoteId"
`;
const selectRelatedNotesFromInitiatorIdsQuery = `
SELECT "note"."id" AS "id", "note"."replyId" AS "replyId", "note"."renoteId" AS "renoteId", "note"."id" AS "initiatorId"
FROM "note" "note" WHERE "note"."id" IN (:...initiatorIds)
`;
const recursiveQuery = `(${selectRelatedNotesFromInitiatorIdsQuery}) UNION (${unionQuery})`;
const removableInitiatorNotesQuery = this.notesRepository.createQueryBuilder('note')
.select('rn."initiatorId"')
.innerJoin('related_notes', 'rn', 'note.id = rn.id')
.groupBy('rn."initiatorId"')
.having(`bool_and(${removeCondition})`);
const notesQuery = this.notesRepository.createQueryBuilder('note')
.addCommonTableExpression(recursiveQuery, 'related_notes', { recursive: true })
.select('note.id', 'id')
.addSelect('rn."initiatorId"')
.innerJoin('related_notes', 'rn', 'note.id = rn.id')
.where(`rn."initiatorId" IN (${removableInitiatorNotesQuery.getQuery()})`)
.distinctOn(['note.id']);
//#endregion
// A note tree can be deleted if there are no unremovable rows with the same rootId.
//
// `candidate_notes` will have the following structure after recursive query (some columns omitted):
// After performing a LEFT JOIN with `candidate_notes` as `unremovable`,
// the note tree containing unremovable notes will be anti-joined.
// For removable rows, the `unremovable` columns will have `NULL` values.
// | id | rootId | isRemovable |
// |-----|--------|-------------|
// | aaa | aaa | TRUE |
// | bbb | aaa | FALSE |
// | ccc | aaa | FALSE |
// | ddd | ddd | TRUE |
// | eee | ddd | TRUE |
// | fff | fff | TRUE |
// | ggg | ggg | FALSE |
//
const candidateNotesQuery = this.db.createQueryBuilder()
.select(`"${candidateNotesCteName}"."id"`, 'id')
.addSelect('unremovable."id" IS NULL', 'isRemovable')
.addSelect(`BOOL_OR("${candidateNotesCteName}"."isBase")`, 'isBase')
.addCommonTableExpression(
`((SELECT "base".* FROM (${candidateNotesQueryBase.orderBy('note.id', 'ASC').limit(currentLimit).getQuery()}) AS "base") UNION ${candidateNotesQueryInductive.getQuery()})`,
candidateNotesCteName,
{ recursive: true },
)
.from(candidateNotesCteName, candidateNotesCteName)
.leftJoin(candidateNotesCteName, 'unremovable', `unremovable."rootId" = "${candidateNotesCteName}"."rootId" AND unremovable."isRemovable" = FALSE`)
.groupBy(`"${candidateNotesCteName}"."id"`)
.addGroupBy('unremovable."id" IS NULL');
const stats = {
deletedCount: 0,
@ -123,74 +174,107 @@ export class CleanRemoteNotesProcessorService {
newest: null as number | null,
};
// The date limit for the newest note to be considered for deletion.
// All notes newer than this limit will always be retained.
const newestLimit = this.idService.gen(Date.now() - (1000 * 60 * 60 * 24 * this.meta.remoteNotesCleaningExpiryDaysForEachNotes));
let cursor = '0'; // oldest note ID to start from
while (true) {
let lowThroughputWarned = false;
let transientErrors = 0;
for (;;) {
//#region check time
const batchBeginAt = Date.now();
const elapsed = batchBeginAt - startAt;
const progress = this.computeProgress(minId, newestLimit, cursorLeft > minId ? cursorLeft : minId);
if (elapsed >= maxDuration) {
this.logger.info(`Reached maximum duration of ${maxDuration}ms, stopping...`);
job.log('Reached maximum duration, stopping cleaning.');
job.log(`Reached maximum duration of ${maxDuration}ms, stopping... (last cursor: ${cursorLeft}, final progress ${progress}%)`);
job.updateProgress(100);
break;
}
job.updateProgress((elapsed / maxDuration) * 100);
const wallClockUsage = elapsed / maxDuration;
if (wallClockUsage > 0.5 && progress < 50 && !lowThroughputWarned) {
const msg = `Not projected to finish in time! (wall clock usage ${wallClockUsage * 100}% at ${progress}%, current limit ${currentLimit})`;
this.logger.warn(msg);
job.log(msg);
lowThroughputWarned = true;
}
job.updateProgress(progress);
//#endregion
// First, we fetch the initiator notes that are older than the newestLimit.
const initiatorNotes: { id: MiNote['id'] }[] = await initiatorQuery.setParameters({ cursor, newestLimit }).getRawMany();
const queryBegin = performance.now();
let noteIds = null;
// update the cursor to the newest initiatorId found in the fetched notes.
const newCursor = initiatorNotes.reduce((max, note) => note.id > max ? note.id : max, cursor);
try {
noteIds = await candidateNotesQuery.setParameters(
{ newestLimit, cursorLeft },
).getRawMany<{ id: MiNote['id'], isRemovable: boolean, isBase: boolean }>();
} catch (e) {
if (currentLimit > minimumLimit && e instanceof QueryFailedError && e.driverError?.code === '57014') {
// Statement timeout (maybe suddenly hit a large note tree), reduce the limit and try again
// continuous failures will eventually converge to currentLimit == minimumLimit and then throw
currentLimit = Math.max(minimumLimit, Math.floor(currentLimit * 0.25));
continue;
}
throw e;
}
if (initiatorNotes.length === 0 || cursor === newCursor || newCursor >= newestLimit) {
// If no notes were found or the cursor did not change, we can stop.
job.log('No more notes to clean. (no initiator notes found or cursor did not change.)');
if (noteIds.length === 0) {
job.log('No more notes to clean.');
break;
}
const notes: { id: MiNote['id'], initiatorId: MiNote['id'] }[] = await notesQuery.setParameters({
initiatorIds: initiatorNotes.map(note => note.id),
newestLimit,
}).getRawMany();
const queryDuration = performance.now() - queryBegin;
// try to adjust such that each query takes about 1~5 seconds and reasonable NodeJS heap so the task stays responsive
// this should not oscillate..
if (queryDuration > 5000 || noteIds.length > 5000) {
currentLimit = Math.floor(currentLimit * 0.5);
} else if (queryDuration < 1000 && noteIds.length < 1000) {
currentLimit = Math.floor(currentLimit * 1.5);
}
// clamp to a sane range
currentLimit = Math.min(Math.max(currentLimit, minimumLimit), 5000);
cursor = newCursor;
const deletableNoteIds = noteIds.filter(result => result.isRemovable).map(result => result.id);
if (deletableNoteIds.length > 0) {
try {
await this.notesRepository.delete(deletableNoteIds);
if (notes.length > 0) {
await this.notesRepository.delete(notes.map(note => note.id));
for (const { id } of notes) {
const t = this.idService.parse(id).date.getTime();
if (stats.oldest === null || t < stats.oldest) {
stats.oldest = t;
for (const id of deletableNoteIds) {
const t = this.idService.parse(id).date.getTime();
if (stats.oldest === null || t < stats.oldest) {
stats.oldest = t;
}
if (stats.newest === null || t > stats.newest) {
stats.newest = t;
}
}
if (stats.newest === null || t > stats.newest) {
stats.newest = t;
stats.deletedCount += deletableNoteIds.length;
} catch (e) {
// check for integrity violation errors (class 23) that might have occurred between the check and the delete
// we can safely continue to the next batch
if (e instanceof QueryFailedError && e.driverError?.code?.startsWith('23')) {
transientErrors++;
job.log(`Error deleting notes: ${e} (transient race condition?)`);
} else {
throw e;
}
}
stats.deletedCount += notes.length;
}
job.log(`Deleted ${notes.length} from ${initiatorNotes.length} initiators; ${Date.now() - batchBeginAt}ms`);
cursorLeft = noteIds.filter(result => result.isBase).reduce((max, { id }) => id > max ? id : max, cursorLeft);
if (initiatorNotes.length < MAX_NOTE_COUNT_PER_QUERY) {
// If we fetched less than the maximum, it means there are no more notes to process.
job.log(`No more notes to clean. (fewer than MAX_NOTE_COUNT_PER_QUERY =${MAX_NOTE_COUNT_PER_QUERY}.)`);
break;
job.log(`Deleted ${noteIds.length} notes; ${Date.now() - batchBeginAt}ms`);
if (process.env.NODE_ENV !== 'test') {
await setTimeout(Math.min(1000 * 5, queryDuration)); // Wait a moment to avoid overwhelming the db
}
};
await setTimeout(1000 * 5); // Wait a moment to avoid overwhelming the db
if (transientErrors > 0) {
const msg = `${transientErrors} transient errors occurred while cleaning remote notes. You may need a second pass to complete the cleaning.`;
this.logger.warn(msg);
job.log(msg);
}
this.logger.succ('cleaning of remote notes completed.');
return {
@ -198,6 +282,7 @@ export class CleanRemoteNotesProcessorService {
oldest: stats.oldest,
newest: stats.newest,
skipped: false,
transientErrors,
};
}
}

View file

@ -6,7 +6,7 @@
import { Inject, Injectable } from '@nestjs/common';
import { MoreThan } from 'typeorm';
import { DI } from '@/di-symbols.js';
import type { DriveFilesRepository, NotesRepository, UserProfilesRepository, UsersRepository } from '@/models/_.js';
import type { DriveFilesRepository, NotesRepository, PagesRepository, UserProfilesRepository, UsersRepository } from '@/models/_.js';
import type Logger from '@/logger.js';
import { DriveService } from '@/core/DriveService.js';
import type { MiDriveFile } from '@/models/DriveFile.js';
@ -14,6 +14,7 @@ import type { MiNote } from '@/models/Note.js';
import { EmailService } from '@/core/EmailService.js';
import { bindThis } from '@/decorators.js';
import { SearchService } from '@/core/SearchService.js';
import { PageService } from '@/core/PageService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbUserDeleteJobData } from '../types.js';
@ -35,7 +36,11 @@ export class DeleteAccountProcessorService {
@Inject(DI.driveFilesRepository)
private driveFilesRepository: DriveFilesRepository,
@Inject(DI.pagesRepository)
private pagesRepository: PagesRepository,
private driveService: DriveService,
private pageService: PageService,
private emailService: EmailService,
private queueLoggerService: QueueLoggerService,
private searchService: SearchService,
@ -112,6 +117,28 @@ export class DeleteAccountProcessorService {
this.logger.succ('All of files deleted');
}
{
// delete pages. Necessary for decrementing pageCount of notes.
while (true) {
const pages = await this.pagesRepository.find({
where: {
userId: user.id,
},
take: 100,
order: {
id: 1,
},
});
if (pages.length === 0) {
break;
}
for (const page of pages) {
await this.pageService.delete(user, page.id);
}
}
}
{ // Send email notification
const profile = await this.userProfilesRepository.findOneByOrFail({ userId: user.id });
if (profile.email && profile.emailVerified) {

View file

@ -5,12 +5,13 @@
import ms from 'ms';
import { Inject, Injectable } from '@nestjs/common';
import type { DriveFilesRepository, PagesRepository } from '@/models/_.js';
import { IdService } from '@/core/IdService.js';
import { MiPage, pageNameSchema } from '@/models/Page.js';
import type { DriveFilesRepository, MiDriveFile, PagesRepository } from '@/models/_.js';
import { pageNameSchema } from '@/models/Page.js';
import { Endpoint } from '@/server/api/endpoint-base.js';
import { PageEntityService } from '@/core/entities/PageEntityService.js';
import { DI } from '@/di-symbols.js';
import { PageService } from '@/core/PageService.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { ApiError } from '../../error.js';
export const meta = {
@ -77,11 +78,11 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
@Inject(DI.driveFilesRepository)
private driveFilesRepository: DriveFilesRepository,
private pageService: PageService,
private pageEntityService: PageEntityService,
private idService: IdService,
) {
super(meta, paramDef, async (ps, me) => {
let eyeCatchingImage = null;
let eyeCatchingImage: MiDriveFile | null = null;
if (ps.eyeCatchingImageId != null) {
eyeCatchingImage = await this.driveFilesRepository.findOneBy({
id: ps.eyeCatchingImageId,
@ -102,24 +103,20 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
}
});
const page = await this.pagesRepository.insertOne(new MiPage({
id: this.idService.gen(),
updatedAt: new Date(),
title: ps.title,
name: ps.name,
summary: ps.summary,
content: ps.content,
variables: ps.variables,
script: ps.script,
eyeCatchingImageId: eyeCatchingImage ? eyeCatchingImage.id : null,
userId: me.id,
visibility: 'public',
alignCenter: ps.alignCenter,
hideTitleWhenPinned: ps.hideTitleWhenPinned,
font: ps.font,
}));
try {
const page = await this.pageService.create(me, {
...ps,
eyeCatchingImage,
summary: ps.summary ?? null,
});
return await this.pageEntityService.pack(page);
return await this.pageEntityService.pack(page);
} catch (err) {
if (err instanceof IdentifiableError && err.id === '1a79e38e-3d83-4423-845b-a9d83ff93b61') {
throw new ApiError(meta.errors.nameAlreadyExists);
}
throw err;
}
});
}
}

View file

@ -4,12 +4,14 @@
*/
import { Inject, Injectable } from '@nestjs/common';
import type { PagesRepository, UsersRepository } from '@/models/_.js';
import type { MiDriveFile, PagesRepository, UsersRepository } from '@/models/_.js';
import { Endpoint } from '@/server/api/endpoint-base.js';
import { DI } from '@/di-symbols.js';
import { ModerationLogService } from '@/core/ModerationLogService.js';
import { RoleService } from '@/core/RoleService.js';
import { ApiError } from '../../error.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { PageService } from '@/core/PageService.js';
export const meta = {
tags: ['pages'],
@ -44,36 +46,17 @@ export const paramDef = {
@Injectable()
export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-disable-line import/no-default-export
constructor(
@Inject(DI.pagesRepository)
private pagesRepository: PagesRepository,
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
private moderationLogService: ModerationLogService,
private roleService: RoleService,
private pageService: PageService,
) {
super(meta, paramDef, async (ps, me) => {
const page = await this.pagesRepository.findOneBy({ id: ps.pageId });
if (page == null) {
throw new ApiError(meta.errors.noSuchPage);
}
if (!await this.roleService.isModerator(me) && page.userId !== me.id) {
throw new ApiError(meta.errors.accessDenied);
}
await this.pagesRepository.delete(page.id);
if (page.userId !== me.id) {
const user = await this.usersRepository.findOneByOrFail({ id: page.userId });
this.moderationLogService.log(me, 'deletePage', {
pageId: page.id,
pageUserId: page.userId,
pageUserUsername: user.username,
page,
});
try {
await this.pageService.delete(me, ps.pageId);
} catch (err) {
if (err instanceof IdentifiableError) {
if (err.id === '66aefd3c-fdb2-4a71-85ae-cc18bea85d3f') throw new ApiError(meta.errors.noSuchPage);
if (err.id === 'd0017699-8256-46f1-aed4-bc03bed73616') throw new ApiError(meta.errors.accessDenied);
}
throw err;
}
});
}

View file

@ -4,13 +4,14 @@
*/
import ms from 'ms';
import { Not } from 'typeorm';
import { Inject, Injectable } from '@nestjs/common';
import type { PagesRepository, DriveFilesRepository } from '@/models/_.js';
import type { DriveFilesRepository, MiDriveFile } from '@/models/_.js';
import { Endpoint } from '@/server/api/endpoint-base.js';
import { DI } from '@/di-symbols.js';
import { ApiError } from '../../error.js';
import { pageNameSchema } from '@/models/Page.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { PageService } from '@/core/PageService.js';
export const meta = {
tags: ['pages'],
@ -75,57 +76,37 @@ export const paramDef = {
@Injectable()
export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-disable-line import/no-default-export
constructor(
@Inject(DI.pagesRepository)
private pagesRepository: PagesRepository,
@Inject(DI.driveFilesRepository)
private driveFilesRepository: DriveFilesRepository,
private pageService: PageService,
) {
super(meta, paramDef, async (ps, me) => {
const page = await this.pagesRepository.findOneBy({ id: ps.pageId });
if (page == null) {
throw new ApiError(meta.errors.noSuchPage);
}
if (page.userId !== me.id) {
throw new ApiError(meta.errors.accessDenied);
}
try {
let eyeCatchingImage: MiDriveFile | null | undefined | string = ps.eyeCatchingImageId;
if (eyeCatchingImage != null) {
eyeCatchingImage = await this.driveFilesRepository.findOneBy({
id: eyeCatchingImage,
userId: me.id,
});
if (ps.eyeCatchingImageId != null) {
const eyeCatchingImage = await this.driveFilesRepository.findOneBy({
id: ps.eyeCatchingImageId,
userId: me.id,
});
if (eyeCatchingImage == null) {
throw new ApiError(meta.errors.noSuchFile);
}
}
if (ps.name != null) {
await this.pagesRepository.findBy({
id: Not(ps.pageId),
userId: me.id,
name: ps.name,
}).then(result => {
if (result.length > 0) {
throw new ApiError(meta.errors.nameAlreadyExists);
if (eyeCatchingImage == null) {
throw new ApiError(meta.errors.noSuchFile);
}
});
}
}
await this.pagesRepository.update(page.id, {
updatedAt: new Date(),
title: ps.title,
name: ps.name,
summary: ps.summary === undefined ? page.summary : ps.summary,
content: ps.content,
variables: ps.variables,
script: ps.script,
alignCenter: ps.alignCenter,
hideTitleWhenPinned: ps.hideTitleWhenPinned,
font: ps.font,
eyeCatchingImageId: ps.eyeCatchingImageId,
});
await this.pageService.update(me, ps.pageId, {
...ps,
eyeCatchingImage,
});
} catch (err) {
if (err instanceof IdentifiableError) {
if (err.id === '66aefd3c-fdb2-4a71-85ae-cc18bea85d3f') throw new ApiError(meta.errors.noSuchPage);
if (err.id === 'd0017699-8256-46f1-aed4-bc03bed73616') throw new ApiError(meta.errors.accessDenied);
if (err.id === 'd05bfe24-24b6-4ea2-a3ec-87cc9bf4daa4') throw new ApiError(meta.errors.nameAlreadyExists);
}
throw err;
}
});
}
}

View file

@ -40,6 +40,7 @@ describe('NoteCreateService', () => {
renoteCount: 0,
repliesCount: 0,
clippedCount: 0,
pageCount: 0,
reactions: {},
visibility: 'public',
uri: null,

View file

@ -23,6 +23,7 @@ const base: MiNote = {
renoteCount: 0,
repliesCount: 0,
clippedCount: 0,
pageCount: 0,
reactions: {},
visibility: 'public',
uri: null,

View file

@ -158,6 +158,7 @@ describe('CleanRemoteNotesProcessorService', () => {
oldest: null,
newest: null,
skipped: true,
transientErrors: 0,
});
});
@ -172,6 +173,7 @@ describe('CleanRemoteNotesProcessorService', () => {
oldest: null,
newest: null,
skipped: false,
transientErrors: 0,
});
}, 3000);
@ -199,6 +201,7 @@ describe('CleanRemoteNotesProcessorService', () => {
oldest: expect.any(Number),
newest: expect.any(Number),
skipped: false,
transientErrors: 0,
});
// Check side-by-side from all notes
@ -278,6 +281,24 @@ describe('CleanRemoteNotesProcessorService', () => {
expect(remainingNote).not.toBeNull();
});
// ページ
test('should not delete note that is embedded in a page', async () => {
const job = createMockJob();
// Create old remote note that is embedded in a page
const clippedNote = await createNote({
pageCount: 1, // Embedded in a page
}, bob, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000);
const result = await service.process(job as any);
expect(result.deletedCount).toBe(0);
expect(result.skipped).toBe(false);
const remainingNote = await notesRepository.findOneBy({ id: clippedNote.id });
expect(remainingNote).not.toBeNull();
});
// 古いreply, renoteが含まれている時の挙動
test('should handle reply/renote relationships correctly', async () => {
const job = createMockJob();

View file

@ -4,7 +4,7 @@ SPDX-License-Identifier: AGPL-3.0-only
-->
<template>
<div :class="$style.root">
<div v-if="form.modified.value" :class="$style.root">
<div :class="$style.text">{{ i18n.tsx.thereAreNChanges({ n: form.modifiedCount.value }) }}</div>
<div style="margin-left: auto;" class="_buttons">
<MkButton danger rounded @click="form.discard"><i class="ti ti-x"></i> {{ i18n.ts.discard }}</MkButton>
@ -16,16 +16,11 @@ SPDX-License-Identifier: AGPL-3.0-only
<script lang="ts" setup>
import { } from 'vue';
import MkButton from './MkButton.vue';
import type { useForm } from '@/composables/use-form.js';
import { i18n } from '@/i18n.js';
const props = withDefaults(defineProps<{
form: {
modifiedCount: {
value: number;
};
discard: () => void;
save: () => void;
};
form: ReturnType<typeof useForm>;
canSaving?: boolean;
}>(), {
canSaving: true,

View file

@ -47,7 +47,7 @@ SPDX-License-Identifier: AGPL-3.0-only
<MkNote v-else :class="$style.note" :note="note" :withHardMute="true" :data-scroll-anchor="note.id"/>
</template>
</component>
<button v-show="paginator.canFetchOlder.value" key="_more_" :disabled="paginator.fetchingOlder.value" class="_button" :class="$style.more" @click="paginator.fetchOlder">
<button v-show="paginator.canFetchOlder.value" key="_more_" v-appear="prefer.s.enableInfiniteScroll ? paginator.fetchOlder : null" :disabled="paginator.fetchingOlder.value" class="_button" :class="$style.more" @click="paginator.fetchOlder">
<div v-if="!paginator.fetchingOlder.value">{{ i18n.ts.loadMore }}</div>
<MkLoading v-else :inline="true"/>
</button>

View file

@ -33,7 +33,7 @@ SPDX-License-Identifier: AGPL-3.0-only
<XNotification v-else :class="$style.content" :notification="notification" :withTime="true" :full="true"/>
</div>
</component>
<button v-show="paginator.canFetchOlder.value" key="_more_" :disabled="paginator.fetchingOlder.value" class="_button" :class="$style.more" @click="paginator.fetchOlder">
<button v-show="paginator.canFetchOlder.value" key="_more_" v-appear="prefer.s.enableInfiniteScroll ? paginator.fetchOlder : null" :disabled="paginator.fetchingOlder.value" class="_button" :class="$style.more" @click="paginator.fetchOlder">
<div v-if="!paginator.fetchingOlder.value">{{ i18n.ts.loadMore }}</div>
<MkLoading v-else/>
</button>

View file

@ -3,6 +3,7 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { throttle } from 'throttle-debounce';
import type { Directive } from 'vue';
export default {
@ -10,12 +11,14 @@ export default {
const fn = binding.value;
if (fn == null) return;
const observer = new IntersectionObserver(entries => {
const check = throttle(1000, (entries) => {
if (entries.some(entry => entry.isIntersecting)) {
fn();
}
});
const observer = new IntersectionObserver(check);
observer.observe(src);
src._observer_ = observer;

View file

@ -111,6 +111,9 @@ SPDX-License-Identifier: AGPL-3.0-only
<div>
<a style="display: inline-block;" class="purpledotdigital" title="Purple Dot Digital" href="https://purpledotdigital.com/" target="_blank"><img style="width: 100%;" src="https://assets.misskey-hub.net/sponsors/purple-dot-digital.jpg" alt="Purple Dot Digital"></a>
</div>
<div>
<a style="display: inline-block;" class="sads-llc" title="合同会社サッズ" href="https://sads-llc.co.jp/" target="_blank"><img style="width: 100%;" src="https://assets.misskey-hub.net/sponsors/sads-llc.png" alt="合同会社サッズ"></a>
</div>
</div>
</FormSection>
<FormSection>
@ -403,6 +406,7 @@ const patrons = [
'東雲 琥珀',
'ほとラズ',
'スズカケン',
'蒼井よみこ',
];
const thereIsTreasure = ref($i && !claimedAchievements.includes('foundTreasure'));

View file

@ -14,7 +14,7 @@ SPDX-License-Identifier: AGPL-3.0-only
<template v-else-if="botProtectionForm.savedState.provider === 'turnstile'" #suffix>Turnstile</template>
<template v-else-if="botProtectionForm.savedState.provider === 'testcaptcha'" #suffix>testCaptcha</template>
<template v-else #suffix>{{ i18n.ts.none }} ({{ i18n.ts.notRecommended }})</template>
<template #footer>
<template v-if="botProtectionForm.modified.value" #footer>
<MkFormFooter :canSaving="canSaving" :form="botProtectionForm"/>
</template>

View file

@ -1,7 +1,7 @@
{
"type": "module",
"name": "misskey-js",
"version": "2025.8.0-alpha.10",
"version": "2025.8.0-alpha.12",
"description": "Misskey SDK for JavaScript",
"license": "MIT",
"main": "./built/index.js",