diff --git a/.config/docker_example.yml b/.config/docker_example.yml index af0a90dc95..6946954ce5 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -95,15 +95,13 @@ redis: # #prefix: example-prefix # #db: 1 -# ┌─────────────────────────────┐ -#───┘ Elasticsearch configuration └───────────────────────────── +# ┌───────────────────────────┐ +#───┘ MeiliSearch configuration └───────────────────────────── -#elasticsearch: -# host: localhost -# port: 9200 -# ssl: false -# user: -# pass: +#meilisearch: +# host: meilisearch +# port: 7700 +# apiKey: '' # ┌───────────────┐ #───┘ ID generation └─────────────────────────────────────────── diff --git a/.config/example.yml b/.config/example.yml index 8111b1992e..5861176677 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -95,15 +95,13 @@ redis: # #prefix: example-prefix # #db: 1 -# ┌─────────────────────────────┐ -#───┘ Elasticsearch configuration └───────────────────────────── +# ┌───────────────────────────┐ +#───┘ MeiliSearch configuration └───────────────────────────── -#elasticsearch: +#meilisearch: # host: localhost -# port: 9200 -# ssl: false -# user: -# pass: +# port: 7700 +# apiKey: '' # ┌───────────────┐ #───┘ ID generation └─────────────────────────────────────────── diff --git a/.devcontainer/devcontainer.yml b/.devcontainer/devcontainer.yml index 2af306e3da..e1b89c25bd 100644 --- a/.devcontainer/devcontainer.yml +++ b/.devcontainer/devcontainer.yml @@ -95,15 +95,13 @@ redis: # #prefix: example-prefix # #db: 1 -# ┌─────────────────────────────┐ -#───┘ Elasticsearch configuration └───────────────────────────── +# ┌───────────────────────────┐ +#───┘ MeiliSearch configuration └───────────────────────────── -#elasticsearch: -# host: localhost -# port: 9200 -# ssl: false -# user: -# pass: +#meilisearch: +# host: meilisearch +# port: 7700 +# apiKey: '' # ┌───────────────┐ #───┘ ID generation └─────────────────────────────────────────── diff --git a/.dockerignore b/.dockerignore index 151ede038e..1de0c7982b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,7 +8,6 @@ build/ built/ db/ docker-compose.yml -elasticsearch/ node_modules/ packages/*/node_modules redis/ diff --git a/.gitignore b/.gitignore index fbe2245502..537232d37f 100644 --- a/.gitignore +++ b/.gitignore @@ -44,7 +44,7 @@ built /data /.cache-loader /db -/elasticsearch +/meili_data npm-debug.log *.pem run.bat diff --git a/CHANGELOG.md b/CHANGELOG.md index f3adee8f52..72a9355565 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - Node.js 18.6.0以上が必要になりました ### General +- Meilisearchを全文検索に使用できるようになりました - 新規登録前に簡潔なルールをユーザーに表示できる、サーバールール機能を追加 - ユーザーへの自分用メモ機能 * ユーザーに対して、自分だけが見られるメモを追加できるようになりました。 diff --git a/chart/files/default.yml b/chart/files/default.yml index 1888669245..f50c38d57e 100644 --- a/chart/files/default.yml +++ b/chart/files/default.yml @@ -116,15 +116,13 @@ redis: # #prefix: example-prefix # #db: 1 -# ┌─────────────────────────────┐ -#───┘ Elasticsearch configuration └───────────────────────────── +# ┌───────────────────────────┐ +#───┘ MeiliSearch configuration └───────────────────────────── -#elasticsearch: +#meilisearch: # host: localhost -# port: 9200 -# ssl: false -# user: -# pass: +# port: 7700 +# apiKey: '' # ┌───────────────┐ #───┘ ID generation └─────────────────────────────────────────── diff --git a/docker-compose.yml.example b/docker-compose.yml.example index b0c4a914d5..a0061c5c20 100644 --- a/docker-compose.yml.example +++ b/docker-compose.yml.example @@ -7,7 +7,7 @@ services: links: - db - redis -# - es +# - meilisearch depends_on: db: condition: service_healthy @@ -48,16 +48,18 @@ services: interval: 5s retries: 20 -# es: +# meilisearch: # restart: always -# image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.4.2 +# image: getmeili/meilisearch:v1.1.1 # environment: -# - "ES_JAVA_OPTS=-Xms512m -Xmx512m" -# - "TAKE_FILE_OWNERSHIP=111" +# - MEILI_NO_ANALYTICS=true +# - MEILI_ENV=production +# env_file: +# - .config/meilisearch.env # networks: # - internal_network # volumes: -# - ./elasticsearch:/usr/share/elasticsearch/data +# - ./meili_data:/meili_data networks: internal_network: diff --git a/packages/backend/package.json b/packages/backend/package.json index 9b20c121eb..08557d415e 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -91,6 +91,7 @@ "jsdom": "21.1.1", "json5": "2.2.3", "jsonld": "8.1.1", + "meilisearch": "0.32.3", "jsrsasign": "10.8.6", "mfm-js": "0.23.3", "mime-types": "2.1.35", diff --git a/packages/backend/src/GlobalModule.ts b/packages/backend/src/GlobalModule.ts index 4574429c43..2f4862285d 100644 --- a/packages/backend/src/GlobalModule.ts +++ b/packages/backend/src/GlobalModule.ts @@ -2,6 +2,7 @@ import { setTimeout } from 'node:timers/promises'; import { Global, Inject, Module } from '@nestjs/common'; import * as Redis from 'ioredis'; import { DataSource } from 'typeorm'; +import { MeiliSearch } from 'meilisearch'; import { DI } from './di-symbols.js'; import { loadConfig } from './config.js'; import { createPostgresDataSource } from './postgres.js'; @@ -22,6 +23,21 @@ const $db: Provider = { inject: [DI.config], }; +const $meilisearch: Provider = { + provide: DI.meilisearch, + useFactory: (config) => { + if (config.meilisearch) { + return new MeiliSearch({ + host: `http://${config.meilisearch.host}:${config.meilisearch.port}`, + apiKey: config.meilisearch.apiKey, + }); + } else { + return null; + } + }, + inject: [DI.config], +}; + const $redis: Provider = { provide: DI.redis, useFactory: (config) => { @@ -73,8 +89,8 @@ const $redisForSub: Provider = { @Global() @Module({ imports: [RepositoryModule], - providers: [$config, $db, $redis, $redisForPub, $redisForSub], - exports: [$config, $db, $redis, $redisForPub, $redisForSub, RepositoryModule], + providers: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub], + exports: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, RepositoryModule], }) export class GlobalModule implements OnApplicationShutdown { constructor( diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 4499475ee9..7354268a4d 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -57,13 +57,10 @@ export type Source = { db?: number; prefix?: string; }; - elasticsearch: { + meilisearch?: { host: string; - port: number; - ssl?: boolean; - user?: string; - pass?: string; - index?: string; + port: string; + apiKey: string; }; proxy?: string; @@ -139,6 +136,7 @@ const path = process.env.MISSKEY_CONFIG_YML : process.env.NODE_ENV === 'test' ? resolve(dir, 'test.yml') : resolve(dir, 'default.yml'); + export function loadConfig() { const meta = JSON.parse(fs.readFileSync(`${_dirname}/../../../built/meta.json`, 'utf-8')); const clientManifestExists = fs.existsSync(_dirname + '/../../../built/_vite_/manifest.json'); diff --git a/packages/backend/src/core/CoreModule.ts b/packages/backend/src/core/CoreModule.ts index 8775536e4a..d3a1b1b024 100644 --- a/packages/backend/src/core/CoreModule.ts +++ b/packages/backend/src/core/CoreModule.ts @@ -50,6 +50,7 @@ import { WebhookService } from './WebhookService.js'; import { ProxyAccountService } from './ProxyAccountService.js'; import { UtilityService } from './UtilityService.js'; import { FileInfoService } from './FileInfoService.js'; +import { SearchService } from './SearchService.js'; import { ChartLoggerService } from './chart/ChartLoggerService.js'; import FederationChart from './chart/charts/federation.js'; import NotesChart from './chart/charts/notes.js'; @@ -171,6 +172,8 @@ const $VideoProcessingService: Provider = { provide: 'VideoProcessingService', u const $WebhookService: Provider = { provide: 'WebhookService', useExisting: WebhookService }; const $UtilityService: Provider = { provide: 'UtilityService', useExisting: UtilityService }; const $FileInfoService: Provider = { provide: 'FileInfoService', useExisting: FileInfoService }; +const $SearchService: Provider = { provide: 'SearchService', useExisting: SearchService }; + const $ChartLoggerService: Provider = { provide: 'ChartLoggerService', useExisting: ChartLoggerService }; const $FederationChart: Provider = { provide: 'FederationChart', useExisting: FederationChart }; const $NotesChart: Provider = { provide: 'NotesChart', useExisting: NotesChart }; @@ -295,6 +298,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting WebhookService, UtilityService, FileInfoService, + SearchService, ChartLoggerService, FederationChart, NotesChart, @@ -413,6 +417,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting $WebhookService, $UtilityService, $FileInfoService, + $SearchService, $ChartLoggerService, $FederationChart, $NotesChart, @@ -532,6 +537,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting WebhookService, UtilityService, FileInfoService, + SearchService, FederationChart, NotesChart, UsersChart, @@ -649,6 +655,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting $WebhookService, $UtilityService, $FileInfoService, + $SearchService, $FederationChart, $NotesChart, $UsersChart, diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 50081f831b..364976e4a7 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -46,6 +46,7 @@ import { bindThis } from '@/decorators.js'; import { DB_MAX_NOTE_TEXT_LENGTH } from '@/const.js'; import { RoleService } from '@/core/RoleService.js'; import { MetaService } from '@/core/MetaService.js'; +import { SearchService } from '@/core/SearchService.js'; const mutedWordsCache = new MemorySingleCache<{ userId: UserProfile['userId']; mutedWords: UserProfile['mutedWords']; }[]>(1000 * 60 * 5); @@ -198,6 +199,7 @@ export class NoteCreateService implements OnApplicationShutdown { private apRendererService: ApRendererService, private roleService: RoleService, private metaService: MetaService, + private searchService: SearchService, private notesChart: NotesChart, private perUserNotesChart: PerUserNotesChart, private activeUsersChart: ActiveUsersChart, @@ -728,17 +730,9 @@ export class NoteCreateService implements OnApplicationShutdown { @bindThis private index(note: Note) { - if (note.text == null || this.config.elasticsearch == null) return; - /* - es!.index({ - index: this.config.elasticsearch.index ?? 'misskey_note', - id: note.id.toString(), - body: { - text: normalizeForSearch(note.text), - userId: note.userId, - userHost: note.userHost, - }, - });*/ + if (note.text == null && note.cw == null) return; + + this.searchService.indexNote(note); } @bindThis diff --git a/packages/backend/src/core/SearchService.ts b/packages/backend/src/core/SearchService.ts new file mode 100644 index 0000000000..67332581f7 --- /dev/null +++ b/packages/backend/src/core/SearchService.ts @@ -0,0 +1,166 @@ +import { Inject, Injectable } from '@nestjs/common'; +import { In } from 'typeorm'; +import { DI } from '@/di-symbols.js'; +import type { Config } from '@/config.js'; +import { bindThis } from '@/decorators.js'; +import { Note } from '@/models/entities/Note.js'; +import { User } from '@/models/index.js'; +import type { NotesRepository } from '@/models/index.js'; +import { sqlLikeEscape } from '@/misc/sql-like-escape.js'; +import { QueryService } from '@/core/QueryService.js'; +import { IdService } from '@/core/IdService.js'; +import type { Index, MeiliSearch } from 'meilisearch'; + +type K = string; +type V = string | number | boolean; +type Q = + { op: '=', k: K, v: V } | + { op: '!=', k: K, v: V } | + { op: '>', k: K, v: number } | + { op: '<', k: K, v: number } | + { op: '>=', k: K, v: number } | + { op: '<=', k: K, v: number } | + { op: 'and', qs: Q[] } | + { op: 'or', qs: Q[] } | + { op: 'not', q: Q }; + +function compileValue(value: V): string { + if (typeof value === 'string') { + return `'${value}'`; // TODO: escape + } else if (typeof value === 'number') { + return value.toString(); + } else if (typeof value === 'boolean') { + return value.toString(); + } + throw new Error('unrecognized value'); +} + +function compileQuery(q: Q): string { + switch (q.op) { + case '=': return `(${q.k} = ${compileValue(q.v)})`; + case '!=': return `(${q.k} != ${compileValue(q.v)})`; + case '>': return `(${q.k} > ${compileValue(q.v)})`; + case '<': return `(${q.k} < ${compileValue(q.v)})`; + case '>=': return `(${q.k} >= ${compileValue(q.v)})`; + case '<=': return `(${q.k} <= ${compileValue(q.v)})`; + case 'and': return q.qs.length === 0 ? '' : `(${ q.qs.map(_q => compileQuery(_q)).join(' AND ') })`; + case 'or': return q.qs.length === 0 ? '' : `(${ q.qs.map(_q => compileQuery(_q)).join(' OR ') })`; + case 'not': return `(NOT ${compileQuery(q.q)})`; + default: throw new Error('unrecognized query operator'); + } +} + +@Injectable() +export class SearchService { + private meilisearchNoteIndex: Index | null = null; + + constructor( + @Inject(DI.config) + private config: Config, + + @Inject(DI.meilisearch) + private meilisearch: MeiliSearch | null, + + @Inject(DI.notesRepository) + private notesRepository: NotesRepository, + + private queryService: QueryService, + private idService: IdService, + ) { + if (meilisearch) { + this.meilisearchNoteIndex = meilisearch.index('notes'); + this.meilisearchNoteIndex.updateSettings({ + searchableAttributes: [ + 'text', + 'cw', + ], + sortableAttributes: [ + 'createdAt', + ], + filterableAttributes: [ + 'createdAt', + 'userId', + 'userHost', + 'channelId', + ], + typoTolerance: { + enabled: false, + }, + pagination: { + maxTotalHits: 10000, + }, + }); + } + } + + @bindThis + public async indexNote(note: Note): Promise { + if (this.meilisearch) { + this.meilisearchNoteIndex!.addDocuments([{ + id: note.id, + createdAt: note.createdAt.getTime(), + userId: note.userId, + userHost: note.userHost, + channelId: note.channelId, + cw: note.cw, + text: note.text, + }], { + primaryKey: 'id', + }); + } + } + + @bindThis + public async searchNote(q: string, me: User | null, opts: { + userId?: Note['userId'] | null; + channelId?: Note['channelId'] | null; + }, pagination: { + untilId?: Note['id']; + sinceId?: Note['id']; + limit?: number; + }): Promise { + if (this.meilisearch) { + const filter: Q = { + op: 'and', + qs: [], + }; + if (pagination.untilId) filter.qs.push({ op: '<', k: 'createdAt', v: this.idService.parse(pagination.untilId).date.getTime() }); + if (pagination.sinceId) filter.qs.push({ op: '>', k: 'createdAt', v: this.idService.parse(pagination.sinceId).date.getTime() }); + if (opts.userId) filter.qs.push({ op: '=', k: 'userId', v: opts.userId }); + if (opts.channelId) filter.qs.push({ op: '=', k: 'channelId', v: opts.channelId }); + const res = await this.meilisearchNoteIndex!.search(q, { + sort: ['createdAt:desc'], + matchingStrategy: 'all', + attributesToRetrieve: ['id', 'createdAt'], + filter: compileQuery(filter), + limit: pagination.limit, + }); + if (res.hits.length === 0) return []; + return await this.notesRepository.findBy({ + id: In(res.hits.map(x => x.id)), + }); + } else { + const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), pagination.sinceId, pagination.untilId); + + if (opts.userId) { + query.andWhere('note.userId = :userId', { userId: opts.userId }); + } else if (opts.channelId) { + query.andWhere('note.channelId = :channelId', { channelId: opts.channelId }); + } + + query + .andWhere('note.text ILIKE :q', { q: `%${ sqlLikeEscape(q) }%` }) + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser'); + + this.queryService.generateVisibilityQuery(query, me); + if (me) this.queryService.generateMutedUserQuery(query, me); + if (me) this.queryService.generateBlockedUserQuery(query, me); + + return await query.take(pagination.limit).getMany(); + } + } +} diff --git a/packages/backend/src/di-symbols.ts b/packages/backend/src/di-symbols.ts index 190d8d65c2..c06c7a7159 100644 --- a/packages/backend/src/di-symbols.ts +++ b/packages/backend/src/di-symbols.ts @@ -1,6 +1,7 @@ export const DI = { config: Symbol('config'), db: Symbol('db'), + meilisearch: Symbol('meilisearch'), redis: Symbol('redis'), redisForPub: Symbol('redisForPub'), redisForSub: Symbol('redisForSub'), diff --git a/packages/backend/src/server/api/endpoints/meta.ts b/packages/backend/src/server/api/endpoints/meta.ts index a5cb3fa7ee..584ea07c3b 100644 --- a/packages/backend/src/server/api/endpoints/meta.ts +++ b/packages/backend/src/server/api/endpoints/meta.ts @@ -201,10 +201,6 @@ export const meta = { type: 'boolean', optional: false, nullable: false, }, - elasticsearch: { - type: 'boolean', - optional: false, nullable: false, - }, hcaptcha: { type: 'boolean', optional: false, nullable: false, @@ -331,7 +327,6 @@ export default class extends Endpoint { response.features = { registration: !instance.disableRegistration, emailRequiredForSignup: instance.emailRequiredForSignup, - elasticsearch: this.config.elasticsearch ? true : false, hcaptcha: instance.enableHcaptcha, recaptcha: instance.enableRecaptcha, turnstile: instance.enableTurnstile, diff --git a/packages/backend/src/server/api/endpoints/notes/search.ts b/packages/backend/src/server/api/endpoints/notes/search.ts index fb5abd917f..990ba526d9 100644 --- a/packages/backend/src/server/api/endpoints/notes/search.ts +++ b/packages/backend/src/server/api/endpoints/notes/search.ts @@ -1,11 +1,10 @@ import { Inject, Injectable } from '@nestjs/common'; import type { NotesRepository } from '@/models/index.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; -import { QueryService } from '@/core/QueryService.js'; +import { SearchService } from '@/core/SearchService.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import type { Config } from '@/config.js'; import { DI } from '@/di-symbols.js'; -import { sqlLikeEscape } from '@/misc/sql-like-escape.js'; import { RoleService } from '@/core/RoleService.js'; import { ApiError } from '../../error.js'; @@ -61,11 +60,8 @@ export default class extends Endpoint { @Inject(DI.config) private config: Config, - @Inject(DI.notesRepository) - private notesRepository: NotesRepository, - private noteEntityService: NoteEntityService, - private queryService: QueryService, + private searchService: SearchService, private roleService: RoleService, ) { super(meta, paramDef, async (ps, me) => { @@ -74,27 +70,14 @@ export default class extends Endpoint { throw new ApiError(meta.errors.unavailable); } - const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId); - - if (ps.userId) { - query.andWhere('note.userId = :userId', { userId: ps.userId }); - } else if (ps.channelId) { - query.andWhere('note.channelId = :channelId', { channelId: ps.channelId }); - } - - query - .andWhere('note.text ILIKE :q', { q: `%${ sqlLikeEscape(ps.query) }%` }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser'); - - this.queryService.generateVisibilityQuery(query, me); - if (me) this.queryService.generateMutedUserQuery(query, me); - if (me) this.queryService.generateBlockedUserQuery(query, me); - - const notes = await query.take(ps.limit).getMany(); + const notes = await this.searchService.searchNote(ps.query, me, { + userId: ps.userId, + channelId: ps.channelId, + }, { + untilId: ps.untilId, + sinceId: ps.sinceId, + limit: ps.limit, + }); return await this.noteEntityService.packMany(notes, me); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0010581416..31f6b919d2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -229,6 +229,9 @@ importers: jsrsasign: specifier: 10.8.6 version: 10.8.6 + meilisearch: + specifier: 0.32.3 + version: 0.32.3 mfm-js: specifier: 0.23.3 version: 0.23.3 @@ -9582,7 +9585,6 @@ packages: node-fetch: 2.6.7 transitivePeerDependencies: - encoding - dev: true /cross-spawn@5.1.0: resolution: {integrity: sha512-pTgQJ5KC0d2hcY8eyL1IzlBPYjTkyH72XRZPnLyKus2mBfNjQs3klqbJU2VILqZryAZUt9JOb3h/mWMy23/f5A==} @@ -14496,6 +14498,14 @@ packages: engines: {node: '>= 0.6'} dev: true + /meilisearch@0.32.3: + resolution: {integrity: sha512-EOgfBuRE5SiIPIpEDYe2HO0D7a4z5bexIgaAdJFma/dH5hx1kwO+u/qb2g3qKyjG+iA3l8MlmTj/Xd72uahaAw==} + dependencies: + cross-fetch: 3.1.5 + transitivePeerDependencies: + - encoding + dev: false + /memoizerific@1.11.3: resolution: {integrity: sha512-/EuHYwAPdLtXwAwSZkh/Gutery6pD2KYd44oQLhAvQp/50mpyduZh8Q7PYHXTCJ+wuXxt7oij2LXyIJOOYFPog==} dependencies: @@ -14657,6 +14667,7 @@ packages: /minimist@1.2.7: resolution: {integrity: sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g==} + dev: false /minimist@1.2.8: resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} @@ -19700,7 +19711,7 @@ packages: axios: 0.27.2(debug@4.3.4) joi: 17.7.0 lodash: 4.17.21 - minimist: 1.2.7 + minimist: 1.2.8 rxjs: 7.8.1 transitivePeerDependencies: - debug