From c932f7a25bfd0944b21fc026a647fa1d04750418 Mon Sep 17 00:00:00 2001 From: syuilo Date: Tue, 12 Mar 2019 00:34:19 +0900 Subject: [PATCH] Resolve #1736 --- .../views/components/settings/profile.vue | 3 +- src/queue/index.ts | 10 +++ src/queue/processors/db/import-following.ts | 62 ++++++++++++++++++ src/queue/processors/db/index.ts | 2 + .../api/endpoints/i/import-following.ts | 64 +++++++++++++++++++ 5 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 src/queue/processors/db/import-following.ts create mode 100644 src/server/api/endpoints/i/import-following.ts diff --git a/src/client/app/common/views/components/settings/profile.vue b/src/client/app/common/views/components/settings/profile.vue index d5ae8591ae..97ad7dec6b 100644 --- a/src/client/app/common/views/components/settings/profile.vue +++ b/src/client/app/common/views/components/settings/profile.vue @@ -101,7 +101,7 @@ {{ $t('export') }} - {{ $t('import') }} + {{ $t('import') }} @@ -301,6 +301,7 @@ export default Vue.extend({ doImport() { this.$chooseDriveFile().then(file => { this.$root.api( + this.exportTarget == 'following' ? 'i/import-following' : this.exportTarget == 'user-lists' ? 'i/import-user-lists' : null, { fileId: file.id diff --git a/src/queue/index.ts b/src/queue/index.ts index 09e0ad59c9..44b24b66cb 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -146,6 +146,16 @@ export function createExportUserListsJob(user: ILocalUser) { }); } +export function createImportFollowingJob(user: ILocalUser, fileId: IDriveFile['_id']) { + return dbQueue.add('importFollowing', { + user: user, + fileId: fileId + }, { + removeOnComplete: true, + removeOnFail: true + }); +} + export function createImportUserListsJob(user: ILocalUser, fileId: IDriveFile['_id']) { return dbQueue.add('importUserLists', { user: user, diff --git a/src/queue/processors/db/import-following.ts b/src/queue/processors/db/import-following.ts new file mode 100644 index 0000000000..8ab20aa8bb --- /dev/null +++ b/src/queue/processors/db/import-following.ts @@ -0,0 +1,62 @@ +import * as Bull from 'bull'; +import * as mongo from 'mongodb'; + +import { queueLogger } from '../../logger'; +import User from '../../../models/user'; +import config from '../../../config'; +import follow from '../../../services/following/create'; +import DriveFile from '../../../models/drive-file'; +import { getOriginalUrl } from '../../../misc/get-drive-file-url'; +import parseAcct from '../../../misc/acct/parse'; +import resolveUser from '../../../remote/resolve-user'; +import { downloadTextFile } from '../../../misc/download-text-file'; +import Following from '../../../models/following'; + +const logger = queueLogger.createSubLogger('import-following'); + +export async function importFollowing(job: Bull.Job, done: any): Promise { + logger.info(`Importing following of ${job.data.user._id} ...`); + + const user = await User.findOne({ + _id: new mongo.ObjectID(job.data.user._id.toString()) + }); + + const file = await DriveFile.findOne({ + _id: new mongo.ObjectID(job.data.fileId.toString()) + }); + + const url = getOriginalUrl(file); + + const csv = await downloadTextFile(url); + + for (const line of csv.trim().split('\n')) { + const { username, host } = parseAcct(line.trim()); + + let target = host === config.host ? await User.findOne({ + host: null, + usernameLower: username.toLowerCase() + }) : await User.findOne({ + host: host, + usernameLower: username.toLowerCase() + }); + + if (host == null && target == null) continue; + + if (target == null) { + target = await resolveUser(username, host); + } + + // Check if already following + const exist = await Following.findOne({ + followerId: user._id, + followeeId: target._id + }); + + if (exist) continue; + + follow(user, target); + } + + logger.succ('Imported'); + done(); +} diff --git a/src/queue/processors/db/index.ts b/src/queue/processors/db/index.ts index 4a97a1c884..1bc9a9af7c 100644 --- a/src/queue/processors/db/index.ts +++ b/src/queue/processors/db/index.ts @@ -6,6 +6,7 @@ import { exportFollowing } from './export-following'; import { exportMute } from './export-mute'; import { exportBlocking } from './export-blocking'; import { exportUserLists } from './export-user-lists'; +import { importFollowing } from './import-following'; import { importUserLists } from './import-user-lists'; const jobs = { @@ -16,6 +17,7 @@ const jobs = { exportMute, exportBlocking, exportUserLists, + importFollowing, importUserLists } as any; diff --git a/src/server/api/endpoints/i/import-following.ts b/src/server/api/endpoints/i/import-following.ts new file mode 100644 index 0000000000..f188291bc2 --- /dev/null +++ b/src/server/api/endpoints/i/import-following.ts @@ -0,0 +1,64 @@ +import $ from 'cafy'; +import ID, { transform } from '../../../../misc/cafy-id'; +import define from '../../define'; +import { createImportFollowingJob } from '../../../../queue'; +import ms = require('ms'); +import DriveFile from '../../../../models/drive-file'; +import { ApiError } from '../../error'; + +export const meta = { + secure: true, + requireCredential: true, + limit: { + duration: ms('1hour'), + max: 1, + }, + + params: { + fileId: { + validator: $.type(ID), + transform: transform, + } + }, + + errors: { + noSuchFile: { + message: 'No such file.', + code: 'NO_SUCH_FILE', + id: 'b98644cf-a5ac-4277-a502-0b8054a709a3' + }, + + unexpectedFileType: { + message: 'We need csv file.', + code: 'UNEXPECTED_FILE_TYPE', + id: '660f3599-bce0-4f95-9dde-311fd841c183' + }, + + tooBigFile: { + message: 'That file is too big.', + code: 'TOO_BIG_FILE', + id: 'dee9d4ed-ad07-43ed-8b34-b2856398bc60' + }, + + emptyFile: { + message: 'That file is empty.', + code: 'EMPTY_FILE', + id: '31a1b42c-06f7-42ae-8a38-a661c5c9f691' + }, + } +}; + +export default define(meta, async (ps, user) => { + const file = await DriveFile.findOne({ + _id: ps.fileId + }); + + if (file == null) throw new ApiError(meta.errors.noSuchFile); + //if (!file.contentType.endsWith('/csv')) throw new ApiError(meta.errors.unexpectedFileType); + if (file.length > 50000) throw new ApiError(meta.errors.tooBigFile); + if (file.length === 0) throw new ApiError(meta.errors.emptyFile); + + createImportFollowingJob(user, file._id); + + return; +});