From 0c57d2a1b424d3e92d8d685d7f343c3b8c0687c9 Mon Sep 17 00:00:00 2001 From: Maya Minatsuki Date: Thu, 16 Jul 2020 16:49:27 +0900 Subject: [PATCH] Add block list import function. --- .../views/components/settings/profile.vue | 3 +- src/queue/index.ts | 10 +++ src/queue/processors/db/import-blocking.ts | 72 +++++++++++++++++++ src/queue/processors/db/index.ts | 2 + src/server/api/endpoints/i/import-blocking.ts | 59 +++++++++++++++ 5 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 src/queue/processors/db/import-blocking.ts create mode 100644 src/server/api/endpoints/i/import-blocking.ts diff --git a/src/client/app/common/views/components/settings/profile.vue b/src/client/app/common/views/components/settings/profile.vue index 0c291f9029..b9437d676e 100644 --- a/src/client/app/common/views/components/settings/profile.vue +++ b/src/client/app/common/views/components/settings/profile.vue @@ -121,7 +121,7 @@ {{ $t('export') }} - {{ $t('import') }} + {{ $t('import') }} @@ -375,6 +375,7 @@ export default Vue.extend({ this.$chooseDriveFile().then(file => { this.$root.api( this.exportTarget == 'following' ? 'i/import-following' : + this.exportTarget == 'blocking' ? 'i/import-blocking' : this.exportTarget == 'user-lists' ? 'i/import-user-lists' : null, { fileId: file.id diff --git a/src/queue/index.ts b/src/queue/index.ts index 163c57d691..7c68a3a135 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -187,6 +187,16 @@ export function createImportFollowingJob(user: ILocalUser, fileId: DriveFile['id }); } +export function createImportBlockingJob(user: ILocalUser, fileId: DriveFile['id']) { + return dbQueue.add('importBlocking', { + user: user, + fileId: fileId + }, { + removeOnComplete: true, + removeOnFail: true + }); +} + export function createImportUserListsJob(user: ILocalUser, fileId: DriveFile['id']) { return dbQueue.add('importUserLists', { user: user, diff --git a/src/queue/processors/db/import-blocking.ts b/src/queue/processors/db/import-blocking.ts new file mode 100644 index 0000000000..57d79c5d71 --- /dev/null +++ b/src/queue/processors/db/import-blocking.ts @@ -0,0 +1,72 @@ +import * as Bull from 'bull'; + +import { queueLogger } from '../../logger'; +import block from '../../../services/blocking/create'; +import parseAcct from '../../../misc/acct/parse'; +import { resolveUser } from '../../../remote/resolve-user'; +import { downloadTextFile } from '../../../misc/download-text-file'; +import { isSelfHost, toPuny } from '../../../misc/convert-host'; +import { Users, DriveFiles } from '../../../models'; + +const logger = queueLogger.createSubLogger('import-blocking'); + +export async function importBlocking(job: Bull.Job, done: any): Promise { + logger.info(`Importing blocking of ${job.data.user.id} ...`); + + const user = await Users.findOne(job.data.user.id); + if (user == null) { + done(); + return; + } + + const file = await DriveFiles.findOne({ + id: job.data.fileId + }); + if (file == null) { + done(); + return; + } + + const csv = await downloadTextFile(file.url); + + let linenum = 0; + + for (const line of csv.trim().split('\n')) { + linenum++; + + try { + const acct = line.split(',')[0].trim(); + const { username, host } = parseAcct(acct); + + let target = isSelfHost(host!) ? await Users.findOne({ + host: null, + usernameLower: username.toLowerCase() + }) : await Users.findOne({ + host: toPuny(host!), + usernameLower: username.toLowerCase() + }); + + if (host == null && target == null) continue; + + if (target == null) { + target = await resolveUser(username, host); + } + + if (target == null) { + throw `cannot resolve user: @${username}@${host}`; + } + + // skip myself + if (target.id === job.data.user.id) continue; + + logger.info(`Block[${linenum}] ${target.id} ...`); + + block(user, target); + } catch (e) { + logger.warn(`Error in line:${linenum} ${e}`); + } + } + + logger.succ('Imported'); + done(); +} diff --git a/src/queue/processors/db/index.ts b/src/queue/processors/db/index.ts index 921cdf7ab1..078d6b02e5 100644 --- a/src/queue/processors/db/index.ts +++ b/src/queue/processors/db/index.ts @@ -6,6 +6,7 @@ import { exportMute } from './export-mute'; import { exportBlocking } from './export-blocking'; import { exportUserLists } from './export-user-lists'; import { importFollowing } from './import-following'; +import { importBlocking } from './import-blocking'; import { importUserLists } from './import-user-lists'; const jobs = { @@ -16,6 +17,7 @@ const jobs = { exportBlocking, exportUserLists, importFollowing, + importBlocking, importUserLists } as any; diff --git a/src/server/api/endpoints/i/import-blocking.ts b/src/server/api/endpoints/i/import-blocking.ts new file mode 100644 index 0000000000..51fa5da032 --- /dev/null +++ b/src/server/api/endpoints/i/import-blocking.ts @@ -0,0 +1,59 @@ +import $ from 'cafy'; +import { ID } from '../../../../misc/cafy-id'; +import define from '../../define'; +import { createImportBlockingJob } from '../../../../queue'; +import ms = require('ms'); +import { ApiError } from '../../error'; +import { DriveFiles } from '../../../../models'; + +export const meta = { + secure: true, + requireCredential: true, + limit: { + duration: ms('1hour'), + max: 1, + }, + + params: { + fileId: { + validator: $.type(ID), + } + }, + + errors: { + noSuchFile: { + message: 'No such file.', + code: 'NO_SUCH_FILE', + id: '91235249-f185-43a4-bbdd-3d77d21cc989' + }, + + unexpectedFileType: { + message: 'We need csv file.', + code: 'UNEXPECTED_FILE_TYPE', + id: '2ae6ec7e-14ba-4f94-aaab-53f5593796b5' + }, + + tooBigFile: { + message: 'That file is too big.', + code: 'TOO_BIG_FILE', + id: 'a8ac0052-c404-4561-ab05-86b0b5e52c65' + }, + + emptyFile: { + message: 'That file is empty.', + code: 'EMPTY_FILE', + id: 'd650cf08-e9b9-4e3e-a337-7e0598c7b7d3' + }, + } +}; + +export default define(meta, async (ps, user) => { + const file = await DriveFiles.findOne(ps.fileId); + + if (file == null) throw new ApiError(meta.errors.noSuchFile); + //if (!file.type.endsWith('/csv')) throw new ApiError(meta.errors.unexpectedFileType); + if (file.size > 50000) throw new ApiError(meta.errors.tooBigFile); + if (file.size === 0) throw new ApiError(meta.errors.emptyFile); + + createImportBlockingJob(user, file.id); +});