diff --git a/.env.default b/.env.default index c40424f..d789b37 100644 --- a/.env.default +++ b/.env.default @@ -86,6 +86,7 @@ MINIO_USE_SSL=disabled MINIO_ACCESS_KEY= MINIO_SECRET_KEY= +MINIO_ADMIN_BUCKET=yourapp-admin MINIO_IMAGE_BUCKET=yourapp-images MINIO_VIDEO_BUCKET=yourapp-videos MINIO_ATTACHMENT_BUCKET=yourapp-attachments diff --git a/app/controllers/admin/attachment.js b/app/controllers/admin/attachment.js index 27163eb..a521f30 100644 --- a/app/controllers/admin/attachment.js +++ b/app/controllers/admin/attachment.js @@ -26,8 +26,7 @@ class AttachmentAdminController extends SiteController { router.post('/:attachmentId', this.postUpdateAttachment.bind(this)); - router.get('/create', this.getAttachmentEditor.bind(this)); - router.get('/:attachmentId', this.getAttachmentEditor.bind(this)); + router.get('/:attachmentId', this.getAttachmentView.bind(this)); router.get('/', this.getDashboard.bind(this)); @@ -83,8 +82,8 @@ class AttachmentAdminController extends SiteController { } } - async getAttachmentEditor (req, res) { - res.render('admin/attachment/editor'); + async getAttachmentView (req, res) { + res.render('admin/attachment/view'); } async getDashboard (req, res, next) { diff --git a/app/controllers/admin/image.js b/app/controllers/admin/image.js index 3e2df35..af3c2c0 100644 --- a/app/controllers/admin/image.js +++ b/app/controllers/admin/image.js @@ -6,7 +6,7 @@ const express = require('express'); -const { SiteController } = require('../../../lib/site-lib'); +const { SiteController, SiteError } = require('../../../lib/site-lib'); class ImageAdminController extends SiteController { @@ -24,6 +24,7 @@ class ImageAdminController extends SiteController { router.param('imageId', this.populateImageId.bind(this)); + router.get('/:imageId/archive-user', this.getUserArchiveView.bind(this)); router.get('/:imageId', this.getImageView.bind(this)); router.get('/', this.getDashboard.bind(this)); @@ -40,6 +41,9 @@ class ImageAdminController extends SiteController { } = this.dtp.services; try { res.locals.image = await imageService.getImageById(imageId); + if (!res.locals.image) { + throw new SiteError(404, 'Image not found'); + } return next(); } catch (error) { loganService.sendRequestEvent(module.exports, req, { @@ -52,6 +56,16 @@ class ImageAdminController extends SiteController { } } + async getUserArchiveView (req, res, next) { + const { image: imageService } = this.dtp.services; + try { + res.locals.imageHistory = await imageService.getRecentImagesForOwner(res.locals.image.owner, 10); + res.render('admin/image/archive-user'); + } catch (error) { + return next(error); + } + } + async getImageView (req, res) { res.render('admin/image/view'); } diff --git a/app/controllers/admin/user.js b/app/controllers/admin/user.js index b851b7e..dd1cf2d 100644 --- a/app/controllers/admin/user.js +++ b/app/controllers/admin/user.js @@ -15,6 +15,14 @@ class UserAdminController extends SiteController { } async start ( ) { + const { jobQueue: jobQueueService } = this.dtp.services; + + this.jobQueues = { }; + this.jobQueues.reeeper = await jobQueueService.getJobQueue( + 'reeeper', + this.dtp.config.jobQueues.reeeper, + ); + const router = express.Router(); router.use(async (req, res, next) => { res.locals.currentView = 'admin'; @@ -23,11 +31,26 @@ class UserAdminController extends SiteController { }); router.param('localUserId', this.populateLocalUserId.bind(this)); + router.param('archiveJobId', this.populateArchiveJobId.bind(this)); + router.param('archiveId', this.populateArchiveId.bind(this)); + + router.post('/local/:localUserId/archive', this.postArchiveLocalUser.bind(this)); router.post('/local/:localUserId', this.postUpdateLocalUser.bind(this)); + + router.get('/local/:localUserId/archive/confirm', this.getArchiveLocalUserConfirm.bind(this)); router.get('/local/:localUserId', this.getLocalUserView.bind(this)); + router.get('/archive/job/:archiveJobId', this.getUserArchiveJobView.bind(this)); + + router.post('/archive/:archiveId/action', this.postArchiveAction.bind(this)); + router.get('/archive/:archiveId/file', this.getUserArchiveFile.bind(this)); + router.get('/archive/:archiveId', this.getUserArchiveView.bind(this)); + + router.get('/archive', this.getUserArchiveIndex.bind(this)); + router.get('/', this.getHomeView.bind(this)); + return router; } @@ -44,6 +67,68 @@ class UserAdminController extends SiteController { } } + async populateArchiveJobId (req, res, next, archiveJobId) { + try { + res.locals.job = await this.jobQueues.reeeper.getJob(archiveJobId); + if (!res.locals.job) { + throw new SiteError(404, 'Job not found'); + } + return next(); + } catch (error) { + this.log.error('failed to populate Bull queue job', { archiveJobId, error }); + return next(error); + } + } + + async populateArchiveId (req, res, next, archiveId) { + const { user: userService } = this.dtp.services; + try { + res.locals.archive = await userService.getArchiveById(archiveId); + if (!res.locals.archive) { + throw new SiteError(404, 'Archive not found'); + } + return next(); + } catch (error) { + this.log.error('failed to populate UserArchive', { archiveId, error }); + return next(error); + } + } + + async postArchiveLocalUser (req, res, next) { + const { + logan: loganService, + user: userService, + } = this.dtp.services; + try { + const user = await userService.getLocalUserAccount(req.body.userId); + if (!user) { + throw new SiteError(404, 'User not found'); + } + res.locals.job = await userService.archiveLocalUser(user); + loganService.sendRequestEvent(module.exports, req, { + level: 'info', + event: 'postArchiveUser', + data: { + job: res.locals.job.id, + user: user, + }, + }); + res.redirect(`/admin/user/archive/job/${res.locals.job.id}`); + } catch (error) { + loganService.sendRequestEvent(module.exports, req, { + level: 'error', + event: 'postArchiveUser', + data: { + offender: { + _id: req.body.userId, + }, + error, + }, + }); + return next(error); + } + } + async postUpdateLocalUser (req, res, next) { const { logan: loganService, @@ -100,6 +185,133 @@ class UserAdminController extends SiteController { } } + async getUserArchiveJobView (req, res) { + res.locals.adminView = 'user-archive'; + res.render('admin/user/archive/job'); + } + + async getArchiveLocalUserConfirm (req, res) { + res.locals.adminView = 'user-archive'; + res.render('admin/user/archive/confirm'); + } + + async postArchiveAction (req, res, next) { + const { + logan: loganService, + user: userService, + } = this.dtp.services; + try { + switch (req.body.action) { + case 'update': + loganService.sendRequestEvent(module.exports, req, { + level: 'info', + event: 'postArchiveAction', + message: 'updating user archive record', + data: { + archive: { + _id: res.locals.archive._id, + user: { + _id: res.locals.archive.user._id, + username: res.locals.archive.user.username, + }, + }, + }, + }); + await userService.updateArchive(res.locals.archive, req.body); + return res.redirect(`/admin/user/archive/${res.locals.archive._id}`); + + case 'delete-file': + loganService.sendRequestEvent(module.exports, req, { + level: 'info', + event: 'postArchiveAction', + message: 'removing user archive file', + data: { + archive: { + _id: res.locals.archive._id, + user: { + _id: res.locals.archive.user._id, + username: res.locals.archive.user.username, + }, + }, + }, + }); + await userService.deleteArchiveFile(res.locals.archive); + return res.redirect(`/admin/user/archive/${res.locals.archive._id}`); + + case 'delete': + loganService.sendRequestEvent(module.exports, req, { + level: 'info', + event: 'postArchiveAction', + message: 'removing user archive', + data: { + archive: { + _id: res.locals.archive._id, + user: { + _id: res.locals.archive.user._id, + username: res.locals.archive.user.username, + }, + }, + }, + }); + await userService.deleteArchive(res.locals.archive); + return res.redirect(`/admin/user/archive`); + + default: + // unknown/invalid action + break; + } + + throw new SiteError(400, `Invalid user archive action: ${req.body.action}`); + + } catch (error) { + this.log.error('failed to delete archive file', { error }); + return next(error); + } + } + + async getUserArchiveFile (req, res, next) { + const { minio: minioService } = this.dtp.services; + try { + res.locals.adminView = 'user-archive'; + + this.log.debug('archive', { archive: res.locals.archive }); + const stream = await minioService.openDownloadStream({ + bucket: res.locals.archive.archive.bucket, + key: res.locals.archive.archive.key, + }); + + res.status(200); + res.set('Content-Type', 'application/zip'); + res.set('Content-Size', res.locals.archive.archive.size); + res.set('Content-Disposition', `attachment; filename="user-${res.locals.archive.user._id}.zip"`); + + stream.pipe(res); + } catch (error) { + this.log.error('failed to stream user archive file', { error }); + return next(error); + } + } + + async getUserArchiveView (req, res) { + res.locals.adminView = 'user-archive'; + res.render('admin/user/archive/view'); + } + + async getUserArchiveIndex (req, res, next) { + const { user: userService } = this.dtp.services; + try { + res.locals.adminView = 'user-archive'; + + res.locals.pagination = this.getPaginationParameters(req, 20); + res.locals.archive = await userService.getArchives(res.locals.pagination); + + res.render('admin/user/archive/index'); + } catch (error) { + this.log.error('failed to render the User archives index', { error }); + return next(error); + } + } + async getHomeView (req, res, next) { const { user: userService } = this.dtp.services; try { diff --git a/app/models/user-archive.js b/app/models/user-archive.js new file mode 100644 index 0000000..842fc67 --- /dev/null +++ b/app/models/user-archive.js @@ -0,0 +1,29 @@ +// user-archive.js +// Copyright (C) 2022 DTP Technologies, LLC +// License: Apache-2.0 + +'use strict'; + +const mongoose = require('mongoose'); + +const Schema = mongoose.Schema; + +const UserArchiveSchema = new Schema({ + created: { type: Date, default: Date.now, required: true, index: -1 }, + user: { + _id: { type: Schema.ObjectId, required: true, index: 1 }, + email: { type: String }, + username: { type: String, required: true }, + }, + archive: { + bucket: { type: String, required: true }, + key: { type: String, required: true }, + etag: { type: String, required: true }, + size: { type: Number, required: true }, + }, + notes: { type: String }, +}); + +module.exports = (conn) => { + return conn.model('UserArchive', UserArchiveSchema); +}; \ No newline at end of file diff --git a/app/services/display-engine.js b/app/services/display-engine.js index 56296eb..66ba9b4 100644 --- a/app/services/display-engine.js +++ b/app/services/display-engine.js @@ -132,10 +132,6 @@ class DisplayEngineService extends SiteService { this.templates = { }; } - async start ( ) { } - - async stop ( ) { } - loadTemplate (name, pugScript) { const scriptFile = path.join(this.dtp.config.root, 'app', 'views', pugScript); this.templates[name] = pug.compileFile(scriptFile); diff --git a/app/services/image.js b/app/services/image.js index bc32251..f7f6068 100644 --- a/app/services/image.js +++ b/app/services/image.js @@ -90,11 +90,11 @@ class ImageService extends SiteService { return image; } - async getRecentImagesForOwner(owner) { + async getRecentImagesForOwner(owner, limit = 10) { const images = await SiteImage .find({ owner: owner._id }) .sort({ created: -1 }) - .limit(10) + .limit(limit) .populate(this.populateImage) .lean(); return images; @@ -112,6 +112,15 @@ class ImageService extends SiteService { return { images, totalImageCount }; } + async downloadImage (image, filename) { + const { minio: minioService } = this.dtp.services; + return minioService.downloadFile({ + bucket: image.file.bucket, + key: image.file.key, + filePath: filename, + }); + } + async deleteImage(image) { const { minio: minioService } = this.dtp.services; diff --git a/app/services/minio.js b/app/services/minio.js index d386cb2..60057ca 100644 --- a/app/services/minio.js +++ b/app/services/minio.js @@ -24,7 +24,6 @@ class MinioService extends SiteService { accessKey: process.env.MINIO_ACCESS_KEY, secretKey: process.env.MINIO_SECRET_KEY, }; - this.log.debug('MinIO config', { minioConfig }); this.minio = new Minio.Client(minioConfig); } diff --git a/app/services/session.js b/app/services/session.js index d4a8cd3..747f6a8 100644 --- a/app/services/session.js +++ b/app/services/session.js @@ -18,7 +18,7 @@ class SessionService extends SiteService { async start ( ) { await super.start(); - this.log.info(`starting ${module.exports.name} service`); + passport.serializeUser(this.serializeUser.bind(this)); passport.deserializeUser(this.deserializeUser.bind(this)); } diff --git a/app/services/sms.js b/app/services/sms.js index 93c26c3..fa110e6 100644 --- a/app/services/sms.js +++ b/app/services/sms.js @@ -18,7 +18,6 @@ class SmsService extends SiteService { async start ( ) { await super.start(); - this.log.info(`starting ${module.exports.name} service`); } async stop ( ) { diff --git a/app/services/user.js b/app/services/user.js index 7348d33..f8d0555 100644 --- a/app/services/user.js +++ b/app/services/user.js @@ -11,6 +11,7 @@ const mongoose = require('mongoose'); const User = mongoose.model('User'); const CoreUser = mongoose.model('CoreUser'); const UserBlock = mongoose.model('UserBlock'); +const UserArchive = mongoose.model('UserArchive'); const passport = require('passport'); const PassportLocal = require('passport-local'); @@ -47,13 +48,20 @@ class UserService extends SiteService { async start ( ) { await super.start(); - this.log.info(`starting ${module.exports.name} service`); this.registerPassportLocal(); - if (process.env.DTP_ADMIN === 'enabled') { this.registerPassportAdmin(); } + + const { jobQueue: jobQueueService } = this.dtp.services; + this.jobQueues = { }; + + this.log.info('connecting to job queue', { name: 'reeeper', config: this.dtp.config.jobQueues.reeeper }); + this.jobQueues.reeeper = jobQueueService.getJobQueue( + 'reeeper', + this.dtp.config.jobQueues.reeeper, + ); } async stop ( ) { @@ -870,6 +878,95 @@ class UserService extends SiteService { } } } + + /** + * Create a job to archive and ban a User (local). The job will immediately + * disable the specified user, create a .zip file of their content on storage. + * Once the worker confirms that the archive file is on storage, it creates a + * UserArchive record for it, then completely bans the User. That removes all + * of the User's content. + * + * It then removes the User record entirely. + * + * @param {User} user the User to be archived + * @returns the newly created Bull queue job + */ + async archiveLocalUser (user) { + return this.jobQueues.reeeper.add('archive-user-local', { userId: user._id }); + } + + /** + * Update a UserArchive document + * @param {UserArchive} archive the existing archive to be updated + * @param {*} archiveDefinition new values to be applied + */ + async updateArchive (archive, archiveDefinition) { + const update = { $set: { }, $unset: { } }; + + archiveDefinition.notes = archiveDefinition.notes.trim(); + if (archiveDefinition.notes && (archiveDefinition.notes.length > 0)) { + update.$set.notes = archiveDefinition.notes; + } else { + update.$unset.notes = 1; + } + + await UserArchive.updateOne({ _id: archive._id }, update); + } + + /** + * Fetch an Array of UserArchive documents with pagination. + * @param {DtpPagination} pagination self explanatory + * @returns Array of UserArchive documents (can be empty) + */ + async getArchives (pagination) { + const search = { }; + const archives = await UserArchive + .find(search) + .sort({ created: -1 }) + .skip(pagination.skip) + .limit(pagination.cpp) + .lean(); + const totalArchiveCount = await UserArchive.estimatedDocumentCount(); + return { archives, totalArchiveCount }; + } + + /** + * Fetch a UserArchive record. This does not fetch the archive file. + * @param {UserArchive} archiveId the ID of the archive to fetch + * @returns the requested UserArchive, or null/undefined. + */ + async getArchiveById (archiveId) { + const archive = await UserArchive.findOne({ _id: archiveId }).lean(); + return archive; + } + + /** + * Removes the .zip file attached to a UserArchive. + * @param {UserArchive} archive the archive for which an associated .zip file + * is to be removed + */ + async deleteArchiveFile (archive) { + const { minio: minioService } = this.dtp.services; + if (!archive.archive || !archive.archive.bucket || !archive.archive.key) { + return; // no archive file present, abort + } + await minioService.removeObject(archive.archive.bucket, archive.archive.key); + await UserArchive.updateOne( + { _id: archive._id }, + { + $unset: { archive: 1 }, + }, + ); + } + + /** + * Removes a UserArchive and any attached data. + * @param {UserArchive} archive the UserArchive to be removed. + */ + async deleteArchive (archive) { + await this.deleteArchiveFile(archive); + await UserArchive.deleteOne({ _id: archive._id }); + } } module.exports = { diff --git a/app/views/admin/components/menu.pug b/app/views/admin/components/menu.pug index 9f6b90a..bc3c940 100644 --- a/app/views/admin/components/menu.pug +++ b/app/views/admin/components/menu.pug @@ -51,6 +51,12 @@ ul(uk-nav).uk-nav-default i.fas.fa-user span.uk-margin-small-left Users + li(class={ 'uk-active': (adminView === 'user-archive') }) + a(href="/admin/user/archive") + span.nav-item-icon + i.fas.fa-file-archive + span.uk-margin-small-left User Archive + li(class={ 'uk-active': (adminView === 'content-report') }) a(href="/admin/content-report") span.nav-item-icon diff --git a/app/views/admin/image/archive-user.pug b/app/views/admin/image/archive-user.pug new file mode 100644 index 0000000..e52098b --- /dev/null +++ b/app/views/admin/image/archive-user.pug @@ -0,0 +1,28 @@ +extends ../layouts/main +block content + + include ../user/components/list-item + + form(method="POST", action=`/admin/user/local/${image.owner._id}/archive`).uk-form + input(type="hidden", name="userId", value= image.owner._id) + .uk-card.uk-card-default.uk-card-small + .uk-card-header + h1.uk-card-title Archive Local User + .uk-card-body + p This action will pull all images from storage into an archive file, place the archive file on storage, delete all the image records and storage data, then ban the User. The archive is produced first because images would be deleted during the ban. So, the archive is made, then the user is banned. + + p These are the #{numeral(imageHistory.length).format('0,0')} most recent images uploaded by #{image.owner.username}. + + div(uk-grid) + each image in imageHistory + .uk-width-medium + .uk-margin-small(uk-lightbox) + a(href=`/image/${image._id}`, data-type="image", data-caption=`id: ${image._id}`) + div + img(src= `/image/${image._id}`).responsive + .uk-card-footer.uk-flex.uk-flex-middle + .uk-width-expand + +renderBackButton() + + .uk-width-auto + button(type="submit").uk-button.uk-button-danger.uk-border-rounded Archive User \ No newline at end of file diff --git a/app/views/admin/user/archive/confirm.pug b/app/views/admin/user/archive/confirm.pug new file mode 100644 index 0000000..9f83eaa --- /dev/null +++ b/app/views/admin/user/archive/confirm.pug @@ -0,0 +1,28 @@ +extends ../../layouts/main +block content + + include ../../user/components/list-item + + form(method="POST", action=`/admin/user/local/${userAccount._id}/archive`).uk-form + input(type="hidden", name="userId", value= userAccount._id) + .uk-card.uk-card-default.uk-card-small + .uk-card-header + h1.uk-card-title + span + i.fas.fa-id-card + span.uk-margin-small-left Archive Local User + .uk-card-body + .uk-margin + +renderUserListItem(userAccount) + + .uk-margin + p This action will archive #{userAccount.displayName || userAccount.username}'s content to a .zip file, place the .zip file on storage, create a UserArchive record for this User account, ban this User account, and remove this User account from the database. + + p #{userAccount.displayName || userAccount.username}'s email address and username will become locked, and will remain unavailable for use for as long as this archive exists. + + .uk-card-footer.uk-flex.uk-flex-middle + .uk-width-expand + +renderBackButton() + + .uk-width-auto + button(type="submit").uk-button.uk-button-danger.uk-border-rounded Archive User \ No newline at end of file diff --git a/app/views/admin/user/archive/index.pug b/app/views/admin/user/archive/index.pug new file mode 100644 index 0000000..798399d --- /dev/null +++ b/app/views/admin/user/archive/index.pug @@ -0,0 +1,41 @@ +extends ../../layouts/main +block content + + include ../components/list-item + include ../../../components/pagination-bar + + .uk-card.uk-card-default.uk-card-small.uk-margin + .uk-card-header + h1.uk-card-title + span + i.fas.fa-id-card + span.uk-margin-small-left User Archives + + .uk-card-body + if Array.isArray(archive.archives) && (archive.archives.length > 0) + table.uk-table.uk-table-divider.uk-table-justify + thead + tr + th Username + th User ID + th Created + th Archive + tbody + each record in archive.archives + tr + td= record.user.username + td= record.user._id + td= moment(record.created).format('MMMM DD, YYYY, [at] h:mm a') + td + span + i.fas.fa-file-archive + a(href=`/admin/user/archive/${record._id}`).uk-margin-small-left View Archive + else + div There are no user archives. + + if Array.isArray(archive.archives) && (archive.archives.length > 0) + .uk-card-footer + +renderPaginationBar('/admin/user/archive', archive.totalArchiveCount) + + .uk-margin + .uk-text-small.uk-text-muted.uk-text-center User accounts referenced on this page have been removed from the database and are no longer able to use #{site.name}. \ No newline at end of file diff --git a/app/views/admin/user/archive/job.pug b/app/views/admin/user/archive/job.pug new file mode 100644 index 0000000..fbb33cc --- /dev/null +++ b/app/views/admin/user/archive/job.pug @@ -0,0 +1,7 @@ +extends ../../layouts/main +block content + + include ../components/list-item + + h1 User Archive Job + pre= JSON.stringify(job, null, 2) \ No newline at end of file diff --git a/app/views/admin/user/archive/view.pug b/app/views/admin/user/archive/view.pug new file mode 100644 index 0000000..a0122fe --- /dev/null +++ b/app/views/admin/user/archive/view.pug @@ -0,0 +1,83 @@ +extends ../../layouts/main +block content + + include ../components/list-item + + form(method="POST", action=`/admin/user/archive/${archive._id}/action`).uk-form + .uk-card.uk-card-default.uk-card-small + .uk-card-header + h1.uk-card-title + span + i.fas.fa-id-card + span.uk-margin-small-left User Archive + + .uk-card-body + .uk-margin + div(uk-grid) + .uk-width-auto + .uk-form-label Archive ID + .uk-text-bold= archive._id + .uk-width-auto + .uk-form-label Created + .uk-text-bold= moment(archive.created).format('MMMM DD, YYYY, [at] h:mm:ss a') + .uk-width-auto + .uk-form-label User + .uk-text-bold= archive.user.username + .uk-width-auto + .uk-form-label User ID + .uk-text-bold= archive.user._id + .uk-width-auto + .uk-form-label User email + .uk-text-bold= archive.user.email + + if archive.archive + div(uk-grid) + .uk-width-auto + .uk-form-label Archive file + .uk-text-bold= archive.archive.key.replace(/\/user-archive\//, '') + .uk-width-auto + .uk-form-label Download size + .uk-text-bold= numeral(archive.archive.size).format('0,0.0a') + else + .uk-text-italic (archive file removed) + + .uk-margin + label(for="notes").uk-form-label Notes + textarea(id="notes", name="notes", rows="4", placeholder="Enter notes").uk-textarea.uk-resize-vertical= archive.notes + + .uk-card-footer + div(uk-grid) + .uk-width-expand + div(hidden= !archive.archive, uk-grid) + .uk-width-auto + a(href=`/admin/user/archive/${archive._id}/file`).uk-button.uk-button-default.uk-border-rounded + span + i.fas.fa-download + span.uk-margin-small-left Download#[span(class="uk-visible@s") File] + .uk-width-auto + button( + type="submit", + name="action", + value="delete-file", + uk-tooltip={ title: 'Remove the .zip file attached to the UserArchive' }, + ).uk-button.uk-button-danger.uk-border-rounded + span + i.fas.fa-trash + span.uk-margin-small-left Delete#[span(class="uk-visible@s") File] + + .uk-width-auto + button( + type="submit", + name="action", + value="delete", + uk-tooltip={ title: 'Remove the UserArchive from the database' }, + ).uk-button.uk-button-danger.uk-border-rounded + span + i.fas.fa-save + span.uk-margin-small-left Delete + + .uk-width-auto + button(type="submit", name="action", value="update").uk-button.uk-button-primary.uk-border-rounded + span + i.fas.fa-save + span.uk-margin-small-left Update \ No newline at end of file diff --git a/app/views/admin/user/form.pug b/app/views/admin/user/form.pug index 8564efd..b40c5cf 100644 --- a/app/views/admin/user/form.pug +++ b/app/views/admin/user/form.pug @@ -81,6 +81,8 @@ block content div(uk-grid).uk-grid-small .uk-width-expand +renderBackButton() + .uk-width-auto + a(href=`/admin/user/local/${userAccount._id}/archive/confirm`).uk-button.uk-button-danger.uk-border-rounded Archive User .uk-width-auto button(type="submit", name="action", value="ban").uk-button.uk-button-danger.uk-border-rounded Ban User .uk-width-auto diff --git a/app/views/admin/user/index.pug b/app/views/admin/user/index.pug index 66c17c2..6afbfbc 100644 --- a/app/views/admin/user/index.pug +++ b/app/views/admin/user/index.pug @@ -3,6 +3,19 @@ block content include ../../components/pagination-bar + .uk-margin + div(uk-grid).uk-flex-middle + .uk-width-expand + h1 + span + i.fas.fa-user-cog + span.uk-margin-small-left User Manager + .uk-width-auto + a(href="/admin/user/archive").uk-button.uk-button-default.uk-border-rounded + span.nav-item-icon + i.fas.fa-file-archive + span.uk-margin-small-left Browse Archive + .uk-margin form(method="GET", action="/admin/user").uk-form div(uk-grid).uk-grid-collapse diff --git a/app/workers/reeeper.js b/app/workers/reeeper.js index 0236034..a3de7d7 100644 --- a/app/workers/reeeper.js +++ b/app/workers/reeeper.js @@ -36,6 +36,8 @@ class ReeeperWorker extends SiteWorker { await this.loadProcessor(path.join(__dirname, 'reeeper', 'cron', 'expire-crashed-hosts.js')); await this.loadProcessor(path.join(__dirname, 'reeeper', 'cron', 'expire-announcements.js')); + await this.loadProcessor(path.join(__dirname, 'reeeper', 'job', 'archive-user-local.js')); + await this.startProcessors(); } diff --git a/app/workers/reeeper/job/archive-user-local.js b/app/workers/reeeper/job/archive-user-local.js new file mode 100644 index 0000000..42a967c --- /dev/null +++ b/app/workers/reeeper/job/archive-user-local.js @@ -0,0 +1,381 @@ +// reeeper/job/archive-user-local.js +// Copyright (C) 2022 DTP Technologies, LLC +// License: Apache-2.0 + +'use strict'; + +const path = require('path'); +const fs = require('fs'); + +const util = require('util'); +const execFile = util.promisify(require('child_process').execFile); + +const mime = require('mime'); + +const mongoose = require('mongoose'); +const User = mongoose.model('User'); + +const { SiteWorkerProcess } = require(path.join(__dirname, '..', '..', '..', '..', 'lib', 'site-lib')); + +/** + * A job to archive and ban a User (local). + * + * 1. Immediately disable the specified User + * 2. Create a .zip file of the User's content on storage + * 3. Creates a UserArchive record for the file and User + * 4. Ban the User (removes all of the User's content) + * 5. Remove the User record from the database + */ +class ArchiveUserLocalJob extends SiteWorkerProcess { + + static get COMPONENT ( ) { + return { + name: 'archiveUserLocalJob', + slug: 'archive-user-local-job', + }; + } + + static get JOB_NAME ( ) { return 'Local User Archive'; } + static get JOB_SLUG ( ) { return 'archive-user-local'; } + + constructor (worker) { + super(worker, ArchiveUserLocalJob.COMPONENT); + this.jobs = new Set(); + } + + async start ( ) { + await super.start(); + + this.queue = await this.getJobQueue('reeeper', this.dtp.config.jobQueues.reeeper); + + this.log.info('registering job processor', { queue: this.queue.name, job: ArchiveUserLocalJob.JOB_SLUG }); + this.queue.process(ArchiveUserLocalJob.JOB_SLUG, 1, this.processArchiveUserLocal.bind(this)); + } + + async stop ( ) { + try { + if (this.queue) { + this.log.info('halting job queue', { jobCount: this.jobs.size }); + await this.queue.pause(true, false); + delete this.queue; + } + } catch (error) { + this.log.error('failed to halt job queue', { error }); + // fall through + } finally { + await super.stop(); + } + } + + async processArchiveUserLocal (job) { + const { user: userService } = this.dtp.services; + try { + job.data.archivePath = path.join('/tmp', this.dtp.pkg.name, ArchiveUserLocalJob.JOB_SLUG); + this.jobs.add(job); + + job.data.userId = mongoose.Types.ObjectId(job.data.userId); + job.data.user = await userService.getLocalUserAccount(job.data.userId); + + job.data.workPath = path.join(job.data.archivePath, job.data.userId.toString()); + await fs.promises.mkdir(job.data.workPath, { recursive: true }); + + /* + * Save the User account data + */ + await this.archiveUserData(job); + + /* + * Disable the User account (which destroys their session and cookie(s)) + */ + // await this.disableUser(job); + + /* + * Archive the User's content to the workPath on the local file system. + */ + await this.archiveUserChat(job); + await this.archiveUserComments(job); + await this.archiveUserStickers(job); + await this.archiveUserImages(job); + + /* + * Create the .zip file archive, upload it to storage, and create the + * UserArchive record. + */ + await this.createArchiveFile(job); + + this.log.info('banning user', { + user: { + _id: job.data.userId, + username: job.data.user.username, + }, + }); + // await userService.ban(job.data.user); + + this.log.info('removing user', { + user: { + _id: job.data.userId, + username: job.data.user.username, + }, + }); + // await User.deleteOne({ _id: job.data.userId }); + } catch (error) { + this.log.error('failed to delete attachment', { attachmentId: job.data.attachmentId, error }); + throw error; + } finally { + if (job.data.workPath) { + this.log.info('cleaning up work directory'); + await fs.promises.rm(job.data.workPath, { force: true, recursive: true }); + + delete job.data.workPath; + } + this.jobs.delete(job); + this.log.info('job complete', { job: job.id, name: ArchiveUserLocalJob.JOB_NAME }); + } + } + + async archiveUserData (job) { + // fetch the entire User record (all fields) + job.data.fullUser = await User + .findOne({ _id: job.data.user._id }) + .select('+email +passwordSalt +password +flags +permissions +optIn') + .lean(); + if (!job.data.fullUser) { + throw new Error('user does not exist'); + } + + const userFilename = path.join(job.data.workPath, `user-${job.data.user._id}.json`); + await fs.promises.writeFile(userFilename, JSON.stringify(job.data.fullUser, null, 2)); + } + + async disableUser (job) { + this.log.info('disabling local User account', { + user: { + _id: job.data.userId, + username: job.data.user.username, + }, + }); + await User.updateOne( + { _id: job.data.user._id }, + { + $set: { + 'flags.isAdmin': false, + 'flags.isModerator': false, + 'flags.isEmailVerified': false, + 'permissions.canLogin': false, + 'permissions.canChat': false, + 'permissions.canComment': false, + 'permissions.canReport': false, + 'optIn.system': false, + 'optIn.marketing': false, + }, + }, + ); + } + + async archiveUserChat (job) { + const ChatMessage = mongoose.model('ChatMessage'); + const ChatRoom = mongoose.model('ChatRoom'); + + job.data.chatPath = path.join(job.data.workPath, 'chat'); + await fs.promises.mkdir(job.data.chatPath, { recursive: true }); + + this.log.info('archiving user chat', { + user: { + _id: job.data.user._id, + username: job.data.user.username, + }, + }); + + await ChatRoom + .find({ owner: job.data.user._id }) + + .lean() + .cursor() + .eachAsync(async (room) => { + const roomFilename = path.join(job.data.workPath, 'chat', `room-${room._id}`); + await fs.promises.writeFile(roomFilename, JSON.stringify(room, null, 2)); + }); + + await ChatMessage + .find({ author: job.data.user._id }) + .lean() + .cursor() + .eachAsync(async (message) => { + const messageFilename = path.join(job.data.workPath, 'chat', `message-${message._id}.json`); + await fs.promises.writeFile(messageFilename, JSON.stringify(message, null, 2)); + }); + } + + async archiveUserComments (job) { + const Comment = mongoose.model('Comment'); + + job.data.commentPath = path.join(job.data.workPath, 'comments'); + await fs.promises.mkdir(job.data.commentPath, { recursive: true }); + + this.log.info('archiving user comments', { + user: { + _id: job.data.user._id, + username: job.data.user.username, + }, + }); + + await Comment + .find({ author: job.data.userId }) + .cursor() + .eachAsync(async (comment) => { + const commentFilename = path.join(job.data.imagePath, `comment-${comment._id}.json`); + await fs.promises.writeFile(commentFilename, JSON.stringify(comment, null, 2)); + }); + } + + async archiveUserStickers (job) { + const Sticker = mongoose.model('Sticker'); + const { minio: minioService } = this.dtp.services; + + job.data.stickerPath = path.join(job.data.workPath, 'stickers'); + await fs.promises.mkdir(job.data.stickerPath, { recursive: true }); + + job.data.stickerMediaPath = path.join(job.data.stickerPath, 'media'); + await fs.promises.mkdir(job.data.stickerMediaPath, { recursive: true }); + + this.log.info('archiving user stickers', { + user: { + _id: job.data.user._id, + username: job.data.user.username, + }, + }); + + await Sticker + .find({ owner: job.data.userId }) + .cursor() + .eachAsync(async (sticker) => { + const stickerFilename = path.join(job.data.stickerPath, `sticker-${sticker._id}.json`); + await fs.promises.writeFile(stickerFilename, JSON.stringify(sticker, null, 2)); + + if (sticker.original && sticker.original.bucket && sticker.orignal.key && sticker.encoded.type) { + const originalExt = mime.getExtension(sticker.original.type); + const originalFilename = path.join(job.data.stickerMediaPath, `sticker-${sticker._id}.original.${originalExt}`); + await minioService.downloadFile({ + bucket: sticker.original.bucket, + key: sticker.original.key, + filePath: originalFilename, + }); + } + + if (sticker.encoded && sticker.encoded.bucket && sticker.encoded.key && sticker.encoded.type) { + const encodedExt = mime.getExtension(sticker.encoded.type); + const encodedFilename = path.join(job.data.stickerMediaPath, `sticker-${sticker._id}.encoded.${encodedExt}`); + await minioService.downloadFile({ + bucket: sticker.encoded.bucket, + key: sticker.encoded.key, + filePath: encodedFilename, + }); + } + }); + } + + async archiveUserImages (job) { + const SiteImage = mongoose.model('Image'); + const { image: imageService } = this.dtp.services; + + job.data.imagePath = path.join(job.data.workPath, 'images'); + await fs.promises.mkdir(job.data.imagePath, { recursive: true }); + + this.log.info('archiving user images', { + user: { + _id: job.data.user._id, + username: job.data.user.username, + }, + }); + + await SiteImage + .find({ owner: job.data.user._id }) + .cursor() + .eachAsync(async (image) => { + try { + let imageExt = mime.getExtension(image.type); + const imageFilename = path.join(job.data.imagePath, `image-${image._id}.${imageExt}`); + const metadataFilename = path.join(job.data.imagePath, `image-${image._id}.metadata.json`); + + await imageService.downloadImage(image, imageFilename); + await fs.promises.writeFile(metadataFilename, JSON.stringify(image.metadata, null, 2)); + + } catch (error) { + this.log.error('failed to download image', { + image: { _id: image._id }, + error, + }); + } + }); + } + + async createArchiveFile (job) { + const { minio: minioService } = this.dtp.services; + try { + job.data.zipFilename = path.join(job.data.archivePath, `user-${job.data.userId}.zip`); + const zipArgs = [ + '-r', '-9', + job.data.zipFilename, + `${job.data.userId}`, + ]; + const options = { + cwd: job.data.archivePath, + encoding: 'utf8', + }; + await execFile('/usr/bin/zip', zipArgs, options); + + const zipFileStat = await fs.promises.stat(job.data.zipFilename); + this.log.info('zip archive created', { size: zipFileStat.size }); + + job.data.archiveFile = { + bucket: process.env.MINIO_ADMIN_BUCKET, + key: `/user-archive/user-${job.data.userId}.zip`, + }; + + const response = await minioService.uploadFile({ + bucket: job.data.archiveFile.bucket, + key: job.data.archiveFile.key, + filePath: job.data.zipFilename, + metadata: { + job: { + id: job.id, + }, + user: job.data.user, + } + }); + + this.log.info('creating user archive record', { etag: response.etag, size: zipFileStat.size }); + const UserArchive = mongoose.model('UserArchive'); + await UserArchive.create({ + created: job.data.startTime, + user: { + _id: job.data.userId, + username: job.data.user.username, + email: job.data.fullUser.email, + }, + archive: { + bucket: job.data.archiveFile.bucket, + key: job.data.archiveFile.key, + etag: response.etag, + size: zipFileStat.size, + } + }); + } catch (error) { + this.log.error('failed to create archive .zip file', { + user: { + _id: job.data.userId, + username: job.data.user.username, + }, + }); + throw error; + } finally { + try { + await fs.promises.rm(job.data.zipFilename, { force: true }); + } catch (error) { + this.log.error('failed to remove temp .zip file', { error }); + } + } + } +} + +module.exports = ArchiveUserLocalJob; \ No newline at end of file