user archive

- stores all data related to one user to a .zip file on storage
- bans the user
- removes the user

.env.default

@@ -86,6 +86,7 @@ MINIO_USE_SSL=disabled
 MINIO_ACCESS_KEY=
 MINIO_SECRET_KEY=
 
+MINIO_ADMIN_BUCKET=yourapp-admin
 MINIO_IMAGE_BUCKET=yourapp-images
 MINIO_VIDEO_BUCKET=yourapp-videos
 MINIO_ATTACHMENT_BUCKET=yourapp-attachments

app/controllers/admin/attachment.js

@@ -26,8 +26,7 @@ class AttachmentAdminController extends SiteController {
 
     router.post('/:attachmentId', this.postUpdateAttachment.bind(this));
 
-    router.get('/create', this.getAttachmentEditor.bind(this));
+    router.get('/:attachmentId', this.getAttachmentView.bind(this));
-    router.get('/:attachmentId', this.getAttachmentEditor.bind(this));
 
     router.get('/', this.getDashboard.bind(this));
 
@@ -83,8 +82,8 @@ class AttachmentAdminController extends SiteController {
     }
   }
 
-  async getAttachmentEditor (req, res) {
+  async getAttachmentView (req, res) {
-    res.render('admin/attachment/editor');
+    res.render('admin/attachment/view');
   }
 
   async getDashboard (req, res, next) {

app/controllers/admin/image.js

@@ -6,7 +6,7 @@
 
 const express = require('express');
 
-const { SiteController } = require('../../../lib/site-lib');
+const { SiteController, SiteError } = require('../../../lib/site-lib');
 
 class ImageAdminController extends SiteController {
 
@@ -24,6 +24,7 @@ class ImageAdminController extends SiteController {
 
     router.param('imageId', this.populateImageId.bind(this));
 
+    router.get('/:imageId/archive-user', this.getUserArchiveView.bind(this));
     router.get('/:imageId', this.getImageView.bind(this));
 
     router.get('/', this.getDashboard.bind(this));
@@ -40,6 +41,9 @@ class ImageAdminController extends SiteController {
     } = this.dtp.services;
     try {
       res.locals.image = await imageService.getImageById(imageId);
+      if (!res.locals.image) {
+        throw new SiteError(404, 'Image not found');
+      }
       return next();
     } catch (error) {
       loganService.sendRequestEvent(module.exports, req, {
@@ -52,6 +56,16 @@ class ImageAdminController extends SiteController {
     }
   }
 
+  async getUserArchiveView (req, res, next) {
+    const { image: imageService } = this.dtp.services;
+    try {
+      res.locals.imageHistory = await imageService.getRecentImagesForOwner(res.locals.image.owner, 10);
+      res.render('admin/image/archive-user');
+    } catch (error) {
+      return next(error);
+    }
+  }
+
   async getImageView (req, res) {
     res.render('admin/image/view');
   }

app/controllers/admin/user.js

@@ -15,6 +15,14 @@ class UserAdminController extends SiteController {
   }
 
   async start ( ) {
+    const { jobQueue: jobQueueService } = this.dtp.services;
+
+    this.jobQueues = { };
+    this.jobQueues.reeeper = await jobQueueService.getJobQueue(
+      'reeeper',
+      this.dtp.config.jobQueues.reeeper,
+    );
+
     const router = express.Router();
     router.use(async (req, res, next) => {
       res.locals.currentView = 'admin';
@@ -23,11 +31,26 @@ class UserAdminController extends SiteController {
     });
 
     router.param('localUserId', this.populateLocalUserId.bind(this));
+    router.param('archiveJobId', this.populateArchiveJobId.bind(this));
+    router.param('archiveId', this.populateArchiveId.bind(this));
+
+    router.post('/local/:localUserId/archive', this.postArchiveLocalUser.bind(this));
     router.post('/local/:localUserId', this.postUpdateLocalUser.bind(this));
+
+    router.get('/local/:localUserId/archive/confirm', this.getArchiveLocalUserConfirm.bind(this));
     router.get('/local/:localUserId', this.getLocalUserView.bind(this));
 
+    router.get('/archive/job/:archiveJobId', this.getUserArchiveJobView.bind(this));
+
+    router.post('/archive/:archiveId/action', this.postArchiveAction.bind(this));
+    router.get('/archive/:archiveId/file', this.getUserArchiveFile.bind(this));
+    router.get('/archive/:archiveId', this.getUserArchiveView.bind(this));
+
+    router.get('/archive', this.getUserArchiveIndex.bind(this));
+
     router.get('/', this.getHomeView.bind(this));
 
+
     return router;
   }
 
@@ -44,6 +67,68 @@ class UserAdminController extends SiteController {
     }
   }
 
+  async populateArchiveJobId (req, res, next, archiveJobId) {
+    try {
+      res.locals.job = await this.jobQueues.reeeper.getJob(archiveJobId);
+      if (!res.locals.job) {
+        throw new SiteError(404, 'Job not found');
+      }
+      return next();
+    } catch (error) {
+      this.log.error('failed to populate Bull queue job', { archiveJobId, error });
+      return next(error);
+    }
+  }
+
+  async populateArchiveId (req, res, next, archiveId) {
+    const { user: userService } = this.dtp.services;
+    try {
+      res.locals.archive = await userService.getArchiveById(archiveId);
+      if (!res.locals.archive) {
+        throw new SiteError(404, 'Archive not found');
+      }
+      return next();
+    } catch (error) {
+      this.log.error('failed to populate UserArchive', { archiveId, error });
+      return next(error);
+    }
+  }
+
+  async postArchiveLocalUser (req, res, next) {
+    const {
+      logan: loganService,
+      user: userService,
+    } = this.dtp.services;
+    try {
+      const user = await userService.getLocalUserAccount(req.body.userId);
+      if (!user) {
+        throw new SiteError(404, 'User not found');
+      }
+      res.locals.job = await userService.archiveLocalUser(user);
+      loganService.sendRequestEvent(module.exports, req, {
+        level: 'info',
+        event: 'postArchiveUser',
+        data: {
+          job: res.locals.job.id,
+          user: user,
+        },
+      });
+      res.redirect(`/admin/user/archive/job/${res.locals.job.id}`);
+    } catch (error) {
+      loganService.sendRequestEvent(module.exports, req, {
+        level: 'error',
+        event: 'postArchiveUser',
+        data: {
+          offender: {
+            _id: req.body.userId,
+          },
+          error,
+        },
+      });
+      return next(error);
+    }
+  }
+
   async postUpdateLocalUser (req, res, next) {
     const {
       logan: loganService,
@@ -100,6 +185,133 @@ class UserAdminController extends SiteController {
     }
   }
 
+  async getUserArchiveJobView (req, res) {
+    res.locals.adminView = 'user-archive';
+    res.render('admin/user/archive/job');
+  }
+
+  async getArchiveLocalUserConfirm (req, res) {
+    res.locals.adminView = 'user-archive';
+    res.render('admin/user/archive/confirm');
+  }
+
+  async postArchiveAction (req, res, next) {
+    const {
+      logan: loganService,
+      user: userService,
+    } = this.dtp.services;
+    try {
+      switch (req.body.action) {
+        case 'update':
+          loganService.sendRequestEvent(module.exports, req, {
+            level: 'info',
+            event: 'postArchiveAction',
+            message: 'updating user archive record',
+            data: {
+              archive: {
+                _id: res.locals.archive._id,
+                user: {
+                  _id: res.locals.archive.user._id,
+                  username: res.locals.archive.user.username,
+                },
+              },
+            },
+          });
+          await userService.updateArchive(res.locals.archive, req.body);
+          return res.redirect(`/admin/user/archive/${res.locals.archive._id}`);
+
+        case 'delete-file':
+          loganService.sendRequestEvent(module.exports, req, {
+            level: 'info',
+            event: 'postArchiveAction',
+            message: 'removing user archive file',
+            data: {
+              archive: {
+                _id: res.locals.archive._id,
+                user: {
+                  _id: res.locals.archive.user._id,
+                  username: res.locals.archive.user.username,
+                },
+              },
+            },
+          });
+          await userService.deleteArchiveFile(res.locals.archive);
+          return res.redirect(`/admin/user/archive/${res.locals.archive._id}`);
+
+        case 'delete':
+          loganService.sendRequestEvent(module.exports, req, {
+            level: 'info',
+            event: 'postArchiveAction',
+            message: 'removing user archive',
+            data: {
+              archive: {
+                _id: res.locals.archive._id,
+                user: {
+                  _id: res.locals.archive.user._id,
+                  username: res.locals.archive.user.username,
+                },
+              },
+            },
+          });
+          await userService.deleteArchive(res.locals.archive);
+          return res.redirect(`/admin/user/archive`);
+
+        default:
+          // unknown/invalid action
+          break;
+      }
+
+      throw new SiteError(400, `Invalid user archive action: ${req.body.action}`);
+
+    } catch (error) {
+      this.log.error('failed to delete archive file', { error });
+      return next(error);
+    }
+  }
+
+  async getUserArchiveFile (req, res, next) {
+    const { minio: minioService } = this.dtp.services;
+    try {
+      res.locals.adminView = 'user-archive';
+
+      this.log.debug('archive', { archive: res.locals.archive });
+      const stream = await minioService.openDownloadStream({
+        bucket: res.locals.archive.archive.bucket,
+        key: res.locals.archive.archive.key,
+      });
+
+      res.status(200);
+      res.set('Content-Type', 'application/zip');
+      res.set('Content-Size', res.locals.archive.archive.size);
+      res.set('Content-Disposition', `attachment; filename="user-${res.locals.archive.user._id}.zip"`);
+
+      stream.pipe(res);
+    } catch (error) {
+      this.log.error('failed to stream user archive file', { error });
+      return next(error);
+    }
+  }
+
+  async getUserArchiveView (req, res) {
+    res.locals.adminView = 'user-archive';
+    res.render('admin/user/archive/view');
+  }
+
+  async getUserArchiveIndex (req, res, next) {
+    const { user: userService } = this.dtp.services;
+    try {
+      res.locals.adminView = 'user-archive';
+
+      res.locals.pagination = this.getPaginationParameters(req, 20);
+      res.locals.archive = await userService.getArchives(res.locals.pagination);
+
+      res.render('admin/user/archive/index');
+    } catch (error) {
+      this.log.error('failed to render the User archives index', { error });
+      return next(error);
+    }
+  }
+
   async getHomeView (req, res, next) {
     const { user: userService } = this.dtp.services;
     try {

app/models/user-archive.js

@@ -0,0 +1,29 @@
+// user-archive.js
+// Copyright (C) 2022 DTP Technologies, LLC
+// License: Apache-2.0
+
+'use strict';
+
+const mongoose = require('mongoose');
+
+const Schema = mongoose.Schema;
+
+const UserArchiveSchema = new Schema({
+  created: { type: Date, default: Date.now, required: true, index: -1 },
+  user: {
+    _id: { type: Schema.ObjectId, required: true, index: 1 },
+    email: { type: String },
+    username: { type: String, required: true },
+  },
+  archive: {
+    bucket: { type: String, required: true },
+    key: { type: String, required: true },
+    etag: { type: String, required: true },
+    size: { type: Number, required: true },
+  },
+  notes: { type: String },
+});
+
+module.exports = (conn) => {
+  return conn.model('UserArchive', UserArchiveSchema);
+};

app/services/display-engine.js

@@ -132,10 +132,6 @@ class DisplayEngineService extends SiteService {
     this.templates = { };
   }
 
-  async start ( ) { }
-
-  async stop ( ) { }
-
   loadTemplate (name, pugScript) {
     const scriptFile = path.join(this.dtp.config.root, 'app', 'views', pugScript);
     this.templates[name] = pug.compileFile(scriptFile);

app/services/image.js

@@ -90,11 +90,11 @@ class ImageService extends SiteService {
     return image;
   }
 
-  async getRecentImagesForOwner(owner) {
+  async getRecentImagesForOwner(owner, limit = 10) {
     const images = await SiteImage
       .find({ owner: owner._id })
       .sort({ created: -1 })
-      .limit(10)
+      .limit(limit)
       .populate(this.populateImage)
       .lean();
     return images;
@@ -112,6 +112,15 @@ class ImageService extends SiteService {
     return { images, totalImageCount };
   }
 
+  async downloadImage (image, filename) {
+    const { minio: minioService } = this.dtp.services;
+    return minioService.downloadFile({
+      bucket: image.file.bucket,
+      key: image.file.key,
+      filePath: filename,
+    });
+  }
+
   async deleteImage(image) {
     const { minio: minioService } = this.dtp.services;
 

app/services/minio.js

@@ -24,7 +24,6 @@ class MinioService extends SiteService {
       accessKey: process.env.MINIO_ACCESS_KEY,
       secretKey: process.env.MINIO_SECRET_KEY,
     };
-    this.log.debug('MinIO config', { minioConfig });
     this.minio = new Minio.Client(minioConfig);
   }
 

app/services/session.js

@@ -18,7 +18,7 @@ class SessionService extends SiteService {
 
   async start ( ) {
     await super.start();
-    this.log.info(`starting ${module.exports.name} service`);
+
     passport.serializeUser(this.serializeUser.bind(this));
     passport.deserializeUser(this.deserializeUser.bind(this));
   }

app/services/sms.js

@@ -18,7 +18,6 @@ class SmsService extends SiteService {
 
   async start ( ) {
     await super.start();
-    this.log.info(`starting ${module.exports.name} service`);
   }
 
   async stop ( ) {

app/services/user.js

@@ -11,6 +11,7 @@ const mongoose = require('mongoose');
 const User = mongoose.model('User');
 const CoreUser = mongoose.model('CoreUser');
 const UserBlock = mongoose.model('UserBlock');
+const UserArchive = mongoose.model('UserArchive');
 
 const passport = require('passport');
 const PassportLocal = require('passport-local');
@@ -47,13 +48,20 @@ class UserService extends SiteService {
 
   async start ( ) {
     await super.start();
-    this.log.info(`starting ${module.exports.name} service`);
 
     this.registerPassportLocal();
-
     if (process.env.DTP_ADMIN === 'enabled') {
       this.registerPassportAdmin();
     }
+
+    const { jobQueue: jobQueueService } = this.dtp.services;
+    this.jobQueues = { };
+
+    this.log.info('connecting to job queue', { name: 'reeeper', config: this.dtp.config.jobQueues.reeeper });
+    this.jobQueues.reeeper = jobQueueService.getJobQueue(
+      'reeeper',
+      this.dtp.config.jobQueues.reeeper,
+    );
   }
 
   async stop ( ) {
@@ -870,6 +878,95 @@ class UserService extends SiteService {
       }
     }
   }
+
+  /**
+   * Create a job to archive and ban a User (local). The job will immediately
+   * disable the specified user, create a .zip file of their content on storage.
+   * Once the worker confirms that the archive file is on storage, it creates a
+   * UserArchive record for it, then completely bans the User. That removes all
+   * of the User's content.
+   *
+   * It then removes the User record entirely.
+   *
+   * @param {User} user the User to be archived
+   * @returns the newly created Bull queue job
+   */
+  async archiveLocalUser (user) {
+    return this.jobQueues.reeeper.add('archive-user-local', { userId: user._id });
+  }
+
+  /**
+   * Update a UserArchive document
+   * @param {UserArchive} archive the existing archive to be updated
+   * @param {*} archiveDefinition new values to be applied
+   */
+  async updateArchive (archive, archiveDefinition) {
+    const update = { $set: { }, $unset: { } };
+
+    archiveDefinition.notes = archiveDefinition.notes.trim();
+    if (archiveDefinition.notes && (archiveDefinition.notes.length > 0)) {
+      update.$set.notes = archiveDefinition.notes;
+    } else {
+      update.$unset.notes = 1;
+    }
+
+    await UserArchive.updateOne({ _id: archive._id }, update);
+  }
+
+  /**
+   * Fetch an Array of UserArchive documents with pagination.
+   * @param {DtpPagination} pagination self explanatory
+   * @returns Array of UserArchive documents (can be empty)
+   */
+  async getArchives (pagination) {
+    const search = { };
+    const archives = await UserArchive
+      .find(search)
+      .sort({ created: -1 })
+      .skip(pagination.skip)
+      .limit(pagination.cpp)
+      .lean();
+    const totalArchiveCount = await UserArchive.estimatedDocumentCount();
+    return { archives, totalArchiveCount };
+  }
+
+  /**
+   * Fetch a UserArchive record. This does not fetch the archive file.
+   * @param {UserArchive} archiveId the ID of the archive to fetch
+   * @returns the requested UserArchive, or null/undefined.
+   */
+  async getArchiveById (archiveId) {
+    const archive = await UserArchive.findOne({ _id: archiveId }).lean();
+    return archive;
+  }
+
+  /**
+   * Removes the .zip file attached to a UserArchive.
+   * @param {UserArchive} archive the archive for which an associated .zip file
+   * is to be removed
+   */
+  async deleteArchiveFile (archive) {
+    const { minio: minioService } = this.dtp.services;
+    if (!archive.archive || !archive.archive.bucket || !archive.archive.key) {
+      return; // no archive file present, abort
+    }
+    await minioService.removeObject(archive.archive.bucket, archive.archive.key);
+    await UserArchive.updateOne(
+      { _id: archive._id },
+      {
+        $unset: { archive: 1 },
+      },
+    );
+  }
+
+  /**
+   * Removes a UserArchive and any attached data.
+   * @param {UserArchive} archive the UserArchive to be removed.
+   */
+  async deleteArchive (archive) {
+    await this.deleteArchiveFile(archive);
+    await UserArchive.deleteOne({ _id: archive._id });
+  }
 }
 
 module.exports = {

app/views/admin/components/menu.pug

@@ -51,6 +51,12 @@ ul(uk-nav).uk-nav-default
         i.fas.fa-user
       span.uk-margin-small-left Users
 
+  li(class={ 'uk-active': (adminView === 'user-archive') })
+    a(href="/admin/user/archive")
+      span.nav-item-icon
+        i.fas.fa-file-archive
+      span.uk-margin-small-left User Archive
+
   li(class={ 'uk-active': (adminView === 'content-report') })
     a(href="/admin/content-report")
       span.nav-item-icon

app/views/admin/image/archive-user.pug

@@ -0,0 +1,28 @@
+extends ../layouts/main
+block content
+
+  include ../user/components/list-item
+
+  form(method="POST", action=`/admin/user/local/${image.owner._id}/archive`).uk-form
+    input(type="hidden", name="userId", value= image.owner._id)
+    .uk-card.uk-card-default.uk-card-small
+      .uk-card-header
+        h1.uk-card-title Archive Local User
+      .uk-card-body
+        p This action will pull all images from storage into an archive file, place the archive file on storage, delete all the image records and storage data, then ban the User. The archive is produced first because images would be deleted during the ban. So, the archive is made, then the user is banned.
+
+        p These are the #{numeral(imageHistory.length).format('0,0')} most recent images uploaded by #{image.owner.username}.
+
+        div(uk-grid)
+          each image in imageHistory
+            .uk-width-medium
+              .uk-margin-small(uk-lightbox)
+                a(href=`/image/${image._id}`, data-type="image", data-caption=`id: ${image._id}`)
+                  div
+                    img(src= `/image/${image._id}`).responsive
+      .uk-card-footer.uk-flex.uk-flex-middle
+        .uk-width-expand
+          +renderBackButton()
+
+        .uk-width-auto
+          button(type="submit").uk-button.uk-button-danger.uk-border-rounded Archive User

app/views/admin/user/archive/confirm.pug

@@ -0,0 +1,28 @@
+extends ../../layouts/main
+block content
+
+  include ../../user/components/list-item
+
+  form(method="POST", action=`/admin/user/local/${userAccount._id}/archive`).uk-form
+    input(type="hidden", name="userId", value= userAccount._id)
+    .uk-card.uk-card-default.uk-card-small
+      .uk-card-header
+        h1.uk-card-title
+          span
+            i.fas.fa-id-card
+          span.uk-margin-small-left Archive Local User
+      .uk-card-body
+        .uk-margin
+          +renderUserListItem(userAccount)
+
+        .uk-margin
+          p This action will archive #{userAccount.displayName || userAccount.username}'s content to a .zip file, place the .zip file on storage, create a UserArchive record for this User account, ban this User account, and remove this User account from the database.
+          
+          p #{userAccount.displayName || userAccount.username}'s email address and username will become locked, and will remain unavailable for use for as long as this archive exists.
+
+      .uk-card-footer.uk-flex.uk-flex-middle
+        .uk-width-expand
+          +renderBackButton()
+
+        .uk-width-auto
+          button(type="submit").uk-button.uk-button-danger.uk-border-rounded Archive User

app/views/admin/user/archive/index.pug

@@ -0,0 +1,41 @@
+extends ../../layouts/main
+block content
+
+  include ../components/list-item
+  include ../../../components/pagination-bar
+
+  .uk-card.uk-card-default.uk-card-small.uk-margin
+    .uk-card-header
+      h1.uk-card-title
+        span
+          i.fas.fa-id-card
+        span.uk-margin-small-left User Archives
+
+    .uk-card-body
+      if Array.isArray(archive.archives) && (archive.archives.length > 0)
+        table.uk-table.uk-table-divider.uk-table-justify
+          thead
+            tr
+              th Username
+              th User ID
+              th Created
+              th Archive
+          tbody
+            each record in archive.archives
+              tr
+                td= record.user.username
+                td= record.user._id
+                td= moment(record.created).format('MMMM DD, YYYY, [at] h:mm a')
+                td
+                  span
+                    i.fas.fa-file-archive
+                  a(href=`/admin/user/archive/${record._id}`).uk-margin-small-left View Archive
+      else
+        div There are no user archives.
+
+    if Array.isArray(archive.archives) && (archive.archives.length > 0)
+      .uk-card-footer
+        +renderPaginationBar('/admin/user/archive', archive.totalArchiveCount)
+  
+  .uk-margin
+    .uk-text-small.uk-text-muted.uk-text-center User accounts referenced on this page have been removed from the database and are no longer able to use #{site.name}.

app/views/admin/user/archive/job.pug

@@ -0,0 +1,7 @@
+extends ../../layouts/main
+block content
+
+  include ../components/list-item
+
+  h1 User Archive Job
+  pre= JSON.stringify(job, null, 2)

app/views/admin/user/archive/view.pug

@@ -0,0 +1,83 @@
+extends ../../layouts/main
+block content
+
+  include ../components/list-item
+
+  form(method="POST", action=`/admin/user/archive/${archive._id}/action`).uk-form
+    .uk-card.uk-card-default.uk-card-small
+      .uk-card-header
+        h1.uk-card-title
+          span
+            i.fas.fa-id-card
+          span.uk-margin-small-left User Archive
+
+      .uk-card-body
+        .uk-margin
+          div(uk-grid)
+            .uk-width-auto
+              .uk-form-label Archive ID
+              .uk-text-bold= archive._id
+            .uk-width-auto
+              .uk-form-label Created
+              .uk-text-bold= moment(archive.created).format('MMMM DD, YYYY, [at] h:mm:ss a')
+            .uk-width-auto
+              .uk-form-label User
+              .uk-text-bold= archive.user.username
+            .uk-width-auto
+              .uk-form-label User ID
+              .uk-text-bold= archive.user._id
+            .uk-width-auto
+              .uk-form-label User email
+              .uk-text-bold= archive.user.email
+
+          if archive.archive
+            div(uk-grid)
+              .uk-width-auto
+                .uk-form-label Archive file
+                .uk-text-bold= archive.archive.key.replace(/\/user-archive\//, '')
+              .uk-width-auto
+                .uk-form-label Download size
+                .uk-text-bold= numeral(archive.archive.size).format('0,0.0a')
+          else
+            .uk-text-italic (archive file removed)
+
+        .uk-margin
+          label(for="notes").uk-form-label Notes
+          textarea(id="notes", name="notes", rows="4", placeholder="Enter notes").uk-textarea.uk-resize-vertical= archive.notes
+
+      .uk-card-footer
+        div(uk-grid)
+          .uk-width-expand
+            div(hidden= !archive.archive, uk-grid)
+              .uk-width-auto
+                a(href=`/admin/user/archive/${archive._id}/file`).uk-button.uk-button-default.uk-border-rounded
+                  span
+                    i.fas.fa-download
+                  span.uk-margin-small-left Download#[span(class="uk-visible@s")  File]
+              .uk-width-auto
+                button(
+                  type="submit",
+                  name="action",
+                  value="delete-file",
+                  uk-tooltip={ title: 'Remove the .zip file attached to the UserArchive' },
+                ).uk-button.uk-button-danger.uk-border-rounded
+                  span
+                    i.fas.fa-trash
+                  span.uk-margin-small-left Delete#[span(class="uk-visible@s")  File]
+
+          .uk-width-auto
+            button(
+              type="submit",
+              name="action",
+              value="delete",
+              uk-tooltip={ title: 'Remove the UserArchive from the database' },
+              ).uk-button.uk-button-danger.uk-border-rounded
+              span
+                i.fas.fa-save
+              span.uk-margin-small-left Delete
+
+          .uk-width-auto
+            button(type="submit", name="action", value="update").uk-button.uk-button-primary.uk-border-rounded
+              span
+                i.fas.fa-save
+              span.uk-margin-small-left Update

app/views/admin/user/form.pug

@@ -81,6 +81,8 @@ block content
             div(uk-grid).uk-grid-small
               .uk-width-expand
                 +renderBackButton()
+              .uk-width-auto
+                a(href=`/admin/user/local/${userAccount._id}/archive/confirm`).uk-button.uk-button-danger.uk-border-rounded Archive User
               .uk-width-auto
                 button(type="submit", name="action", value="ban").uk-button.uk-button-danger.uk-border-rounded Ban User
               .uk-width-auto

app/views/admin/user/index.pug

@@ -3,6 +3,19 @@ block content
 
   include ../../components/pagination-bar
 
+  .uk-margin
+    div(uk-grid).uk-flex-middle
+      .uk-width-expand
+        h1
+          span
+            i.fas.fa-user-cog
+          span.uk-margin-small-left User Manager
+      .uk-width-auto
+        a(href="/admin/user/archive").uk-button.uk-button-default.uk-border-rounded
+          span.nav-item-icon
+            i.fas.fa-file-archive
+          span.uk-margin-small-left Browse Archive
+
   .uk-margin
     form(method="GET", action="/admin/user").uk-form
       div(uk-grid).uk-grid-collapse

app/workers/reeeper.js

@@ -36,6 +36,8 @@ class ReeeperWorker extends SiteWorker {
 
     await this.loadProcessor(path.join(__dirname, 'reeeper', 'cron', 'expire-crashed-hosts.js'));
     await this.loadProcessor(path.join(__dirname, 'reeeper', 'cron', 'expire-announcements.js'));
+    await this.loadProcessor(path.join(__dirname, 'reeeper', 'job', 'archive-user-local.js'));
+
     await this.startProcessors();
   }
 

app/workers/reeeper/job/archive-user-local.js

@@ -0,0 +1,381 @@
+// reeeper/job/archive-user-local.js
+// Copyright (C) 2022 DTP Technologies, LLC
+// License: Apache-2.0
+
+'use strict';
+
+const path = require('path');
+const fs = require('fs');
+
+const util = require('util');
+const execFile = util.promisify(require('child_process').execFile);
+
+const mime = require('mime');
+
+const mongoose = require('mongoose');
+const User = mongoose.model('User');
+
+const { SiteWorkerProcess } = require(path.join(__dirname, '..', '..', '..', '..', 'lib', 'site-lib'));
+
+/**
+ * A job to archive and ban a User (local).
+ * 
+ * 1. Immediately disable the specified User
+ * 2. Create a .zip file of the User's content on storage
+ * 3. Creates a UserArchive record for the file and User
+ * 4. Ban the User (removes all of the User's content)
+ * 5. Remove the User record from the database
+ */
+class ArchiveUserLocalJob extends SiteWorkerProcess {
+
+  static get COMPONENT ( ) {
+    return {
+      name: 'archiveUserLocalJob',
+      slug: 'archive-user-local-job',
+    };
+  }
+
+  static get JOB_NAME ( ) { return 'Local User Archive'; }
+  static get JOB_SLUG ( ) { return 'archive-user-local'; }
+
+  constructor (worker) {
+    super(worker, ArchiveUserLocalJob.COMPONENT);
+    this.jobs = new Set();
+  }
+
+  async start ( ) {
+    await super.start();
+
+    this.queue = await this.getJobQueue('reeeper', this.dtp.config.jobQueues.reeeper);
+
+    this.log.info('registering job processor', { queue: this.queue.name, job: ArchiveUserLocalJob.JOB_SLUG });
+    this.queue.process(ArchiveUserLocalJob.JOB_SLUG, 1, this.processArchiveUserLocal.bind(this));
+  }
+
+  async stop ( ) {
+    try {
+      if (this.queue) {
+        this.log.info('halting job queue', { jobCount: this.jobs.size });
+        await this.queue.pause(true, false);
+        delete this.queue;
+      }
+    } catch (error) {
+      this.log.error('failed to halt job queue', { error });
+      // fall through
+    } finally {
+      await super.stop();
+    }
+  }
+
+  async processArchiveUserLocal (job) {
+    const { user: userService } = this.dtp.services;
+    try {
+      job.data.archivePath = path.join('/tmp', this.dtp.pkg.name, ArchiveUserLocalJob.JOB_SLUG);
+      this.jobs.add(job);
+
+      job.data.userId = mongoose.Types.ObjectId(job.data.userId);
+      job.data.user = await userService.getLocalUserAccount(job.data.userId);
+
+      job.data.workPath = path.join(job.data.archivePath, job.data.userId.toString());
+      await fs.promises.mkdir(job.data.workPath, { recursive: true });
+
+      /*
+       * Save the User account data
+       */
+      await this.archiveUserData(job);
+
+      /*
+       * Disable the User account (which destroys their session and cookie(s))
+       */
+      // await this.disableUser(job);
+
+      /*
+       * Archive the User's content to the workPath on the local file system.
+       */
+      await this.archiveUserChat(job);
+      await this.archiveUserComments(job);
+      await this.archiveUserStickers(job);
+      await this.archiveUserImages(job);
+
+      /*
+       * Create the .zip file archive, upload it to storage, and create the
+       * UserArchive record.
+       */
+      await this.createArchiveFile(job);
+
+      this.log.info('banning user', {
+        user: {
+          _id: job.data.userId,
+          username: job.data.user.username,
+        },
+      });
+      // await userService.ban(job.data.user);
+
+      this.log.info('removing user', {
+        user: {
+          _id: job.data.userId,
+          username: job.data.user.username,
+        },
+      });
+      // await User.deleteOne({ _id: job.data.userId });
+    } catch (error) {
+      this.log.error('failed to delete attachment', { attachmentId: job.data.attachmentId, error });
+      throw error;
+    } finally {
+      if (job.data.workPath) {
+        this.log.info('cleaning up work directory');
+        await fs.promises.rm(job.data.workPath, { force: true, recursive: true });
+
+        delete job.data.workPath;
+      }
+      this.jobs.delete(job);
+      this.log.info('job complete', { job: job.id, name: ArchiveUserLocalJob.JOB_NAME });
+    }
+  }
+
+  async archiveUserData (job) {
+    // fetch the entire User record (all fields)
+    job.data.fullUser = await User
+      .findOne({ _id: job.data.user._id })
+      .select('+email +passwordSalt +password +flags +permissions +optIn')
+      .lean();
+    if (!job.data.fullUser) {
+      throw new Error('user does not exist');
+    }
+
+    const userFilename = path.join(job.data.workPath, `user-${job.data.user._id}.json`);
+    await fs.promises.writeFile(userFilename, JSON.stringify(job.data.fullUser, null, 2));
+  }
+
+  async disableUser (job) {
+    this.log.info('disabling local User account', {
+      user: {
+        _id: job.data.userId,
+        username: job.data.user.username,
+      },
+    });
+    await User.updateOne(
+      { _id: job.data.user._id },
+      {
+        $set: {
+          'flags.isAdmin': false,
+          'flags.isModerator': false,
+          'flags.isEmailVerified': false,
+          'permissions.canLogin': false,
+          'permissions.canChat': false,
+          'permissions.canComment': false,
+          'permissions.canReport': false,
+          'optIn.system': false,
+          'optIn.marketing': false,
+        },
+      },
+    );
+  }
+
+  async archiveUserChat (job) {
+    const ChatMessage = mongoose.model('ChatMessage');
+    const ChatRoom = mongoose.model('ChatRoom');
+
+    job.data.chatPath = path.join(job.data.workPath, 'chat');
+    await fs.promises.mkdir(job.data.chatPath, { recursive: true });
+
+    this.log.info('archiving user chat', {
+      user: {
+        _id: job.data.user._id,
+        username: job.data.user.username,
+      },
+    });
+
+    await ChatRoom
+      .find({ owner: job.data.user._id })
+
+      .lean()
+      .cursor()
+      .eachAsync(async (room) => {
+        const roomFilename = path.join(job.data.workPath, 'chat', `room-${room._id}`);
+        await fs.promises.writeFile(roomFilename, JSON.stringify(room, null, 2));
+      });
+
+    await ChatMessage
+      .find({ author: job.data.user._id })
+      .lean()
+      .cursor()
+      .eachAsync(async (message) => {
+        const messageFilename = path.join(job.data.workPath, 'chat', `message-${message._id}.json`);
+        await fs.promises.writeFile(messageFilename, JSON.stringify(message, null, 2));
+      });
+  }
+
+  async archiveUserComments (job) {
+    const Comment = mongoose.model('Comment');
+
+    job.data.commentPath = path.join(job.data.workPath, 'comments');
+    await fs.promises.mkdir(job.data.commentPath, { recursive: true });
+
+    this.log.info('archiving user comments', {
+      user: {
+        _id: job.data.user._id,
+        username: job.data.user.username,
+      },
+    });
+
+    await Comment
+      .find({ author: job.data.userId })
+      .cursor()
+      .eachAsync(async (comment) => {
+        const commentFilename = path.join(job.data.imagePath, `comment-${comment._id}.json`);
+        await fs.promises.writeFile(commentFilename, JSON.stringify(comment, null, 2));
+      });
+  }
+
+  async archiveUserStickers (job) {
+    const Sticker = mongoose.model('Sticker');
+    const { minio: minioService } = this.dtp.services;
+
+    job.data.stickerPath = path.join(job.data.workPath, 'stickers');
+    await fs.promises.mkdir(job.data.stickerPath, { recursive: true });
+
+    job.data.stickerMediaPath = path.join(job.data.stickerPath, 'media');
+    await fs.promises.mkdir(job.data.stickerMediaPath, { recursive: true });
+
+    this.log.info('archiving user stickers', {
+      user: {
+        _id: job.data.user._id,
+        username: job.data.user.username,
+      },
+    });
+
+    await Sticker
+      .find({ owner: job.data.userId })
+      .cursor()
+      .eachAsync(async (sticker) => {
+        const stickerFilename = path.join(job.data.stickerPath, `sticker-${sticker._id}.json`);
+        await fs.promises.writeFile(stickerFilename, JSON.stringify(sticker, null, 2));
+
+        if (sticker.original && sticker.original.bucket && sticker.orignal.key && sticker.encoded.type) {
+          const originalExt = mime.getExtension(sticker.original.type);
+          const originalFilename = path.join(job.data.stickerMediaPath, `sticker-${sticker._id}.original.${originalExt}`);
+          await minioService.downloadFile({
+            bucket: sticker.original.bucket,
+            key: sticker.original.key,
+            filePath: originalFilename,
+          });
+        }
+
+        if (sticker.encoded && sticker.encoded.bucket && sticker.encoded.key && sticker.encoded.type) {
+          const encodedExt = mime.getExtension(sticker.encoded.type);
+          const encodedFilename = path.join(job.data.stickerMediaPath, `sticker-${sticker._id}.encoded.${encodedExt}`);
+          await minioService.downloadFile({
+            bucket: sticker.encoded.bucket,
+            key: sticker.encoded.key,
+            filePath: encodedFilename,
+          });
+        }
+      });
+  }
+
+  async archiveUserImages (job) {
+    const SiteImage = mongoose.model('Image');
+    const { image: imageService } = this.dtp.services;
+
+    job.data.imagePath = path.join(job.data.workPath, 'images');
+    await fs.promises.mkdir(job.data.imagePath, { recursive: true });
+
+    this.log.info('archiving user images', {
+      user: {
+        _id: job.data.user._id,
+        username: job.data.user.username,
+      },
+    });
+
+    await SiteImage
+      .find({ owner: job.data.user._id })
+      .cursor()
+      .eachAsync(async (image) => {
+        try {
+          let imageExt = mime.getExtension(image.type);
+          const imageFilename = path.join(job.data.imagePath, `image-${image._id}.${imageExt}`);
+          const metadataFilename = path.join(job.data.imagePath, `image-${image._id}.metadata.json`);
+
+          await imageService.downloadImage(image, imageFilename);
+          await fs.promises.writeFile(metadataFilename, JSON.stringify(image.metadata, null, 2));
+
+        } catch (error) {
+          this.log.error('failed to download image', {
+            image: { _id: image._id },
+            error,
+          });
+        }
+      });
+  }
+
+  async createArchiveFile (job) {
+    const { minio: minioService } = this.dtp.services;
+    try {
+      job.data.zipFilename = path.join(job.data.archivePath, `user-${job.data.userId}.zip`);
+      const zipArgs = [
+        '-r', '-9', 
+        job.data.zipFilename,
+        `${job.data.userId}`,
+      ];
+      const options = {
+        cwd: job.data.archivePath,
+        encoding: 'utf8',
+      };
+      await execFile('/usr/bin/zip', zipArgs, options);
+
+      const zipFileStat = await fs.promises.stat(job.data.zipFilename);
+      this.log.info('zip archive created', { size: zipFileStat.size });
+
+      job.data.archiveFile = {
+        bucket: process.env.MINIO_ADMIN_BUCKET,
+        key: `/user-archive/user-${job.data.userId}.zip`,
+      };
+
+      const response = await minioService.uploadFile({
+        bucket: job.data.archiveFile.bucket,
+        key: job.data.archiveFile.key,
+        filePath: job.data.zipFilename,
+        metadata: {
+          job: {
+            id: job.id,
+          },
+          user: job.data.user,
+        }
+      });
+
+      this.log.info('creating user archive record', { etag: response.etag, size: zipFileStat.size });
+      const UserArchive = mongoose.model('UserArchive');
+      await UserArchive.create({
+        created: job.data.startTime,
+        user: {
+          _id: job.data.userId,
+          username: job.data.user.username,
+          email: job.data.fullUser.email,
+        },
+        archive: {
+          bucket: job.data.archiveFile.bucket,
+          key: job.data.archiveFile.key,
+          etag: response.etag,
+          size: zipFileStat.size,
+        }
+      });
+    } catch (error) {
+      this.log.error('failed to create archive .zip file', {
+        user: {
+          _id: job.data.userId,
+          username: job.data.user.username,
+        },
+      });
+      throw error;
+    } finally {
+      try {
+        await fs.promises.rm(job.data.zipFilename, { force: true });
+      } catch (error) {
+        this.log.error('failed to remove temp .zip file', { error });
+      }
+    }
+  }
+}
+
+module.exports = ArchiveUserLocalJob;