added chat worker; all workers start

app/workers/chat.js

@@ -0,0 +1,66 @@
+// chat.js
+// Copyright (C) 2022 DTP Technologies, LLC
+// License: Apache-2.0
+
+'use strict';
+
+const path = require('path');
+require('dotenv').config({ path: path.resolve(__dirname, '..', '..', '.env') });
+
+const mongoose = require('mongoose');
+
+const { SiteLog, SiteWorker, SiteAsync } = require(path.join(__dirname, '..', '..', 'lib', 'site-lib'));
+
+module.rootPath = path.resolve(__dirname, '..', '..');
+module.pkg = require(path.resolve(__dirname, '..', '..', 'package.json'));
+
+module.config = {
+  environment: process.env.NODE_ENV,
+  root: module.rootPath,
+  component: { name: 'chatWorker', slug: 'chat-worker' },
+};
+
+module.config.site = require(path.join(module.rootPath, 'config', 'site'));
+
+class ChatWorker extends SiteWorker {
+
+  constructor (dtp) {
+    super(dtp, dtp.config.component);
+  }
+
+  async start ( ) {
+    await super.start();
+
+    await this.loadProcessor(path.join(__dirname, 'chat', 'job', 'chat-room-clear.js'));
+    await this.loadProcessor(path.join(__dirname, 'chat', 'job', 'chat-room-delete.js'));
+
+    await this.startProcessors();
+  }
+
+  async stop ( ) {
+    await super.stop();
+  }
+
+  async deleteChatMessage (message) {
+    const { attachment: attachmentService } = this.dtp.services;
+    const ChatMessage = mongoose.model('ChatMessage');
+
+    await SiteAsync.each(message.attachments, attachmentService.remove.bind(attachmentService), 2);
+    await ChatMessage.deleteOne({ _id: message._id });
+  }
+}
+
+(async ( ) => {
+  try {
+    module.log = new SiteLog(module, module.config.component);
+
+    module.worker = new ChatWorker(module);
+    await module.worker.start();
+
+    module.log.info(`${module.pkg.name} v${module.pkg.version} ${module.config.component.name} started`);
+  } catch (error) {
+    module.log.error('failed to start worker', { component: module.config.component, error });
+    process.exit(-1);
+  }
+
+})();

app/workers/chat/job/chat-room-clear.js

@@ -8,19 +8,10 @@ const path = require('path');
 
 const mongoose = require('mongoose');
 
-const ChatRoom = mongoose.model('ChatRoom');
-const ChatRoomInvite = mongoose.model('ChatRoomInvite');
 const ChatMessage = mongoose.model('ChatMessage');
-const EmojiReaction = mongoose.model('EmojiReaction');
 
-const { SiteWorkerProcess } = require(path.join(__dirname, '..', '..', 'lib', 'site-lib'));
+const { SiteWorkerProcess } = require(path.join(__dirname, '..', '..', '..', '..', 'lib', 'site-lib'));
 
-/**
- * DTP Core Chat sticker processor can receive requests to ingest and delete
- * stickers to be executed as background jobs in a queue. This processor
- * attaches to the `media` queue and registers processors for `sticker-ingest`
- * and `sticker-delete`.
- */
 class ChatRoomClearJob extends SiteWorkerProcess {
 
   static get COMPONENT ( ) {
@@ -37,10 +28,10 @@ class ChatRoomClearJob extends SiteWorkerProcess {
   async start ( ) {
     await super.start();
 
-    const queue = this.getJobQueue('chat');
+    this.queue = await this.getJobQueue('chat');
 
     this.log.info('registering job processor', { queue: this.queue.name, name: 'chat-room-clear' });
-    queue.process('chat-room-clear', this.processChatRoomClear.bind(this));
+    this.queue.process('chat-room-clear', this.processChatRoomClear.bind(this));
   }
 
   async stop ( ) {
@@ -50,7 +41,6 @@ class ChatRoomClearJob extends SiteWorkerProcess {
   async processChatRoomClear (job) {
     const { roomId } = job.data;
     this.log.info('received chat room clear job', { id: job.id, roomId });
-
     await ChatMessage
       .find({ room: roomId })
       .cursor()

app/workers/chat/job/chat-room-delete.js

@@ -13,7 +13,7 @@ const ChatRoomInvite = mongoose.model('ChatRoomInvite');
 const ChatMessage = mongoose.model('ChatMessage');
 const EmojiReaction = mongoose.model('EmojiReaction');
 
-const { SiteWorkerProcess } = require(path.join(__dirname, '..', '..', 'lib', 'site-lib'));
+const { SiteWorkerProcess } = require(path.join(__dirname, '..', '..', '..', '..', 'lib', 'site-lib'));
 
 /**
  * DTP Core Chat sticker processor can receive requests to ingest and delete
@@ -37,10 +37,10 @@ class ChatRoomDeleteJob extends SiteWorkerProcess {
   async start ( ) {
     await super.start();
 
-    const queue = this.getJobQueue('chat');
+    this.queue = await this.getJobQueue('chat');
 
     this.log.info('registering job processor', { queue: this.queue.name, name: 'chat-room-delete' });
-    queue.process('chat-room-delete', this.processChatRoomDelete.bind(this));
+    this.queue.process('chat-room-delete', this.processChatRoomDelete.bind(this));
   }
 
   async stop ( ) {

app/workers/newsletter.js

@@ -7,8 +7,6 @@
 const path = require('path');
 require('dotenv').config({ path: path.resolve(__dirname, '..', '..', '.env') });
 
-const mongoose = require('mongoose');
-
 const { SiteWorker, SiteLog } = require(path.join(__dirname, '..', '..', 'lib', 'site-lib'));
 
 module.pkg = require(path.resolve(__dirname, '..', '..', 'package.json'));
@@ -21,18 +19,16 @@ class NewsletterWorker extends SiteWorker {
 
   constructor (dtp) {
     super(dtp, dtp.config.component);
-    this.newsletters = this.newsletters || { };
+    this.newsletters = { };
   }
 
   async start ( ) {
     await super.start();
     
-    const { jobQueue: jobQueueService } = this.dtp.services;
+    await this.loadProcessor(path.join(__dirname, 'newsletter', 'job', 'transmit.js'));
-    this.jobQueue = await jobQueueService.getJobQueue('newsletter', {
+    await this.loadProcessor(path.join(__dirname, 'newsletter', 'job', 'email-send.js'));
-      attempts: 3,
+
-    });
+    await this.startProcessors();
-    this.jobQueue.process('transmit', this.transmitNewsletter.bind(this));
-    this.jobQueue.process('email-send', this.sendNewsletterEmail.bind(this));
   }
 
   async stop ( ) {
@@ -53,89 +49,6 @@ class NewsletterWorker extends SiteWorker {
     }
     return newsletter;
   }
-
-  async transmitNewsletter (job) {
-    const User = mongoose.model('User');
-    const NewsletterRecipient = mongoose.model('NewsletterRecipient');
-    this.log.info('newsletter email job received', { data: job.data });
-    try {
-      /*
-       * Transmit first to all local user accounts with verified email who've
-       * opted in for receiving marketing email.
-       */
-      await User
-        .find({
-          'flags.isEmailVerified': true,
-          'optIn.marketing': true,
-        })
-        .select('email displayName username username_lc')
-        .lean()
-        .cursor()
-        .eachAsync(async (user) => {
-          try {
-            const jobData = {
-              newsletterId: job.data.newsletterId,
-              recipient: user.email,
-              recipientName: user.displayName || user.username,
-            };
-            const jobOptions = { attempts: 3 };
-            await this.jobQueue.add('email-send', jobData, jobOptions);
-          } catch (error) {
-            this.log.error('failed to create newsletter email job', { error });
-          }
-        }, { parallel: 4 });
-
-      /*
-       * Transmit to all newsletter recipients on file who've joined through the
-       * widget on the site w/o signing up for an account.
-       */
-      await NewsletterRecipient
-        .find({ 'flags.isVerified': true, 'flags.isOptIn': true, 'flags.isRejected': false })
-        .lean()
-        .cursor()
-        .eachAsync(async (recipient) => {
-          try {
-            const jobData = {
-              newsletterId: job.data.newsletterId,
-              recipient: recipient.address,
-            };
-            const jobOptions = { attempts: 3 };
-            await this.jobQueue.add('email-send', jobData, jobOptions);
-          } catch (error) {
-            this.log.error('failed to create newsletter email job', { error });
-          }
-        }, { parallel: 4 });
-    } catch (error) {
-      this.log.error('failed to send newsletter', { newsletterId: job.data.newsletterId, error });
-      throw error;
-    }
-  }
-
-  async sendNewsletterEmail (job) {
-    const { email: emailService } = this.dtp.services;
-    const { newsletterId, recipient } = job.data;
-
-    try {
-      let newsletter = await this.loadNewsletter(newsletterId);
-      if (!newsletter) {
-        throw new Error('newsletter not found');
-      }
-  
-      const result = await emailService.send({
-        from: process.env.DTP_EMAIL_SMTP_FROM || `noreply@${this.dtp.config.site.domainKey}`,
-        to: recipient,
-        subject: newsletter.title,
-        html: newsletter.content.html,
-        text: newsletter.content.text,
-      });
-  
-      job.log(`newsletter email sent: ${result}`);
-      this.log.info('newsletter email sent', { recipient, result });
-    } catch (error) {
-      this.log.error('failed to send newsletter email', { newsletterId, recipient, error });
-      throw error; // throw error to Bull so it can report in job reports
-    }
-  }
 }  
 
 (async ( ) => {

app/workers/newsletter/job/email-send.js

@@ -0,0 +1,63 @@
+// newsletter/job/email-send.js
+// Copyright (C) 2022 DTP Technologies, LLC
+// License: Apache-2.0
+
+'use strict';
+
+const path = require('path');
+
+const { SiteWorkerProcess } = require(path.join(__dirname, '..', '..', '..', '..', 'lib', 'site-lib'));
+
+class NewsletterEmailSendJob extends SiteWorkerProcess {
+
+  static get COMPONENT ( ) {
+    return {
+      name: 'newsletterEmailSendJob',
+      slug: 'newsletter-email-send-job',
+    };
+  }
+
+  constructor (worker) {
+    super(worker, NewsletterEmailSendJob.COMPONENT);
+  }
+
+  async start ( ) {
+    await super.start();
+
+    this.queue = await this.getJobQueue('newsletter');
+
+    this.log.info('registering job processor', { queue: this.queue.name, name: 'email-send' });
+    this.queue.process('email-send', this.processEmailSend.bind(this));
+  }
+
+  async stop ( ) {
+    await super.stop();
+  }
+
+  async processEmailSend (job) {
+    const { email: emailService } = this.dtp.services;
+    const { newsletterId, recipient } = job.data;
+
+    try {
+      let newsletter = await this.worker.loadNewsletter(newsletterId);
+      if (!newsletter) {
+        throw new Error('newsletter not found');
+      }
+  
+      const result = await emailService.send({
+        from: process.env.DTP_EMAIL_SMTP_FROM || `noreply@${this.dtp.config.site.domainKey}`,
+        to: recipient,
+        subject: newsletter.title,
+        html: newsletter.content.html,
+        text: newsletter.content.text,
+      });
+  
+      this.jobLog(job, 'newsletter email sent', { result });
+    } catch (error) {
+      this.log.error('failed to send newsletter email', { newsletterId, recipient, error });
+      throw error; // throw error to Bull so it can report in job reports
+    }
+  }
+}
+
+module.exports = NewsletterEmailSendJob;

app/workers/newsletter/job/transmit.js

@@ -0,0 +1,100 @@
+// newsletter/job/transmit.js
+// Copyright (C) 2022 DTP Technologies, LLC
+// License: Apache-2.0
+
+'use strict';
+
+const path = require('path');
+
+const mongoose = require('mongoose');
+
+const User = mongoose.model('User');
+const NewsletterRecipient = mongoose.model('NewsletterRecipient');
+
+const { SiteWorkerProcess } = require(path.join(__dirname, '..', '..', '..', '..', 'lib', 'site-lib'));
+
+class NewsletterTransmitJob extends SiteWorkerProcess {
+
+  static get COMPONENT ( ) {
+    return {
+      name: 'newsletterTransmitJob',
+      slug: 'newsletter-transmit-job',
+    };
+  }
+
+  constructor (worker) {
+    super(worker, NewsletterTransmitJob.COMPONENT);
+  }
+
+  async start ( ) {
+    await super.start();
+
+    this.queue = await this.getJobQueue('newsletter');
+
+    this.log.info('registering job processor', { queue: this.queue.name, name: 'transmit' });
+    this.queue.process('transmit', this.processTransmit.bind(this));
+  }
+
+  async stop ( ) {
+    await super.stop();
+  }
+
+  async processTransmit (job) {
+    const { newsletterId } = job.data;
+    this.log.info('newsletter email job received', { id: job.id, newsletterId });
+
+    try {
+      /*
+       * Transmit first to all local user accounts with verified email who've
+       * opted in for receiving marketing email.
+       */
+      await User
+        .find({
+          'flags.isEmailVerified': true,
+          'optIn.marketing': true,
+        })
+        .select('email displayName username username_lc')
+        .lean()
+        .cursor()
+        .eachAsync(async (user) => {
+          try {
+            const jobData = {
+              newsletterId: newsletterId,
+              recipient: user.email,
+              recipientName: user.displayName || user.username,
+            };
+            const jobOptions = { attempts: 3 };
+            await this.queue.add('email-send', jobData, jobOptions);
+          } catch (error) {
+            this.log.error('failed to create newsletter email job', { error });
+          }
+        }, { parallel: 4 });
+
+      /*
+       * Transmit to all newsletter recipients on file who've joined through the
+       * widget on the site w/o signing up for an account.
+       */
+      await NewsletterRecipient
+        .find({ 'flags.isVerified': true, 'flags.isOptIn': true, 'flags.isRejected': false })
+        .lean()
+        .cursor()
+        .eachAsync(async (recipient) => {
+          try {
+            const jobData = {
+              newsletterId: newsletterId,
+              recipient: recipient.address,
+            };
+            const jobOptions = { attempts: 3 };
+            await this.queue.add('email-send', jobData, jobOptions);
+          } catch (error) {
+            this.log.error('failed to create newsletter email job', { error });
+          }
+        }, { parallel: 4 });
+    } catch (error) {
+      this.log.error('failed to send newsletter', { newsletterId, error });
+      throw error;
+    }
+  }
+}
+
+module.exports = NewsletterTransmitJob;

app/workers/reeeper/cron/expire-crashed-hosts.js

@@ -57,6 +57,7 @@ class CrashedHostsCron extends SiteWorkerProcess {
 
   async expireCrashedHosts ( ) {
     try {
+      this.log.debug('expiring crashed hosts');
       await NetHost
         .find({ status: 'crashed' })
         .select('_id hostname')

config/job-queues.js

@@ -5,6 +5,10 @@
 'use strict';
 
 module.exports = {
+  'chat': {
+    attempts: 5,
+    removeOnComplete: true,
+  },
   'media': {
     attempts: 3,
     removeOnComplete: true,

lib/site-common.js

@@ -42,6 +42,10 @@ class SiteCommon extends Events {
   }
 
   async getJobQueue (name) {
+    if (this.jobQueues[name]) {
+      return this.jobQueues[name];
+    }
+
     const { jobQueue: jobQueueService } = this.dtp.services;
     const config = this.dtp.config.jobQueues[name];
 

lib/site-worker.js

@@ -70,7 +70,7 @@ class SiteWorker extends SiteCommon {
     const processor = new ProcessorClass(this);
     const { COMPONENT } = ProcessorClass;
 
-    this.log.info('registering worker processor', { component: COMPONENT });
+    this.log.info('loading worker processor', { component: COMPONENT.name });
     this.processors[COMPONENT.name] = processor;
 
     return processor;
@@ -84,8 +84,18 @@ class SiteWorker extends SiteCommon {
   async startProcessors ( ) {
     const slugs = Object.keys(this.processors);
     await SiteAsync.each(slugs, async (slug) => {
-      this.log.info('starting worker processor', { slug });
+      const processor = this.processors[slug];
-      await this.processors[slug].start();
+      try {
+        this.log.info('starting worker processor', {
+          component: processor.component.name,
+        });
+        await processor.start();
+      } catch (error) {
+        this.log.error('failed to start processor', {
+          component: processor.component.name,
+          error,
+        });
+      }
     }, 1);
   }
 
@@ -95,8 +105,18 @@ class SiteWorker extends SiteCommon {
    async stop ( ) {
     const slugs = Object.keys(this.processors);
     await SiteAsync.each(slugs, async (slug) => {
-      this.log.info('stopping worker processor', { slug });
+      const processor = this.processors[slug];
-      await this.processors[slug].stop();
+      try {
+        this.log.info('stopping worker processor', {
+          component: processor.component.name,
+        });
+        await processor.stop();
+      } catch (error) {
+        this.log.error('failed to stop processor', {
+          component: processor.component.name,
+          error,
+        });
+      }
     }, 1);
   }
 }