From 2dcd95c74b6243a9e629bfd7762eb37707c211dd Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 6 Jul 2022 10:09:36 -0400 Subject: [PATCH] large component and worker management and logging refactor --- .vscode/launch.json | 12 ++- app/controllers/admin.js | 4 +- app/controllers/admin/content-report.js | 16 ++-- app/controllers/admin/core-node.js | 14 ++-- app/controllers/admin/host.js | 16 ++-- app/controllers/admin/job-queue.js | 13 ++-- app/controllers/admin/log.js | 17 +++-- app/controllers/admin/settings.js | 13 ++-- app/controllers/admin/user.js | 18 +++-- app/controllers/auth.js | 6 +- app/controllers/email.js | 4 +- app/controllers/hive.js | 6 +- app/controllers/hive/kaleidoscope.js | 18 +++-- app/controllers/home.js | 4 +- app/controllers/image.js | 6 +- app/controllers/manifest.js | 6 +- app/controllers/user.js | 6 +- app/controllers/welcome.js | 4 +- app/models/log.js | 5 +- app/services/log.js | 4 +- app/services/session.js | 11 ++- app/views/admin/log/index.pug | 6 +- app/views/index.pug | 18 ++++- app/views/layouts/main.pug | 1 + app/views/user/components/profile-icon.pug | 4 +- app/views/user/profile.pug | 25 ++---- app/workers/host-services.js | 14 ++-- app/workers/reeeper.js | 16 ++-- app/workers/sample-worker.js | 88 ++++++++++++++++++++++ client/js/index-admin.js | 4 +- client/js/index.js | 4 +- client/js/site-admin-app.js | 4 +- client/js/site-app.js | 4 +- config/http.js | 8 ++ dtp-webapp-cli.js | 9 ++- dtp-webapp.js | 14 ++-- lib/client/js/dtp-log.js | 8 +- lib/client/js/dtp-plugin.js | 8 +- lib/site-common.js | 8 +- lib/site-controller.js | 10 +-- lib/site-ioserver.js | 3 +- lib/site-lib.js | 1 + lib/site-log.js | 22 ++++-- lib/site-platform.js | 8 +- lib/site-service.js | 8 +- lib/site-worker.js | 62 +++++++++++++++ update-deps.js | 14 ++-- 47 files changed, 382 insertions(+), 192 deletions(-) create mode 100644 app/workers/sample-worker.js create mode 100644 config/http.js create mode 100644 lib/site-worker.js diff --git a/.vscode/launch.json b/.vscode/launch.json index 8385797..ca2f26a 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -27,6 +27,16 @@ "program": "${workspaceFolder:dtp-base}/dtp-webapp-cli.js", "console": "integratedTerminal", "args": ["--action=reset-indexes", "all"] - } + }, + { + "type": "pwa-node", + "request": "launch", + "name": "worker:sample", + "skipFiles": [ + "/**" + ], + "program": "${workspaceFolder:dtp-base}/app/workers/sample-worker.js", + "console": "integratedTerminal", + }, ] } \ No newline at end of file diff --git a/app/controllers/admin.js b/app/controllers/admin.js index 78d74a6..ee0c144 100644 --- a/app/controllers/admin.js +++ b/app/controllers/admin.js @@ -4,8 +4,6 @@ 'use strict'; -const DTP_COMPONENT_NAME = 'admin'; - const path = require('path'); const express = require('express'); @@ -17,7 +15,7 @@ const { SiteError, SiteController } = require('../../lib/site-lib'); class AdminController extends SiteController { constructor (dtp) { - super(dtp, DTP_COMPONENT_NAME); + super(dtp, module.exports); } async start ( ) { diff --git a/app/controllers/admin/content-report.js b/app/controllers/admin/content-report.js index 61db82d..b998de1 100644 --- a/app/controllers/admin/content-report.js +++ b/app/controllers/admin/content-report.js @@ -4,8 +4,6 @@ 'use strict'; -const DTP_COMPONENT_NAME = 'admin:content-report'; - const express = require('express'); const multer = require('multer'); @@ -14,7 +12,7 @@ const { /*SiteError,*/ SiteController } = require('../../../lib/site-lib'); class ContentReportController extends SiteController { constructor (dtp) { - super(dtp, DTP_COMPONENT_NAME); + super(dtp, module.exports); } async start ( ) { @@ -89,7 +87,11 @@ class ContentReportController extends SiteController { } } -module.exports = async (dtp) => { - let controller = new ContentReportController(dtp); - return controller; -}; +module.exports = { + name: 'Admin: Content Reports', + slug: 'admin:content-report', + create: async (dtp) => { + let controller = new ContentReportController(dtp); + return controller; + }, +}; \ No newline at end of file diff --git a/app/controllers/admin/core-node.js b/app/controllers/admin/core-node.js index da8d196..9c47987 100644 --- a/app/controllers/admin/core-node.js +++ b/app/controllers/admin/core-node.js @@ -4,8 +4,6 @@ 'use strict'; -const DTP_COMPONENT_NAME = 'admin:core-node'; - const express = require('express'); // const multer = require('multer'); @@ -14,7 +12,7 @@ const { /*SiteError,*/ SiteController } = require('../../../lib/site-lib'); class CoreNodeController extends SiteController { constructor (dtp) { - super(dtp, DTP_COMPONENT_NAME); + super(dtp, module.exports); } async start ( ) { @@ -99,7 +97,11 @@ class CoreNodeController extends SiteController { } } -module.exports = async (dtp) => { - let controller = new CoreNodeController(dtp); - return controller; +module.exports = { + name: 'Admin: Core Node', + slug: 'admin:core-node', + create: async (dtp) => { + let controller = new CoreNodeController(dtp); + return controller; + }, }; \ No newline at end of file diff --git a/app/controllers/admin/host.js b/app/controllers/admin/host.js index 1322eda..bacaf87 100644 --- a/app/controllers/admin/host.js +++ b/app/controllers/admin/host.js @@ -4,8 +4,6 @@ 'use strict'; -const DTP_COMPONENT_NAME = 'admin:host'; - const express = require('express'); const mongoose = require('mongoose'); @@ -17,7 +15,7 @@ const { /*SiteError,*/ SiteController } = require('../../../lib/site-lib'); class HostController extends SiteController { constructor (dtp) { - super(dtp, DTP_COMPONENT_NAME); + super(dtp, module.exports); } async start ( ) { @@ -117,7 +115,11 @@ class HostController extends SiteController { } } -module.exports = async (dtp) => { - let controller = new HostController(dtp); - return controller; -}; +module.exports = { + name: 'Admin: Network Host', + slug: 'admin:host', + create: async (dtp) => { + let controller = new HostController(dtp); + return controller; + }, +}; \ No newline at end of file diff --git a/app/controllers/admin/job-queue.js b/app/controllers/admin/job-queue.js index 69f00f9..15945a9 100644 --- a/app/controllers/admin/job-queue.js +++ b/app/controllers/admin/job-queue.js @@ -4,7 +4,6 @@ 'use strict'; -const DTP_COMPONENT_NAME = 'admin:job-queue'; const express = require('express'); const { SiteController, SiteError } = require('../../../lib/site-lib'); @@ -12,7 +11,7 @@ const { SiteController, SiteError } = require('../../../lib/site-lib'); class JobQueueController extends SiteController { constructor (dtp) { - super(dtp, DTP_COMPONENT_NAME); + super(dtp, module.exports); } async start ( ) { @@ -119,7 +118,11 @@ class JobQueueController extends SiteController { } } -module.exports = async (dtp) => { - let controller = new JobQueueController(dtp); - return controller; +module.exports = { + name: 'Admin: Job Queues', + slug: 'admin:job-queue', + create: async (dtp) => { + let controller = new JobQueueController(dtp); + return controller; + }, }; \ No newline at end of file diff --git a/app/controllers/admin/log.js b/app/controllers/admin/log.js index f1cfc62..1efc33d 100644 --- a/app/controllers/admin/log.js +++ b/app/controllers/admin/log.js @@ -4,7 +4,6 @@ 'use strict'; -const DTP_COMPONENT_NAME = 'admin:log'; const express = require('express'); const { SiteController } = require('../../../lib/site-lib'); @@ -12,7 +11,7 @@ const { SiteController } = require('../../../lib/site-lib'); class LogController extends SiteController { constructor (dtp) { - super(dtp, DTP_COMPONENT_NAME); + super(dtp, module.exports); } async start ( ) { @@ -33,12 +32,12 @@ class LogController extends SiteController { try { res.locals.query = req.query; - res.locals.components = await logService.getComponentNames(); + res.locals.components = await logService.getComponentSlugs(); res.locals.pagination = this.getPaginationParameters(req, 25); const search = { }; if (req.query.component) { - search.componentName = req.query.component; + search.component = { slug: req.query.component }; } res.locals.logs = await logService.getRecords(search, res.locals.pagination); @@ -51,7 +50,11 @@ class LogController extends SiteController { } } -module.exports = async (dtp) => { - let controller = new LogController(dtp); - return controller; +module.exports = { + name: 'Admin: Log', + slug: 'admin:log', + create: async (dtp) => { + let controller = new LogController(dtp); + return controller; + }, }; \ No newline at end of file diff --git a/app/controllers/admin/settings.js b/app/controllers/admin/settings.js index 94c51e0..fb49ef4 100644 --- a/app/controllers/admin/settings.js +++ b/app/controllers/admin/settings.js @@ -4,7 +4,6 @@ 'use strict'; -const DTP_COMPONENT_NAME = 'admin:settings'; const express = require('express'); const { SiteController } = require('../../../lib/site-lib'); @@ -12,7 +11,7 @@ const { SiteController } = require('../../../lib/site-lib'); class SettingsController extends SiteController { constructor (dtp) { - super(dtp, DTP_COMPONENT_NAME); + super(dtp, module.exports); } async start ( ) { @@ -50,7 +49,11 @@ class SettingsController extends SiteController { } } -module.exports = async (dtp) => { - let controller = new SettingsController(dtp); - return controller; +module.exports = { + name: 'Admin: Settings', + slug: 'admin:settings', + create: async (dtp) => { + let controller = new SettingsController(dtp); + return controller; + }, }; \ No newline at end of file diff --git a/app/controllers/admin/user.js b/app/controllers/admin/user.js index 682247d..2500ac1 100644 --- a/app/controllers/admin/user.js +++ b/app/controllers/admin/user.js @@ -4,16 +4,14 @@ 'use strict'; -const DTP_COMPONENT_NAME = 'admin:user'; - const express = require('express'); -const { /*SiteError,*/ SiteController } = require('../../../lib/site-lib'); +const { SiteController } = require('../../../lib/site-lib'); class UserController extends SiteController { constructor (dtp) { - super(dtp, DTP_COMPONENT_NAME); + super(dtp, module.exports); } async start ( ) { @@ -79,7 +77,11 @@ class UserController extends SiteController { } } -module.exports = async (dtp) => { - let controller = new UserController(dtp); - return controller; -}; +module.exports = { + name: 'Admin: User Management', + slug: 'admin:user', + create: async (dtp) => { + let controller = new UserController(dtp); + return controller; + }, +}; \ No newline at end of file diff --git a/app/controllers/auth.js b/app/controllers/auth.js index d6c9887..4674e9f 100644 --- a/app/controllers/auth.js +++ b/app/controllers/auth.js @@ -4,8 +4,6 @@ 'use strict'; -const DTP_COMPONENT_NAME = 'auth'; - const express = require('express'); const mongoose = require('mongoose'); const multer = require('multer'); @@ -19,7 +17,7 @@ const ConnectToken = mongoose.model('ConnectToken'); class AuthController extends SiteController { constructor (dtp) { - super(dtp, DTP_COMPONENT_NAME); + super(dtp, module.exports); } async start ( ) { @@ -27,7 +25,7 @@ class AuthController extends SiteController { coreNode: coreNodeService, limiter: limiterService, } = this.dtp.services; - const upload = multer({ dest: `/tmp/${this.dtp.config.site.domainKey}/uploads/${DTP_COMPONENT_NAME}` }); + const upload = multer({ dest: `/tmp/${this.dtp.config.site.domainKey}/uploads/${this.component.slug}` }); const router = express.Router(); this.dtp.app.use('/auth', router); diff --git a/app/controllers/email.js b/app/controllers/email.js index e152299..f2832bc 100644 --- a/app/controllers/email.js +++ b/app/controllers/email.js @@ -4,8 +4,6 @@ 'use strict'; -const DTP_COMPONENT_NAME = 'email'; - const express = require('express'); const { SiteController/*, SiteError*/ } = require('../../lib/site-lib'); @@ -13,7 +11,7 @@ const { SiteController/*, SiteError*/ } = require('../../lib/site-lib'); class EmailController extends SiteController { constructor (dtp) { - super(dtp, DTP_COMPONENT_NAME); + super(dtp, module.exports); } async start ( ) { diff --git a/app/controllers/hive.js b/app/controllers/hive.js index f0199b2..dc00ce7 100644 --- a/app/controllers/hive.js +++ b/app/controllers/hive.js @@ -4,8 +4,6 @@ 'use strict'; -const DTP_COMPONENT_NAME = 'hive'; - const path = require('path'); const express = require('express'); @@ -14,7 +12,7 @@ const { SiteController } = require('../../lib/site-lib'); class HiveController extends SiteController { constructor (dtp) { - super(dtp, DTP_COMPONENT_NAME); + super(dtp, module.exports); this.services = [ ]; } @@ -45,7 +43,7 @@ class HiveController extends SiteController { async getHiveRoot (req, res) { res.status(200).json({ - component: DTP_COMPONENT_NAME, + component: this.component, host: this.dtp.pkg.name, description: this.dtp.pkg.description, version: this.dtp.pkg.version, diff --git a/app/controllers/hive/kaleidoscope.js b/app/controllers/hive/kaleidoscope.js index cbfe9f2..460a561 100644 --- a/app/controllers/hive/kaleidoscope.js +++ b/app/controllers/hive/kaleidoscope.js @@ -4,8 +4,6 @@ 'use strict'; -const DTP_COMPONENT_NAME = 'hive:kaleidoscope'; - const express = require('express'); const { SiteController } = require('../../../lib/site-lib'); @@ -13,7 +11,7 @@ const { SiteController } = require('../../../lib/site-lib'); class HostController extends SiteController { constructor (dtp) { - super(dtp, DTP_COMPONENT_NAME); + super(dtp, module.exports); this.methods = [ { @@ -58,7 +56,7 @@ class HostController extends SiteController { async getKaleidoscopeRoot (req, res) { res.status(200).json({ - component: DTP_COMPONENT_NAME, + component: this.component, version: this.dtp.pkg.version, services: this.services, methods: this.methods, @@ -66,7 +64,11 @@ class HostController extends SiteController { } } -module.exports = async (dtp) => { - let controller = new HostController(dtp); - return controller; -}; +module.exports = { + name: 'H1V3: Kaleidoscope', + slug: 'hive:kaleidoscope', + create: async (dtp) => { + let controller = new HostController(dtp); + return controller; + }, +}; \ No newline at end of file diff --git a/app/controllers/home.js b/app/controllers/home.js index 2e258e0..a7ffa5d 100644 --- a/app/controllers/home.js +++ b/app/controllers/home.js @@ -4,8 +4,6 @@ 'use strict'; -const DTP_COMPONENT_NAME = 'home'; - const path = require('path'); const express = require('express'); @@ -15,7 +13,7 @@ const { SiteController, SiteError } = require('../../lib/site-lib'); class HomeController extends SiteController { constructor (dtp) { - super(dtp, DTP_COMPONENT_NAME); + super(dtp, module.exports); } async start ( ) { diff --git a/app/controllers/image.js b/app/controllers/image.js index bafdb68..aa55882 100644 --- a/app/controllers/image.js +++ b/app/controllers/image.js @@ -4,8 +4,6 @@ 'use strict'; -const DTP_COMPONENT_NAME = 'image'; - const fs = require('fs'); const express = require('express'); @@ -17,7 +15,7 @@ const { SiteController/*, SiteError*/ } = require('../../lib/site-lib'); class ImageController extends SiteController { constructor (dtp) { - super(dtp, DTP_COMPONENT_NAME); + super(dtp, module.exports); } @@ -29,7 +27,7 @@ class ImageController extends SiteController { dtp.app.use('/image', router); const imageUpload = multer({ - dest: `/tmp/${this.dtp.config.site.domainKey}/uploads/${DTP_COMPONENT_NAME}`, + dest: `/tmp/${this.dtp.config.site.domainKey}/uploads/${this.component.slug}`, limits: { fileSize: 1024 * 1000 * 12, }, diff --git a/app/controllers/manifest.js b/app/controllers/manifest.js index c9edf1c..6c3274f 100644 --- a/app/controllers/manifest.js +++ b/app/controllers/manifest.js @@ -4,8 +4,6 @@ 'use strict'; -const DTP_COMPONENT_NAME = 'manifest'; - const express = require('express'); const { SiteController } = require('../../lib/site-lib'); @@ -13,7 +11,7 @@ const { SiteController } = require('../../lib/site-lib'); class ManifestController extends SiteController { constructor (dtp) { - super(dtp, DTP_COMPONENT_NAME); + super(dtp, module.exports); } async start ( ) { @@ -24,7 +22,7 @@ class ManifestController extends SiteController { dtp.app.use('/manifest.json', router); router.use(async (req, res, next) => { - res.locals.currentView = DTP_COMPONENT_NAME; + res.locals.currentView = this.component.slug; return next(); }); diff --git a/app/controllers/user.js b/app/controllers/user.js index 37541bf..79fe6cf 100644 --- a/app/controllers/user.js +++ b/app/controllers/user.js @@ -4,8 +4,6 @@ 'use strict'; -const DTP_COMPONENT_NAME = 'user'; - const express = require('express'); const mongoose = require('mongoose'); const multer = require('multer'); @@ -15,7 +13,7 @@ const { SiteController, SiteError } = require('../../lib/site-lib'); class UserController extends SiteController { constructor (dtp) { - super(dtp, DTP_COMPONENT_NAME); + super(dtp, module.exports); } async start ( ) { @@ -26,7 +24,7 @@ class UserController extends SiteController { session: sessionService, } = dtp.services; - const upload = multer({ dest: `/tmp/${this.dtp.config.site.domainKey}/uploads/${DTP_COMPONENT_NAME}` }); + const upload = multer({ dest: `/tmp/${this.dtp.config.site.domainKey}/uploads/${this.component.slug}` }); const router = express.Router(); dtp.app.use('/user', router); diff --git a/app/controllers/welcome.js b/app/controllers/welcome.js index 09b54d3..bca5265 100644 --- a/app/controllers/welcome.js +++ b/app/controllers/welcome.js @@ -4,8 +4,6 @@ 'use strict'; -const DTP_COMPONENT_NAME = 'welcome'; - const path = require('path'); const express = require('express'); @@ -16,7 +14,7 @@ const { SiteController/*, SiteError */ } = require('../../lib/site-lib'); class WelcomeController extends SiteController { constructor (dtp) { - super(dtp, DTP_COMPONENT_NAME); + super(dtp, module.exports); } async start ( ) { diff --git a/app/models/log.js b/app/models/log.js index 8b25f15..b2462f6 100644 --- a/app/models/log.js +++ b/app/models/log.js @@ -20,7 +20,10 @@ const LOG_LEVEL_LIST = [ const LogSchema = new Schema({ created: { type: Date, default: Date.now, required: true, index: -1, expires: '7d' }, - componentName: { type: String, required: true, index: 1 }, + component: { + name: { type: String, required: true }, + slug: { type: String, required: true, index: 1 }, + }, level: { type: String, enum: LOG_LEVEL_LIST, required: true, index: true }, message: { type: String }, metadata: { type: Schema.Types.Mixed }, diff --git a/app/services/log.js b/app/services/log.js index 039b4d0..661d22f 100644 --- a/app/services/log.js +++ b/app/services/log.js @@ -26,8 +26,8 @@ class SystemLogService extends SiteService { return logs; } - async getComponentNames ( ) { - return await Log.distinct('componentName'); + async getComponentSlugs ( ) { + return await Log.distinct('component.slug'); } async getTotalCount ( ) { diff --git a/app/services/session.js b/app/services/session.js index 33aad3e..9288d7c 100644 --- a/app/services/session.js +++ b/app/services/session.js @@ -28,15 +28,14 @@ class SessionService extends SiteService { middleware ( ) { return async (req, res, next) => { + res.locals.util = util; + res.locals.user = req.user; res.locals.query = req.query; - if (req.user) { - if (req.user.flags.isAdmin) { - res.locals.config = this.dtp.config; - res.locals.session = req.session; - res.locals.util = util; - } + res.locals.config = this.dtp.config; + if (req.session) { + res.locals.session = req.session; } return next(); diff --git a/app/views/admin/log/index.pug b/app/views/admin/log/index.pug index 802b040..9f11170 100644 --- a/app/views/admin/log/index.pug +++ b/app/views/admin/log/index.pug @@ -19,8 +19,8 @@ block content .uk-width-auto select(id="component", name="component").uk-select - each componentName in components - option(value= componentName, selected= (query.component === componentName))= componentName + each componentSlug in components + option(value= componentSlug, selected= (query.component === componentSlug))= componentSlug .uk-width-auto button(type="submit").uk-button.dtp-button-primary Filter @@ -37,7 +37,7 @@ block content tr td= moment(log.created).format('YYYY-MM-DD hh:mm:ss.SSS') td= log.level - td= log.componentName + td= log.component.slug td div= log.message if log.metadata diff --git a/app/views/index.pug b/app/views/index.pug index 07e9ee2..0d48921 100644 --- a/app/views/index.pug +++ b/app/views/index.pug @@ -1,10 +1,22 @@ extends layouts/main-sidebar block content - h1 Sample DTP Web Application p - img(src="/img/the-bobs.jpg", alt="The Bobs have questions").uk-width-large + img(src="/img/the-bobs.jpg", alt="The Bobs have questions").responsive + h1 Sample DTP Web Application p This application doesn't actually do anything. The Bobs would have questions. - pre= JSON.stringify(user, null, 2) \ No newline at end of file + if user + h2 Current User + pre= JSON.stringify(user, null, 2) + + if session + h2 Session + pre= JSON.stringify(session, null, 2) + + h2 Site Configuration + pre= JSON.stringify(config, null, 2) + + h2 Package Information + pre= JSON.stringify(pkg, null, 2) \ No newline at end of file diff --git a/app/views/layouts/main.pug b/app/views/layouts/main.pug index efc7b6b..8629316 100644 --- a/app/views/layouts/main.pug +++ b/app/views/layouts/main.pug @@ -1,5 +1,6 @@ include ../components/library include ../components/page-sidebar +include ../user/components/profile-icon doctype html html(lang='en') head diff --git a/app/views/user/components/profile-icon.pug b/app/views/user/components/profile-icon.pug index f2e05da..9d440e5 100644 --- a/app/views/user/components/profile-icon.pug +++ b/app/views/user/components/profile-icon.pug @@ -1,7 +1,7 @@ -mixin renderProfileIcon (user, title) +mixin renderProfileIcon (user, title, size) if user.core img( - src=`http://${user.core.meta.domain}/core/user/${user.coreUserId}/picture`, + src=`http://${user.core.meta.domain}/core/user/${user.coreUserId}/picture?s=${size || 'small'}`, title= title, ).site-profile-picture.sb-navbar else diff --git a/app/views/user/profile.pug b/app/views/user/profile.pug index 2dbeec0..6a6e9cf 100644 --- a/app/views/user/profile.pug +++ b/app/views/user/profile.pug @@ -5,22 +5,9 @@ block content section.uk-section.uk-section-default .uk-container - h1= user.displayName || user.username || user.email - p People don't have public profiles on #[+renderSiteLink()]. You must be logged in to view your profile. And, only you can view your profile. - - p Your profile is where you edit your account settings, configure your commenting defaults, and otherwise manage how you use #[+renderSiteLink()]. - - section.uk-section.uk-section-default - .uk-container - .uk-margin - +renderSectionTitle('Comment History') - - if Array.isArray(commentHistory) && (commentHistory.length > 0) - ul.uk-list.uk-list-divider - each comment in commentHistory - li - .uk-margin-small - .uk-text-small commenting on #[a(href=`/post/${comment.resource.slug}?comment=${comment._id}#featured-comment`)= comment.resource.title] - +renderComment(comment) - else - div You haven't written any comments on posts. \ No newline at end of file + div(uk-grid) + .uk-width-auto + +renderProfileIcon(user, user.displayName || user.username, 'large') + .uk-width-expand + h1= user.displayName || user.username || user.email + p Welcome to #[+renderSiteLink()]. This app/site really doesn't do anything at all other than provide everything needed for building distributed communities of web applications that all talk to each other and share information in new and easy ways. \ No newline at end of file diff --git a/app/workers/host-services.js b/app/workers/host-services.js index 6326632..fb90a8f 100644 --- a/app/workers/host-services.js +++ b/app/workers/host-services.js @@ -27,13 +27,17 @@ const { CronJob } = require('cron'); const CRON_TIMEZONE = 'America/New_York'; +module.rootPath = path.resolve(__dirname, '..', '..'); module.pkg = require(path.resolve(__dirname, '..', '..', 'package.json')); module.config = { - componentName: 'host-services', - root: path.resolve(__dirname, '..', '..'), + environment: process.env.NODE_ENV, + root: module.rootPath, + component: { name: 'DTP Host Services', slug: 'host-services' }, + site: require(path.join(module.rootPath, 'config', 'site')), + http: require(path.join(module.rootPath, 'config', 'http')), }; -module.log = new SiteLog(module, module.config.componentName); +module.log = new SiteLog(module, module.config.component); class CacheStats { @@ -655,9 +659,9 @@ module.expireNetHosts = async ( ) => { await module.registerHost(); await module.setHostStatus('active'); - module.log.info(`${module.pkg.name} v${module.pkg.version} ${module.config.componentName} started`); + module.log.info(`${module.pkg.name} v${module.pkg.version} ${module.config.component.name} started`); } catch (error) { - module.log.error('failed to start Host Cache worker', { error }); + module.log.error('failed to start worker', { component: module.config.component, error }); process.exit(-1); } diff --git a/app/workers/reeeper.js b/app/workers/reeeper.js index dacf7f9..4dc1ca6 100644 --- a/app/workers/reeeper.js +++ b/app/workers/reeeper.js @@ -19,13 +19,19 @@ const { CronJob } = require('cron'); const CRON_TIMEZONE = 'America/New_York'; +module.rootPath = path.resolve(__dirname, '..', '..'); module.pkg = require(path.resolve(__dirname, '..', '..', 'package.json')); + module.config = { - componentName: 'reeeper', - root: path.resolve(__dirname, '..', '..'), + environment: process.env.NODE_ENV, + root: module.rootPath, + component: { name: 'DTP Reeeper', slug: 'reeeper' }, }; -module.log = new SiteLog(module, module.config.componentName); +module.config.site = require(path.join(module.rootPath, 'config', 'site')); +module.config.http = require(path.join(module.rootPath, 'config', 'http')); + +module.log = new SiteLog(module, module.config.component); module.expireCrashedHosts = async ( ) => { const NetHost = mongoose.model('NetHost'); @@ -81,9 +87,9 @@ module.expireCrashedHosts = async ( ) => { null, true, CRON_TIMEZONE, ); - module.log.info(`${module.pkg.name} v${module.pkg.version} ${module.config.componentName} started`); + module.log.info(`${module.pkg.name} v${module.pkg.version} ${module.config.component.name} started`); } catch (error) { - module.log.error('failed to start Host Cache worker', { error }); + module.log.error('failed to start worker', { component: module.config.component, error }); process.exit(-1); } diff --git a/app/workers/sample-worker.js b/app/workers/sample-worker.js new file mode 100644 index 0000000..7960069 --- /dev/null +++ b/app/workers/sample-worker.js @@ -0,0 +1,88 @@ +// sample-worker.js +// Copyright (C) 2022 DTP Technologies, LLC +// License: Apache-2.0 + +'use strict'; + +const DTP_COMPONENT = { name: 'Sample Worker', slug: 'sample-worker' }; + +const path = require('path'); +require('dotenv').config({ path: path.resolve(__dirname, '..', '..', '.env') }); + +const { + SiteLog, + SiteWorker, +} = require(path.join(__dirname, '..', '..', 'lib', 'site-lib')); + +const { CronJob } = require('cron'); + +module.rootPath = path.resolve(__dirname, '..', '..'); +module.pkg = require(path.resolve(__dirname, '..', '..', 'package.json')); + +module.config = { + environment: process.env.NODE_ENV, + root: module.rootPath, + component: DTP_COMPONENT, +}; + +module.config.site = require(path.join(module.rootPath, 'config', 'site')); +module.config.http = require(path.join(module.rootPath, 'config', 'http')); + +class SampleWorker extends SiteWorker { + + constructor (dtp) { + super(dtp, { }); + } + + async start ( ) { + const CRON_TIMEZONE = 'America/New_York'; + + await super.start(); + + this.log.info('starting worker job'); + this.job = new CronJob( + '*/5 * * * * *', + this.runJob.bind(this), + null, true, CRON_TIMEZONE, + ); + + const { jobQueue: jobQueueService } = this.dtp.services; + this.sampleJobQueue = jobQueueService.getJobQueue('dtp-sample', this.dtp.config.jobQueues['dtp-sample']); + this.sampleJobQueue.process('dtp-sample', 1, this.processDtpSample.bind(this)); + } + + async stop ( ) { + this.log.info('stopping worker job'); + this.job.stop(); + delete this.job; + } + + async runJob ( ) { + this.log.alert('sample job starting'); + + /* + * Your worker will do interesting things here + */ + + this.log.alert('sample job ending'); + } + + async processDtpSample (job) { + this.log.info('received sample job', { id: job.id, data: job.data }); + } +} + +(async ( ) => { + try { + module.log = new SiteLog(module, module.config.component); + module.worker = new SampleWorker(module); + await module.worker.start(); + } catch (error) { + module.log.error('failed to start worker', { + component: module.config.component, + error, + }); + process.exit(-1); + } + +})(); \ No newline at end of file diff --git a/client/js/index-admin.js b/client/js/index-admin.js index 7914cef..6324dd1 100644 --- a/client/js/index-admin.js +++ b/client/js/index-admin.js @@ -4,7 +4,7 @@ 'use strict'; -const DTP_COMPONENT_NAME = 'SiteAdmin'; +const DTP_COMPONENT = { name: 'Site Admin', slug: 'site-admin' }; const dtp = window.dtp = window.dtp || { }; dtp.admin = dtp.admin || { }; @@ -14,7 +14,7 @@ import DtpWebLog from 'dtp/dtp-log.js'; window.addEventListener('load', async ( ) => { // application console log - dtp.admin.log = new DtpWebLog(DTP_COMPONENT_NAME); + dtp.admin.log = new DtpWebLog(DTP_COMPONENT); dtp.adminApp = new DtpSiteAdminApp(dtp.user); diff --git a/client/js/index.js b/client/js/index.js index a053a12..31619cc 100644 --- a/client/js/index.js +++ b/client/js/index.js @@ -4,7 +4,7 @@ 'use strict'; -const DTP_COMPONENT_NAME = 'Site'; +const DTP_COMPONENT = { name: 'Site', slug: 'site' }; const dtp = window. dtp = window.dtp || { }; @@ -14,7 +14,7 @@ import UIkit from 'uikit'; window.addEventListener('load', async ( ) => { // application console log - dtp.log = new DtpWebLog(DTP_COMPONENT_NAME); + dtp.log = new DtpWebLog(DTP_COMPONENT); // service worker if ('serviceWorker' in navigator) { diff --git a/client/js/site-admin-app.js b/client/js/site-admin-app.js index d42c74a..4c0c4a4 100644 --- a/client/js/site-admin-app.js +++ b/client/js/site-admin-app.js @@ -4,7 +4,7 @@ 'use strict'; -const DTP_COMPONENT_NAME = 'SiteAdminApp'; +const DTP_COMPONENT = { name: 'Site Admin', slug: 'site-admin-app' }; const dtp = window.dtp = window.dtp || { }; const GRID_COLOR = '#a0a0a0'; @@ -28,7 +28,7 @@ import UIkit from 'uikit'; export default class DtpSiteAdminHostStatsApp extends DtpApp { constructor (user) { - super(DTP_COMPONENT_NAME, user); + super(DTP_COMPONENT, user); this.log.debug('constructor', 'app instance created'); } diff --git a/client/js/site-app.js b/client/js/site-app.js index 51f3cd4..ef739cc 100644 --- a/client/js/site-app.js +++ b/client/js/site-app.js @@ -4,7 +4,7 @@ 'use strict'; -const DTP_COMPONENT_NAME = 'SiteApp'; +const DTP_COMPONENT = { name: 'Site App', slug: 'site-app' }; const dtp = window.dtp = window.dtp || { }; import DtpApp from 'dtp/dtp-app.js'; @@ -26,7 +26,7 @@ const CHART_FILL_USER = 'rgb(0, 128, 0)'; export default class DtpSiteApp extends DtpApp { constructor (user) { - super(DTP_COMPONENT_NAME, user); + super(DTP_COMPONENT, user); this.log.debug('constructor', 'app instance created'); this.chat = { diff --git a/config/http.js b/config/http.js new file mode 100644 index 0000000..59e82a1 --- /dev/null +++ b/config/http.js @@ -0,0 +1,8 @@ + +'use strict'; + +module.exports = { + scheme: (process.env.NODE_ENV === 'production') ? 'https' : 'http', + address: process.env.HTTP_BIND_ADDRESS, + port: parseInt(process.env.HTTP_BIND_PORT, 10), +}; \ No newline at end of file diff --git a/dtp-webapp-cli.js b/dtp-webapp-cli.js index 8aff7d6..48fc85f 100644 --- a/dtp-webapp-cli.js +++ b/dtp-webapp-cli.js @@ -4,7 +4,7 @@ 'use strict'; -const DTP_COMPONENT_NAME = 'webapp-cli'; +const DTP_COMPONENT = { name: 'Sample App CLI', slug: 'webapp-cli' }; require('dotenv').config(); @@ -22,12 +22,13 @@ const { module.rootPath = __dirname; module.pkg = require(path.join(module.rootPath, 'package.json')); module.config = { - componentName: DTP_COMPONENT_NAME, + component: DTP_COMPONENT, root: module.rootPath, site: require(path.join(module.rootPath, 'config', 'site')), + http: require(path.join(module.rootPath, 'config', 'http')), }; -module.log = new SiteLog(module, module.config.componentName); +module.log = new SiteLog(module, module.config.component); module.grantPermission = async (target, permission) => { const User = mongoose.model('User'); @@ -220,7 +221,7 @@ module.requestCoreConnect = async (host) => { try { await SitePlatform.startPlatform(module); } catch (error) { - module.log.error(`failed to start DTP ${DTP_COMPONENT_NAME} platform`, { error }); + module.log.error(`failed to start DTP ${module.config.component.slug} platform`, { error }); return; } diff --git a/dtp-webapp.js b/dtp-webapp.js index 4597acc..fca0d9d 100644 --- a/dtp-webapp.js +++ b/dtp-webapp.js @@ -4,7 +4,7 @@ 'use strict'; -const DTP_COMPONENT_NAME = 'webapp'; +const DTP_COMPONENT = { name: 'Sample App', slug: 'webapp' }; require('dotenv').config(); @@ -15,16 +15,14 @@ const { SitePlatform, SiteLog } = require(path.join(__dirname, 'lib', 'site-lib' module.rootPath = __dirname; module.pkg = require(path.join(module.rootPath, 'package.json')); module.config = { - componentName: DTP_COMPONENT_NAME, + environment: process.env.NODE_ENV, root: module.rootPath, + component: DTP_COMPONENT, site: require(path.join(module.rootPath, 'config', 'site')), - http: { - address: process.env.HTTP_BIND_ADDRESS, - port: parseInt(process.env.HTTP_BIND_PORT, 10), - }, + http: require(path.join(module.rootPath, 'config', 'http')), }; -module.log = new SiteLog(module, module.config.componentName); +module.log = new SiteLog(module, module.config.component); module.shutdown = async ( ) => { @@ -63,7 +61,7 @@ module.shutdown = async ( ) => { await SitePlatform.startPlatform(module); await SitePlatform.startWebServer(module); } catch (error) { - module.log.error(`failed to start DTP ${DTP_COMPONENT_NAME}`, { error }); + module.log.error(`failed to start DTP ${module.config.component.name}`, { error }); process.exit(-1); } diff --git a/lib/client/js/dtp-log.js b/lib/client/js/dtp-log.js index acea612..6ee4c4f 100644 --- a/lib/client/js/dtp-log.js +++ b/lib/client/js/dtp-log.js @@ -8,8 +8,8 @@ window.dtp = window.dtp || { }; export default class DtpWebLog { - constructor (componentName, options) { - this.componentName = componentName; + constructor (component, options) { + this.component = component; this.options = Object.assign({ color: { debug: '#808080', @@ -113,13 +113,13 @@ export default class DtpWebLog { if (!this.enabled) { return; } if (data) { console[method]('%c%s%c: %s', - css.label, `${this.componentName}.${event}`, + css.label, `${this.component.slug}.${event}`, css.message, msg, data, ); } else { console[method]('%c%s%c: %s', - css.label, `${this.componentName}.${event}`, + css.label, `${this.component.slug}.${event}`, css.message, msg, ); } diff --git a/lib/client/js/dtp-plugin.js b/lib/client/js/dtp-plugin.js index c395411..d8edbd3 100644 --- a/lib/client/js/dtp-plugin.js +++ b/lib/client/js/dtp-plugin.js @@ -24,19 +24,17 @@ 'use strict'; -const DTP_COMPONENT_NAME = 'dtp:plugin'; - window.dtp = window.dtp || { }; import DtpLog from 'dtp/dtp-log.js'; export default class DtpPlugin { - constructor (app, context, componentName) { + constructor (app, context, component) { this.app = app; this.context = context; - this.componentName = componentName; - this.log = new DtpLog(`${DTP_COMPONENT_NAME}:${componentName}`); + this.component = component; + this.log = new DtpLog(component); } } diff --git a/lib/site-common.js b/lib/site-common.js index 8638739..10cc8c6 100644 --- a/lib/site-common.js +++ b/lib/site-common.js @@ -7,12 +7,18 @@ const path = require('path'); const pug = require('pug'); +const { SiteLog } = require(path.join(__dirname, 'site-log')); + const Events = require('events'); class SiteCommon extends Events { - constructor (dtp) { + constructor (dtp, component) { super(); + this.dtp = dtp; + this.component = component; + + this.log = new SiteLog(dtp, component); this.appTemplateRoot = path.join(this.dtp.config.root, 'app', 'templates'); } diff --git a/lib/site-controller.js b/lib/site-controller.js index 1b0b725..37e487e 100644 --- a/lib/site-controller.js +++ b/lib/site-controller.js @@ -7,18 +7,16 @@ const path = require('path'); const { SiteCommon } = require(path.join(__dirname, 'site-common')); -const { SiteLog } = require(path.join(__dirname, 'site-log')); class SiteController extends SiteCommon { - constructor (dtp, name) { - super(dtp); - this.name = name; - this.log = new SiteLog(dtp, `ctrl:${name}`); + constructor (dtp, component) { + super(dtp, component); } async loadChild (filename) { - let child = await require(filename)(this.dtp); + let component = require(filename); + let child = await component.create(this.dtp); return await child.start(); } diff --git a/lib/site-ioserver.js b/lib/site-ioserver.js index 6cfeb66..5552f94 100644 --- a/lib/site-ioserver.js +++ b/lib/site-ioserver.js @@ -4,6 +4,7 @@ 'use strict'; +const DTP_COMPONENT = { name: 'I/O Server', slug: 'ioserver', prefix: 'srv' }; const path = require('path'); const Redis = require('ioredis'); @@ -23,7 +24,7 @@ class SiteIoServer extends Events { constructor (dtp) { super(); this.dtp = dtp; - this.log = new SiteLog(dtp, 'ioserver'); + this.log = new SiteLog(dtp, DTP_COMPONENT); } async start (httpServer) { diff --git a/lib/site-lib.js b/lib/site-lib.js index 882b531..14cf415 100644 --- a/lib/site-lib.js +++ b/lib/site-lib.js @@ -14,4 +14,5 @@ module.exports = { SiteLog: require(path.join(__dirname, 'site-log')).SiteLog, SiteController: require(path.join(__dirname, 'site-controller')).SiteController, SiteService: require(path.join(__dirname, 'site-service')).SiteService, + SiteWorker: require(path.join(__dirname, 'site-worker')).SiteWorker, }; \ No newline at end of file diff --git a/lib/site-log.js b/lib/site-log.js index f894db0..52d9c90 100644 --- a/lib/site-log.js +++ b/lib/site-log.js @@ -31,9 +31,16 @@ class SiteLog { */ static setModel (model) { LogModel = model; } - constructor (dtp, componentName) { + constructor (dtp, component) { + if (!dtp) { + throw new Error('Must specify DTP module'); + } this.dtp = dtp; - this.componentName = componentName; + + if (!component || !component.slug || !component.name) { + throw new Error('Must specify DTP component'); + } + this.component = component; } async debug (message, metadata) { @@ -75,7 +82,6 @@ class SiteLog { async writeLog (level, message, metadata) { const NOW = new Date(); - const { componentName } = this; if (process.env.DTP_LOG_CONSOLE === 'enabled') { let clevel = level.padEnd(5); @@ -104,22 +110,22 @@ class SiteLog { } const ctimestamp = color.black(moment(NOW).format('YYYY-MM-DD HH:mm:ss.SSS')); - const ccomponentName = color.cyan(componentName); + const ccomponentSlug = color.cyan(this.component.slug); const cmessage = color.darkGray(message); if (metadata) { - console.log(`${ctimestamp} ${clevel} ${ccomponentName} ${cmessage}`, util.inspect(metadata, false, Infinity, true)); + console.log(`${ctimestamp} ${clevel} ${ccomponentSlug} ${cmessage}`, util.inspect(metadata, false, Infinity, true)); } else { - console.log(`${ctimestamp} ${clevel} ${ccomponentName} ${cmessage}`); + console.log(`${ctimestamp} ${clevel} ${ccomponentSlug} ${cmessage}`); } } if (LogModel && (process.env.DTP_LOG_MONGODB === 'enabled')) { - await LogModel.create({ created: NOW, level, componentName, message, metadata }); + await LogModel.create({ created: NOW, level, component: this.component, message, metadata }); } if (LogStream && (process.env.DTP_LOG_FILE === 'enabled')) { const logEntry = { - t: NOW, c: componentName, l: level, m: message, d: metadata, + t: NOW, c: this.component.slug, l: level, m: message, d: metadata, }; LogStream.write(`${JSON.stringify(logEntry)}\n`); } diff --git a/lib/site-platform.js b/lib/site-platform.js index f122b0f..22a8a41 100644 --- a/lib/site-platform.js +++ b/lib/site-platform.js @@ -182,7 +182,7 @@ module.loadControllers = async (dtp) => { module.exports.startPlatform = async (dtp) => { try { - module.log = new SiteLog(module, dtp.config.componentName); + module.log = new SiteLog(module, dtp.config.component); dtp.config.jobQueues = require(path.join(dtp.config.root, 'config', 'job-queues')); await module.connectDatabase(dtp); @@ -192,8 +192,10 @@ module.exports.startPlatform = async (dtp) => { SiteLog.setModel(mongoose.model('Log')); await module.loadServices(dtp); + + module.log.info(`Digital Telepresence Platform v${dtp.pkg.version} started for [${dtp.pkg.name}]`); } catch (error) { - module.log.error('failed to initialize the Digital Telepresence Platform', { error }); + module.log.error('platform failed to start', { error }); return; } }; @@ -372,7 +374,7 @@ module.exports.startWebServer = async (dtp) => { bind: dtp.config.http.address, }); module.http.listen(dtp.config.http.port, dtp.config.http.address, ( ) => { - module.log.info(`${dtp.config.componentName} platform online`, { port: dtp.config.http.port }); + module.log.info(`${dtp.config.component.name} platform online`, { port: dtp.config.http.port }); }); }; diff --git a/lib/site-service.js b/lib/site-service.js index 542089c..161cba6 100644 --- a/lib/site-service.js +++ b/lib/site-service.js @@ -7,15 +7,11 @@ const path = require('path'); const { SiteCommon } = require(path.join(__dirname, 'site-common')); -const { SiteLog } = require(path.join(__dirname, 'site-log')); class SiteService extends SiteCommon { - constructor (dtp, definition) { - super(dtp); - this.slug = definition.slug; - this.name = definition.name; - this.log = new SiteLog(dtp, `svc:${this.slug}`); + constructor (dtp, component) { + super(dtp, component); } async start ( ) { diff --git a/lib/site-worker.js b/lib/site-worker.js new file mode 100644 index 0000000..2b9bd00 --- /dev/null +++ b/lib/site-worker.js @@ -0,0 +1,62 @@ +// site-service.js +// Copyright (C) 2022 DTP Technologies, LLC +// License: Apache-2.0 + +'use strict'; + +const path = require('path'); + +const SitePlatform = require(path.join(__dirname, 'site-platform')); +const { SiteCommon } = require(path.join(__dirname, 'site-common')); + +class SiteWorker extends SiteCommon { + + constructor (dtp, component) { + super(dtp, component); + } + + async start ( ) { + try { + process.on('unhandledRejection', (error, p) => { + this.log.error('Unhandled rejection', { + error: error, + promise: p, + stack: error.stack + }); + }); + + process.on('warning', (error) => { + this.log.alert('warning', { error }); + }); + + process.once('SIGINT', async ( ) => { + this.log.info('SIGINT received'); + this.log.info('requesting shutdown...'); + + await this.stop(); + + const exitCode = await SitePlatform.shutdown(); + process.nextTick(( ) => { + process.exit(exitCode); + }); + }); + + /* + * Site Platform startup + */ + await SitePlatform.startPlatform(this.dtp); + } catch (error) { + this.log.error('failed to start worker', { + component: this.dtp.config.component, + error, + }); + process.exit(-1); + } + } + + async stop ( ) { + + } +} + +module.exports.SiteWorker = SiteWorker; \ No newline at end of file diff --git a/update-deps.js b/update-deps.js index b7e4e4f..a81ac2e 100644 --- a/update-deps.js +++ b/update-deps.js @@ -4,20 +4,22 @@ 'use strict'; -const DTP_COMPONENT_NAME = 'webapp'; - -require('dotenv').config(); - const path = require('path'); +require('dotenv').config(); const { spawn } = require('child_process'); -const { SiteAsync, SiteLog } = require(path.join(__dirname, 'lib', 'site-lib')); +const { SiteLog } = require(path.join(__dirname, 'lib', 'site-lib')); module.rootPath = __dirname; module.pkg = require(path.join(module.rootPath, 'package.json')); module.pinnedPackages = require(path.join(module.rootPath, 'config', 'pinned-packages')); -module.log = new SiteLog(module, DTP_COMPONENT_NAME); + +module.config = { + component: { name: 'Webapp Dependency Updater', slug: 'webapp-update-deps' }, +}; + +module.log = new SiteLog(module, module.config.component); module.runCommand = async (command, args) => { return new Promise((resolve, reject) => {