Browse Source

large component and worker management and logging refactor

develop
Rob Colbert 2 years ago
parent
commit
2dcd95c74b
  1. 12
      .vscode/launch.json
  2. 4
      app/controllers/admin.js
  3. 16
      app/controllers/admin/content-report.js
  4. 14
      app/controllers/admin/core-node.js
  5. 16
      app/controllers/admin/host.js
  6. 13
      app/controllers/admin/job-queue.js
  7. 17
      app/controllers/admin/log.js
  8. 13
      app/controllers/admin/settings.js
  9. 18
      app/controllers/admin/user.js
  10. 6
      app/controllers/auth.js
  11. 4
      app/controllers/email.js
  12. 6
      app/controllers/hive.js
  13. 18
      app/controllers/hive/kaleidoscope.js
  14. 4
      app/controllers/home.js
  15. 6
      app/controllers/image.js
  16. 6
      app/controllers/manifest.js
  17. 6
      app/controllers/user.js
  18. 4
      app/controllers/welcome.js
  19. 5
      app/models/log.js
  20. 4
      app/services/log.js
  21. 11
      app/services/session.js
  22. 6
      app/views/admin/log/index.pug
  23. 18
      app/views/index.pug
  24. 1
      app/views/layouts/main.pug
  25. 4
      app/views/user/components/profile-icon.pug
  26. 25
      app/views/user/profile.pug
  27. 14
      app/workers/host-services.js
  28. 16
      app/workers/reeeper.js
  29. 88
      app/workers/sample-worker.js
  30. 4
      client/js/index-admin.js
  31. 4
      client/js/index.js
  32. 4
      client/js/site-admin-app.js
  33. 4
      client/js/site-app.js
  34. 8
      config/http.js
  35. 9
      dtp-webapp-cli.js
  36. 14
      dtp-webapp.js
  37. 8
      lib/client/js/dtp-log.js
  38. 8
      lib/client/js/dtp-plugin.js
  39. 8
      lib/site-common.js
  40. 10
      lib/site-controller.js
  41. 3
      lib/site-ioserver.js
  42. 1
      lib/site-lib.js
  43. 22
      lib/site-log.js
  44. 8
      lib/site-platform.js
  45. 8
      lib/site-service.js
  46. 62
      lib/site-worker.js
  47. 14
      update-deps.js

12
.vscode/launch.json

@ -27,6 +27,16 @@
"program": "${workspaceFolder:dtp-base}/dtp-webapp-cli.js", "program": "${workspaceFolder:dtp-base}/dtp-webapp-cli.js",
"console": "integratedTerminal", "console": "integratedTerminal",
"args": ["--action=reset-indexes", "all"] "args": ["--action=reset-indexes", "all"]
} },
{
"type": "pwa-node",
"request": "launch",
"name": "worker:sample",
"skipFiles": [
"<node_internals>/**"
],
"program": "${workspaceFolder:dtp-base}/app/workers/sample-worker.js",
"console": "integratedTerminal",
},
] ]
} }

4
app/controllers/admin.js

@ -4,8 +4,6 @@
'use strict'; 'use strict';
const DTP_COMPONENT_NAME = 'admin';
const path = require('path'); const path = require('path');
const express = require('express'); const express = require('express');
@ -17,7 +15,7 @@ const { SiteError, SiteController } = require('../../lib/site-lib');
class AdminController extends SiteController { class AdminController extends SiteController {
constructor (dtp) { constructor (dtp) {
super(dtp, DTP_COMPONENT_NAME); super(dtp, module.exports);
} }
async start ( ) { async start ( ) {

16
app/controllers/admin/content-report.js

@ -4,8 +4,6 @@
'use strict'; 'use strict';
const DTP_COMPONENT_NAME = 'admin:content-report';
const express = require('express'); const express = require('express');
const multer = require('multer'); const multer = require('multer');
@ -14,7 +12,7 @@ const { /*SiteError,*/ SiteController } = require('../../../lib/site-lib');
class ContentReportController extends SiteController { class ContentReportController extends SiteController {
constructor (dtp) { constructor (dtp) {
super(dtp, DTP_COMPONENT_NAME); super(dtp, module.exports);
} }
async start ( ) { async start ( ) {
@ -89,7 +87,11 @@ class ContentReportController extends SiteController {
} }
} }
module.exports = async (dtp) => { module.exports = {
let controller = new ContentReportController(dtp); name: 'Admin: Content Reports',
return controller; slug: 'admin:content-report',
}; create: async (dtp) => {
let controller = new ContentReportController(dtp);
return controller;
},
};

14
app/controllers/admin/core-node.js

@ -4,8 +4,6 @@
'use strict'; 'use strict';
const DTP_COMPONENT_NAME = 'admin:core-node';
const express = require('express'); const express = require('express');
// const multer = require('multer'); // const multer = require('multer');
@ -14,7 +12,7 @@ const { /*SiteError,*/ SiteController } = require('../../../lib/site-lib');
class CoreNodeController extends SiteController { class CoreNodeController extends SiteController {
constructor (dtp) { constructor (dtp) {
super(dtp, DTP_COMPONENT_NAME); super(dtp, module.exports);
} }
async start ( ) { async start ( ) {
@ -99,7 +97,11 @@ class CoreNodeController extends SiteController {
} }
} }
module.exports = async (dtp) => { module.exports = {
let controller = new CoreNodeController(dtp); name: 'Admin: Core Node',
return controller; slug: 'admin:core-node',
create: async (dtp) => {
let controller = new CoreNodeController(dtp);
return controller;
},
}; };

16
app/controllers/admin/host.js

@ -4,8 +4,6 @@
'use strict'; 'use strict';
const DTP_COMPONENT_NAME = 'admin:host';
const express = require('express'); const express = require('express');
const mongoose = require('mongoose'); const mongoose = require('mongoose');
@ -17,7 +15,7 @@ const { /*SiteError,*/ SiteController } = require('../../../lib/site-lib');
class HostController extends SiteController { class HostController extends SiteController {
constructor (dtp) { constructor (dtp) {
super(dtp, DTP_COMPONENT_NAME); super(dtp, module.exports);
} }
async start ( ) { async start ( ) {
@ -117,7 +115,11 @@ class HostController extends SiteController {
} }
} }
module.exports = async (dtp) => { module.exports = {
let controller = new HostController(dtp); name: 'Admin: Network Host',
return controller; slug: 'admin:host',
}; create: async (dtp) => {
let controller = new HostController(dtp);
return controller;
},
};

13
app/controllers/admin/job-queue.js

@ -4,7 +4,6 @@
'use strict'; 'use strict';
const DTP_COMPONENT_NAME = 'admin:job-queue';
const express = require('express'); const express = require('express');
const { SiteController, SiteError } = require('../../../lib/site-lib'); const { SiteController, SiteError } = require('../../../lib/site-lib');
@ -12,7 +11,7 @@ const { SiteController, SiteError } = require('../../../lib/site-lib');
class JobQueueController extends SiteController { class JobQueueController extends SiteController {
constructor (dtp) { constructor (dtp) {
super(dtp, DTP_COMPONENT_NAME); super(dtp, module.exports);
} }
async start ( ) { async start ( ) {
@ -119,7 +118,11 @@ class JobQueueController extends SiteController {
} }
} }
module.exports = async (dtp) => { module.exports = {
let controller = new JobQueueController(dtp); name: 'Admin: Job Queues',
return controller; slug: 'admin:job-queue',
create: async (dtp) => {
let controller = new JobQueueController(dtp);
return controller;
},
}; };

17
app/controllers/admin/log.js

@ -4,7 +4,6 @@
'use strict'; 'use strict';
const DTP_COMPONENT_NAME = 'admin:log';
const express = require('express'); const express = require('express');
const { SiteController } = require('../../../lib/site-lib'); const { SiteController } = require('../../../lib/site-lib');
@ -12,7 +11,7 @@ const { SiteController } = require('../../../lib/site-lib');
class LogController extends SiteController { class LogController extends SiteController {
constructor (dtp) { constructor (dtp) {
super(dtp, DTP_COMPONENT_NAME); super(dtp, module.exports);
} }
async start ( ) { async start ( ) {
@ -33,12 +32,12 @@ class LogController extends SiteController {
try { try {
res.locals.query = req.query; res.locals.query = req.query;
res.locals.components = await logService.getComponentNames(); res.locals.components = await logService.getComponentSlugs();
res.locals.pagination = this.getPaginationParameters(req, 25); res.locals.pagination = this.getPaginationParameters(req, 25);
const search = { }; const search = { };
if (req.query.component) { if (req.query.component) {
search.componentName = req.query.component; search.component = { slug: req.query.component };
} }
res.locals.logs = await logService.getRecords(search, res.locals.pagination); res.locals.logs = await logService.getRecords(search, res.locals.pagination);
@ -51,7 +50,11 @@ class LogController extends SiteController {
} }
} }
module.exports = async (dtp) => { module.exports = {
let controller = new LogController(dtp); name: 'Admin: Log',
return controller; slug: 'admin:log',
create: async (dtp) => {
let controller = new LogController(dtp);
return controller;
},
}; };

13
app/controllers/admin/settings.js

@ -4,7 +4,6 @@
'use strict'; 'use strict';
const DTP_COMPONENT_NAME = 'admin:settings';
const express = require('express'); const express = require('express');
const { SiteController } = require('../../../lib/site-lib'); const { SiteController } = require('../../../lib/site-lib');
@ -12,7 +11,7 @@ const { SiteController } = require('../../../lib/site-lib');
class SettingsController extends SiteController { class SettingsController extends SiteController {
constructor (dtp) { constructor (dtp) {
super(dtp, DTP_COMPONENT_NAME); super(dtp, module.exports);
} }
async start ( ) { async start ( ) {
@ -50,7 +49,11 @@ class SettingsController extends SiteController {
} }
} }
module.exports = async (dtp) => { module.exports = {
let controller = new SettingsController(dtp); name: 'Admin: Settings',
return controller; slug: 'admin:settings',
create: async (dtp) => {
let controller = new SettingsController(dtp);
return controller;
},
}; };

18
app/controllers/admin/user.js

@ -4,16 +4,14 @@
'use strict'; 'use strict';
const DTP_COMPONENT_NAME = 'admin:user';
const express = require('express'); const express = require('express');
const { /*SiteError,*/ SiteController } = require('../../../lib/site-lib'); const { SiteController } = require('../../../lib/site-lib');
class UserController extends SiteController { class UserController extends SiteController {
constructor (dtp) { constructor (dtp) {
super(dtp, DTP_COMPONENT_NAME); super(dtp, module.exports);
} }
async start ( ) { async start ( ) {
@ -79,7 +77,11 @@ class UserController extends SiteController {
} }
} }
module.exports = async (dtp) => { module.exports = {
let controller = new UserController(dtp); name: 'Admin: User Management',
return controller; slug: 'admin:user',
}; create: async (dtp) => {
let controller = new UserController(dtp);
return controller;
},
};

6
app/controllers/auth.js

@ -4,8 +4,6 @@
'use strict'; 'use strict';
const DTP_COMPONENT_NAME = 'auth';
const express = require('express'); const express = require('express');
const mongoose = require('mongoose'); const mongoose = require('mongoose');
const multer = require('multer'); const multer = require('multer');
@ -19,7 +17,7 @@ const ConnectToken = mongoose.model('ConnectToken');
class AuthController extends SiteController { class AuthController extends SiteController {
constructor (dtp) { constructor (dtp) {
super(dtp, DTP_COMPONENT_NAME); super(dtp, module.exports);
} }
async start ( ) { async start ( ) {
@ -27,7 +25,7 @@ class AuthController extends SiteController {
coreNode: coreNodeService, coreNode: coreNodeService,
limiter: limiterService, limiter: limiterService,
} = this.dtp.services; } = this.dtp.services;
const upload = multer({ dest: `/tmp/${this.dtp.config.site.domainKey}/uploads/${DTP_COMPONENT_NAME}` }); const upload = multer({ dest: `/tmp/${this.dtp.config.site.domainKey}/uploads/${this.component.slug}` });
const router = express.Router(); const router = express.Router();
this.dtp.app.use('/auth', router); this.dtp.app.use('/auth', router);

4
app/controllers/email.js

@ -4,8 +4,6 @@
'use strict'; 'use strict';
const DTP_COMPONENT_NAME = 'email';
const express = require('express'); const express = require('express');
const { SiteController/*, SiteError*/ } = require('../../lib/site-lib'); const { SiteController/*, SiteError*/ } = require('../../lib/site-lib');
@ -13,7 +11,7 @@ const { SiteController/*, SiteError*/ } = require('../../lib/site-lib');
class EmailController extends SiteController { class EmailController extends SiteController {
constructor (dtp) { constructor (dtp) {
super(dtp, DTP_COMPONENT_NAME); super(dtp, module.exports);
} }
async start ( ) { async start ( ) {

6
app/controllers/hive.js

@ -4,8 +4,6 @@
'use strict'; 'use strict';
const DTP_COMPONENT_NAME = 'hive';
const path = require('path'); const path = require('path');
const express = require('express'); const express = require('express');
@ -14,7 +12,7 @@ const { SiteController } = require('../../lib/site-lib');
class HiveController extends SiteController { class HiveController extends SiteController {
constructor (dtp) { constructor (dtp) {
super(dtp, DTP_COMPONENT_NAME); super(dtp, module.exports);
this.services = [ ]; this.services = [ ];
} }
@ -45,7 +43,7 @@ class HiveController extends SiteController {
async getHiveRoot (req, res) { async getHiveRoot (req, res) {
res.status(200).json({ res.status(200).json({
component: DTP_COMPONENT_NAME, component: this.component,
host: this.dtp.pkg.name, host: this.dtp.pkg.name,
description: this.dtp.pkg.description, description: this.dtp.pkg.description,
version: this.dtp.pkg.version, version: this.dtp.pkg.version,

18
app/controllers/hive/kaleidoscope.js

@ -4,8 +4,6 @@
'use strict'; 'use strict';
const DTP_COMPONENT_NAME = 'hive:kaleidoscope';
const express = require('express'); const express = require('express');
const { SiteController } = require('../../../lib/site-lib'); const { SiteController } = require('../../../lib/site-lib');
@ -13,7 +11,7 @@ const { SiteController } = require('../../../lib/site-lib');
class HostController extends SiteController { class HostController extends SiteController {
constructor (dtp) { constructor (dtp) {
super(dtp, DTP_COMPONENT_NAME); super(dtp, module.exports);
this.methods = [ this.methods = [
{ {
@ -58,7 +56,7 @@ class HostController extends SiteController {
async getKaleidoscopeRoot (req, res) { async getKaleidoscopeRoot (req, res) {
res.status(200).json({ res.status(200).json({
component: DTP_COMPONENT_NAME, component: this.component,
version: this.dtp.pkg.version, version: this.dtp.pkg.version,
services: this.services, services: this.services,
methods: this.methods, methods: this.methods,
@ -66,7 +64,11 @@ class HostController extends SiteController {
} }
} }
module.exports = async (dtp) => { module.exports = {
let controller = new HostController(dtp); name: 'H1V3: Kaleidoscope',
return controller; slug: 'hive:kaleidoscope',
}; create: async (dtp) => {
let controller = new HostController(dtp);
return controller;
},
};

4
app/controllers/home.js

@ -4,8 +4,6 @@
'use strict'; 'use strict';
const DTP_COMPONENT_NAME = 'home';
const path = require('path'); const path = require('path');
const express = require('express'); const express = require('express');
@ -15,7 +13,7 @@ const { SiteController, SiteError } = require('../../lib/site-lib');
class HomeController extends SiteController { class HomeController extends SiteController {
constructor (dtp) { constructor (dtp) {
super(dtp, DTP_COMPONENT_NAME); super(dtp, module.exports);
} }
async start ( ) { async start ( ) {

6
app/controllers/image.js

@ -4,8 +4,6 @@
'use strict'; 'use strict';
const DTP_COMPONENT_NAME = 'image';
const fs = require('fs'); const fs = require('fs');
const express = require('express'); const express = require('express');
@ -17,7 +15,7 @@ const { SiteController/*, SiteError*/ } = require('../../lib/site-lib');
class ImageController extends SiteController { class ImageController extends SiteController {
constructor (dtp) { constructor (dtp) {
super(dtp, DTP_COMPONENT_NAME); super(dtp, module.exports);
} }
@ -29,7 +27,7 @@ class ImageController extends SiteController {
dtp.app.use('/image', router); dtp.app.use('/image', router);
const imageUpload = multer({ const imageUpload = multer({
dest: `/tmp/${this.dtp.config.site.domainKey}/uploads/${DTP_COMPONENT_NAME}`, dest: `/tmp/${this.dtp.config.site.domainKey}/uploads/${this.component.slug}`,
limits: { limits: {
fileSize: 1024 * 1000 * 12, fileSize: 1024 * 1000 * 12,
}, },

6
app/controllers/manifest.js

@ -4,8 +4,6 @@
'use strict'; 'use strict';
const DTP_COMPONENT_NAME = 'manifest';
const express = require('express'); const express = require('express');
const { SiteController } = require('../../lib/site-lib'); const { SiteController } = require('../../lib/site-lib');
@ -13,7 +11,7 @@ const { SiteController } = require('../../lib/site-lib');
class ManifestController extends SiteController { class ManifestController extends SiteController {
constructor (dtp) { constructor (dtp) {
super(dtp, DTP_COMPONENT_NAME); super(dtp, module.exports);
} }
async start ( ) { async start ( ) {
@ -24,7 +22,7 @@ class ManifestController extends SiteController {
dtp.app.use('/manifest.json', router); dtp.app.use('/manifest.json', router);
router.use(async (req, res, next) => { router.use(async (req, res, next) => {
res.locals.currentView = DTP_COMPONENT_NAME; res.locals.currentView = this.component.slug;
return next(); return next();
}); });

6
app/controllers/user.js

@ -4,8 +4,6 @@
'use strict'; 'use strict';
const DTP_COMPONENT_NAME = 'user';
const express = require('express'); const express = require('express');
const mongoose = require('mongoose'); const mongoose = require('mongoose');
const multer = require('multer'); const multer = require('multer');
@ -15,7 +13,7 @@ const { SiteController, SiteError } = require('../../lib/site-lib');
class UserController extends SiteController { class UserController extends SiteController {
constructor (dtp) { constructor (dtp) {
super(dtp, DTP_COMPONENT_NAME); super(dtp, module.exports);
} }
async start ( ) { async start ( ) {
@ -26,7 +24,7 @@ class UserController extends SiteController {
session: sessionService, session: sessionService,
} = dtp.services; } = dtp.services;
const upload = multer({ dest: `/tmp/${this.dtp.config.site.domainKey}/uploads/${DTP_COMPONENT_NAME}` }); const upload = multer({ dest: `/tmp/${this.dtp.config.site.domainKey}/uploads/${this.component.slug}` });
const router = express.Router(); const router = express.Router();
dtp.app.use('/user', router); dtp.app.use('/user', router);

4
app/controllers/welcome.js

@ -4,8 +4,6 @@
'use strict'; 'use strict';
const DTP_COMPONENT_NAME = 'welcome';
const path = require('path'); const path = require('path');
const express = require('express'); const express = require('express');
@ -16,7 +14,7 @@ const { SiteController/*, SiteError */ } = require('../../lib/site-lib');
class WelcomeController extends SiteController { class WelcomeController extends SiteController {
constructor (dtp) { constructor (dtp) {
super(dtp, DTP_COMPONENT_NAME); super(dtp, module.exports);
} }
async start ( ) { async start ( ) {

5
app/models/log.js

@ -20,7 +20,10 @@ const LOG_LEVEL_LIST = [
const LogSchema = new Schema({ const LogSchema = new Schema({
created: { type: Date, default: Date.now, required: true, index: -1, expires: '7d' }, created: { type: Date, default: Date.now, required: true, index: -1, expires: '7d' },
componentName: { type: String, required: true, index: 1 }, component: {
name: { type: String, required: true },
slug: { type: String, required: true, index: 1 },
},
level: { type: String, enum: LOG_LEVEL_LIST, required: true, index: true }, level: { type: String, enum: LOG_LEVEL_LIST, required: true, index: true },
message: { type: String }, message: { type: String },
metadata: { type: Schema.Types.Mixed }, metadata: { type: Schema.Types.Mixed },

4
app/services/log.js

@ -26,8 +26,8 @@ class SystemLogService extends SiteService {
return logs; return logs;
} }
async getComponentNames ( ) { async getComponentSlugs ( ) {
return await Log.distinct('componentName'); return await Log.distinct('component.slug');
} }
async getTotalCount ( ) { async getTotalCount ( ) {

11
app/services/session.js

@ -28,15 +28,14 @@ class SessionService extends SiteService {
middleware ( ) { middleware ( ) {
return async (req, res, next) => { return async (req, res, next) => {
res.locals.util = util;
res.locals.user = req.user; res.locals.user = req.user;
res.locals.query = req.query; res.locals.query = req.query;
if (req.user) { res.locals.config = this.dtp.config;
if (req.user.flags.isAdmin) { if (req.session) {
res.locals.config = this.dtp.config; res.locals.session = req.session;
res.locals.session = req.session;
res.locals.util = util;
}
} }
return next(); return next();

6
app/views/admin/log/index.pug

@ -19,8 +19,8 @@ block content
.uk-width-auto .uk-width-auto
select(id="component", name="component").uk-select select(id="component", name="component").uk-select
each componentName in components each componentSlug in components
option(value= componentName, selected= (query.component === componentName))= componentName option(value= componentSlug, selected= (query.component === componentSlug))= componentSlug
.uk-width-auto .uk-width-auto
button(type="submit").uk-button.dtp-button-primary Filter button(type="submit").uk-button.dtp-button-primary Filter
@ -37,7 +37,7 @@ block content
tr tr
td= moment(log.created).format('YYYY-MM-DD hh:mm:ss.SSS') td= moment(log.created).format('YYYY-MM-DD hh:mm:ss.SSS')
td= log.level td= log.level
td= log.componentName td= log.component.slug
td td
div= log.message div= log.message
if log.metadata if log.metadata

18
app/views/index.pug

@ -1,10 +1,22 @@
extends layouts/main-sidebar extends layouts/main-sidebar
block content block content
h1 Sample DTP Web Application
p p
img(src="/img/the-bobs.jpg", alt="The Bobs have questions").uk-width-large img(src="/img/the-bobs.jpg", alt="The Bobs have questions").responsive
h1 Sample DTP Web Application
p This application doesn't actually do anything. The Bobs would have questions. p This application doesn't actually do anything. The Bobs would have questions.
pre= JSON.stringify(user, null, 2) if user
h2 Current User
pre= JSON.stringify(user, null, 2)
if session
h2 Session
pre= JSON.stringify(session, null, 2)
h2 Site Configuration
pre= JSON.stringify(config, null, 2)
h2 Package Information
pre= JSON.stringify(pkg, null, 2)

1
app/views/layouts/main.pug

@ -1,5 +1,6 @@
include ../components/library include ../components/library
include ../components/page-sidebar include ../components/page-sidebar
include ../user/components/profile-icon
doctype html doctype html
html(lang='en') html(lang='en')
head head

4
app/views/user/components/profile-icon.pug

@ -1,7 +1,7 @@
mixin renderProfileIcon (user, title) mixin renderProfileIcon (user, title, size)
if user.core if user.core
img( img(
src=`http://${user.core.meta.domain}/core/user/${user.coreUserId}/picture`, src=`http://${user.core.meta.domain}/core/user/${user.coreUserId}/picture?s=${size || 'small'}`,
title= title, title= title,
).site-profile-picture.sb-navbar ).site-profile-picture.sb-navbar
else else

25
app/views/user/profile.pug

@ -5,22 +5,9 @@ block content
section.uk-section.uk-section-default section.uk-section.uk-section-default
.uk-container .uk-container
h1= user.displayName || user.username || user.email div(uk-grid)
p People don't have public profiles on #[+renderSiteLink()]. You must be logged in to view your profile. And, only you can view your profile. .uk-width-auto
+renderProfileIcon(user, user.displayName || user.username, 'large')
p Your profile is where you edit your account settings, configure your commenting defaults, and otherwise manage how you use #[+renderSiteLink()]. .uk-width-expand
h1= user.displayName || user.username || user.email
section.uk-section.uk-section-default p Welcome to #[+renderSiteLink()]. This app/site really doesn't do anything at all other than provide everything needed for building distributed communities of web applications that all talk to each other and share information in new and easy ways.
.uk-container
.uk-margin
+renderSectionTitle('Comment History')
if Array.isArray(commentHistory) && (commentHistory.length > 0)
ul.uk-list.uk-list-divider
each comment in commentHistory
li
.uk-margin-small
.uk-text-small commenting on #[a(href=`/post/${comment.resource.slug}?comment=${comment._id}#featured-comment`)= comment.resource.title]
+renderComment(comment)
else
div You haven't written any comments on posts.

14
app/workers/host-services.js

@ -27,13 +27,17 @@ const { CronJob } = require('cron');
const CRON_TIMEZONE = 'America/New_York'; const CRON_TIMEZONE = 'America/New_York';
module.rootPath = path.resolve(__dirname, '..', '..');
module.pkg = require(path.resolve(__dirname, '..', '..', 'package.json')); module.pkg = require(path.resolve(__dirname, '..', '..', 'package.json'));
module.config = { module.config = {
componentName: 'host-services', environment: process.env.NODE_ENV,
root: path.resolve(__dirname, '..', '..'), root: module.rootPath,
component: { name: 'DTP Host Services', slug: 'host-services' },
site: require(path.join(module.rootPath, 'config', 'site')),
http: require(path.join(module.rootPath, 'config', 'http')),
}; };
module.log = new SiteLog(module, module.config.componentName); module.log = new SiteLog(module, module.config.component);
class CacheStats { class CacheStats {
@ -655,9 +659,9 @@ module.expireNetHosts = async ( ) => {
await module.registerHost(); await module.registerHost();
await module.setHostStatus('active'); await module.setHostStatus('active');
module.log.info(`${module.pkg.name} v${module.pkg.version} ${module.config.componentName} started`); module.log.info(`${module.pkg.name} v${module.pkg.version} ${module.config.component.name} started`);
} catch (error) { } catch (error) {
module.log.error('failed to start Host Cache worker', { error }); module.log.error('failed to start worker', { component: module.config.component, error });
process.exit(-1); process.exit(-1);
} }

16
app/workers/reeeper.js

@ -19,13 +19,19 @@ const { CronJob } = require('cron');
const CRON_TIMEZONE = 'America/New_York'; const CRON_TIMEZONE = 'America/New_York';
module.rootPath = path.resolve(__dirname, '..', '..');
module.pkg = require(path.resolve(__dirname, '..', '..', 'package.json')); module.pkg = require(path.resolve(__dirname, '..', '..', 'package.json'));
module.config = { module.config = {
componentName: 'reeeper', environment: process.env.NODE_ENV,
root: path.resolve(__dirname, '..', '..'), root: module.rootPath,
component: { name: 'DTP Reeeper', slug: 'reeeper' },
}; };
module.log = new SiteLog(module, module.config.componentName); module.config.site = require(path.join(module.rootPath, 'config', 'site'));
module.config.http = require(path.join(module.rootPath, 'config', 'http'));
module.log = new SiteLog(module, module.config.component);
module.expireCrashedHosts = async ( ) => { module.expireCrashedHosts = async ( ) => {
const NetHost = mongoose.model('NetHost'); const NetHost = mongoose.model('NetHost');
@ -81,9 +87,9 @@ module.expireCrashedHosts = async ( ) => {
null, true, CRON_TIMEZONE, null, true, CRON_TIMEZONE,
); );
module.log.info(`${module.pkg.name} v${module.pkg.version} ${module.config.componentName} started`); module.log.info(`${module.pkg.name} v${module.pkg.version} ${module.config.component.name} started`);
} catch (error) { } catch (error) {
module.log.error('failed to start Host Cache worker', { error }); module.log.error('failed to start worker', { component: module.config.component, error });
process.exit(-1); process.exit(-1);
} }

88
app/workers/sample-worker.js

@ -0,0 +1,88 @@
// sample-worker.js
// Copyright (C) 2022 DTP Technologies, LLC
// License: Apache-2.0
'use strict';
const DTP_COMPONENT = { name: 'Sample Worker', slug: 'sample-worker' };
const path = require('path');
require('dotenv').config({ path: path.resolve(__dirname, '..', '..', '.env') });
const {
SiteLog,
SiteWorker,
} = require(path.join(__dirname, '..', '..', 'lib', 'site-lib'));
const { CronJob } = require('cron');
module.rootPath = path.resolve(__dirname, '..', '..');
module.pkg = require(path.resolve(__dirname, '..', '..', 'package.json'));
module.config = {
environment: process.env.NODE_ENV,
root: module.rootPath,
component: DTP_COMPONENT,
};
module.config.site = require(path.join(module.rootPath, 'config', 'site'));
module.config.http = require(path.join(module.rootPath, 'config', 'http'));
class SampleWorker extends SiteWorker {
constructor (dtp) {
super(dtp, { });
}
async start ( ) {
const CRON_TIMEZONE = 'America/New_York';
await super.start();
this.log.info('starting worker job');
this.job = new CronJob(
'*/5 * * * * *',
this.runJob.bind(this),
null, true, CRON_TIMEZONE,
);
const { jobQueue: jobQueueService } = this.dtp.services;
this.sampleJobQueue = jobQueueService.getJobQueue('dtp-sample', this.dtp.config.jobQueues['dtp-sample']);
this.sampleJobQueue.process('dtp-sample', 1, this.processDtpSample.bind(this));
}
async stop ( ) {
this.log.info('stopping worker job');
this.job.stop();
delete this.job;
}
async runJob ( ) {
this.log.alert('sample job starting');
/*
* Your worker will do interesting things here
*/
this.log.alert('sample job ending');
}
async processDtpSample (job) {
this.log.info('received sample job', { id: job.id, data: job.data });
}
}
(async ( ) => {
try {
module.log = new SiteLog(module, module.config.component);
module.worker = new SampleWorker(module);
await module.worker.start();
} catch (error) {
module.log.error('failed to start worker', {
component: module.config.component,
error,
});
process.exit(-1);
}
})();

4
client/js/index-admin.js

@ -4,7 +4,7 @@
'use strict'; 'use strict';
const DTP_COMPONENT_NAME = 'SiteAdmin'; const DTP_COMPONENT = { name: 'Site Admin', slug: 'site-admin' };
const dtp = window.dtp = window.dtp || { }; const dtp = window.dtp = window.dtp || { };
dtp.admin = dtp.admin || { }; dtp.admin = dtp.admin || { };
@ -14,7 +14,7 @@ import DtpWebLog from 'dtp/dtp-log.js';
window.addEventListener('load', async ( ) => { window.addEventListener('load', async ( ) => {
// application console log // application console log
dtp.admin.log = new DtpWebLog(DTP_COMPONENT_NAME); dtp.admin.log = new DtpWebLog(DTP_COMPONENT);
dtp.adminApp = new DtpSiteAdminApp(dtp.user); dtp.adminApp = new DtpSiteAdminApp(dtp.user);

4
client/js/index.js

@ -4,7 +4,7 @@
'use strict'; 'use strict';
const DTP_COMPONENT_NAME = 'Site'; const DTP_COMPONENT = { name: 'Site', slug: 'site' };
const dtp = window. dtp = window.dtp || { }; const dtp = window. dtp = window.dtp || { };
@ -14,7 +14,7 @@ import UIkit from 'uikit';
window.addEventListener('load', async ( ) => { window.addEventListener('load', async ( ) => {
// application console log // application console log
dtp.log = new DtpWebLog(DTP_COMPONENT_NAME); dtp.log = new DtpWebLog(DTP_COMPONENT);
// service worker // service worker
if ('serviceWorker' in navigator) { if ('serviceWorker' in navigator) {

4
client/js/site-admin-app.js

@ -4,7 +4,7 @@
'use strict'; 'use strict';
const DTP_COMPONENT_NAME = 'SiteAdminApp'; const DTP_COMPONENT = { name: 'Site Admin', slug: 'site-admin-app' };
const dtp = window.dtp = window.dtp || { }; const dtp = window.dtp = window.dtp || { };
const GRID_COLOR = '#a0a0a0'; const GRID_COLOR = '#a0a0a0';
@ -28,7 +28,7 @@ import UIkit from 'uikit';
export default class DtpSiteAdminHostStatsApp extends DtpApp { export default class DtpSiteAdminHostStatsApp extends DtpApp {
constructor (user) { constructor (user) {
super(DTP_COMPONENT_NAME, user); super(DTP_COMPONENT, user);
this.log.debug('constructor', 'app instance created'); this.log.debug('constructor', 'app instance created');
} }

4
client/js/site-app.js

@ -4,7 +4,7 @@
'use strict'; 'use strict';
const DTP_COMPONENT_NAME = 'SiteApp'; const DTP_COMPONENT = { name: 'Site App', slug: 'site-app' };
const dtp = window.dtp = window.dtp || { }; const dtp = window.dtp = window.dtp || { };
import DtpApp from 'dtp/dtp-app.js'; import DtpApp from 'dtp/dtp-app.js';
@ -26,7 +26,7 @@ const CHART_FILL_USER = 'rgb(0, 128, 0)';
export default class DtpSiteApp extends DtpApp { export default class DtpSiteApp extends DtpApp {
constructor (user) { constructor (user) {
super(DTP_COMPONENT_NAME, user); super(DTP_COMPONENT, user);
this.log.debug('constructor', 'app instance created'); this.log.debug('constructor', 'app instance created');
this.chat = { this.chat = {

8
config/http.js

@ -0,0 +1,8 @@
'use strict';
module.exports = {
scheme: (process.env.NODE_ENV === 'production') ? 'https' : 'http',
address: process.env.HTTP_BIND_ADDRESS,
port: parseInt(process.env.HTTP_BIND_PORT, 10),
};

9
dtp-webapp-cli.js

@ -4,7 +4,7 @@
'use strict'; 'use strict';
const DTP_COMPONENT_NAME = 'webapp-cli'; const DTP_COMPONENT = { name: 'Sample App CLI', slug: 'webapp-cli' };
require('dotenv').config(); require('dotenv').config();
@ -22,12 +22,13 @@ const {
module.rootPath = __dirname; module.rootPath = __dirname;
module.pkg = require(path.join(module.rootPath, 'package.json')); module.pkg = require(path.join(module.rootPath, 'package.json'));
module.config = { module.config = {
componentName: DTP_COMPONENT_NAME, component: DTP_COMPONENT,
root: module.rootPath, root: module.rootPath,
site: require(path.join(module.rootPath, 'config', 'site')), site: require(path.join(module.rootPath, 'config', 'site')),
http: require(path.join(module.rootPath, 'config', 'http')),
}; };
module.log = new SiteLog(module, module.config.componentName); module.log = new SiteLog(module, module.config.component);
module.grantPermission = async (target, permission) => { module.grantPermission = async (target, permission) => {
const User = mongoose.model('User'); const User = mongoose.model('User');
@ -220,7 +221,7 @@ module.requestCoreConnect = async (host) => {
try { try {
await SitePlatform.startPlatform(module); await SitePlatform.startPlatform(module);
} catch (error) { } catch (error) {
module.log.error(`failed to start DTP ${DTP_COMPONENT_NAME} platform`, { error }); module.log.error(`failed to start DTP ${module.config.component.slug} platform`, { error });
return; return;
} }

14
dtp-webapp.js

@ -4,7 +4,7 @@
'use strict'; 'use strict';
const DTP_COMPONENT_NAME = 'webapp'; const DTP_COMPONENT = { name: 'Sample App', slug: 'webapp' };
require('dotenv').config(); require('dotenv').config();
@ -15,16 +15,14 @@ const { SitePlatform, SiteLog } = require(path.join(__dirname, 'lib', 'site-lib'
module.rootPath = __dirname; module.rootPath = __dirname;
module.pkg = require(path.join(module.rootPath, 'package.json')); module.pkg = require(path.join(module.rootPath, 'package.json'));
module.config = { module.config = {
componentName: DTP_COMPONENT_NAME, environment: process.env.NODE_ENV,
root: module.rootPath, root: module.rootPath,
component: DTP_COMPONENT,
site: require(path.join(module.rootPath, 'config', 'site')), site: require(path.join(module.rootPath, 'config', 'site')),
http: { http: require(path.join(module.rootPath, 'config', 'http')),
address: process.env.HTTP_BIND_ADDRESS,
port: parseInt(process.env.HTTP_BIND_PORT, 10),
},
}; };
module.log = new SiteLog(module, module.config.componentName); module.log = new SiteLog(module, module.config.component);
module.shutdown = async ( ) => { module.shutdown = async ( ) => {
@ -63,7 +61,7 @@ module.shutdown = async ( ) => {
await SitePlatform.startPlatform(module); await SitePlatform.startPlatform(module);
await SitePlatform.startWebServer(module); await SitePlatform.startWebServer(module);
} catch (error) { } catch (error) {
module.log.error(`failed to start DTP ${DTP_COMPONENT_NAME}`, { error }); module.log.error(`failed to start DTP ${module.config.component.name}`, { error });
process.exit(-1); process.exit(-1);
} }

8
lib/client/js/dtp-log.js

@ -8,8 +8,8 @@ window.dtp = window.dtp || { };
export default class DtpWebLog { export default class DtpWebLog {
constructor (componentName, options) { constructor (component, options) {
this.componentName = componentName; this.component = component;
this.options = Object.assign({ this.options = Object.assign({
color: { color: {
debug: '#808080', debug: '#808080',
@ -113,13 +113,13 @@ export default class DtpWebLog {
if (!this.enabled) { return; } if (!this.enabled) { return; }
if (data) { if (data) {
console[method]('%c%s%c: %s', console[method]('%c%s%c: %s',
css.label, `${this.componentName}.${event}`, css.label, `${this.component.slug}.${event}`,
css.message, msg, css.message, msg,
data, data,
); );
} else { } else {
console[method]('%c%s%c: %s', console[method]('%c%s%c: %s',
css.label, `${this.componentName}.${event}`, css.label, `${this.component.slug}.${event}`,
css.message, msg, css.message, msg,
); );
} }

8
lib/client/js/dtp-plugin.js

@ -24,19 +24,17 @@
'use strict'; 'use strict';
const DTP_COMPONENT_NAME = 'dtp:plugin';
window.dtp = window.dtp || { }; window.dtp = window.dtp || { };
import DtpLog from 'dtp/dtp-log.js'; import DtpLog from 'dtp/dtp-log.js';
export default class DtpPlugin { export default class DtpPlugin {
constructor (app, context, componentName) { constructor (app, context, component) {
this.app = app; this.app = app;
this.context = context; this.context = context;
this.componentName = componentName; this.component = component;
this.log = new DtpLog(`${DTP_COMPONENT_NAME}:${componentName}`); this.log = new DtpLog(component);
} }
} }

8
lib/site-common.js

@ -7,12 +7,18 @@
const path = require('path'); const path = require('path');
const pug = require('pug'); const pug = require('pug');
const { SiteLog } = require(path.join(__dirname, 'site-log'));
const Events = require('events'); const Events = require('events');
class SiteCommon extends Events { class SiteCommon extends Events {
constructor (dtp) { constructor (dtp, component) {
super(); super();
this.dtp = dtp; this.dtp = dtp;
this.component = component;
this.log = new SiteLog(dtp, component);
this.appTemplateRoot = path.join(this.dtp.config.root, 'app', 'templates'); this.appTemplateRoot = path.join(this.dtp.config.root, 'app', 'templates');
} }

10
lib/site-controller.js

@ -7,18 +7,16 @@
const path = require('path'); const path = require('path');
const { SiteCommon } = require(path.join(__dirname, 'site-common')); const { SiteCommon } = require(path.join(__dirname, 'site-common'));
const { SiteLog } = require(path.join(__dirname, 'site-log'));
class SiteController extends SiteCommon { class SiteController extends SiteCommon {
constructor (dtp, name) { constructor (dtp, component) {
super(dtp); super(dtp, component);
this.name = name;
this.log = new SiteLog(dtp, `ctrl:${name}`);
} }
async loadChild (filename) { async loadChild (filename) {
let child = await require(filename)(this.dtp); let component = require(filename);
let child = await component.create(this.dtp);
return await child.start(); return await child.start();
} }

3
lib/site-ioserver.js

@ -4,6 +4,7 @@
'use strict'; 'use strict';
const DTP_COMPONENT = { name: 'I/O Server', slug: 'ioserver', prefix: 'srv' };
const path = require('path'); const path = require('path');
const Redis = require('ioredis'); const Redis = require('ioredis');
@ -23,7 +24,7 @@ class SiteIoServer extends Events {
constructor (dtp) { constructor (dtp) {
super(); super();
this.dtp = dtp; this.dtp = dtp;
this.log = new SiteLog(dtp, 'ioserver'); this.log = new SiteLog(dtp, DTP_COMPONENT);
} }
async start (httpServer) { async start (httpServer) {

1
lib/site-lib.js

@ -14,4 +14,5 @@ module.exports = {
SiteLog: require(path.join(__dirname, 'site-log')).SiteLog, SiteLog: require(path.join(__dirname, 'site-log')).SiteLog,
SiteController: require(path.join(__dirname, 'site-controller')).SiteController, SiteController: require(path.join(__dirname, 'site-controller')).SiteController,
SiteService: require(path.join(__dirname, 'site-service')).SiteService, SiteService: require(path.join(__dirname, 'site-service')).SiteService,
SiteWorker: require(path.join(__dirname, 'site-worker')).SiteWorker,
}; };

22
lib/site-log.js

@ -31,9 +31,16 @@ class SiteLog {
*/ */
static setModel (model) { LogModel = model; } static setModel (model) { LogModel = model; }
constructor (dtp, componentName) { constructor (dtp, component) {
if (!dtp) {
throw new Error('Must specify DTP module');
}
this.dtp = dtp; this.dtp = dtp;
this.componentName = componentName;
if (!component || !component.slug || !component.name) {
throw new Error('Must specify DTP component');
}
this.component = component;
} }
async debug (message, metadata) { async debug (message, metadata) {
@ -75,7 +82,6 @@ class SiteLog {
async writeLog (level, message, metadata) { async writeLog (level, message, metadata) {
const NOW = new Date(); const NOW = new Date();
const { componentName } = this;
if (process.env.DTP_LOG_CONSOLE === 'enabled') { if (process.env.DTP_LOG_CONSOLE === 'enabled') {
let clevel = level.padEnd(5); let clevel = level.padEnd(5);
@ -104,22 +110,22 @@ class SiteLog {
} }
const ctimestamp = color.black(moment(NOW).format('YYYY-MM-DD HH:mm:ss.SSS')); const ctimestamp = color.black(moment(NOW).format('YYYY-MM-DD HH:mm:ss.SSS'));
const ccomponentName = color.cyan(componentName); const ccomponentSlug = color.cyan(this.component.slug);
const cmessage = color.darkGray(message); const cmessage = color.darkGray(message);
if (metadata) { if (metadata) {
console.log(`${ctimestamp} ${clevel} ${ccomponentName} ${cmessage}`, util.inspect(metadata, false, Infinity, true)); console.log(`${ctimestamp} ${clevel} ${ccomponentSlug} ${cmessage}`, util.inspect(metadata, false, Infinity, true));
} else { } else {
console.log(`${ctimestamp} ${clevel} ${ccomponentName} ${cmessage}`); console.log(`${ctimestamp} ${clevel} ${ccomponentSlug} ${cmessage}`);
} }
} }
if (LogModel && (process.env.DTP_LOG_MONGODB === 'enabled')) { if (LogModel && (process.env.DTP_LOG_MONGODB === 'enabled')) {
await LogModel.create({ created: NOW, level, componentName, message, metadata }); await LogModel.create({ created: NOW, level, component: this.component, message, metadata });
} }
if (LogStream && (process.env.DTP_LOG_FILE === 'enabled')) { if (LogStream && (process.env.DTP_LOG_FILE === 'enabled')) {
const logEntry = { const logEntry = {
t: NOW, c: componentName, l: level, m: message, d: metadata, t: NOW, c: this.component.slug, l: level, m: message, d: metadata,
}; };
LogStream.write(`${JSON.stringify(logEntry)}\n`); LogStream.write(`${JSON.stringify(logEntry)}\n`);
} }

8
lib/site-platform.js

@ -182,7 +182,7 @@ module.loadControllers = async (dtp) => {
module.exports.startPlatform = async (dtp) => { module.exports.startPlatform = async (dtp) => {
try { try {
module.log = new SiteLog(module, dtp.config.componentName); module.log = new SiteLog(module, dtp.config.component);
dtp.config.jobQueues = require(path.join(dtp.config.root, 'config', 'job-queues')); dtp.config.jobQueues = require(path.join(dtp.config.root, 'config', 'job-queues'));
await module.connectDatabase(dtp); await module.connectDatabase(dtp);
@ -192,8 +192,10 @@ module.exports.startPlatform = async (dtp) => {
SiteLog.setModel(mongoose.model('Log')); SiteLog.setModel(mongoose.model('Log'));
await module.loadServices(dtp); await module.loadServices(dtp);
module.log.info(`Digital Telepresence Platform v${dtp.pkg.version} started for [${dtp.pkg.name}]`);
} catch (error) { } catch (error) {
module.log.error('failed to initialize the Digital Telepresence Platform', { error }); module.log.error('platform failed to start', { error });
return; return;
} }
}; };
@ -372,7 +374,7 @@ module.exports.startWebServer = async (dtp) => {
bind: dtp.config.http.address, bind: dtp.config.http.address,
}); });
module.http.listen(dtp.config.http.port, dtp.config.http.address, ( ) => { module.http.listen(dtp.config.http.port, dtp.config.http.address, ( ) => {
module.log.info(`${dtp.config.componentName} platform online`, { port: dtp.config.http.port }); module.log.info(`${dtp.config.component.name} platform online`, { port: dtp.config.http.port });
}); });
}; };

8
lib/site-service.js

@ -7,15 +7,11 @@
const path = require('path'); const path = require('path');
const { SiteCommon } = require(path.join(__dirname, 'site-common')); const { SiteCommon } = require(path.join(__dirname, 'site-common'));
const { SiteLog } = require(path.join(__dirname, 'site-log'));
class SiteService extends SiteCommon { class SiteService extends SiteCommon {
constructor (dtp, definition) { constructor (dtp, component) {
super(dtp); super(dtp, component);
this.slug = definition.slug;
this.name = definition.name;
this.log = new SiteLog(dtp, `svc:${this.slug}`);
} }
async start ( ) { async start ( ) {

62
lib/site-worker.js

@ -0,0 +1,62 @@
// site-service.js
// Copyright (C) 2022 DTP Technologies, LLC
// License: Apache-2.0
'use strict';
const path = require('path');
const SitePlatform = require(path.join(__dirname, 'site-platform'));
const { SiteCommon } = require(path.join(__dirname, 'site-common'));
class SiteWorker extends SiteCommon {
constructor (dtp, component) {
super(dtp, component);
}
async start ( ) {
try {
process.on('unhandledRejection', (error, p) => {
this.log.error('Unhandled rejection', {
error: error,
promise: p,
stack: error.stack
});
});
process.on('warning', (error) => {
this.log.alert('warning', { error });
});
process.once('SIGINT', async ( ) => {
this.log.info('SIGINT received');
this.log.info('requesting shutdown...');
await this.stop();
const exitCode = await SitePlatform.shutdown();
process.nextTick(( ) => {
process.exit(exitCode);
});
});
/*
* Site Platform startup
*/
await SitePlatform.startPlatform(this.dtp);
} catch (error) {
this.log.error('failed to start worker', {
component: this.dtp.config.component,
error,
});
process.exit(-1);
}
}
async stop ( ) {
}
}
module.exports.SiteWorker = SiteWorker;

14
update-deps.js

@ -4,20 +4,22 @@
'use strict'; 'use strict';
const DTP_COMPONENT_NAME = 'webapp';
require('dotenv').config();
const path = require('path'); const path = require('path');
require('dotenv').config();
const { spawn } = require('child_process'); const { spawn } = require('child_process');
const { SiteAsync, SiteLog } = require(path.join(__dirname, 'lib', 'site-lib')); const { SiteLog } = require(path.join(__dirname, 'lib', 'site-lib'));
module.rootPath = __dirname; module.rootPath = __dirname;
module.pkg = require(path.join(module.rootPath, 'package.json')); module.pkg = require(path.join(module.rootPath, 'package.json'));
module.pinnedPackages = require(path.join(module.rootPath, 'config', 'pinned-packages')); module.pinnedPackages = require(path.join(module.rootPath, 'config', 'pinned-packages'));
module.log = new SiteLog(module, DTP_COMPONENT_NAME);
module.config = {
component: { name: 'Webapp Dependency Updater', slug: 'webapp-update-deps' },
};
module.log = new SiteLog(module, module.config.component);
module.runCommand = async (command, args) => { module.runCommand = async (command, args) => {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {

Loading…
Cancel
Save