Browse Source

migrate to the use of dtp-logan-api

- convert LoganService to use the API module
- add logan worker to transmit Logan events from job queue
- added logan worker config to supervisord directory
- updated the other configs
develop
Rob Colbert 10 months ago
parent
commit
26df84d0c8
  1. 10
      .env.default
  2. 2
      .jshintrc
  3. 92
      app/services/logan.js
  4. 92
      app/workers/logan.js
  5. 1
      package.json
  6. 4
      supervisord/dtp-base-host-services.conf
  7. 13
      supervisord/dtp-base-logan.conf
  8. 4
      supervisord/dtp-base-newsletter.conf
  9. 4
      supervisord/dtp-base-newsroom.conf
  10. 2
      supervisord/dtp-base.conf
  11. 42
      yarn.lock

10
.env.default

@ -116,3 +116,13 @@ DTP_LOG_INFO=enabled
DTP_LOG_WARN=enabled
DTP_LOG_HTTP_FORMAT=combined
#
# DTP Logan Integration
#
DTP_LOGAN=disabled
DTP_LOGAN_API_KEY=########-####-####-####-############
DTP_LOGAN_SCHEME=https
DTP_LOGAN_HOST=logan.digitaltelepresence.com
DTP_LOGAN_QUEUE_NAME=logan

2
.jshintrc

@ -10,7 +10,7 @@
"undef": true,
"unused": true,
"futurehostile": true,
"esversion": 9,
"esversion": 11,
"mocha": true,
"globals": {
"markdown": true,

92
app/services/logan.js

@ -4,9 +4,7 @@
'use strict';
const os = require('os');
const { SiteService, SiteError } = require('../../lib/site-lib');
const { SiteService } = require('../../lib/site-lib');
class LoganService extends SiteService {
@ -16,56 +14,56 @@ class LoganService extends SiteService {
async start ( ) {
await super.start();
}
async sendRequestEvent (component, req, event) {
if (req.user) {
event.data = event.data || { };
event.data.user = {
_id: req.user._id,
username: req.user.username,
};
}
event.ip = req.ip;
return this.sendEvent(component, event);
}
async sendEvent (component, event) {
try {
event.host = os.hostname();
event['component.slug'] = component.slug;
event['component.name'] = component.className || component.name;
this.log[event.level]('application event', { event });
const { LoganClient } = await import('dtp-logan-api');
if (process.env.DTP_LOGAN !== 'enabled') {
return;
}
this.log.info('creating Logan client');
this.logan = new LoganClient();
const loganScheme = process.env.DTP_LOGAN_SCHEME || 'http';
const loganUrl = `${loganScheme}://${process.env.DTP_LOGAN_HOST}/api/event`;
const payload = JSON.stringify(event);
const response = await fetch(loganUrl, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Content-Length': payload.length,
'X-LogAn-Auth': process.env.DTP_LOGAN_API_KEY,
this.log.info('initializing Logan client');
await this.logan.initialize({
log: this.log,
api: {
enabled: process.env.DTP_LOGAN === 'enabled',
key: process.env.DTP_LOGAN_API_KEY,
scheme: process.env.DTP_LOGAN_SCHEME,
host: process.env.DTP_LOGAN_HOST,
},
request: {
userField: 'user',
userIdField: '_id',
usernameField: 'username',
},
flags: {
includeHostname: true,
includeClientIpAddress: true,
includeUser: true,
},
queue: {
enabled: true,
name: process.env.DTP_LOGAN_QUEUE_NAME || 'logan',
redis: {
host: process.env.REDIS_HOST,
port: process.env.REDIS_PORT,
username: process.env.REDIS_USERNAME, // requires Redis >= 6
password: process.env.REDIS_PASSWORD,
keyPrefix: process.env.REDIS_PREFIX,
},
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: true,
},
body: payload,
});
},
});
}
const json = await response.json();
if (!json.success) {
throw new SiteError(500, json.message);
}
async sendRequestEvent (component, req, event) {
return this.logan.sendRequestEvent(component, req, event);
}
return json;
} catch (error) {
this.log.error('failed to send LOGAN event', { event, error });
// fall through
}
async sendEvent (component, event) {
return this.logan.sendEvent(component, event);
}
}

92
app/workers/logan.js

@ -0,0 +1,92 @@
'use strict';
require('dotenv').config();
const path = require('path');
const {
SiteLog,
SiteWorker,
} = require(path.join(__dirname, '..', '..', 'lib', 'site-lib'));
module.rootPath = path.resolve(__dirname, '..', '..');
module.pkg = require(path.resolve(__dirname, '..', '..', 'package.json'));
module.config = {
environment: process.env.NODE_ENV,
root: module.rootPath,
component: { name: 'LoganSiteWorker', slug: 'logan-site-worker' },
site: require(path.join(module.rootPath, 'config', 'site')),
};
class LoganSiteWorker extends SiteWorker {
constructor (dtp) {
super(dtp, dtp.config.component);
}
async start ( ) {
await super.start();
const { LoganWorker } = await import('dtp-logan-api');
this.log.info('creating Logan worker');
this.loganWorker = new LoganWorker();
this.log.info('initializing Logan worker');
await this.loganWorker.initialize({
api: {
enabled: process.env.DTP_LOGAN === 'enabled',
key: process.env.DTP_LOGAN_API_KEY,
scheme: process.env.DTP_LOGAN_SCHEME,
host: process.env.DTP_LOGAN_HOST,
},
queue: {
enabled: true,
name: 'logan',
redis: {
host: process.env.REDIS_HOST,
port: process.env.REDIS_PORT,
username: process.env.REDIS_USERNAME, // requires Redis >= 6
password: process.env.REDIS_PASSWORD,
keyPrefix: process.env.REDIS_PREFIX,
},
defaultJobOptions: {
attempts: 3,
priority: 10,
removeOnComplete: true,
removeOnFail: true,
},
},
});
}
async stop ( ) {
if (this.loganWorker) {
await this.loganWorker.close();
delete this.loganWorker;
}
await super.stop();
}
}
(async ( ) => {
module.log = new SiteLog(module, module.config.component);
if (!process.env.DTP_LOGAN_API_KEY) {
console.log('Must define DTP_LOGAN_API_KEY environment variable to run test');
process.exit(-1);
}
try {
module.worker = new LoganSiteWorker(module);
await module.worker.start();
module.log.info(`${module.pkg.name} v${module.pkg.version} ${module.config.component.name} started`);
} catch (error) {
module.log.error('failed to start worker', { component: module.config.component, error });
process.exit(-1);
}
})();

1
package.json

@ -29,6 +29,7 @@
"disposable-email-provider-domains": "^1.0.9",
"dotenv": "^16.0.0",
"dtp-jshint-reporter": "git+https://git.digitaltelepresence.com/digital-telepresence/dtp-jshint-reporter.git#master",
"dtp-logan-api": "^0.3.2",
"ein-validator": "^1.0.1",
"email-domain-check": "^1.1.4",
"email-validator": "^2.0.4",

4
supervisord/dtp-base-host-services.conf

@ -1,7 +1,7 @@
[program:dtp-base:host-services]
numprocs=1
process_name=%(program_name)s_%(process_num)02d
command=/home/dtp/.nvm/versions/node/v16.13.0/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/host-services.js
command=/home/dtp/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/host-services.js
directory=/home/dtp/live/dtp-base
autostart=true
autorestart=true
@ -10,4 +10,4 @@ stopsignal=INT
stderr_logfile=/var/log/dtp-base/host-services.err.log
stdout_logfile=/var/log/dtp-base/host-services.out.log
user=dtp
environment=HOME='/home/dtp/live/dtp-base',HTTP_BIND_PORT=30%(process_num)02d,NODE_ENV=production,LOGNAME=host-services
environment=HOME='/home/dtp/live/dtp-base',NODE_ENV=production,LOGNAME=host-services

13
supervisord/dtp-base-logan.conf

@ -0,0 +1,13 @@
[program:dtp-base:logan]
numprocs=1
process_name=%(program_name)s_%(process_num)02d
command=/home/dtp/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/logan.js
directory=/home/dtp/live/dtp-base
autostart=true
autorestart=true
startretries=3
stopsignal=INT
stdout_logfile=/var/log/dtp-base/logan.out.log
stderr_logfile=/var/log/dtp-base/logan.err.log
user=dtp
environment=HOME='/home/dtp/live/dtp-base',NODE_ENV=production,LOGNAME=logan

4
supervisord/dtp-base-newsletter.conf

@ -1,7 +1,7 @@
[program:dtp-base:newsletter]
numprocs=1
process_name=%(program_name)s_%(process_num)02d
command=/home/dtp/.nvm/versions/node/v16.13.0/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/newsletter.js
command=/home/dtp/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/newsletter.js
directory=/home/dtp/live/dtp-base
autostart=true
autorestart=true
@ -10,4 +10,4 @@ stopsignal=INT
stderr_logfile=/var/log/dtp-base/newsletter.err.log
stdout_logfile=/var/log/dtp-base/newsletter.out.log
user=dtp
environment=HOME='/home/dtp/live/dtp-base',HTTP_BIND_PORT=30%(process_num)02d,NODE_ENV=production,LOGNAME=newsletter
environment=HOME='/home/dtp/live/dtp-base',NODE_ENV=production,LOGNAME=newsletter

4
supervisord/dtp-base-newsroom.conf

@ -1,7 +1,7 @@
[program:dtp-base:newsroom]
numprocs=1
process_name=%(program_name)s_%(process_num)02d
command=/home/dtp/.nvm/versions/node/v16.13.0/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/newsroom.js
command=/home/dtp/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/newsroom.js
directory=/home/dtp/live/dtp-base
autostart=true
autorestart=true
@ -10,4 +10,4 @@ stopsignal=INT
stderr_logfile=/var/log/dtp-base/newsroom.err.log
stdout_logfile=/var/log/dtp-base/newsroom.out.log
user=dtp
environment=HOME='/home/dtp/live/dtp-base',HTTP_BIND_PORT=30%(process_num)02d,NODE_ENV=production,LOGNAME=newsroom
environment=HOME='/home/dtp/live/dtp-base',NODE_ENV=production,LOGNAME=newsroom

2
supervisord/dtp-base.conf

@ -1,7 +1,7 @@
[program:dtp-base]
numprocs=1
process_name=%(program_name)s_%(process_num)02d
command=/home/dtp/.nvm/versions/node/v16.13.0/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 dtp-webapp.js
command=/home/dtp/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 dtp-webapp.js
directory=/home/dtp/live/dtp-base
autostart=true
autorestart=true

42
yarn.lock

@ -2076,6 +2076,20 @@ builtin-modules@^3.1.0:
resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-3.2.0.tgz#45d5db99e7ee5e6bc4f362e008bf917ab5049887"
integrity sha512-lGzLKcioL90C7wMczpkY0n/oART3MbBa8R9OFGE1rJxoVI86u4WAGfEk8Wjv10eKSyTHVGkSo3bvBylCEtk7LA==
bull@^4.10.4:
version "4.10.4"
resolved "https://registry.yarnpkg.com/bull/-/bull-4.10.4.tgz#db39ee0c3bfbe3b76f1f35db800501de5bba4f84"
integrity sha512-o9m/7HjS/Or3vqRd59evBlWCXd9Lp+ALppKseoSKHaykK46SmRjAilX98PgmOz1yeVaurt8D5UtvEt4bUjM3eA==
dependencies:
cron-parser "^4.2.1"
debuglog "^1.0.0"
get-port "^5.1.1"
ioredis "^5.0.0"
lodash "^4.17.21"
msgpackr "^1.5.2"
semver "^7.3.2"
uuid "^8.3.0"
bull@^4.7.0:
version "4.7.0"
resolved "https://registry.yarnpkg.com/bull/-/bull-4.7.0.tgz#89442d4676117edd9f9a1359bb0edfb489595e70"
@ -2940,6 +2954,11 @@ denque@^2.0.1:
resolved "https://registry.yarnpkg.com/denque/-/denque-2.0.1.tgz#bcef4c1b80dc32efe97515744f21a4229ab8934a"
integrity sha512-tfiWc6BQLXNLpNiR5iGd0Ocu3P3VpxfzFiqubLgMfhfOw9WyvgJBd46CClNn9k3qfbjvT//0cf7AlYRX/OslMQ==
denque@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1"
integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==
depd@~1.1.2:
version "1.1.2"
resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9"
@ -3106,6 +3125,14 @@ drange@^1.0.2:
dependencies:
chalk "^4.1.1"
dtp-logan-api@^0.3.2:
version "0.3.2"
resolved "https://registry.yarnpkg.com/dtp-logan-api/-/dtp-logan-api-0.3.2.tgz#354c184ce9ea8a31c57bf0592278f5861fe672d2"
integrity sha512-7mORjmktZY0xpUNecXVLiHvRkBbJxQkjRom2QFzb+5NLQxml2u28cPegINNIlnFKjduXme7a+MQm0msuCmHYtA==
dependencies:
bull "^4.10.4"
ioredis "^5.3.2"
duplexer3@^0.1.4:
version "0.1.4"
resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.4.tgz#ee01dd1cac0ed3cbc7fdbea37dc0a8f1ce002ce2"
@ -4627,6 +4654,21 @@ ioredis@^4.28.5:
redis-parser "^3.0.0"
standard-as-callback "^2.1.0"
ioredis@^5.0.0, ioredis@^5.3.2:
version "5.3.2"
resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-5.3.2.tgz#9139f596f62fc9c72d873353ac5395bcf05709f7"
integrity sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==
dependencies:
"@ioredis/commands" "^1.1.1"
cluster-key-slot "^1.1.0"
debug "^4.3.4"
denque "^2.1.0"
lodash.defaults "^4.2.0"
lodash.isarguments "^3.1.0"
redis-errors "^1.2.0"
redis-parser "^3.0.0"
standard-as-callback "^2.1.0"
ioredis@^5.2.2:
version "5.2.2"
resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-5.2.2.tgz#212467e04f6779b4e0e800cece7bb7d3d7b546d2"

Loading…
Cancel
Save