Browse Source

Merge branch 'develop' of git.digitaltelepresence.com:digital-telepresence/dtp-base into develop

develop
Rob Colbert 1 year ago
parent
commit
8370ab34e3
  1. 1
      .vscode/launch.json
  2. 4
      app/controllers/admin/newsroom.js
  3. 2
      app/controllers/newsroom.js
  4. 18
      app/controllers/welcome.js
  5. 27
      app/services/feed.js
  6. 15
      app/views/admin/newsroom/editor.pug
  7. 30
      app/views/components/library.pug
  8. 33
      app/views/components/page-sidebar.pug
  9. 2
      app/views/components/social-card/facebook.pug
  10. 9
      app/views/newsroom/components/feed-entry-list-item.pug
  11. 32
      app/views/newsroom/index.pug
  12. 22
      app/workers/newsroom.js
  13. 20
      app/workers/newsroom/cron/update-feeds.js
  14. 52
      app/workers/newsroom/job/update-feed.js
  15. 27
      client/js/site-admin-app.js
  16. 4
      config/job-queues.js
  17. 6
      config/limiter.js
  18. 34
      lib/site-platform.js
  19. 3
      restart-production
  20. 4
      start-local
  21. 14
      start-production
  22. 14
      stop-production
  23. 13
      supervisord/dtp-base-newsroom.conf

1
.vscode/launch.json

@ -15,6 +15,7 @@
"console": "integratedTerminal",
"env": {
"HTTP_BIND_PORT": "3000",
"HTTPS_ENABLE": "enabled",
"HTTPS_BIND_PORT": "3400"
}
},

4
app/controllers/admin/newsroom.js

@ -106,7 +106,7 @@ class NewsroomAdminController extends SiteController {
await feedService.update(res.locals.feed, req.body);
res.redirect('/admin/newsroom');
} catch (error) {
this.log.error('failed to create feed', { error });
this.log.error('failed to update feed', { error });
return next(error);
}
}
@ -115,7 +115,7 @@ class NewsroomAdminController extends SiteController {
const { feed: feedService } = this.dtp.services;
try {
res.locals.feed = await feedService.create(req.body);
res.redirect(`/admin/newsroom/${res.locals.feed._id}`);
res.redirect('/admin/newsroom');
} catch (error) {
this.log.error('failed to create feed', { error });
return next(error);

2
app/controllers/newsroom.js

@ -69,7 +69,7 @@ class NewsroomController extends SiteController {
const { feed: feedService } = this.dtp.services;
try {
res.locals.pagination = this.getPaginationParameters(req, 10);
res.locals.newsroom = await feedService.getFeeds(res.locals.pagination);
res.locals.newsroom = await feedService.getFeeds(res.locals.pagination, { withEntries: true });
res.render('newsroom/index');
} catch (error) {
this.log.error('failed to present newsroom home', { error });

18
app/controllers/welcome.js

@ -23,17 +23,24 @@ class WelcomeController extends SiteController {
captcha.loadFont(path.join(this.dtp.config.root, 'client', 'fonts', 'Dirty Sweb.ttf'));
function preventUserAccess (req, res, next) {
if (req.user) {
return res.redirect(301, '/');
}
return next();
}
const router = express.Router();
this.dtp.app.use('/welcome', welcomeLimiter, async (req, res, next) => {
res.locals.currentView = 'welcome';
return next();
}, router);
router.get('/core-member', this.getWelcomeCoreMember.bind(this));
router.get('/core-member', preventUserAccess, this.getWelcomeCoreMember.bind(this));
router.get('/signup/captcha', this.getSignupCaptcha.bind(this));
router.get('/signup', this.getSignupView.bind(this));
router.get('/login', this.getLoginView.bind(this));
router.get('/', this.getHomeView.bind(this));
router.get('/signup', preventUserAccess, this.getSignupView.bind(this));
router.get('/login', preventUserAccess, this.getLoginView.bind(this));
router.get('/', preventUserAccess, this.getHomeView.bind(this));
return router;
}
@ -43,6 +50,9 @@ class WelcomeController extends SiteController {
}
async getSignupCaptcha (req, res) {
if (!req.session || !req.session.captcha || !req.session.captcha.signup) {
return res.status(500).end('Session is not in a valid state for generating a captcha image');
}
const signupCaptcha = captcha(req.session.captcha.signup, {
color: false,
noise: 3,

27
app/services/feed.js

@ -8,7 +8,7 @@ const mongoose = require('mongoose');
const Feed = mongoose.model('Feed');
const FeedEntry = mongoose.model('FeedEntry');
const { SiteService, SiteError } = require('../../lib/site-lib');
const { SiteService, SiteError, SiteAsync } = require('../../lib/site-lib');
const { read: feedReader } = require('feed-reader');
class FeedService extends SiteService {
@ -23,6 +23,10 @@ class FeedService extends SiteService {
];
}
async start ( ) {
this.jobQueue = await this.getJobQueue('newsroom', this.dtp.config.jobQueues.newsroom);
}
async create (feedDefinition) {
feedDefinition.url = feedDefinition.url.trim();
const feedContent = await this.load(feedDefinition.url);
@ -40,6 +44,8 @@ class FeedService extends SiteService {
feed.published = feedContent.published;
await feed.save();
this.jobQueue.add('update-feed', { feedId: feed._id });
return feed.toObject();
}
@ -61,9 +67,12 @@ class FeedService extends SiteService {
updateOp.$set.generator = feedDefinition.generator || feedContent.generator;
await Feed.updateOne({ _id: feed._id }, updateOp);
this.jobQueue.add('update-feed', { feedId: feed._id });
}
async getFeeds (pagination) {
async getFeeds (pagination, options) {
options = Object.assign({ withEntries: false, entryCount: 3 }, options);
pagination = Object.assign({ skip: 0, cpp: 10 }, pagination);
const feeds = await Feed
.find()
@ -71,7 +80,21 @@ class FeedService extends SiteService {
.skip(pagination.skip)
.limit(pagination.cpp)
.lean();
if (options.withEntries) {
await SiteAsync.each(feeds, async (feed) => {
try {
feed.recent = await this.getFeedEntries(feed, { skip: 0, cpp: options.entryCount });
this.log.debug('feed entries', { count: feed.recent.entries.length });
} catch (error) {
this.log.error('failed to populate recent entries for feed', { feedId: feed._id, error });
// fall through
}
}, 2);
}
const totalFeedCount = await Feed.countDocuments();
return { feeds, totalFeedCount };
}

15
app/views/admin/newsroom/editor.pug

@ -33,7 +33,14 @@ block content
textarea(id="description", name="description", rows="4", placeholder="Enter feed description").uk-textarea.uk-resize-vertical= feed ? feed.description : undefined
.uk-card-footer
button(type="submit").uk-button.uk-button-primary.uk-border-rounded= feed ? 'Update Feed' : 'Add Feed'
if feed
pre= JSON.stringify(feed, null, 2)
div(uk-grid).uk-flex-right.uk-flex-middle
if feed
.uk-width-auto
button(
type="button",
data-feed-id= feed._id,
data-feed-title= feed.title,
onclick="return dtp.adminApp.removeNewsroomFeed(event);",
).uk-button.uk-button-danger.uk-border-rounded Remove Feed
.uk-width-auto
button(type="submit").uk-button.uk-button-primary.uk-border-rounded= feed ? 'Update Feed' : 'Add Feed'

30
app/views/components/library.pug

@ -10,6 +10,30 @@ include section-title
return (value < 1000) ? numeral(value).format('0,0') : numeral(value).format('0,0.0a');
}
function formatBitRate (value) {
if ((value !== 0) && !value) { return '---'; }
return numeral(value).format('0,0.0ib').slice(0, -2) + 'bps';
}
function formatDataSize (value) {
if ((value !== 0) && !value) { return '---'; }
return numeral(value).format('0,0.0b');
}
function formatTimestamp (value, withFractional = false) {
if (value !== 0 && !value) {
return withFractional ? '-:--:--.---' : '-:--:--';
}
return numeral(value).format(withFractional ? 'h:mm:ss.sss' : 'h:mm:ss');
}
function formatDuration (value) {
let duration = formatTimestamp(value);
if (duration.startsWith('0:')) { duration = duration.slice(2); }
if (duration.startsWith('0')) { duration = duration.slice(1); }
return duration;
}
function displayIntegerValue (value) {
return numeral(value).format(value > 1000 ? '0,0.0a' : '0,0');
}
@ -26,11 +50,13 @@ mixin renderCell (label, value, className)
div(class=className)!= value
.uk-text-muted.uk-text-small= label
mixin renderBackButton ( )
mixin renderBackButton (options)
- options = Object.assign({ includeLabel: true, label: 'Back' }, options)
button(type="button", onclick="window.history.back();").uk-button.uk-button-default
span
i.fas.fa-chevron-left
span(class="uk-visible@s").uk-margin-small-left Back
if options.includeLabel
span(class="uk-visible@s").uk-margin-small-left= options.label
mixin renderUserLink (user)
if user.coreUserId

33
app/views/components/page-sidebar.pug

@ -1,4 +1,5 @@
include ../announcement/components/announcement
include ../newsroom/components/feed-entry-list-item
mixin renderPageSidebar ( )
if Array.isArray(announcements) && (announcements.length > 0)
@ -8,17 +9,18 @@ mixin renderPageSidebar ( )
+renderAnnouncement(announcement)
if Array.isArray(featuredDestinations) && (featuredDestinations.length >0)
+renderSectionTitle('Featured Destinations', {
label: 'see all',
title: 'Select a destination and be present',
url: '/destination',
})
.sidebar-widget
ul.uk-list.uk-list-divider
each destination in featuredDestinations
li
a(href=`http://${destination.site.domain}`).uk-display-block.uk-link-reset
+renderDestinationListItem(destination)
.uk-margin
+renderSectionTitle('Featured Destinations', {
label: 'see all',
title: 'Select a destination and be present',
url: '/destination',
})
.sidebar-widget
ul.uk-list.uk-list-divider
each destination in featuredDestinations
li
a(href=`http://${destination.site.domain}`).uk-display-block.uk-link-reset
+renderDestinationListItem(destination)
.uk-margin
+renderSectionTitle('DTP Newsfeed', {
@ -31,11 +33,4 @@ mixin renderPageSidebar ( )
ul.uk-list
each entry in newsfeed.entries
li
div
a(href= entry.link, target="_blank")= entry.title
.uk-text-small
div(uk-grid).uk-grid-small
.uk-width-expand
a(href= entry.feed.link, target="_blank").uk-link-reset= entry.feed.title
.uk-width-auto
div= moment(entry.published).fromNow()
+renderNewsroomFeedEntryListItem(entry)

2
app/views/components/social-card/facebook.pug

@ -2,7 +2,7 @@ block facebook-card
meta(property='og:site_name', content= site.name)
meta(property='og:type', content='website')
meta(property='og:image', content= `https://${site.domain}/img/social-cards/${site.domainKey}.png?v=${pkg.version}`)
meta(property='og:url', content= `https://${site.domain}${dtp.request.url}`)
meta(property='og:url', content= `https://${site.domain}${request.url}`)
meta(property='og:title', content= pageTitle || site.name)
meta(property='og:description', content= pageDescription || site.description)
meta(property='og:image:alt', content= `${site.name} | ${site.description}`)

9
app/views/newsroom/components/feed-entry-list-item.pug

@ -0,0 +1,9 @@
mixin renderNewsroomFeedEntryListItem (entry)
.uk-text-bold
a(href= entry.link, target="_blank").uk-link-reset= entry.title
.uk-text-small
div(uk-grid).uk-grid-small
.uk-width-expand
a(href= entry.feed.link, target="_blank").uk-link-reset= entry.feed.title
.uk-width-auto
div= moment(entry.published).fromNow()

32
app/views/newsroom/index.pug

@ -8,12 +8,30 @@ block content
if Array.isArray(newsroom.feeds) && (newsroom.feeds.length > 0)
div(uk-grid).uk-grid-match
each feed in newsroom.feeds
.uk-width-1-3
.uk-tile.uk-tile-secondary.uk-padding-small.uk-border-rounded
.uk-text-bold
a(href=`/newsroom/${feed._id}`)= feed.title
.uk-text-small.uk-text-muted
div last update #{moment(feed.published).fromNow()}
div= feed.description
div(class="uk-width-1-1 uk-width-1-2@s uk-width-1-3@m uk-width-1-4@l")
.uk-card.uk-card-secondary.uk-card-small.uk-border-rounded
.uk-card-header(style="border-bottom: solid 1px #808080;")
div(uk-grid).uk-grid-small.uk-flex-middle
.uk-width-expand
h2.uk-card-title.uk-margin-remove
a(href=`/newsroom/${feed._id}`, uk-tooltip=`See all for ${feed.title}`)= feed.title
.uk-width-auto
a(href=`/newsroom/${feed._id}`, uk-tooltip=`See all for ${feed.title}`).uk-text-small ALL
.uk-text-small.uk-text-muted
div last update #{moment(feed.published).fromNow()}
//- div= feed.description
.uk-card-body
if Array.isArray(feed.recent.entries) && (feed.recent.entries.length > 0)
ul.uk-list.uk-list-divider
each entry in feed.recent.entries
li
.uk-text-truncate
a(href= entry.link, uk-tooltip= entry.title, target="_blank").uk-link-reset= entry.title
.uk-article-meta= moment(entry.published).fromNow()
else
div No recent posts
else
div There are no configured news feeds.

22
app/workers/newsroom.js

@ -8,7 +8,11 @@ const path = require('path');
require('dotenv').config({ path: path.resolve(__dirname, '..', '..', '.env') });
const mongoose = require('mongoose');
const { read: feedReader } = require('feed-reader');
const {
SiteAsync,
SiteLog,
SiteWorker,
} = require(path.join(__dirname, '..', '..', 'lib', 'site-lib'));
@ -35,6 +39,7 @@ class NewsroomWorker extends SiteWorker {
await super.start();
await this.loadProcessor(path.join(__dirname, 'newsroom', 'cron', 'update-feeds.js'));
await this.loadProcessor(path.join(__dirname, 'newsroom', 'job', 'update-feed.js'));
await this.startProcessors();
}
@ -42,6 +47,23 @@ class NewsroomWorker extends SiteWorker {
async stop ( ) {
await super.stop();
}
async updateFeed (feed) {
const Feed = mongoose.model('Feed');
const NOW = new Date();
const { feed: feedService } = this.dtp.services;
try {
this.log.info('loading latest feed data', { feedId: feed._id, title: feed.title });
const response = await feedReader(feed.url);
await SiteAsync.each(response.entries, async (entry) => {
await Feed.updateOne({ _id: feed._id }, { $set: { published: feed.published || NOW }});
await feedService.createEntry(feed, entry);
}, 4);
this.log.info('feed updated', { entries: response.entries.length });
} catch (error) {
this.log.error('failed to update feed', { feedId: feed._id, title: feed.title, error });
}
}
}
(async ( ) => {

20
app/workers/newsroom/cron/update-feeds.js

@ -10,8 +10,6 @@ const mongoose = require('mongoose');
const Feed = mongoose.model('Feed');
const { CronJob } = require('cron');
const { read: feedReader } = require('feed-reader');
const { SiteAsync } = require('../../../../lib/site-lib');
const { SiteWorkerProcess } = require(path.join(__dirname, '..', '..', '..', '..', 'lib', 'site-lib'));
@ -58,28 +56,12 @@ class UpdateFeedsCron extends SiteWorkerProcess {
.lean()
.cursor()
.eachAsync(async (feed) => {
await this.updateFeed(feed);
await this.worker.updateFeed(feed);
}, 4);
} catch (error) {
this.log.error('failed to update feeds', { error });
}
}
async updateFeed (feed) {
const NOW = new Date();
const { feed: feedService } = this.dtp.services;
try {
this.log.info('loading latest feed data', { feedId: feed._id, title: feed.title });
const response = await feedReader(feed.url);
await SiteAsync.each(response.entries, async (entry) => {
await Feed.updateOne({ _id: feed._id }, { $set: { published: feed.published || NOW }});
await feedService.createEntry(feed, entry);
}, 4);
this.log.info('feed updated', { entries: response.entries.length });
} catch (error) {
this.log.error('failed to update feed', { feedId: feed._id, title: feed.title, error });
}
}
}
module.exports = UpdateFeedsCron;

52
app/workers/newsroom/job/update-feed.js

@ -0,0 +1,52 @@
// newsroom/job/update-feed.js
// Copyright (C) 2022 DTP Technologies, LLC
// License: Apache-2.0
'use strict';
const path = require('path');
const { SiteWorkerProcess } = require(path.join(__dirname, '..', '..', '..', '..', 'lib', 'site-lib'));
class UpdateFeedJob extends SiteWorkerProcess {
static get COMPONENT ( ) {
return {
name: 'newsroomUpdateFeedJob',
slug: 'newsroom-update-feed-job',
};
}
constructor (worker) {
super(worker, UpdateFeedJob.COMPONENT);
}
async start ( ) {
await super.start();
this.queue = await this.getJobQueue('newsroom', this.dtp.config.jobQueues.newsroom);
this.log.info('registering job processor', { queue: this.queue.name, name: 'update-feed' });
this.queue.process('update-feed', this.processUpdateFeed.bind(this));
}
async stop ( ) {
await super.stop();
}
async processUpdateFeed (job) {
const { feed: feedService } = this.dtp.services;
const { feedId } = job.data;
this.log.info('newsroom feed update job received', { id: job.id, feedId });
try {
const feed = await feedService.getById(feedId);
await this.worker.updateFeed(feed);
} catch (error) {
this.log.error('failed to update newsroom feed', { feedId, error });
throw error;
}
}
}
module.exports = UpdateFeedJob;

27
client/js/site-admin-app.js

@ -367,6 +367,33 @@ export default class DtpSiteAdminHostStatsApp extends DtpApp {
return false;
}
async removeNewsroomFeed (event) {
event.preventDefault();
event.stopPropagation();
const target = event.currentTarget || event.target;
const feedId = target.getAttribute('data-feed-id');
const feedTitle = target.getAttribute('data-feed-title');
try {
await UIkit.modal.confirm(`Are you sure you want to remove feed "${feedTitle}"?`);
} catch (error) {
// canceled
return false;
}
try {
const response = await fetch(`/admin/newsroom/${feedId}`, { method: 'DELETE' });
await this.processResponse(response);
} catch (error) {
UIkit.modal.alert(`Failed to remove feed: ${error.message}`);
}
return false;
}
}
dtp.DtpSiteAdminHostStatsApp = DtpSiteAdminHostStatsApp;

4
config/job-queues.js

@ -17,6 +17,10 @@ module.exports = {
attempts: 3,
removeOnComplete: true,
},
'newsroom': {
attempts: 3,
removeOnComplete: true,
},
'reeeper': {
attempts: 3,
removeOnComplete: true,

6
config/limiter.js

@ -117,6 +117,9 @@ module.exports = {
},
},
/*
* CommentController
*/
comment: {
deleteComment: {
total: 1,
@ -305,6 +308,9 @@ module.exports = {
},
},
/*
* WelcomeController
*/
welcome: {
total: 12,
expire: ONE_MINUTE,

34
lib/site-platform.js

@ -342,9 +342,7 @@ module.exports.startWebServer = async (dtp) => {
module.app.use(async (req, res, next) => {
const { cache: cacheService } = dtp.services;
try {
res.locals.dtp = {
request: req,
};
res.locals.request = req;
const settingsKey = `settings:${dtp.config.site.domainKey}:site`;
res.locals.site = Object.assign({ }, dtp.config.site);
@ -360,6 +358,16 @@ module.exports.startWebServer = async (dtp) => {
}
});
/*
* Call out to application to register their custom middleware at the right
* point in the processing chain.
*/
module.log.debug('typeof dtp.config.registerMiddleware', { type: (typeof dtp.config.registerMiddleware) });
if (dtp.config && (typeof dtp.config.registerMiddleware === 'function')) {
module.log.info('registering custom application middleware');
await dtp.config.registerMiddleware(dtp, module.app);
}
/*
* System Init
*/
@ -383,12 +391,7 @@ module.exports.startWebServer = async (dtp) => {
if (process.env.HTTP_ENABLE === 'enabled') {
if (process.env.HTTP_REDIRECT_SSL === 'enabled') {
module.log.info('creating HTTP SSL redirect app');
module.redirectApp = express();
module.redirectApp.use((req, res) => {
module.log.info('redirecting to SSL', { host: req.host, url: req.url });
res.redirect(`https://${process.env.DTP_SITE_DOMAIN}${req.url}`);
});
await module.createSslRedirectApp(dtp);
await module.createHttpServer(dtp, module.redirectApp);
} else {
await module.createHttpServer(dtp, module.app);
@ -435,6 +438,19 @@ module.createHttpsServer = async (dtp, app) => {
return module.https;
};
module.createSslRedirectApp = async (/* dtp */) => {
module.log.info('creating HTTP SSL redirect app');
module.redirectApp = express();
module.redirectApp.use((req, res) => {
module.log.info('redirecting to SSL', { host: req.host, url: req.url });
res.redirect(`https://${process.env.DTP_SITE_DOMAIN}${req.url}`);
});
return module.redirectApp;
};
module.startHttpServer = async (dtp, server, config) => {
return new Promise((resolve, reject) => {
module.log.info('starting HTTP server', { port: config.port, bind: config.address });

3
restart-production

@ -0,0 +1,3 @@
#!/bin/bash
./stop-production
./start-production

4
start-local

@ -11,6 +11,8 @@ forever start --killSignal=SIGINT app/workers/host-services.js
forever start --killSignal=SIGINT app/workers/reeeper.js
forever start --killSignal=SIGINT app/workers/newsletter.js
forever start --killSignal=SIGINT app/workers/newsroom.js
forever start --killSignal=SIGINT app/workers/media.js
forever start --killSignal=SIGINT app/workers/chat.js
@ -18,6 +20,8 @@ minio server ./data/minio --address ":9000" --console-address ":9001"
forever stop app/workers/chat.js
forever stop app/workers/media.js
forever stop app/workers/newsroom.js
forever stop app/workers/newsletter.js
forever stop app/workers/reeeper.js

14
start-production

@ -1,8 +1,10 @@
#!/bin/bash
sudo supervisorctl start core-host-services:*
sudo supervisorctl start core-reeeper:*
sudo supervisorctl start core-newsletter:*
sudo supervisorctl start core-media:*
sudo supervisorctl start core-chat:*
sudo supervisorctl start core:*
sudo supervisorctl start \
core-host-services:* \
core-reeeper:* \
core-newsletter:* \
core-newsroom:* \
core-media:* \
core-chat:* \
core:*

14
stop-production

@ -1,8 +1,10 @@
#!/bin/bash
sudo supervisorctl stop core:*
sudo supervisorctl stop core-chat:*
sudo supervisorctl stop core-media:*
sudo supervisorctl stop core-newsletter:*
sudo supervisorctl stop core-reeeper:*
sudo supervisorctl stop core-host-services:*
sudo supervisorctl stop \
core:* \
core-chat:* \
core-media:* \
core-newsroom:* \
core-newsletter:* \
core-reeeper:* \
core-host-services:*

13
supervisord/dtp-base-newsroom.conf

@ -0,0 +1,13 @@
[program:dtp-base:newsroom]
numprocs=1
process_name=%(program_name)s_%(process_num)02d
command=/home/dtp/.nvm/versions/node/v16.13.0/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/newsroom.js
directory=/home/dtp/live/dtp-base
autostart=true
autorestart=true
startretries=3
stopsignal=INT
stderr_logfile=/var/log/dtp-base/newsroom.err.log
stdout_logfile=/var/log/dtp-base/newsroom.out.log
user=dtp
environment=HOME='/home/dtp/live/dtp-base',HTTP_BIND_PORT=30%(process_num)02d,NODE_ENV=production,LOGNAME=newsroom
Loading…
Cancel
Save