From 18f49c842a7ab0144f26773b8d9f6e6b7fe81256 Mon Sep 17 00:00:00 2001 From: rob Date: Sun, 23 Oct 2022 11:26:03 -0400 Subject: [PATCH 01/23] factoring evolution down from Sites to Base --- .vscode/launch.json | 1 + app/views/components/library.pug | 24 +++++++++++++ app/views/components/page-sidebar.pug | 10 ++---- app/views/components/social-card/facebook.pug | 2 +- .../components/feed-entry-list-item.pug | 9 +++++ config/limiter.js | 6 ++++ lib/site-platform.js | 34 ++++++++++++++----- 7 files changed, 68 insertions(+), 18 deletions(-) create mode 100644 app/views/newsroom/components/feed-entry-list-item.pug diff --git a/.vscode/launch.json b/.vscode/launch.json index 7fa2197..ff2e38a 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -15,6 +15,7 @@ "console": "integratedTerminal", "env": { "HTTP_BIND_PORT": "3010", + "HTTPS_ENABLE": "enabled", "HTTPS_BIND_PORT": "3410", } }, diff --git a/app/views/components/library.pug b/app/views/components/library.pug index d6ca080..20eb6b0 100644 --- a/app/views/components/library.pug +++ b/app/views/components/library.pug @@ -10,6 +10,30 @@ include section-title return (value < 1000) ? numeral(value).format('0,0') : numeral(value).format('0,0.0a'); } + function formatBitRate (value) { + if ((value !== 0) && !value) { return '---'; } + return numeral(value).format('0,0.0ib').slice(0, -2) + 'bps'; + } + + function formatDataSize (value) { + if ((value !== 0) && !value) { return '---'; } + return numeral(value).format('0,0.0b'); + } + + function formatTimestamp (value, withFractional = false) { + if (value !== 0 && !value) { + return withFractional ? '-:--:--.---' : '-:--:--'; + } + return numeral(value).format(withFractional ? 'h:mm:ss.sss' : 'h:mm:ss'); + } + + function formatDuration (value) { + let duration = formatTimestamp(value); + if (duration.startsWith('0:')) { duration = duration.slice(2); } + if (duration.startsWith('0')) { duration = duration.slice(1); } + return duration; + } + function displayIntegerValue (value) { return numeral(value).format(value > 1000 ? '0,0.0a' : '0,0'); } diff --git a/app/views/components/page-sidebar.pug b/app/views/components/page-sidebar.pug index 85b9383..623e0d1 100644 --- a/app/views/components/page-sidebar.pug +++ b/app/views/components/page-sidebar.pug @@ -1,4 +1,5 @@ include ../announcement/components/announcement +include ../newsroom/components/feed-entry-list-item mixin renderPageSidebar ( ) if Array.isArray(announcements) && (announcements.length > 0) @@ -17,14 +18,7 @@ mixin renderPageSidebar ( ) ul.uk-list each entry in newsfeed.entries li - div - a(href= entry.link, target="_blank").uk-link-reset= entry.title - .uk-text-small - div(uk-grid).uk-grid-small - .uk-width-expand - a(href= entry.feed.link, target="_blank").uk-link-reset= entry.feed.title - .uk-width-auto - div= moment(entry.published).fromNow() + +renderNewsroomFeedEntryListItem(entry) .uk-margin +renderSectionTitle('Widget', { diff --git a/app/views/components/social-card/facebook.pug b/app/views/components/social-card/facebook.pug index 20cb5fa..d3a7432 100644 --- a/app/views/components/social-card/facebook.pug +++ b/app/views/components/social-card/facebook.pug @@ -2,7 +2,7 @@ block facebook-card meta(property='og:site_name', content= site.name) meta(property='og:type', content='website') meta(property='og:image', content= `https://${site.domain}/img/social-cards/${site.domainKey}.png?v=${pkg.version}`) - meta(property='og:url', content= `https://${site.domain}${dtp.request.url}`) + meta(property='og:url', content= `https://${site.domain}${request.url}`) meta(property='og:title', content= pageTitle || site.name) meta(property='og:description', content= pageDescription || site.description) meta(property='og:image:alt', content= `${site.name} | ${site.description}`) diff --git a/app/views/newsroom/components/feed-entry-list-item.pug b/app/views/newsroom/components/feed-entry-list-item.pug new file mode 100644 index 0000000..da32d31 --- /dev/null +++ b/app/views/newsroom/components/feed-entry-list-item.pug @@ -0,0 +1,9 @@ +mixin renderNewsroomFeedEntryListItem (entry) + .uk-text-bold + a(href= entry.link, target="_blank").uk-link-reset= entry.title + .uk-text-small + div(uk-grid).uk-grid-small + .uk-width-expand + a(href= entry.feed.link, target="_blank").uk-link-reset= entry.feed.title + .uk-width-auto + div= moment(entry.published).fromNow() \ No newline at end of file diff --git a/config/limiter.js b/config/limiter.js index 63397cf..e49b258 100644 --- a/config/limiter.js +++ b/config/limiter.js @@ -117,6 +117,9 @@ module.exports = { }, }, + /* + * CommentController + */ comment: { deleteComment: { total: 1, @@ -305,6 +308,9 @@ module.exports = { }, }, + /* + * WelcomeController + */ welcome: { total: 12, expire: ONE_MINUTE, diff --git a/lib/site-platform.js b/lib/site-platform.js index 9080985..5d628a7 100644 --- a/lib/site-platform.js +++ b/lib/site-platform.js @@ -341,9 +341,7 @@ module.exports.startWebServer = async (dtp) => { module.app.use(async (req, res, next) => { const { cache: cacheService } = dtp.services; try { - res.locals.dtp = { - request: req, - }; + res.locals.request = req; const settingsKey = `settings:${dtp.config.site.domainKey}:site`; res.locals.site = Object.assign({ }, dtp.config.site); @@ -359,6 +357,16 @@ module.exports.startWebServer = async (dtp) => { } }); + /* + * Call out to application to register their custom middleware at the right + * point in the processing chain. + */ + module.log.debug('typeof dtp.config.registerMiddleware', { type: (typeof dtp.config.registerMiddleware) }); + if (dtp.config && (typeof dtp.config.registerMiddleware === 'function')) { + module.log.info('registering custom application middleware'); + await dtp.config.registerMiddleware(dtp, module.app); + } + /* * System Init */ @@ -382,12 +390,7 @@ module.exports.startWebServer = async (dtp) => { if (process.env.HTTP_ENABLE === 'enabled') { if (process.env.HTTP_REDIRECT_SSL === 'enabled') { - module.log.info('creating HTTP SSL redirect app'); - module.redirectApp = express(); - module.redirectApp.use((req, res) => { - module.log.info('redirecting to SSL', { host: req.host, url: req.url }); - res.redirect(`https://${process.env.DTP_SITE_DOMAIN}${req.url}`); - }); + await module.createSslRedirectApp(dtp); await module.createHttpServer(dtp, module.redirectApp); } else { await module.createHttpServer(dtp, module.app); @@ -434,6 +437,19 @@ module.createHttpsServer = async (dtp, app) => { return module.https; }; +module.createSslRedirectApp = async (/* dtp */) => { + module.log.info('creating HTTP SSL redirect app'); + + module.redirectApp = express(); + + module.redirectApp.use((req, res) => { + module.log.info('redirecting to SSL', { host: req.host, url: req.url }); + res.redirect(`https://${process.env.DTP_SITE_DOMAIN}${req.url}`); + }); + + return module.redirectApp; +}; + module.startHttpServer = async (dtp, server, config) => { return new Promise((resolve, reject) => { module.log.info('starting HTTP server', { port: config.port, bind: config.address }); From 94fdb7282adb67d2a0d727ea1b96b9cf0849fc67 Mon Sep 17 00:00:00 2001 From: rob Date: Sun, 23 Oct 2022 11:26:24 -0400 Subject: [PATCH 02/23] v0.4.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9a0cb9a..940e7a8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dtp-base", - "version": "0.4.1", + "version": "0.4.2", "description": "Open source web app engine for the Digital Telepresence Platform.", "main": "dtp-webapp.js", "author": "DTP Technologies, LLC", From b598be48686c9e84b470f02eafe8080d6d94e640 Mon Sep 17 00:00:00 2001 From: rob Date: Sun, 23 Oct 2022 11:40:13 -0400 Subject: [PATCH 03/23] add newsroom worker to start-local and supervisord --- start-local | 4 ++++ supervisord/dtp-base-newsroom.conf | 13 +++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 supervisord/dtp-base-newsroom.conf diff --git a/start-local b/start-local index dde9911..7a87f10 100755 --- a/start-local +++ b/start-local @@ -11,6 +11,8 @@ forever start --killSignal=SIGINT app/workers/host-services.js forever start --killSignal=SIGINT app/workers/reeeper.js forever start --killSignal=SIGINT app/workers/newsletter.js +forever start --killSignal=SIGINT app/workers/newsroom.js + forever start --killSignal=SIGINT app/workers/media.js forever start --killSignal=SIGINT app/workers/chat.js @@ -18,6 +20,8 @@ minio server ./data/minio --address ":9010" --console-address ":9011" forever stop app/workers/chat.js forever stop app/workers/media.js + +forever stop app/workers/newsroom.js forever stop app/workers/newsletter.js forever stop app/workers/reeeper.js diff --git a/supervisord/dtp-base-newsroom.conf b/supervisord/dtp-base-newsroom.conf new file mode 100644 index 0000000..50b4b8c --- /dev/null +++ b/supervisord/dtp-base-newsroom.conf @@ -0,0 +1,13 @@ +[program:dtp-base:newsroom] +numprocs=1 +process_name=%(program_name)s_%(process_num)02d +command=/home/dtp/.nvm/versions/node/v16.13.0/bin/node --optimize_for_size --max_old_space_size=1024 --gc_interval=100 app/workers/newsroom.js +directory=/home/dtp/live/dtp-base +autostart=true +autorestart=true +startretries=3 +stopsignal=INT +stderr_logfile=/var/log/dtp-base/newsroom.err.log +stdout_logfile=/var/log/dtp-base/newsroom.out.log +user=dtp +environment=HOME='/home/dtp/live/dtp-base',HTTP_BIND_PORT=30%(process_num)02d,NODE_ENV=production,LOGNAME=newsroom \ No newline at end of file From a9f0fe52833b9f45d0f11ec943f6d2525261aaf2 Mon Sep 17 00:00:00 2001 From: rob Date: Sun, 23 Oct 2022 11:40:20 -0400 Subject: [PATCH 04/23] v0.4.3 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 940e7a8..59dd2a8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dtp-base", - "version": "0.4.2", + "version": "0.4.3", "description": "Open source web app engine for the Digital Telepresence Platform.", "main": "dtp-webapp.js", "author": "DTP Technologies, LLC", From 767fad47c4675f940600f2f8fad8adeb133eba61 Mon Sep 17 00:00:00 2001 From: rob Date: Sun, 23 Oct 2022 11:44:56 -0400 Subject: [PATCH 05/23] put newsroom feed in .sidebar-widget content class --- app/views/components/page-sidebar.pug | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/app/views/components/page-sidebar.pug b/app/views/components/page-sidebar.pug index 623e0d1..537d622 100644 --- a/app/views/components/page-sidebar.pug +++ b/app/views/components/page-sidebar.pug @@ -14,11 +14,12 @@ mixin renderPageSidebar ( ) title: 'Browse all news feeds', url: '/newsroom', }) - if Array.isArray(newsfeed.entries) && (newsfeed.entries.length > 0) - ul.uk-list - each entry in newsfeed.entries - li - +renderNewsroomFeedEntryListItem(entry) + .sidebar-widget + if Array.isArray(newsfeed.entries) && (newsfeed.entries.length > 0) + ul.uk-list + each entry in newsfeed.entries + li + +renderNewsroomFeedEntryListItem(entry) .uk-margin +renderSectionTitle('Widget', { From 6283264c305e3b7177b9710350d7c9d731cadcc8 Mon Sep 17 00:00:00 2001 From: rob Date: Sun, 23 Oct 2022 11:45:04 -0400 Subject: [PATCH 06/23] v0.4.4 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 59dd2a8..3947f67 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dtp-base", - "version": "0.4.3", + "version": "0.4.4", "description": "Open source web app engine for the Digital Telepresence Platform.", "main": "dtp-webapp.js", "author": "DTP Technologies, LLC", From a364597d5399133e632e04eeea1c4e19daf8b8eb Mon Sep 17 00:00:00 2001 From: rob Date: Sun, 23 Oct 2022 12:20:36 -0400 Subject: [PATCH 07/23] add error message for captcha generation --- app/controllers/welcome.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/controllers/welcome.js b/app/controllers/welcome.js index 8fcc5f1..b6bdef3 100644 --- a/app/controllers/welcome.js +++ b/app/controllers/welcome.js @@ -43,6 +43,9 @@ class WelcomeController extends SiteController { } async getSignupCaptcha (req, res) { + if (!req.session || !req.session.captcha || !req.session.captcha.signup) { + return res.status(500).end('Session is not in a valid state for generating a captcha image'); + } const signupCaptcha = captcha(req.session.captcha.signup, { color: false, noise: 3, From a32c140c3ae197e98f33aa0e1c5fcf4b08817585 Mon Sep 17 00:00:00 2001 From: rob Date: Sun, 23 Oct 2022 12:20:45 -0400 Subject: [PATCH 08/23] v0.4.5 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 3947f67..5c91317 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dtp-base", - "version": "0.4.4", + "version": "0.4.5", "description": "Open source web app engine for the Digital Telepresence Platform.", "main": "dtp-webapp.js", "author": "DTP Technologies, LLC", From 2b359c5b3ee4ea2537546e650633dfd17f24e727 Mon Sep 17 00:00:00 2001 From: rob Date: Sun, 30 Oct 2022 22:03:24 -0400 Subject: [PATCH 09/23] environment control scripts updated --- restart-production | 3 +++ start-production | 18 ++++++++---------- stop-production | 18 ++++++++---------- 3 files changed, 19 insertions(+), 20 deletions(-) create mode 100644 restart-production diff --git a/restart-production b/restart-production new file mode 100644 index 0000000..51076c2 --- /dev/null +++ b/restart-production @@ -0,0 +1,3 @@ +#!/bin/bash +./stop-production +./start-production \ No newline at end of file diff --git a/start-production b/start-production index 965260a..e876e83 100755 --- a/start-production +++ b/start-production @@ -1,12 +1,10 @@ #!/bin/bash -# -# WORKERS -# -forever start --killSignal=SIGINT app/workers/reeeper.js -forever start --killSignal=SIGINT app/workers/host-services.js - -# -# APP HOSTS -# -forever start --killSignal=SIGINT dtp-libertylinks.js \ No newline at end of file +sudo supervisorctl start \ + base-host-services:* \ + base-reeeper:* \ + base-newsletter:* \ + base-newsroom:* \ + base-media:* \ + base-chat:* \ + base:* \ No newline at end of file diff --git a/stop-production b/stop-production index 692e5b2..12b4516 100755 --- a/stop-production +++ b/stop-production @@ -1,12 +1,10 @@ #!/bin/bash -# -# APP HOSTS -# -forever stop dtp-libertylinks.js - -# -# WORKERS -# -forever stop app/workers/host-services.js -forever stop app/workers/reeeper.js \ No newline at end of file +sudo supervisorctl stop \ + base:* + base-chat:* \ + base-media:* \ + base-newsroom:* \ + base-newsletter:* \ + base-reeeper:* \ + base-host-services:* \ \ No newline at end of file From a13e722db349334c95313dba5d8ba28b571834b8 Mon Sep 17 00:00:00 2001 From: rob Date: Sun, 30 Oct 2022 22:03:38 -0400 Subject: [PATCH 10/23] v0.4.6 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5c91317..4125475 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dtp-base", - "version": "0.4.5", + "version": "0.4.6", "description": "Open source web app engine for the Digital Telepresence Platform.", "main": "dtp-webapp.js", "author": "DTP Technologies, LLC", From 40ab2d2dfe9d373769fcd71a619c0bfc9705a52f Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 31 Oct 2022 04:12:56 -0400 Subject: [PATCH 11/23] newsroom updates --- app/controllers/admin/newsroom.js | 2 +- app/controllers/newsroom.js | 2 +- app/services/feed.js | 23 ++++++++++++++++++++-- app/views/admin/newsroom/editor.pug | 15 +++++++++++---- app/views/newsroom/index.pug | 30 +++++++++++++++++++++++------ client/js/site-admin-app.js | 27 ++++++++++++++++++++++++++ 6 files changed, 85 insertions(+), 14 deletions(-) diff --git a/app/controllers/admin/newsroom.js b/app/controllers/admin/newsroom.js index dea2a9e..d8cb8ee 100644 --- a/app/controllers/admin/newsroom.js +++ b/app/controllers/admin/newsroom.js @@ -115,7 +115,7 @@ class NewsroomAdminController extends SiteController { const { feed: feedService } = this.dtp.services; try { res.locals.feed = await feedService.create(req.body); - res.redirect(`/admin/newsroom/${res.locals.feed._id}`); + res.redirect('/admin/newsroom'); } catch (error) { this.log.error('failed to create feed', { error }); return next(error); diff --git a/app/controllers/newsroom.js b/app/controllers/newsroom.js index fb47135..265e018 100644 --- a/app/controllers/newsroom.js +++ b/app/controllers/newsroom.js @@ -69,7 +69,7 @@ class NewsroomController extends SiteController { const { feed: feedService } = this.dtp.services; try { res.locals.pagination = this.getPaginationParameters(req, 10); - res.locals.newsroom = await feedService.getFeeds(res.locals.pagination); + res.locals.newsroom = await feedService.getFeeds(res.locals.pagination, { withEntries: true }); res.render('newsroom/index'); } catch (error) { this.log.error('failed to present newsroom home', { error }); diff --git a/app/services/feed.js b/app/services/feed.js index f1b3f88..4f9a2e0 100644 --- a/app/services/feed.js +++ b/app/services/feed.js @@ -8,7 +8,7 @@ const mongoose = require('mongoose'); const Feed = mongoose.model('Feed'); const FeedEntry = mongoose.model('FeedEntry'); -const { SiteService, SiteError } = require('../../lib/site-lib'); +const { SiteService, SiteError, SiteAsync } = require('../../lib/site-lib'); const { read: feedReader } = require('feed-reader'); class FeedService extends SiteService { @@ -23,6 +23,10 @@ class FeedService extends SiteService { ]; } + async start ( ) { + this.jobQueue = this.getJobQueue('newsroom', this.dtp.config.jobQueues.newsroom); + } + async create (feedDefinition) { feedDefinition.url = feedDefinition.url.trim(); const feedContent = await this.load(feedDefinition.url); @@ -63,7 +67,8 @@ class FeedService extends SiteService { await Feed.updateOne({ _id: feed._id }, updateOp); } - async getFeeds (pagination) { + async getFeeds (pagination, options) { + options = Object.assign({ withEntries: false, entryCount: 3 }, options); pagination = Object.assign({ skip: 0, cpp: 10 }, pagination); const feeds = await Feed .find() @@ -71,7 +76,21 @@ class FeedService extends SiteService { .skip(pagination.skip) .limit(pagination.cpp) .lean(); + + if (options.withEntries) { + await SiteAsync.each(feeds, async (feed) => { + try { + feed.recent = await this.getFeedEntries(feed, { skip: 0, cpp: options.entryCount }); + this.log.debug('feed entries', { count: feed.recent.entries.length }); + } catch (error) { + this.log.error('failed to populate recent entries for feed', { feedId: feed._id, error }); + // fall through + } + }, 2); + } + const totalFeedCount = await Feed.countDocuments(); + return { feeds, totalFeedCount }; } diff --git a/app/views/admin/newsroom/editor.pug b/app/views/admin/newsroom/editor.pug index 06700b3..55c5eb2 100644 --- a/app/views/admin/newsroom/editor.pug +++ b/app/views/admin/newsroom/editor.pug @@ -33,7 +33,14 @@ block content textarea(id="description", name="description", rows="4", placeholder="Enter feed description").uk-textarea.uk-resize-vertical= feed ? feed.description : undefined .uk-card-footer - button(type="submit").uk-button.uk-button-primary.uk-border-rounded= feed ? 'Update Feed' : 'Add Feed' - - if feed - pre= JSON.stringify(feed, null, 2) \ No newline at end of file + div(uk-grid).uk-flex-right.uk-flex-middle + if feed + .uk-width-auto + button( + type="button", + data-feed-id= feed._id, + data-feed-title= feed.title, + onclick="return dtp.adminApp.removeNewsroomFeed(event);", + ).uk-button.uk-button-danger.uk-border-rounded Remove Feed + .uk-width-auto + button(type="submit").uk-button.uk-button-primary.uk-border-rounded= feed ? 'Update Feed' : 'Add Feed' \ No newline at end of file diff --git a/app/views/newsroom/index.pug b/app/views/newsroom/index.pug index 5119ff8..fe54702 100644 --- a/app/views/newsroom/index.pug +++ b/app/views/newsroom/index.pug @@ -9,11 +9,29 @@ block content div(uk-grid).uk-grid-match each feed in newsroom.feeds .uk-width-1-3 - .uk-tile.uk-tile-secondary.uk-padding-small.uk-border-rounded - .uk-text-bold - a(href=`/newsroom/${feed._id}`)= feed.title - .uk-text-small.uk-text-muted - div last update #{moment(feed.published).fromNow()} - div= feed.description + .uk-card.uk-card-secondary.uk-card-small.uk-border-rounded + .uk-card-header(style="border-bottom: solid 1px #808080;") + div(uk-grid).uk-grid-small.uk-flex-middle + .uk-width-expand + h2.uk-card-title.uk-margin-remove + a(href=`/newsroom/${feed._id}`, uk-tooltip=`See all for ${feed.title}`)= feed.title + .uk-width-auto + a(href=`/newsroom/${feed._id}`, uk-tooltip=`See all for ${feed.title}`).uk-text-small ALL + + .uk-text-small.uk-text-muted + div last update #{moment(feed.published).fromNow()} + //- div= feed.description + + .uk-card-body + if Array.isArray(feed.recent.entries) && (feed.recent.entries.length > 0) + ul.uk-list.uk-list-divider + each entry in feed.recent.entries + li + .uk-text-truncate + a(href= entry.link, uk-tooltip= entry.title, target="_blank").uk-link-reset= entry.title + .uk-article-meta= moment(entry.published).fromNow() + + else + div No recent posts else div There are no configured news feeds. \ No newline at end of file diff --git a/client/js/site-admin-app.js b/client/js/site-admin-app.js index 46c1910..a18aba2 100644 --- a/client/js/site-admin-app.js +++ b/client/js/site-admin-app.js @@ -367,6 +367,33 @@ export default class DtpSiteAdminHostStatsApp extends DtpApp { return false; } + + async removeNewsroomFeed (event) { + event.preventDefault(); + event.stopPropagation(); + + const target = event.currentTarget || event.target; + const feedId = target.getAttribute('data-feed-id'); + const feedTitle = target.getAttribute('data-feed-title'); + + try { + await UIkit.modal.confirm(`Are you sure you want to remove feed "${feedTitle}"?`); + } catch (error) { + // canceled + return false; + } + + try { + const response = await fetch(`/admin/newsroom/${feedId}`, { method: 'DELETE' }); + await this.processResponse(response); + } catch (error) { + UIkit.modal.alert(`Failed to remove feed: ${error.message}`); + } + + return false; + } + + } dtp.DtpSiteAdminHostStatsApp = DtpSiteAdminHostStatsApp; \ No newline at end of file From 19b888298a1f2e7995b5ad92f9d14553c7b0facb Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 31 Oct 2022 04:13:03 -0400 Subject: [PATCH 12/23] v0.4.7 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4125475..f5900c1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dtp-base", - "version": "0.4.6", + "version": "0.4.7", "description": "Open source web app engine for the Digital Telepresence Platform.", "main": "dtp-webapp.js", "author": "DTP Technologies, LLC", From dda92c8eec9a54f79e6e249135448e717ec3d167 Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 31 Oct 2022 04:34:21 -0400 Subject: [PATCH 13/23] add job to update feed after create or update --- app/controllers/admin/newsroom.js | 2 +- app/services/feed.js | 6 ++- app/workers/newsroom.js | 22 +++++++++ app/workers/newsroom/cron/update-feeds.js | 20 +------- app/workers/newsroom/job/update-feed.js | 56 +++++++++++++++++++++++ config/job-queues.js | 4 ++ 6 files changed, 89 insertions(+), 21 deletions(-) create mode 100644 app/workers/newsroom/job/update-feed.js diff --git a/app/controllers/admin/newsroom.js b/app/controllers/admin/newsroom.js index d8cb8ee..e2811e2 100644 --- a/app/controllers/admin/newsroom.js +++ b/app/controllers/admin/newsroom.js @@ -106,7 +106,7 @@ class NewsroomAdminController extends SiteController { await feedService.update(res.locals.feed, req.body); res.redirect('/admin/newsroom'); } catch (error) { - this.log.error('failed to create feed', { error }); + this.log.error('failed to update feed', { error }); return next(error); } } diff --git a/app/services/feed.js b/app/services/feed.js index 4f9a2e0..baefc5d 100644 --- a/app/services/feed.js +++ b/app/services/feed.js @@ -24,7 +24,7 @@ class FeedService extends SiteService { } async start ( ) { - this.jobQueue = this.getJobQueue('newsroom', this.dtp.config.jobQueues.newsroom); + this.jobQueue = await this.getJobQueue('newsroom', this.dtp.config.jobQueues.newsroom); } async create (feedDefinition) { @@ -44,6 +44,8 @@ class FeedService extends SiteService { feed.published = feedContent.published; await feed.save(); + this.jobQueue.add('update-feed', { feedId: feed._id }); + return feed.toObject(); } @@ -65,6 +67,8 @@ class FeedService extends SiteService { updateOp.$set.generator = feedDefinition.generator || feedContent.generator; await Feed.updateOne({ _id: feed._id }, updateOp); + + this.jobQueue.add('update-feed', { feedId: feed._id }); } async getFeeds (pagination, options) { diff --git a/app/workers/newsroom.js b/app/workers/newsroom.js index ad4dc02..6ca4fdc 100644 --- a/app/workers/newsroom.js +++ b/app/workers/newsroom.js @@ -8,7 +8,11 @@ const path = require('path'); require('dotenv').config({ path: path.resolve(__dirname, '..', '..', '.env') }); +const mongoose = require('mongoose'); +const { read: feedReader } = require('feed-reader'); + const { + SiteAsync, SiteLog, SiteWorker, } = require(path.join(__dirname, '..', '..', 'lib', 'site-lib')); @@ -35,6 +39,7 @@ class NewsroomWorker extends SiteWorker { await super.start(); await this.loadProcessor(path.join(__dirname, 'newsroom', 'cron', 'update-feeds.js')); + await this.loadProcessor(path.join(__dirname, 'newsroom', 'job', 'update-feed.js')); await this.startProcessors(); } @@ -42,6 +47,23 @@ class NewsroomWorker extends SiteWorker { async stop ( ) { await super.stop(); } + + async updateFeed (feed) { + const Feed = mongoose.model('Feed'); + const NOW = new Date(); + const { feed: feedService } = this.dtp.services; + try { + this.log.info('loading latest feed data', { feedId: feed._id, title: feed.title }); + const response = await feedReader(feed.url); + await SiteAsync.each(response.entries, async (entry) => { + await Feed.updateOne({ _id: feed._id }, { $set: { published: feed.published || NOW }}); + await feedService.createEntry(feed, entry); + }, 4); + this.log.info('feed updated', { entries: response.entries.length }); + } catch (error) { + this.log.error('failed to update feed', { feedId: feed._id, title: feed.title, error }); + } + } } (async ( ) => { diff --git a/app/workers/newsroom/cron/update-feeds.js b/app/workers/newsroom/cron/update-feeds.js index e2278da..b4d6b5e 100644 --- a/app/workers/newsroom/cron/update-feeds.js +++ b/app/workers/newsroom/cron/update-feeds.js @@ -10,8 +10,6 @@ const mongoose = require('mongoose'); const Feed = mongoose.model('Feed'); const { CronJob } = require('cron'); -const { read: feedReader } = require('feed-reader'); -const { SiteAsync } = require('../../../../lib/site-lib'); const { SiteWorkerProcess } = require(path.join(__dirname, '..', '..', '..', '..', 'lib', 'site-lib')); @@ -58,28 +56,12 @@ class UpdateFeedsCron extends SiteWorkerProcess { .lean() .cursor() .eachAsync(async (feed) => { - await this.updateFeed(feed); + await this.worker.updateFeed(feed); }, 4); } catch (error) { this.log.error('failed to update feeds', { error }); } } - - async updateFeed (feed) { - const NOW = new Date(); - const { feed: feedService } = this.dtp.services; - try { - this.log.info('loading latest feed data', { feedId: feed._id, title: feed.title }); - const response = await feedReader(feed.url); - await SiteAsync.each(response.entries, async (entry) => { - await Feed.updateOne({ _id: feed._id }, { $set: { published: feed.published || NOW }}); - await feedService.createEntry(feed, entry); - }, 4); - this.log.info('feed updated', { entries: response.entries.length }); - } catch (error) { - this.log.error('failed to update feed', { feedId: feed._id, title: feed.title, error }); - } - } } module.exports = UpdateFeedsCron; \ No newline at end of file diff --git a/app/workers/newsroom/job/update-feed.js b/app/workers/newsroom/job/update-feed.js new file mode 100644 index 0000000..8200527 --- /dev/null +++ b/app/workers/newsroom/job/update-feed.js @@ -0,0 +1,56 @@ +// newsroom/job/update-feed.js +// Copyright (C) 2022 DTP Technologies, LLC +// License: Apache-2.0 + +'use strict'; + +const path = require('path'); + +const mongoose = require('mongoose'); + +const Feed = mongoose.model('Feed'); + +const { SiteWorkerProcess } = require(path.join(__dirname, '..', '..', '..', '..', 'lib', 'site-lib')); + +class UpdateFeedJob extends SiteWorkerProcess { + + static get COMPONENT ( ) { + return { + name: 'newsroomUpdateFeedJob', + slug: 'newsroom-update-feed-job', + }; + } + + constructor (worker) { + super(worker, UpdateFeedJob.COMPONENT); + } + + async start ( ) { + await super.start(); + + this.queue = await this.getJobQueue('newsroom', this.dtp.config.jobQueues.newsroom); + + this.log.info('registering job processor', { queue: this.queue.name, name: 'update-feed' }); + this.queue.process('update-feed', this.processUpdateFeed.bind(this)); + } + + async stop ( ) { + await super.stop(); + } + + async processUpdateFeed (job) { + const { feed: feedService } = this.dtp.services; + const { feedId } = job.data; + this.log.info('newsroom feed update job received', { id: job.id, feedId }); + + try { + const feed = await feedService.getById(feedId); + await this.worker.updateFeed(feed); + } catch (error) { + this.log.error('failed to update newsroom feed', { feedId, error }); + throw error; + } + } +} + +module.exports = UpdateFeedJob; \ No newline at end of file diff --git a/config/job-queues.js b/config/job-queues.js index 0c55228..c5077ae 100644 --- a/config/job-queues.js +++ b/config/job-queues.js @@ -17,6 +17,10 @@ module.exports = { attempts: 3, removeOnComplete: true, }, + 'newsroom': { + attempts: 3, + removeOnComplete: true, + }, 'reeeper': { attempts: 3, removeOnComplete: true, From adfc32d5599fec0bde93bc28352845a5c9fcac5f Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 31 Oct 2022 04:34:30 -0400 Subject: [PATCH 14/23] v0.5.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f5900c1..f5a13e3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dtp-base", - "version": "0.4.7", + "version": "0.5.0", "description": "Open source web app engine for the Digital Telepresence Platform.", "main": "dtp-webapp.js", "author": "DTP Technologies, LLC", From 494f788fd7af37b6fee8c72e2d1e445babb0a18d Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 31 Oct 2022 04:45:19 -0400 Subject: [PATCH 15/23] remove unused things --- app/workers/newsroom/job/update-feed.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/app/workers/newsroom/job/update-feed.js b/app/workers/newsroom/job/update-feed.js index 8200527..6c96106 100644 --- a/app/workers/newsroom/job/update-feed.js +++ b/app/workers/newsroom/job/update-feed.js @@ -6,10 +6,6 @@ const path = require('path'); -const mongoose = require('mongoose'); - -const Feed = mongoose.model('Feed'); - const { SiteWorkerProcess } = require(path.join(__dirname, '..', '..', '..', '..', 'lib', 'site-lib')); class UpdateFeedJob extends SiteWorkerProcess { From 07a887ef9db0e5090234b0bc39a8e625c782e84e Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 31 Oct 2022 04:45:26 -0400 Subject: [PATCH 16/23] v0.5.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f5a13e3..c364825 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dtp-base", - "version": "0.5.0", + "version": "0.5.1", "description": "Open source web app engine for the Digital Telepresence Platform.", "main": "dtp-webapp.js", "author": "DTP Technologies, LLC", From b350b2632f3d5d53367a3cff3f468d609fa7611c Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 31 Oct 2022 11:55:08 -0400 Subject: [PATCH 17/23] newsroom index is now responsive --- app/views/newsroom/index.pug | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/newsroom/index.pug b/app/views/newsroom/index.pug index fe54702..27fc24a 100644 --- a/app/views/newsroom/index.pug +++ b/app/views/newsroom/index.pug @@ -8,7 +8,7 @@ block content if Array.isArray(newsroom.feeds) && (newsroom.feeds.length > 0) div(uk-grid).uk-grid-match each feed in newsroom.feeds - .uk-width-1-3 + div(class="uk-width-1-1 uk-width-1-2@s uk-width-1-3@m uk-width-1-4@l") .uk-card.uk-card-secondary.uk-card-small.uk-border-rounded .uk-card-header(style="border-bottom: solid 1px #808080;") div(uk-grid).uk-grid-small.uk-flex-middle From 965e7d03f9762aa96f0565efffd2d55332dd6428 Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 31 Oct 2022 11:55:15 -0400 Subject: [PATCH 18/23] v0.5.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c364825..c16103a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dtp-base", - "version": "0.5.1", + "version": "0.5.2", "description": "Open source web app engine for the Digital Telepresence Platform.", "main": "dtp-webapp.js", "author": "DTP Technologies, LLC", From fd532d9aaab2a038613376cf447fd9401f84f168 Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 2 Nov 2022 14:12:10 -0400 Subject: [PATCH 19/23] add label options to +renderBackButton --- app/views/components/library.pug | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/views/components/library.pug b/app/views/components/library.pug index 20eb6b0..e53b6f9 100644 --- a/app/views/components/library.pug +++ b/app/views/components/library.pug @@ -50,11 +50,13 @@ mixin renderCell (label, value, className) div(class=className)!= value .uk-text-muted.uk-text-small= label -mixin renderBackButton ( ) +mixin renderBackButton (options) + - options = Object.assign({ includeLabel: true, label: 'Back' }, options) button(type="button", onclick="window.history.back();").uk-button.uk-button-default span i.fas.fa-chevron-left - span(class="uk-visible@s").uk-margin-small-left Back + if options.includeLabel + span(class="uk-visible@s").uk-margin-small-left= options.label mixin renderUserLink (user) if user.coreUserId From 08d70c35a7720f6da97e5e433094f05a3d4f0dbb Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 2 Nov 2022 14:18:57 -0400 Subject: [PATCH 20/23] v0.5.3 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c16103a..bfdf0b3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dtp-base", - "version": "0.5.2", + "version": "0.5.3", "description": "Open source web app engine for the Digital Telepresence Platform.", "main": "dtp-webapp.js", "author": "DTP Technologies, LLC", From 830c2f8ba809b75272e57761a3daa117b28fd0bc Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 2 Nov 2022 19:14:43 -0400 Subject: [PATCH 21/23] v0.5.4 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index bfdf0b3..e3ae702 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dtp-base", - "version": "0.5.3", + "version": "0.5.4", "description": "Open source web app engine for the Digital Telepresence Platform.", "main": "dtp-webapp.js", "author": "DTP Technologies, LLC", From 5d12e3913f10ad5946d96b788af5f40aaca8fb03 Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 4 Nov 2022 08:00:04 -0400 Subject: [PATCH 22/23] prevent logged-in access of the Welcome flow --- app/controllers/welcome.js | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/app/controllers/welcome.js b/app/controllers/welcome.js index b6bdef3..eb24032 100644 --- a/app/controllers/welcome.js +++ b/app/controllers/welcome.js @@ -23,17 +23,24 @@ class WelcomeController extends SiteController { captcha.loadFont(path.join(this.dtp.config.root, 'client', 'fonts', 'Dirty Sweb.ttf')); + function preventUserAccess (req, res, next) { + if (req.user) { + return res.redirect(301, '/'); + } + return next(); + } + const router = express.Router(); this.dtp.app.use('/welcome', welcomeLimiter, async (req, res, next) => { res.locals.currentView = 'welcome'; return next(); }, router); - router.get('/core-member', this.getWelcomeCoreMember.bind(this)); + router.get('/core-member', preventUserAccess, this.getWelcomeCoreMember.bind(this)); router.get('/signup/captcha', this.getSignupCaptcha.bind(this)); - router.get('/signup', this.getSignupView.bind(this)); - router.get('/login', this.getLoginView.bind(this)); - router.get('/', this.getHomeView.bind(this)); + router.get('/signup', preventUserAccess, this.getSignupView.bind(this)); + router.get('/login', preventUserAccess, this.getLoginView.bind(this)); + router.get('/', preventUserAccess, this.getHomeView.bind(this)); return router; } From 689f15b78bf86eebfe0055342d3065dad47745c6 Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 4 Nov 2022 08:00:14 -0400 Subject: [PATCH 23/23] v0.5.5 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e3ae702..a5c2c4a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dtp-base", - "version": "0.5.4", + "version": "0.5.5", "description": "Open source web app engine for the Digital Telepresence Platform.", "main": "dtp-webapp.js", "author": "DTP Technologies, LLC",