Browse Source

letting prettier do its thing on the code base

master
Rob Colbert 3 months ago
parent
commit
efb1412fa5
  1. 1
      package.json
  2. 10
      pnpm-lock.yaml
  3. 102
      src/app/controllers/auth.ts
  4. 41
      src/app/controllers/home.ts
  5. 91
      src/app/controllers/lib/populators.ts
  6. 48
      src/app/controllers/manifest.ts
  7. 79
      src/app/controllers/user.ts
  8. 50
      src/app/controllers/welcome.ts
  9. 36
      src/app/models/broadcast-show.ts
  10. 14
      src/app/models/connect-token.ts
  11. 8
      src/app/models/csrf-token.ts
  12. 45
      src/app/models/email-blacklist.ts
  13. 4
      src/app/models/email-log.ts
  14. 17
      src/app/models/email-verify.ts
  15. 15
      src/app/models/episode.ts
  16. 4
      src/app/models/feed-item.ts
  17. 2
      src/app/models/feed.ts
  18. 31
      src/app/models/image.ts
  19. 13
      src/app/models/lib/email-blacklist-flags.ts
  20. 17
      src/app/models/lib/image-file.ts
  21. 2
      src/app/models/lib/image-metadata.ts
  22. 13
      src/app/models/lib/process-endpoint.ts
  23. 20
      src/app/models/lib/process-stats.ts
  24. 18
      src/app/models/lib/speech-personality.ts
  25. 11
      src/app/models/lib/user-apis.ts
  26. 11
      src/app/models/lib/user-flags.ts
  27. 17
      src/app/models/lib/user-permissions.ts
  28. 44
      src/app/models/process-stats.ts
  29. 20
      src/app/models/process.ts
  30. 56
      src/app/models/user.ts
  31. 38
      src/app/models/video.ts
  32. 46
      src/app/services/cache.ts
  33. 66
      src/app/services/crypto.ts
  34. 29
      src/app/services/csrf-token.ts
  35. 29
      src/app/services/display-engine.ts
  36. 225
      src/app/services/email.ts
  37. 78
      src/app/services/feed.ts
  38. 28
      src/app/services/image.ts
  39. 32
      src/app/services/job-queue.ts
  40. 20
      src/app/services/minio.ts
  41. 74
      src/app/services/openai.ts
  42. 131
      src/app/services/process.ts
  43. 168
      src/app/services/session.ts
  44. 21
      src/app/services/sidebar.ts
  45. 15
      src/app/services/text.ts
  46. 100
      src/app/services/user.ts
  47. 79
      src/app/services/video.ts
  48. 2
      src/browsersync.ts
  49. 5
      src/client/js/lib/app.ts
  50. 25
      src/client/js/lib/log.ts
  51. 7
      src/client/js/newsroom-app.ts
  52. 40
      src/config/env.ts
  53. 46
      src/lib/core/base.ts
  54. 33
      src/lib/core/component-registry.ts
  55. 4
      src/lib/core/component.ts
  56. 23
      src/lib/core/log-file.ts
  57. 10
      src/lib/core/log-transport-console.ts
  58. 25
      src/lib/core/log-transport-file.ts
  59. 8
      src/lib/core/log-transport.ts
  60. 6
      src/lib/core/log.ts
  61. 27
      src/lib/core/platform.ts
  62. 51
      src/lib/core/process.ts
  63. 23
      src/lib/core/service.ts
  64. 1
      src/lib/core/unzalgo.ts
  65. 17
      src/lib/core/worker.ts
  66. 45
      src/lib/web/controller.ts
  67. 100
      src/lib/web/display-list.ts
  68. 3
      src/lib/web/error.ts
  69. 6
      src/lib/web/file.ts
  70. 188
      src/lib/web/server.ts
  71. 58
      src/newsroom-web.ts
  72. 41
      src/release.ts
  73. 23
      src/speechgen.ts
  74. 30
      src/workers/newsroom.ts
  75. 71
      src/workers/newsroom/fetch-news.ts

1
package.json

@ -107,6 +107,7 @@
"globals": "^15.14.0",
"less": "^4.2.2",
"nodemon": "^3.1.9",
"prettier": "^3.4.2",
"simple-git": "^3.27.0",
"ts-node": "^10.9.2",
"tsc-watch": "^6.2.1",

10
pnpm-lock.yaml

@ -249,6 +249,9 @@ importers:
nodemon:
specifier: ^3.1.9
version: 3.1.9
prettier:
specifier: ^3.4.2
version: 3.4.2
simple-git:
specifier: ^3.27.0
version: 3.27.0
@ -2254,6 +2257,11 @@ packages:
resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==}
engines: {node: '>= 0.8.0'}
[email protected]:
resolution: {integrity: sha512-e9MewbtFo+Fevyuxn/4rrcDAaq0IYxPGLvObpQjiZBMAzB9IGmzlnG9RZy3FFas+eBMu2vA0CszMeduow5dIuQ==}
engines: {node: '>=14'}
hasBin: true
[email protected]:
resolution: {integrity: sha512-kCLsENsJ6h5Bcq106Q3YMSxuz2q3jtIXP7fgDB/+jZjUsZjRjAoL9Lr1TVwAEcugufVBhr5Mfd9L7P6d+SR+Yw==}
@ -4965,6 +4973,8 @@ snapshots:
[email protected]: {}
[email protected]: {}
[email protected]: {}
[email protected]: {}

102
src/app/controllers/auth.ts

@ -10,11 +10,7 @@ import passport from "passport";
import svgCaptcha from "svg-captcha";
import {
WebServer,
WebController,
WebError,
} from "../../lib/dtplib.js";
import { WebServer, WebController, WebError } from "../../lib/dtplib.js";
import { UserService } from "../services/user.js";
import CsrfTokenService from "../services/csrf-token.js";
@ -23,19 +19,24 @@ import { ConnectToken } from "../models/connect-token.js";
import { IUser } from "app/models/user.js";
export default class AuthController extends WebController {
static get name ( ) : string { return "AuthController"; }
static get slug ( ) : string { return "auth"; }
static get name(): string {
return "AuthController";
}
static get slug(): string {
return "auth";
}
constructor (server: WebServer) {
constructor(server: WebServer) {
super(server, AuthController);
}
get route ( ) : string {
get route(): string {
return "/auth";
}
async start ( ) : Promise<void> {
const csrfTokenService = this.platform.components.getService<CsrfTokenService>("csrfToken");
async start(): Promise<void> {
const csrfTokenService =
this.platform.components.getService<CsrfTokenService>("csrfToken");
const signupToken: RequestHandler = csrfTokenService.middleware({
name: "user-create",
allowReuse: false,
@ -50,20 +51,21 @@ export default class AuthController extends WebController {
this.router.post("/signup", signupToken, this.postSignup.bind(this));
this.router.post("/login", loginToken, this.postLogin.bind(this));
this.router.get(
'/google',
(req, res, next) => {
passport.authenticate("google", {
session: true,
scope: ["email", "profile"],
})(req, res, next);
}
);
this.router.get("/google", (req, res, next) => {
passport.authenticate("google", {
session: true,
scope: ["email", "profile"],
})(req, res, next);
});
this.router.get(
'/google/callback',
"/google/callback",
passport.authenticate("google"),
async (req: Request, res: Response, next: NextFunction) : Promise<void> => {
async (
req: Request,
res: Response,
next: NextFunction
): Promise<void> => {
assert(req.user, "User is required");
req.login(req.user, async (error: Error | null) => {
if (error) {
@ -75,7 +77,7 @@ export default class AuthController extends WebController {
res.locals.serviceName = "Google";
res.locals.user = req.user;
res.render('welcome/connected');
res.render("welcome/connected");
});
}
);
@ -86,8 +88,13 @@ export default class AuthController extends WebController {
this.router.get("/logout", this.getLogout.bind(this));
}
async postSignup (req: Request, res: Response, next: NextFunction) : Promise<void> {
const userService = this.platform.components.getService<UserService>("user");
async postSignup(
req: Request,
res: Response,
next: NextFunction
): Promise<void> {
const userService =
this.platform.components.getService<UserService>("user");
try {
assert(req.session, "Active HTTP session is required");
@ -100,12 +107,12 @@ export default class AuthController extends WebController {
const user: IUser = await userService.login(
req.body.email as string,
req.body.password as string,
req.body.password as string
);
req.logIn(user, (error: Error | null) => {
if (error) {
this.log.error('failed to regenerate user session', { error });
this.log.error("failed to regenerate user session", { error });
return next(error);
}
return res.redirect(303, "/");
@ -116,13 +123,17 @@ export default class AuthController extends WebController {
}
}
async postLogin (req: Request, res: Response, next: NextFunction) : Promise<void> {
async postLogin(
req: Request,
res: Response,
next: NextFunction
): Promise<void> {
try {
assert(req.session, "An active session is required");
const redirectUri = req.session.loginReturnTo || '/';
const redirectUri = req.session.loginReturnTo || "/";
this.log.debug('starting passport authenticate', { redirectUri });
passport.authenticate('mt-local', (error: Error, user: IUser) : void => {
this.log.debug("starting passport authenticate", { redirectUri });
passport.authenticate("mt-local", (error: Error, user: IUser): void => {
if (error) {
req.session.loginResult = error.toString();
return next(error);
@ -131,7 +142,9 @@ export default class AuthController extends WebController {
req.session.loginResult = "No user account";
return res.redirect("/welcome/login");
}
this.log.info("establishing authenticated user session", { user: { _id: user._id } });
this.log.info("establishing authenticated user session", {
user: { _id: user._id },
});
req.login(user, async (error: Error | null) => {
if (error) {
this.log.error("failed to start user session", { error });
@ -146,7 +159,11 @@ export default class AuthController extends WebController {
}
}
async getSocketToken (req: Request, res: Response, next: NextFunction) : Promise<void> {
async getSocketToken(
req: Request,
res: Response,
next: NextFunction
): Promise<void> {
try {
if (!req.session || !req.user) {
res.status(403).json({
@ -157,7 +174,7 @@ export default class AuthController extends WebController {
}
const token = await ConnectToken.create({
created: new Date(),
userType: 'User',
userType: "User",
user: req.user._id,
token: uuidv4(),
});
@ -171,8 +188,13 @@ export default class AuthController extends WebController {
}
}
async getSignupForm (req: Request, res: Response, next: NextFunction) : Promise<void> {
const csrfTokenService = this.platform.components.getService<CsrfTokenService>("csrfToken");
async getSignupForm(
req: Request,
res: Response,
next: NextFunction
): Promise<void> {
const csrfTokenService =
this.platform.components.getService<CsrfTokenService>("csrfToken");
try {
assert(req.session, "HTTP session is required");
@ -201,11 +223,15 @@ export default class AuthController extends WebController {
}
}
async getLoginForm (_req: Request, res: Response) : Promise<void> {
async getLoginForm(_req: Request, res: Response): Promise<void> {
res.render("auth/login");
}
async getLogout (req: Request, res: Response, next: NextFunction) : Promise<void> {
async getLogout(
req: Request,
res: Response,
next: NextFunction
): Promise<void> {
try {
if (!req.session) {
return next(new WebError(400, "You are not currently logged in."));

41
src/app/controllers/home.ts

@ -6,37 +6,46 @@ import { NextFunction, Request, Response } from "express";
import { SidebarService } from "../services/sidebar.js";
import {
WebServer,
WebController,
} from '../../lib/dtplib.js';
import { WebServer, WebController } from "../../lib/dtplib.js";
import { FeedService } from "app/services/feed.js";
export class HomeController extends WebController {
static get name(): string {
return "HomeController";
}
static get slug(): string {
return "home";
}
static get name ( ) : string { return 'HomeController'; }
static get slug ( ) : string { return 'home'; }
constructor (server: WebServer) {
constructor(server: WebServer) {
super(server, HomeController);
}
get route ( ) : string { return '/'; }
get route(): string {
return "/";
}
async start ( ) : Promise<void> {
const sidebarService = this.platform.components.getService<SidebarService>("sidebar");
this.router.get('/', sidebarService.middleware(), this.getHome.bind(this));
async start(): Promise<void> {
const sidebarService =
this.platform.components.getService<SidebarService>("sidebar");
this.router.get("/", sidebarService.middleware(), this.getHome.bind(this));
}
async getHome (req: Request, res: Response, next: NextFunction) : Promise<void> {
async getHome(
req: Request,
res: Response,
next: NextFunction
): Promise<void> {
const feedService = this.getService<FeedService>("feed");
try {
res.locals.pagination = this.getPaginationParameters(req, 20);
res.locals.feedLibrary = await feedService.getUnifiedFeed(res.locals.pagination);
res.locals.feedLibrary = await feedService.getUnifiedFeed(
res.locals.pagination
);
if (req.user) {
return res.render('home-auth');
return res.render("home-auth");
}
return res.render('home');
return res.render("home");
} catch (error) {
this.log.error("failed to present the Home view", { error });
return next(error);

91
src/app/controllers/lib/populators.ts

@ -7,37 +7,54 @@ import assert from "node:assert";
import { Types } from "mongoose";
import { Request, Response, NextFunction, RequestHandler } from "express";
import {
WebController,
WebError,
} from "../../../lib/dtplib.js";
import { WebController, WebError } from "../../../lib/dtplib.js";
import { ImageService } from "../../services/image.js";
import { UserService } from "../../services/user.js";
export function populateAccountByEmail (controller: WebController) : RequestHandler {
const userService = controller.platform.components.getService<UserService>("user");
return async function (_req: Request, res: Response, next: NextFunction, emailAddress?: string) : Promise<void> {
export function populateAccountByEmail(
controller: WebController
): RequestHandler {
const userService =
controller.platform.components.getService<UserService>("user");
return async function (
_req: Request,
res: Response,
next: NextFunction,
emailAddress?: string
): Promise<void> {
assert(emailAddress, "Email address is required");
try {
res.locals.userProfile = await userService.getAccountByEmailAddress(emailAddress);
res.locals.userProfile =
await userService.getAccountByEmailAddress(emailAddress);
if (!res.locals.userProfile) {
return next(new WebError(404, "User Account not found"));
}
return next();
} catch (error) {
controller.log.error("failed to populate user account by email address", { emailAddress, error });
controller.log.error("failed to populate user account by email address", {
emailAddress,
error,
});
return next(error);
}
};
}
export function populateAccountById (controller: WebController) : RequestHandler {
const userService = controller.platform.components.getService<UserService>("user");
return async function (_req: Request, res: Response, next: NextFunction, userId?: string) : Promise<void> {
export function populateAccountById(controller: WebController): RequestHandler {
const userService =
controller.platform.components.getService<UserService>("user");
return async function (
_req: Request,
res: Response,
next: NextFunction,
userId?: string
): Promise<void> {
assert(userId, "User ID is required");
try {
res.locals.userProfile = await userService.getAccountById(Types.ObjectId.createFromHexString(userId));
res.locals.userProfile = await userService.getAccountById(
Types.ObjectId.createFromHexString(userId)
);
if (!res.locals.userProfile) {
return next(new WebError(404, "User profile not found"));
}
@ -49,9 +66,15 @@ export function populateAccountById (controller: WebController) : RequestHandler
};
}
export function populateImageId (controller: WebController) : RequestHandler {
const imageService = controller.platform.components.getService<ImageService>("image");
return async function (_req: Request, res: Response, next: NextFunction, imageId?: string) : Promise<void> {
export function populateImageId(controller: WebController): RequestHandler {
const imageService =
controller.platform.components.getService<ImageService>("image");
return async function (
_req: Request,
res: Response,
next: NextFunction,
imageId?: string
): Promise<void> {
assert(imageId, "imageId parameter required");
try {
res.locals.image = await imageService.getById(
@ -68,29 +91,47 @@ export function populateImageId (controller: WebController) : RequestHandler {
};
}
export function populateUserByEmail (controller: WebController) : RequestHandler {
const userService = controller.platform.components.getService<UserService>("user");
return async function (_req: Request, res: Response, next: NextFunction, emailAddress?: string) : Promise<void> {
export function populateUserByEmail(controller: WebController): RequestHandler {
const userService =
controller.platform.components.getService<UserService>("user");
return async function (
_req: Request,
res: Response,
next: NextFunction,
emailAddress?: string
): Promise<void> {
assert(emailAddress, "Email address is required");
try {
res.locals.userProfile = await userService.getByEmailAddress(emailAddress);
res.locals.userProfile =
await userService.getByEmailAddress(emailAddress);
if (!res.locals.userProfile) {
return next(new WebError(404, "User profile not found"));
}
return next();
} catch (error) {
controller.log.error("failed to populate user by email address", { emailAddress, error });
controller.log.error("failed to populate user by email address", {
emailAddress,
error,
});
return next(error);
}
};
}
export function populateUserById (controller: WebController) : RequestHandler {
const userService = controller.platform.components.getService<UserService>("user");
return async function (_req: Request, res: Response, next: NextFunction, userId?: string) : Promise<void> {
export function populateUserById(controller: WebController): RequestHandler {
const userService =
controller.platform.components.getService<UserService>("user");
return async function (
_req: Request,
res: Response,
next: NextFunction,
userId?: string
): Promise<void> {
assert(userId, "User ID is required");
try {
res.locals.userProfile = await userService.getById(Types.ObjectId.createFromHexString(userId));
res.locals.userProfile = await userService.getById(
Types.ObjectId.createFromHexString(userId)
);
if (!res.locals.userProfile) {
return next(new WebError(404, "User profile not found"));
}

48
src/app/controllers/manifest.ts

@ -6,10 +6,7 @@ import env from "../../config/env.js";
import { NextFunction, Request, Response } from "express";
import {
WebServer,
WebController,
} from '../../lib/dtplib.js';
import { WebServer, WebController } from "../../lib/dtplib.js";
type AppManifestIcon = {
src: string;
@ -62,43 +59,52 @@ type AppManifest = {
};
export class ManifestController extends WebController {
static get name(): string {
return "ManifestController";
}
static get slug(): string {
return "manifest";
}
static get name ( ) : string { return 'ManifestController'; }
static get slug ( ) : string { return 'manifest'; }
constructor (server: WebServer) {
constructor(server: WebServer) {
super(server, ManifestController);
}
get route ( ) : string { return '/manifest.json'; }
get route(): string {
return "/manifest.json";
}
async start ( ) : Promise<void> {
this.router.get('/', this.getManifest.bind(this));
async start(): Promise<void> {
this.router.get("/", this.getManifest.bind(this));
}
async getManifest (_req: Request, res: Response, next: NextFunction) : Promise<void> {
const DEFAULT_THEME_COLOR = '#4a4a4a';
const DEFAULT_BACKGROUND_COLOR = '#1a1a1a';
async getManifest(
_req: Request,
res: Response,
next: NextFunction
): Promise<void> {
const DEFAULT_THEME_COLOR = "#4a4a4a";
const DEFAULT_BACKGROUND_COLOR = "#1a1a1a";
try {
const manifest: AppManifest = {
id: '/',
id: "/",
theme_color: DEFAULT_THEME_COLOR,
background_color: DEFAULT_BACKGROUND_COLOR,
display: 'fullscreen',
scope: '/',
start_url: '/',
display: "fullscreen",
scope: "/",
start_url: "/",
name: env.site.name,
short_name: env.site.shortName,
description: env.site.description,
icons: [ ],
screenshots: [ ],
icons: [],
screenshots: [],
};
[512, 384, 256, 192, 144, 96, 72, 48, 32, 16].forEach((size) => {
manifest.icons.push({
src: `/img/icon/${env.site.domainKey}/icon-${size}x${size}.png`,
sizes: `${size}x${size}`,
type: 'image/png'
type: "image/png",
});
});

79
src/app/controllers/user.ts

@ -13,15 +13,15 @@ import { DisplayEngineService } from "../services/display-engine.js";
import { SidebarService } from "../services/sidebar.js";
import { SessionService } from "../services/session.js";
import {
WebServer,
WebController,
} from "../../lib/dtplib.js";
import { WebServer, WebController } from "../../lib/dtplib.js";
export default class UserController extends WebController {
static get name ( ) : string { return "UserController"; }
static get slug ( ) : string { return "user"; }
static get name(): string {
return "UserController";
}
static get slug(): string {
return "user";
}
constructor(server: WebServer) {
super(server, UserController);
@ -32,10 +32,16 @@ export default class UserController extends WebController {
}
async start(): Promise<void> {
const displayEngineService = this.platform.components.getService<DisplayEngineService>("displayEngine");
const sessionService = this.platform.components.getService<SessionService>("session");
const userService = this.platform.components.getService<UserService>("user");
const sidebarService = this.platform.components.getService<SidebarService>("sidebar");
const displayEngineService =
this.platform.components.getService<DisplayEngineService>(
"displayEngine"
);
const sessionService =
this.platform.components.getService<SessionService>("session");
const userService =
this.platform.components.getService<UserService>("user");
const sidebarService =
this.platform.components.getService<SidebarService>("sidebar");
displayEngineService.loadTemplate(
"profile-picture-form",
@ -76,7 +82,7 @@ export default class UserController extends WebController {
"/:userId/password",
requireLogin,
requireAccountOwner,
this.postUserPasswordChange.bind(this),
this.postUserPasswordChange.bind(this)
);
this.router.post(
@ -84,17 +90,21 @@ export default class UserController extends WebController {
requireLogin,
requireAccountOwner,
imageUpload.single("image"),
this.postUserProfilePicture.bind(this),
this.postUserProfilePicture.bind(this)
);
this.router.post(
"/:userId",
requireLogin,
requireAccountOwner,
this.postUserUpdate.bind(this),
this.postUserUpdate.bind(this)
);
this.router.get("/:userId/settings", sidebar, this.getSettingsView.bind(this));
this.router.get(
"/:userId/settings",
sidebar,
this.getSettingsView.bind(this)
);
this.router.get("/:userId", sidebar, this.getProfileView.bind(this));
this.router.get("/", requireLogin, this.getHome.bind(this));
@ -103,11 +113,11 @@ export default class UserController extends WebController {
"/:userId/picture",
requireLogin,
requireAccountOwner,
this.deleteProfilePicture.bind(this),
this.deleteProfilePicture.bind(this)
);
}
async postUserPasswordChange (req: Request, res: Response) : Promise<void> {
async postUserPasswordChange(req: Request, res: Response): Promise<void> {
const userService = this.getService<UserService>("user");
try {
assert(req.user, "Login required");
@ -129,9 +139,13 @@ export default class UserController extends WebController {
}
}
async postUserProfilePicture(req: Request, res: Response) : Promise<void> {
const displayEngineService = this.platform.components.getService<DisplayEngineService>("displayEngine");
const userService = this.platform.components.getService<UserService>("user");
async postUserProfilePicture(req: Request, res: Response): Promise<void> {
const displayEngineService =
this.platform.components.getService<DisplayEngineService>(
"displayEngine"
);
const userService =
this.platform.components.getService<UserService>("user");
try {
assert(req.user, "Login required");
assert(req.file, "Image file not uploaded");
@ -157,12 +171,12 @@ export default class UserController extends WebController {
this.log.error("failed to process user account update", { error });
this.renderErrorJSON(
res,
new Error("failed to process user account update", { cause: error }),
new Error("failed to process user account update", { cause: error })
);
}
}
async postUserUpdate (req: Request, res: Response) : Promise<void> {
async postUserUpdate(req: Request, res: Response): Promise<void> {
const userService = this.getService<UserService>("user");
try {
assert(req.user, "Login required");
@ -172,12 +186,16 @@ export default class UserController extends WebController {
this.log.error("failed to process user account update", { error });
this.renderErrorJSON(
res,
new Error("failed to process user account update", { cause: error }),
new Error("failed to process user account update", { cause: error })
);
}
}
async getSettingsView(_req: Request, res: Response, next: NextFunction): Promise<void> {
async getSettingsView(
_req: Request,
res: Response,
next: NextFunction
): Promise<void> {
try {
assert(res.locals.userProfile, "User profile is missing");
res.render("user/settings");
@ -187,7 +205,11 @@ export default class UserController extends WebController {
}
}
async getProfileView(req: Request, res: Response, next: NextFunction): Promise<void> {
async getProfileView(
req: Request,
res: Response,
next: NextFunction
): Promise<void> {
try {
assert(res.locals.userProfile, "User profile is missing");
res.locals.pagination = this.getPaginationParameters(req, 20);
@ -198,12 +220,13 @@ export default class UserController extends WebController {
}
}
async getHome(_req: Request, res: Response) : Promise<void> {
async getHome(_req: Request, res: Response): Promise<void> {
return res.render("user/home");
}
async deleteProfilePicture (req: Request, res: Response) : Promise<void> {
const displayEngineService = this.getService<DisplayEngineService>("displayEngine");
async deleteProfilePicture(req: Request, res: Response): Promise<void> {
const displayEngineService =
this.getService<DisplayEngineService>("displayEngine");
const userService = this.getService<UserService>("user");
try {
assert(req.user, "Login required");

50
src/app/controllers/welcome.ts

@ -9,25 +9,31 @@ import { Request, Response, NextFunction } from "express";
import svgCaptcha from "svg-captcha";
import {
WebServer,
WebController,
} from '../../lib/dtplib.js';
import { WebServer, WebController } from "../../lib/dtplib.js";
import CsrfTokenService from "app/services/csrf-token.js";
export class WelcomeController extends WebController {
static get name(): string {
return "WelcomeController";
}
static get slug(): string {
return "welcome";
}
static get name ( ) : string { return 'WelcomeController'; }
static get slug ( ) : string { return 'welcome'; }
constructor (server: WebServer) {
constructor(server: WebServer) {
super(server, WelcomeController);
}
get route ( ) : string { return '/welcome'; }
get route(): string {
return "/welcome";
}
async start ( ) : Promise<void> {
function preventUserAccess(req: Request, res: Response, next: NextFunction) {
async start(): Promise<void> {
function preventUserAccess(
req: Request,
res: Response,
next: NextFunction
) {
if (req.user) {
return res.redirect(301, "/");
}
@ -36,11 +42,15 @@ export class WelcomeController extends WebController {
if (env.user.signupEnabled) {
this.router.get("/signup/captcha", this.getSignupCaptcha.bind(this));
this.router.get("/signup", preventUserAccess, this.getSignupView.bind(this));
this.router.get(
"/signup",
preventUserAccess,
this.getSignupView.bind(this)
);
}
this.router.get("/login", preventUserAccess, this.getLoginView.bind(this));
this.router.get('/', preventUserAccess, this.getHome.bind(this));
this.router.get("/", preventUserAccess, this.getHome.bind(this));
}
async getSignupCaptcha(req: Request, res: Response): Promise<void> {
@ -53,7 +63,7 @@ export class WelcomeController extends WebController {
height: 80,
});
req.session.captcha = req.session.captcha || { };
req.session.captcha = req.session.captcha || {};
req.session.captcha.signup = signupCaptcha.text;
res.set("Content-Type", "image/svg+xml");
@ -62,8 +72,9 @@ export class WelcomeController extends WebController {
res.status(200).send(signupCaptcha.data);
}
async getSignupView (req: Request, res: Response) {
const csrfTokenService = this.platform.components.getService<CsrfTokenService>("csrfToken");
async getSignupView(req: Request, res: Response) {
const csrfTokenService =
this.platform.components.getService<CsrfTokenService>("csrfToken");
assert(req.session, "The request must provide a session");
res.locals.csrfTokenSignup = await csrfTokenService.create(req, {
@ -76,7 +87,8 @@ export class WelcomeController extends WebController {
}
async getLoginView(req: Request, res: Response) {
const csrfTokenService = this.platform.components.getService<CsrfTokenService>("csrfToken");
const csrfTokenService =
this.platform.components.getService<CsrfTokenService>("csrfToken");
assert(req.session, "The request must provide a session");
res.locals.csrfTokenLogin = await csrfTokenService.create(req, {
@ -92,8 +104,8 @@ export class WelcomeController extends WebController {
res.render("welcome/login");
}
async getHome (_req: Request, res: Response) : Promise<void> {
res.render('welcome/home');
async getHome(_req: Request, res: Response): Promise<void> {
res.render("welcome/home");
}
}

36
src/app/models/broadcast-show.ts

@ -7,7 +7,10 @@ import { Schema, Types, model } from "mongoose";
import { IEpisode } from "./episode";
import { HumanGender } from "./lib/human-gender.ts";
import { ISpeechPersonality, SpeechPersonalitySchema } from "./lib/speech-personality.ts";
import {
ISpeechPersonality,
SpeechPersonalitySchema,
} from "./lib/speech-personality.ts";
export enum BroadcastShowStatus {
Offline = "offline",
@ -45,8 +48,8 @@ export const BroadcastShowProducerSchema = new Schema({
});
export interface IBroadcastShow {
_id: Types.ObjectId; // MongoDB concern
__v: number; // MongoDB concern
_id: Types.ObjectId; // MongoDB concern
__v: number; // MongoDB concern
status: BroadcastShowStatus;
title: string;
@ -57,13 +60,30 @@ export interface IBroadcastShow {
}
export const BroadcastShowSchema = new Schema<IBroadcastShow>({
status: { type: String, enum: BroadcastShowStatus, default: BroadcastShowStatus.Offline, required: true },
status: {
type: String,
enum: BroadcastShowStatus,
default: BroadcastShowStatus.Offline,
required: true,
},
title: { type: String, required: true },
description: { type: String, required: true },
producers: { type: [BroadcastShowProducerSchema], default: [ ], required: true },
hosts: { type: [BroadcastShowHostSchema], default: [ ], required: true },
recentEpisodes: { type: [Types.ObjectId], default: [ ], required: true, ref: 'Episode' },
producers: {
type: [BroadcastShowProducerSchema],
default: [],
required: true,
},
hosts: { type: [BroadcastShowHostSchema], default: [], required: true },
recentEpisodes: {
type: [Types.ObjectId],
default: [],
required: true,
ref: "Episode",
},
});
export const BroadcastShow = model<IBroadcastShow>("BroadcastShow", BroadcastShowSchema);
export const BroadcastShow = model<IBroadcastShow>(
"BroadcastShow",
BroadcastShowSchema
);
export default BroadcastShow;

14
src/app/models/connect-token.ts

@ -2,7 +2,7 @@
// Copyright (C) 2025 DTP Technologies, LLC
// All Rights Reserved
'use strict';
"use strict";
import { Schema, Types, model } from "mongoose";
@ -15,11 +15,17 @@ export interface IConnectToken {
}
const ConnectTokenSchema = new Schema<IConnectToken>({
created: { type: Date, default: Date.now, required: true, index: -1, expires: '1m' },
user: { type: Schema.ObjectId, required: true, index: true, ref: 'User' },
created: {
type: Date,
default: Date.now,
required: true,
index: -1,
expires: "1m",
},
user: { type: Schema.ObjectId, required: true, index: true, ref: "User" },
token: { type: String, required: true },
claimed: { type: Date },
});
export const ConnectToken = model('ConnectToken', ConnectTokenSchema);
export const ConnectToken = model("ConnectToken", ConnectTokenSchema);
export default ConnectToken;

8
src/app/models/csrf-token.ts

@ -19,7 +19,13 @@ export interface ICsrfToken {
}
const CsrfTokenSchema = new Schema<ICsrfToken>({
created: { type: Date, required: true, default: Date.now, index: -1, expires: "24h" },
created: {
type: Date,
required: true,
default: Date.now,
index: -1,
expires: "24h",
},
expires: { type: Date, required: true, default: Date.now, index: -1 },
claimed: { type: Date },
token: { type: String, required: true, index: 1 },

45
src/app/models/email-blacklist.ts

@ -2,9 +2,12 @@
// Copyright (C) 2025 DTP Technologies, LLC
// All Rights Reserved
import { Schema, Types, model } from 'mongoose';
import { Schema, Types, model } from "mongoose";
import { IEmailBlacklistFlags, EmailBlacklistFlagsSchema } from './lib/email-blacklist-flags.js';
import {
IEmailBlacklistFlags,
EmailBlacklistFlagsSchema,
} from "./lib/email-blacklist-flags.js";
export interface IEmailBlacklist {
_id: Types.ObjectId;
@ -14,19 +17,37 @@ export interface IEmailBlacklist {
}
const EmailBlacklistSchema = new Schema<IEmailBlacklist>({
created: { type: Date, required: true, default: Date.now, index: -1, expires: '30d' },
email: { type: String, required: true, lowercase: true, maxLength: 255, unique: true },
created: {
type: Date,
required: true,
default: Date.now,
index: -1,
expires: "30d",
},
email: {
type: String,
required: true,
lowercase: true,
maxLength: 255,
unique: true,
},
flags: { type: EmailBlacklistFlagsSchema, required: true },
});
EmailBlacklistSchema.index({
email: 1,
'flags.isVerified': 1,
}, {
partialFilterExpression: {
'flags.isVerified': true,
EmailBlacklistSchema.index(
{
email: 1,
"flags.isVerified": 1,
},
});
{
partialFilterExpression: {
"flags.isVerified": true,
},
}
);
export const EmailBlacklist = model<IEmailBlacklist>('EmailBlacklist', EmailBlacklistSchema);
export const EmailBlacklist = model<IEmailBlacklist>(
"EmailBlacklist",
EmailBlacklistSchema
);
export default EmailBlacklist;

4
src/app/models/email-log.ts

@ -16,12 +16,12 @@ export interface IEmailLog {
const EmailLogSchema = new Schema<IEmailLog>({
created: { type: Date, default: Date.now, required: true, index: -1 },
from: { type: String, required: true, },
from: { type: String, required: true },
to: { type: String, required: true },
to_lc: { type: String, required: true, lowercase: true, index: 1 },
subject: { type: String, required: true },
messageId: { type: String },
});
export const EmailLog = model<IEmailLog>('EmailLog', EmailLogSchema);
export const EmailLog = model<IEmailLog>("EmailLog", EmailLogSchema);
export default EmailLog;

17
src/app/models/email-verify.ts

@ -4,7 +4,7 @@
import { Schema, Types, model } from "mongoose";
import { IUser } from './user.js';
import { IUser } from "./user.js";
export interface IEmailVerify {
_id: Types.ObjectId;
@ -15,11 +15,20 @@ export interface IEmailVerify {
}
const EmailVerifySchema = new Schema({
created: { type: Date, default: Date.now, required: true, index: -1, expires: '30d' },
created: {
type: Date,
default: Date.now,
required: true,
index: -1,
expires: "30d",
},
verified: { type: Date },
user: { type: Types.ObjectId, required: true, index: 1, ref: 'User' },
user: { type: Types.ObjectId, required: true, index: 1, ref: "User" },
token: { type: String, required: true },
});
export const EmailVerify = model<IEmailVerify>('EmailVerify', EmailVerifySchema);
export const EmailVerify = model<IEmailVerify>(
"EmailVerify",
EmailVerifySchema
);
export default EmailVerify;

15
src/app/models/episode.ts

@ -9,8 +9,8 @@ import { IVideo } from "./video";
import { IFeedItem } from "./feed-item";
export interface IEpisode {
_id: Types.ObjectId; // MongoDB concern
__v: number; // MongoDB concern
_id: Types.ObjectId; // MongoDB concern
__v: number; // MongoDB concern
created: Date;
show: IBroadcastShow;
@ -23,11 +23,16 @@ export interface IEpisode {
export const EpisodeSchema = new Schema<IEpisode>({
created: { type: Date, default: Date.now, required: true, index: -1 },
show: { type: Schema.ObjectId, required: true, index: 1, ref: "BroadcastShow" },
show: {
type: Schema.ObjectId,
required: true,
index: 1,
ref: "BroadcastShow",
},
title: { type: String, required: true },
description: { type: String, required: true },
video: { type: Schema.ObjectId, ref: 'Video' },
feedItems: { type: [Schema.ObjectId], ref: 'FeedItem' },
video: { type: Schema.ObjectId, ref: "Video" },
feedItems: { type: [Schema.ObjectId], ref: "FeedItem" },
});
export const Episode = model<IEpisode>("Episode", EpisodeSchema);

4
src/app/models/feed-item.ts

@ -19,7 +19,7 @@ export interface IFeedItem {
body?: string;
summary?: string;
videos: Array<IVideo | Types.ObjectId>
videos: Array<IVideo | Types.ObjectId>;
}
export const FeedItemSchema = new Schema<IFeedItem>({
@ -30,7 +30,7 @@ export const FeedItemSchema = new Schema<IFeedItem>({
description: { type: String },
body: { type: String },
summary: { type: String },
videos: { type: [Types.ObjectId], default: [ ], ref: "Video" },
videos: { type: [Types.ObjectId], default: [], ref: "Video" },
});
export const FeedItem = model<IFeedItem>("FeedItem", FeedItemSchema);

2
src/app/models/feed.ts

@ -27,7 +27,7 @@ const FeedSchema = new Schema<IFeed>({
description: { type: String, required: true },
url: { type: String, required: true },
web: { type: String, required: true },
latestItem: { type: Schema.ObjectId, ref: 'FeedItem' },
latestItem: { type: Schema.ObjectId, ref: "FeedItem" },
});
export const Feed = model<IFeed>("Feed", FeedSchema);

31
src/app/models/image.ts

@ -2,11 +2,11 @@
// Copyright (C) 2025 DTP Technologies, LLC
// All Rights Reserved
import { Schema, Types, model } from 'mongoose';
import { Schema, Types, model } from "mongoose";
import { IImageFile, ImageFileSchema } from './lib/image-file.js';
import { IImageMetadata, ImageMetadataSchema } from './lib/image-metadata.js';
import { IUser } from './user.js';
import { IImageFile, ImageFileSchema } from "./lib/image-file.js";
import { IImageMetadata, ImageMetadataSchema } from "./lib/image-metadata.js";
import { IUser } from "./user.js";
export interface IImage {
_id: Types.ObjectId;
@ -21,7 +21,7 @@ export interface IImage {
export const ImageSchema = new Schema<IImage>({
created: { type: Date, default: Date.now, required: true, index: -1 },
owner: { type: Types.ObjectId, required: true, index: 1, ref: 'User' },
owner: { type: Types.ObjectId, required: true, index: 1, ref: "User" },
caption: { type: String, maxLength: 300 },
type: { type: String, required: true },
size: { type: Number, required: true },
@ -29,15 +29,18 @@ export const ImageSchema = new Schema<IImage>({
metadata: { type: ImageMetadataSchema },
});
ImageSchema.index({
'flags.isPendingAttachment': 1,
created: -1,
}, {
partialFilterExpression: {
'flags.isPendingAttachment': true,
ImageSchema.index(
{
"flags.isPendingAttachment": 1,
created: -1,
},
name: 'img_pendattach_idx',
});
{
partialFilterExpression: {
"flags.isPendingAttachment": true,
},
name: "img_pendattach_idx",
}
);
export const Image = model<IImage>('Image', ImageSchema);
export const Image = model<IImage>("Image", ImageSchema);
export default Image;

13
src/app/models/lib/email-blacklist-flags.ts

@ -2,14 +2,17 @@
// Copyright (C) 2025 DTP Technologies, LLC
// All Rights Reserved
'use strict';
"use strict";
import { Schema } from 'mongoose';
import { Schema } from "mongoose";
export interface IEmailBlacklistFlags {
isVerified: boolean;
}
export const EmailBlacklistFlagsSchema = new Schema<IEmailBlacklistFlags>({
isVerified: { type: Boolean, default: false, required: true },
}, { _id: false });
export const EmailBlacklistFlagsSchema = new Schema<IEmailBlacklistFlags>(
{
isVerified: { type: Boolean, default: false, required: true },
},
{ _id: false }
);

17
src/app/models/lib/image-file.ts

@ -2,14 +2,17 @@
// Copyright (C) 2025 DTP Technologies, LLC
// All Rights Reserved
import { Schema } from 'mongoose';
import { Schema } from "mongoose";
export interface IImageFile {
bucket: string,
key: string,
bucket: string;
key: string;
}
export const ImageFileSchema = new Schema<IImageFile>({
bucket: { type: String, required: true },
key: { type: String, required: true },
}, { _id: false });
export const ImageFileSchema = new Schema<IImageFile>(
{
bucket: { type: String, required: true },
key: { type: String, required: true },
},
{ _id: false }
);

2
src/app/models/lib/image-metadata.ts

@ -2,7 +2,7 @@
// Copyright (C) 2025 DTP Technologies, LLC
// All Rights Reserved
import { Schema } from 'mongoose';
import { Schema } from "mongoose";
export interface IImageMetadata {
format: string | undefined;

13
src/app/models/lib/process-endpoint.ts

@ -10,8 +10,11 @@ export interface IDtpProcessEndpoint {
port: number;
}
export const DtpProcessEndpointSchema = new Schema<IDtpProcessEndpoint>({
hostname: { type: String, required: true },
ip: { type: String, required: true },
port: { type: Number, required: true },
}, { _id: false });
export const DtpProcessEndpointSchema = new Schema<IDtpProcessEndpoint>(
{
hostname: { type: String, required: true },
ip: { type: String, required: true },
port: { type: Number, required: true },
},
{ _id: false }
);

20
src/app/models/lib/process-stats.ts

@ -21,14 +21,20 @@ export interface IDtpWebProcessStats {
webRequestCount: number;
webErrorCount: number;
}
export const DtpWebProcessStatsSchema = new Schema<IDtpWebProcessStats>({
webRequestCount: { type: Number, default: 0, required: true },
webErrorCount: { type: Number, default: 0, required: true },
}, { _id: false });
export const DtpWebProcessStatsSchema = new Schema<IDtpWebProcessStats>(
{
webRequestCount: { type: Number, default: 0, required: true },
webErrorCount: { type: Number, default: 0, required: true },
},
{ _id: false }
);
export interface IDtpWorkerProcessStats {
jobRunCount: number;
}
export const DtpWorkerProcessStatsSchema = new Schema<IDtpWorkerProcessStats>({
jobRunCount: { type: Number, default: 0, required: true },
}, { _id: false });
export const DtpWorkerProcessStatsSchema = new Schema<IDtpWorkerProcessStats>(
{
jobRunCount: { type: Number, default: 0, required: true },
},
{ _id: false }
);

18
src/app/models/lib/speech-personality.ts

@ -5,15 +5,15 @@
import { Schema } from "mongoose";
export enum SpeechVoice {
Allow = 'alloy',
Ash = 'ash',
Coral = 'coral',
Echo = 'echo',
Fable = 'fable',
Onyx = 'onyx',
Nova = 'nova',
Sage = 'sage',
Shimmer = 'shimmer',
Allow = "alloy",
Ash = "ash",
Coral = "coral",
Echo = "echo",
Fable = "fable",
Onyx = "onyx",
Nova = "nova",
Sage = "sage",
Shimmer = "shimmer",
}
export interface ISpeechPersonality {

11
src/app/models/lib/user-apis.ts

@ -9,10 +9,13 @@ export interface IUserApiGoogle {
refreshToken: string;
}
export const UserApiGoogleSchema = new Schema<IUserApiGoogle>({
accessToken: { type: String, required: true },
refreshToken: { type: String, required: true },
}, { _id: false });
export const UserApiGoogleSchema = new Schema<IUserApiGoogle>(
{
accessToken: { type: String, required: true },
refreshToken: { type: String, required: true },
},
{ _id: false }
);
export interface IUserApis {
google: IUserApiGoogle;

11
src/app/models/lib/user-flags.ts

@ -9,7 +9,10 @@ export interface IUserFlags {
isEmailVerified: boolean;
}
export const UserFlagsSchema = new Schema<IUserFlags>({
isAdmin: { type: Boolean, default: false, required: true },
isEmailVerified: { type: Boolean, default: false, required: true },
}, { _id: false });
export const UserFlagsSchema = new Schema<IUserFlags>(
{
isAdmin: { type: Boolean, default: false, required: true },
isEmailVerified: { type: Boolean, default: false, required: true },
},
{ _id: false }
);

17
src/app/models/lib/user-permissions.ts

@ -2,7 +2,7 @@
// Copyright (C) 2025 DTP Technologies, LLC
// All Rights Reserved
import { Schema } from 'mongoose';
import { Schema } from "mongoose";
export interface IUserPermissions {
canLogin: boolean;
@ -11,9 +11,12 @@ export interface IUserPermissions {
canManageContent: boolean;
}
export const UserPermissionsSchema = new Schema<IUserPermissions>({
canLogin: { type: Boolean, default: true, required: true },
canComment: { type: Boolean, default: true, required: true },
canManageFeeds: { type: Boolean, default: true, required: true },
canManageContent: { type: Boolean, default: true, required: true },
}, { _id: false });
export const UserPermissionsSchema = new Schema<IUserPermissions>(
{
canLogin: { type: Boolean, default: true, required: true },
canComment: { type: Boolean, default: true, required: true },
canManageFeeds: { type: Boolean, default: true, required: true },
canManageContent: { type: Boolean, default: true, required: true },
},
{ _id: false }
);

44
src/app/models/process-stats.ts

@ -5,7 +5,14 @@
import { Types, Schema, model } from "mongoose";
import { IDtpProcess } from "./process";
import { DtpProcessStatsSchema, DtpWebProcessStatsSchema, DtpWorkerProcessStatsSchema, IDtpProcessStats, IDtpWebProcessStats, IDtpWorkerProcessStats } from "./lib/process-stats.js";
import {
DtpProcessStatsSchema,
DtpWebProcessStatsSchema,
DtpWorkerProcessStatsSchema,
IDtpProcessStats,
IDtpWebProcessStats,
IDtpWorkerProcessStats,
} from "./lib/process-stats.js";
/*
* Process stats history
@ -16,14 +23,16 @@ export interface IDtpProcessStatsHistory {
process: IDtpProcess | Types.ObjectId;
processStats: IDtpProcessStats;
}
export const DtpProcessStatsHistorySchema = new Schema<IDtpProcessStatsHistory>({
created: { type: Date, required: true, index: -1 },
processStats: { type: DtpProcessStatsSchema, required: true },
});
export const DtpProcessStatsHistorySchema = new Schema<IDtpProcessStatsHistory>(
{
created: { type: Date, required: true, index: -1 },
processStats: { type: DtpProcessStatsSchema, required: true },
}
);
export const DtpProcessStatsHistory = model<IDtpProcessStatsHistory>(
"DtpProcessStatsHistory",
DtpProcessStatsHistorySchema,
DtpProcessStatsHistorySchema
);
export default DtpProcessStatsHistory;
@ -34,9 +43,10 @@ export default DtpProcessStatsHistory;
export interface IDtpWebProcessStatsHistory extends IDtpProcessStatsHistory {
webStats: IDtpWebProcessStats;
}
export const DtpWebProcessStatsHistorySchema = new Schema<IDtpWebProcessStatsHistory>({
webStats: { type: DtpWebProcessStatsSchema, required: true },
});
export const DtpWebProcessStatsHistorySchema =
new Schema<IDtpWebProcessStatsHistory>({
webStats: { type: DtpWebProcessStatsSchema, required: true },
});
export const DtpWebProcessStatsHistory = DtpProcessStatsHistory.discriminator(
"DtpWebProcessStatsHistory",
DtpWebProcessStatsHistorySchema
@ -49,10 +59,12 @@ export const DtpWebProcessStatsHistory = DtpProcessStatsHistory.discriminator(
export interface IDtpWorkerProcessStatsHistory extends IDtpProcessStatsHistory {
workerStats: IDtpWorkerProcessStats;
}
export const DtpWorkerProcessStatsHistorySchema = new Schema<IDtpWorkerProcessStatsHistory>({
workerStats: { type: DtpWorkerProcessStatsSchema, required: true },
});
export const DtpWorkerProcessStatsHistory = DtpProcessStatsHistory.discriminator(
"DtpWorkerProcessStatsHistory",
DtpWorkerProcessStatsHistorySchema
);
export const DtpWorkerProcessStatsHistorySchema =
new Schema<IDtpWorkerProcessStatsHistory>({
workerStats: { type: DtpWorkerProcessStatsSchema, required: true },
});
export const DtpWorkerProcessStatsHistory =
DtpProcessStatsHistory.discriminator(
"DtpWorkerProcessStatsHistory",
DtpWorkerProcessStatsHistorySchema
);

20
src/app/models/process.ts

@ -8,10 +8,8 @@ import { DtpProcessStatus } from "./lib/process-status.js";
import {
IDtpProcessStats,
DtpProcessStatsSchema,
IDtpWebProcessStats,
DtpWebProcessStatsSchema,
IDtpWorkerProcessStats,
DtpWorkerProcessStatsSchema,
} from "./lib/process-stats.js";
@ -27,8 +25,8 @@ export enum DtpProcessType {
}
export interface IDtpProcess {
_id: Types.ObjectId; // MongoDB concern
__v: number; // MongoDB concern (optimistic consistency)
_id: Types.ObjectId; // MongoDB concern
__v: number; // MongoDB concern (optimistic consistency)
created: Date;
updated: Date;
@ -39,7 +37,13 @@ export interface IDtpProcess {
}
export const DtpProcessSchema = new Schema<IDtpProcess>({
created: { type: Date, required: true, default: Date.now, index: -1, expires: "24h" },
created: {
type: Date,
required: true,
default: Date.now,
index: -1,
expires: "24h",
},
updated: { type: Date, required: true, default: Date.now, index: -1 },
type: { type: String, enum: DtpProcessType, required: true, index: 1 },
status: { type: String, enum: DtpProcessStatus, required: true, index: 1 },
@ -54,10 +58,10 @@ export interface IDtpWebProcess extends IDtpProcess {
webStats: IDtpWebProcessStats;
}
export const DtpWebProcess = DtpProcess.discriminator(
'DtpWebProcess',
"DtpWebProcess",
new Schema<IDtpWebProcess>({
webStats: { type: DtpWebProcessStatsSchema, required: true },
}),
})
);
export interface IDtpWorkerProcess extends IDtpProcess {
@ -66,6 +70,6 @@ export interface IDtpWorkerProcess extends IDtpProcess {
export const DtpWorkerProcess = DtpProcess.discriminator(
"DtpWorkerProcess",
new Schema<IDtpWorkerProcess>({
workerStats: { type: DtpWorkerProcessStatsSchema, required: true }
workerStats: { type: DtpWorkerProcessStatsSchema, required: true },
})
);

56
src/app/models/user.ts

@ -5,13 +5,16 @@
import { Schema, Types, model } from "mongoose";
import { IUserFlags, UserFlagsSchema } from "./lib/user-flags.js";
import { IUserPermissions, UserPermissionsSchema } from "./lib/user-permissions.js";
import {
IUserPermissions,
UserPermissionsSchema,
} from "./lib/user-permissions.js";
import { IUserApis, UserApisSchema } from "./lib/user-apis.js";
import { IImage } from "./image.js";
export interface IUser {
_id: Types.ObjectId; // MongoDB concern (efficient record ID)
__v: number; // MongoDB concern (optimistic consistency)
_id: Types.ObjectId; // MongoDB concern (efficient record ID)
__v: number; // MongoDB concern (optimistic consistency)
created: Date;
updated?: Date;
@ -33,21 +36,34 @@ export interface IUser {
notes?: string;
}
const UserSchema = new Schema<IUser>({
created: { type: Date, default: Date.now, required: true, index: -1 },
updated: { type: Date, index: -1 },
email: { type: String, required: true, lowercase: true, unique: true },
email_lc: { type: String, required: true, lowercase: true, unique: true },
displayName: { type: String, maxLength: 60 },
picture: { type: Types.ObjectId, ref: 'Image' },
passwordSalt: { type: String, required: true, select: false },
password: { type: String, required: true, select: false },
apis: { type: UserApisSchema, default: { }, select: false },
flags: { type: UserFlagsSchema, default: { }, required: true, select: false },
permissions: { type: UserPermissionsSchema, default: { }, required: true, select: false },
theme: { type: String, default: 'dtp-light', required: true },
notes: { type: String, select: false },
}, { optimisticConcurrency: true });
export const User = model<IUser>('User', UserSchema);
const UserSchema = new Schema<IUser>(
{
created: { type: Date, default: Date.now, required: true, index: -1 },
updated: { type: Date, index: -1 },
email: { type: String, required: true, lowercase: true, unique: true },
email_lc: { type: String, required: true, lowercase: true, unique: true },
displayName: { type: String, maxLength: 60 },
picture: { type: Types.ObjectId, ref: "Image" },
passwordSalt: { type: String, required: true, select: false },
password: { type: String, required: true, select: false },
apis: { type: UserApisSchema, default: {}, select: false },
flags: {
type: UserFlagsSchema,
default: {},
required: true,
select: false,
},
permissions: {
type: UserPermissionsSchema,
default: {},
required: true,
select: false,
},
theme: { type: String, default: "dtp-light", required: true },
notes: { type: String, select: false },
},
{ optimisticConcurrency: true }
);
export const User = model<IUser>("User", UserSchema);
export default User;

38
src/app/models/video.ts

@ -30,23 +30,29 @@ export interface IVideoMetadata {
fps: number;
bitRate: number;
}
export const VideoMetadataSchema = new Schema<IVideoMetadata>({
width: { type: Number, required: true },
height: { type: Number, required: true },
fps: { type: Number, required: true },
bitRate: { type: Number, required: true },
}, { _id: false });
export const VideoMetadataSchema = new Schema<IVideoMetadata>(
{
width: { type: Number, required: true },
height: { type: Number, required: true },
fps: { type: Number, required: true },
bitRate: { type: Number, required: true },
},
{ _id: false }
);
export interface IAudioMetadata {
channelCount: number;
sampleRate: number;
bitRate: number;
}
export const AudioMetadataSchema = new Schema<IAudioMetadata>({
channelCount: { type: Number, required: true },
sampleRate: { type: Number, required: true },
bitRate: { type: Number, required: true },
}, { _id: false });
export const AudioMetadataSchema = new Schema<IAudioMetadata>(
{
channelCount: { type: Number, required: true },
sampleRate: { type: Number, required: true },
bitRate: { type: Number, required: true },
},
{ _id: false }
);
export interface IVideo {
_id: Types.ObjectId;
@ -61,12 +67,18 @@ export interface IVideo {
metadata: {
video: IVideoMetadata;
audio: IAudioMetadata;
}
};
}
export const VideoSchema = new Schema<IVideo>({
created: { type: Date, default: Date.now, required: true, index: -1 },
status: { type: String, enum: VideoStatus, default: VideoStatus.Pending, required: true, index: 1 },
status: {
type: String,
enum: VideoStatus,
default: VideoStatus.Pending,
required: true,
index: 1,
},
title: { type: String, required: true },
description: { type: String, required: true },
file: { type: MediaFileSchema },

46
src/app/services/cache.ts

@ -2,58 +2,58 @@
// Copyright (C) 2025 DTP Technologies, LLC
// All Rights Reserved
import assert from 'node:assert';
import assert from "node:assert";
import { Redis } from 'ioredis';
import { Redis } from "ioredis";
import { DtpService, DtpPlatform } from '../../lib/dtplib.js';
import { DtpService, DtpPlatform } from "../../lib/dtplib.js";
export class CacheService extends DtpService {
static get name ( ) { return 'CacheService'; }
static get slug ( ) { return 'cache'; }
static get name() {
return "CacheService";
}
static get slug() {
return "cache";
}
redis: Redis;
constructor (platform: DtpPlatform) {
constructor(platform: DtpPlatform) {
super(platform, CacheService);
assert(platform.redis, 'Redis connection required');
assert(platform.redis, "Redis connection required");
this.redis = platform.redis;
}
async set (name: string, value: string) {
async set(name: string, value: string) {
return this.redis.set(name, value);
}
async setEx (
async setEx(
name: string,
seconds: string | number,
value: string | number | Buffer,
) : Promise<"OK"> {
value: string | number | Buffer
): Promise<"OK"> {
return this.redis.setex(name, seconds, value);
}
async get (name: string) : Promise<string | null> {
async get(name: string): Promise<string | null> {
return this.redis.get(name);
}
async setObject (
name: string,
value: object,
) : Promise<"OK"> {
async setObject(name: string, value: object): Promise<"OK"> {
return this.redis.set(name, JSON.stringify(value));
}
async setObjectEx (
async setObjectEx(
name: string,
seconds: number,
value: object,
) : Promise<"OK"> {
value: object
): Promise<"OK"> {
return this.redis.setex(name, seconds, JSON.stringify(value));
}
async getObject (name: string) : Promise<object | null> {
async getObject(name: string): Promise<object | null> {
const value = await this.redis.get(name);
if (!value) {
return null;
@ -61,11 +61,11 @@ export class CacheService extends DtpService {
return JSON.parse(value);
}
async del (name: string) : Promise<number> {
async del(name: string): Promise<number> {
return this.redis.del(name);
}
getKeys (pattern: string) : Promise<Array<string> > {
getKeys(pattern: string): Promise<Array<string>> {
return this.redis.keys(pattern);
}
}

66
src/app/services/crypto.ts

@ -3,11 +3,11 @@
// All Rights Reserved
import assert from "node:assert";
import crypto from 'node:crypto';
import crypto from "node:crypto";
import env from '../../config/env.js';
import env from "../../config/env.js";
import { DtpPlatform, DtpService } from '../../lib/dtplib.js';
import { DtpPlatform, DtpService } from "../../lib/dtplib.js";
export type EncryptedMessage = {
iv: string;
@ -15,41 +15,43 @@ export type EncryptedMessage = {
};
export class CryptoService extends DtpService {
static get name() {
return "CryptoService";
}
static get slug() {
return "crypto";
}
static get name ( ) { return 'CryptoService'; }
static get slug ( ) { return 'crypto'; }
constructor (platform: DtpPlatform) {
constructor(platform: DtpPlatform) {
super(platform, CryptoService);
}
async start ( ) {
async start() {
await super.start();
assert(env.user.passwordSalt, "Password salt is required");
}
maskPassword (passwordSalt: string, password: string) : string {
maskPassword(passwordSalt: string, password: string): string {
assert(env.user.passwordSalt, "Password salt is required");
const hash = crypto.createHash('sha256');
const hash = crypto.createHash("sha256");
hash.update(env.user.passwordSalt);
hash.update(passwordSalt);
hash.update(password);
return hash.digest('hex');
return hash.digest("hex");
}
createHash (content: string | Buffer, algorithm: string = 'sha256') : string {
createHash(content: string | Buffer, algorithm: string = "sha256"): string {
const hash = crypto.createHash(algorithm);
hash.update(content);
return hash.digest('hex');
return hash.digest("hex");
}
hash32 (text: string) : string {
function hashToHex (hash: number) : string {
hash32(text: string): string {
function hashToHex(hash: number): string {
let hashHex = hash.toString(16);
if (hashHex[0] === '-') {
if (hashHex[0] === "-") {
hashHex = hashHex.slice(1);
}
return hashHex;
@ -62,44 +64,44 @@ export class CryptoService extends DtpService {
for (let i = text.length - 1; i >= 0; --i) {
const chr = text.charCodeAt(i);
hash = ((hash << 5) - hash) + chr;
hash = (hash << 5) - hash + chr;
hash |= 0;
}
return hashToHex(hash);
}
createProof (secret: string | Buffer, challenge: string | Buffer) : string {
const hash = crypto.createHash('sha256');
createProof(secret: string | Buffer, challenge: string | Buffer): string {
const hash = crypto.createHash("sha256");
hash.update(secret);
hash.update(challenge);
return hash.digest('hex');
return hash.digest("hex");
}
encrypt (key: string, data: string | Buffer) : EncryptedMessage {
encrypt(key: string, data: string | Buffer): EncryptedMessage {
const iv = crypto.randomBytes(16);
const cipher = crypto.createCipheriv(
'aes-256-cbc',
Buffer.from(key, 'hex'),
iv,
"aes-256-cbc",
Buffer.from(key, "hex"),
iv
);
let encryptedData = cipher.update(data);
encryptedData = Buffer.concat([encryptedData, cipher.final()]);
return {
iv: iv.toString('hex'),
data: encryptedData.toString('hex'),
iv: iv.toString("hex"),
data: encryptedData.toString("hex"),
};
}
decrypt (key: string, message: EncryptedMessage) : Buffer {
decrypt(key: string, message: EncryptedMessage): Buffer {
const decipher = crypto.createDecipheriv(
'aes-256-cbc',
Buffer.from(key, 'hex'),
Buffer.from(message.iv, 'hex'),
"aes-256-cbc",
Buffer.from(key, "hex"),
Buffer.from(message.iv, "hex")
);
const encryptedData = Buffer.from(message.data, 'hex');
const encryptedData = Buffer.from(message.data, "hex");
const decryptedData = decipher.update(encryptedData);
return Buffer.concat([decryptedData, decipher.final()]);
}

29
src/app/services/csrf-token.ts

@ -8,11 +8,7 @@ import dayjs from "dayjs";
import CsrfToken, { ICsrfToken } from "../models/csrf-token.js";
import {
DtpPlatform,
DtpService,
WebError,
} from "../../lib/dtplib.js";
import { DtpPlatform, DtpService, WebError } from "../../lib/dtplib.js";
export interface CsrfTokenOptions {
name: string;
@ -21,15 +17,23 @@ export interface CsrfTokenOptions {
}
export class CsrfTokenService extends DtpService {
static get name ( ) { return "CsrfTokenService"; }
static get slug ( ) { return "csrfToken"; }
static get name() {
return "CsrfTokenService";
}
static get slug() {
return "csrfToken";
}
constructor(platform: DtpPlatform) {
super(platform, CsrfTokenService);
}
middleware (options: CsrfTokenOptions) : RequestHandler {
return async (req: Request, _res: Response, next: NextFunction) : Promise<void> => {
middleware(options: CsrfTokenOptions): RequestHandler {
return async (
req: Request,
_res: Response,
next: NextFunction
): Promise<void> => {
const requestToken = req.body[`csrf-token-${options.name}`];
if (!requestToken) {
this.log.error("missing CSRF token", { name: options.name });
@ -44,7 +48,12 @@ export class CsrfTokenService extends DtpService {
return next(new WebError(401, "CSRF request token client mismatch"));
}
if (token.claimed && !options.allowReuse) {
return next(new WebError(401, "Your request can't be accepted. Please refresh the page and try again."));
return next(
new WebError(
401,
"Your request can't be accepted. Please refresh the page and try again."
)
);
}
if (token.user) {

29
src/app/services/display-engine.ts

@ -4,9 +4,9 @@
import env from "../../config/env.js";
import path from 'node:path';
import path from "node:path";
import pug from 'pug';
import pug from "pug";
import {
DtpPlatform,
@ -17,35 +17,38 @@ import {
} from "../../lib/dtplib.js";
export class DisplayEngineService extends DtpService {
static get name ( ) { return 'DisplayEngineService'; }
static get slug ( ) { return 'displayEngine'; }
static get name() {
return "DisplayEngineService";
}
static get slug() {
return "displayEngine";
}
templates: Record<string, pug.compileTemplate>;
constructor (platform: DtpPlatform) {
constructor(platform: DtpPlatform) {
super(platform, DisplayEngineService);
this.templates = { };
this.templates = {};
}
loadTemplate (name: string, pugScript: string) : void {
loadTemplate(name: string, pugScript: string): void {
if (this.templates[name]) {
throw new Error(`Template already registered for name ${name}`);
}
const scriptFile = path.join(env.src, 'app', 'views', pugScript);
const scriptFile = path.join(env.src, "app", "views", pugScript);
this.templates[name] = pug.compileFile(scriptFile);
}
renderTemplate (name: string, viewModel: WebViewModel) : string {
renderTemplate(name: string, viewModel: WebViewModel): string {
const template = this.templates[name];
if (!template) {
this.log.error('view engine template undefined', { name });
throw new WebError(500, 'Unknown display engine template');
this.log.error("view engine template undefined", { name });
throw new WebError(500, "Unknown display engine template");
}
return template(viewModel);
}
createDisplayList (name?: string) : WebDisplayList {
createDisplayList(name?: string): WebDisplayList {
return new WebDisplayList(name);
}
}

225
src/app/services/email.ts

@ -14,25 +14,25 @@ const require = createRequire(import.meta.url); // jshint ignore:line
import nodemailer from "nodemailer";
import Mail from "nodemailer/lib/mailer";
import mongoose from 'mongoose';
import Bull from 'bull';
import pug from 'pug';
import mongoose from "mongoose";
import Bull from "bull";
import pug from "pug";
import dayjs from 'dayjs';
import emailValidator from 'email-validator';
const emailDomainCheck = require('email-domain-check');
import numeral from 'numeral';
import dayjs from "dayjs";
import emailValidator from "email-validator";
const emailDomainCheck = require("email-domain-check");
import numeral from "numeral";
import { IEmailBlacklist } from '../../app/models/email-blacklist.js';
const EmailBlacklist = mongoose.model<IEmailBlacklist>('EmailBlacklist');
import { IEmailBlacklist } from "../../app/models/email-blacklist.js";
const EmailBlacklist = mongoose.model<IEmailBlacklist>("EmailBlacklist");
import { IEmailVerify } from '../../app/models/email-verify.js';
const EmailVerify = mongoose.model<IEmailVerify>('EmailVerify');
import { IEmailVerify } from "../../app/models/email-verify.js";
const EmailVerify = mongoose.model<IEmailVerify>("EmailVerify");
import { IEmailLog } from '../../app/models/email-log.js';
const EmailLog = mongoose.model<IEmailLog>('EmailLog');
import { IEmailLog } from "../../app/models/email-log.js";
const EmailLog = mongoose.model<IEmailLog>("EmailLog");
import { IUser } from '../../app/models/user.js';
import { IUser } from "../../app/models/user.js";
import {
DtpTools,
@ -40,11 +40,11 @@ import {
DtpService,
WebError,
WebViewModel,
} from '../../lib/dtplib.js';
} from "../../lib/dtplib.js";
import { UserService } from './user.js';
import { JobQueueService } from './job-queue.js';
import { CacheService } from './cache.js';
import { UserService } from "./user.js";
import { JobQueueService } from "./job-queue.js";
import { CacheService } from "./cache.js";
export type EmailAddress = string | Mail.Address;
@ -58,48 +58,56 @@ export type EmailMessage = {
};
export class EmailService extends DtpService {
static get name ( ) { return 'EmailService'; }
static get slug ( ) { return 'email'; }
static get name() {
return "EmailService";
}
static get slug() {
return "email";
}
transport?: nodemailer.Transporter;
emailQueue?: Bull.Queue;
disposableEmailDomains?: Array<string>;
templates: Record<string, Record<string, pug.compileTemplate> > = { };
templates: Record<string, Record<string, pug.compileTemplate>> = {};
populateEmailVerify: Array<mongoose.PopulateOptions> = [ ];
populateEmailVerify: Array<mongoose.PopulateOptions> = [];
constructor (platform: DtpPlatform) {
constructor(platform: DtpPlatform) {
super(platform, EmailService);
}
async start ( ) {
async start() {
await super.start();
const domainFile = path.resolve(
env.root,
'node_modules',
'disposable-email-provider-domains',
'data',
'domains.json',
"node_modules",
"disposable-email-provider-domains",
"data",
"domains.json"
);
this.disposableEmailDomains = await DtpTools.readJsonFile(domainFile) as Array<string>;
this.disposableEmailDomains = (await DtpTools.readJsonFile(
domainFile
)) as Array<string>;
const userService = this.platform.components.getService<UserService>('user');
const userService =
this.platform.components.getService<UserService>("user");
this.populateEmailVerify = [
{
path: 'user',
path: "user",
select: userService.USER_SELECT,
},
];
if (!env.email.enabled) {
this.log.info("MT_EMAIL_SERVICE is disabled, the system can't send email and will not try.");
this.log.info(
"MT_EMAIL_SERVICE is disabled, the system can't send email and will not try."
);
return;
}
this.log.info('creating SMTP transport', {
this.log.info("creating SMTP transport", {
host: env.email.smtp.host,
port: env.email.smtp.port,
});
@ -119,22 +127,23 @@ export class EmailService extends DtpService {
this.templates = {
html: {
welcome: this.loadAppTemplate('html', 'welcome.pug'),
passwordReset: this.loadAppTemplate('html', 'password-reset.pug'),
marketingBlast: this.loadAppTemplate('html', 'newsletter.pug'),
userEmail: this.loadAppTemplate('html', 'user-email.pug'),
welcome: this.loadAppTemplate("html", "welcome.pug"),
passwordReset: this.loadAppTemplate("html", "password-reset.pug"),
marketingBlast: this.loadAppTemplate("html", "newsletter.pug"),
userEmail: this.loadAppTemplate("html", "user-email.pug"),
},
text: {
welcome: this.loadAppTemplate('text', 'welcome.pug'),
passwordReset: this.loadAppTemplate('text', 'password-reset.pug'),
marketingBlast: this.loadAppTemplate('text', 'newsletter.pug'),
userEmail: this.loadAppTemplate('text', 'user-email.pug'),
welcome: this.loadAppTemplate("text", "welcome.pug"),
passwordReset: this.loadAppTemplate("text", "password-reset.pug"),
marketingBlast: this.loadAppTemplate("text", "newsletter.pug"),
userEmail: this.loadAppTemplate("text", "user-email.pug"),
},
};
const jobQueueService = this.platform.components.getService<JobQueueService>('jobQueue');
this.emailQueue = jobQueueService.getJobQueue('email', env.jobQueues.email);
const jobQueueService =
this.platform.components.getService<JobQueueService>("jobQueue");
this.emailQueue = jobQueueService.getJobQueue("email", env.jobQueues.email);
}
/**
@ -148,11 +157,11 @@ export class EmailService extends DtpService {
* @returns Promise<string> A promise that resolves the rendered HTML or text
* content.
*/
async renderTemplate (
async renderTemplate(
templateId: string,
templateType: string,
templateModel: WebViewModel,
) : Promise<string> {
templateModel: WebViewModel
): Promise<string> {
const templateMap = this.templates[templateType];
if (!templateMap) {
throw new Error(`Unknown email template type: ${templateType}`);
@ -163,12 +172,16 @@ export class EmailService extends DtpService {
throw new WebError(500, `Unknown email template: ${templateId}`);
}
const cacheService = this.platform.components.getService<CacheService>('cache');
const cacheService =
this.platform.components.getService<CacheService>("cache");
const settingsKey = `settings:${env.site.domainKey}:site`;
const adminSettings = await cacheService.getObject(settingsKey) as Record<string, unknown>;
const adminSettings = (await cacheService.getObject(settingsKey)) as Record<
string,
unknown
>;
const siteSettings = Object.assign({ }, env.site); // defaults and .env
const siteSettings = Object.assign({}, env.site); // defaults and .env
templateModel.site = Object.assign(siteSettings, adminSettings); // admin overrides
return template(templateModel);
@ -179,21 +192,23 @@ export class EmailService extends DtpService {
* The message specifies to, from, subject, html, and text content.
* @param message NodeMailerMessage The message to be sent.
*/
async send (message: EmailMessage) : Promise<IEmailLog> {
async send(message: EmailMessage): Promise<IEmailLog> {
if (!this.transport) {
throw new WebError(500, 'Email transport required');
throw new WebError(500, "Email transport required");
}
const NOW = new Date();
this.log.info('sending email', { to: message.to });
this.log.info("sending email", { to: message.to });
const response = await this.transport.sendMail(message);
this.log.debug('email sent', { to: message.to, response });
this.log.debug("email sent", { to: message.to, response });
const from = (typeof message.from === 'object') ? message.from.address : message.from;
const to = (typeof message.to === 'object') ? message.to.address : message.to;
const from =
typeof message.from === "object" ? message.from.address : message.from;
const to = typeof message.to === "object" ? message.to.address : message.to;
const log = await EmailLog.create({
created: NOW,
from, to,
from,
to,
to_lc: to.toLowerCase(),
subject: message.subject,
messageId: response.messageId,
@ -202,7 +217,7 @@ export class EmailService extends DtpService {
return log.toObject();
}
async sendWelcomeEmail (user: IUser) : Promise<void> {
async sendWelcomeEmail(user: IUser): Promise<void> {
await this.removeVerificationTokensForUser(user);
const verifyToken = await this.createVerificationToken(user);
const templateModel: WebViewModel = {
@ -211,7 +226,7 @@ export class EmailService extends DtpService {
emailVerifyToken: verifyToken.token,
};
const emailName = user.email.split('@')[0];
const emailName = user.email.split("@")[0];
assert(emailName, "recipient email address is malformed");
const message: EmailMessage = {
@ -221,18 +236,21 @@ export class EmailService extends DtpService {
address: user.email,
},
subject: `Welcome to ${env.site.name}!`,
html: await this.renderTemplate('welcome', 'html', templateModel),
text: await this.renderTemplate('welcome', 'text', templateModel),
html: await this.renderTemplate("welcome", "html", templateModel),
text: await this.renderTemplate("welcome", "text", templateModel),
};
await this.send(message);
}
async sendNewsletter (message: EmailMessage) : Promise<void> {
async sendNewsletter(message: EmailMessage): Promise<void> {
if (!this.emailQueue) {
throw new WebError(500, 'Email job queue is required for newsletter services');
throw new WebError(
500,
"Email job queue is required for newsletter services"
);
}
this.log.info('mass mail definition', { message });
await this.emailQueue.add('email-newsletter', { message });
this.log.info("mass mail definition", { message });
await this.emailQueue.add("email-newsletter", { message });
}
/**
@ -241,18 +259,18 @@ export class EmailService extends DtpService {
* the address is rejected by the system.
* @param emailAddress string The email address to be checked.
*/
async checkEmailAddress (emailAddress: string) : Promise<string> {
this.log.debug('validating email address', { emailAddress });
async checkEmailAddress(emailAddress: string): Promise<string> {
this.log.debug("validating email address", { emailAddress });
emailAddress = emailAddress.trim();
if (!emailValidator.validate(emailAddress)) {
throw new Error('Email address is invalid');
throw new Error("Email address is invalid");
}
const domainCheck = await emailDomainCheck(emailAddress);
this.log.debug('email domain check', { domainCheck });
this.log.debug("email domain check", { domainCheck });
if (!domainCheck) {
throw new Error('Email address is invalid');
throw new Error("Email address is invalid");
}
await this.isEmailBlacklisted(emailAddress);
@ -266,26 +284,34 @@ export class EmailService extends DtpService {
* @param emailAddress string The email address to be checked
* @returns boolean true if the address is blacklisted; false otherwise.
*/
async isEmailBlacklisted (emailAddress: string) : Promise<boolean> {
async isEmailBlacklisted(emailAddress: string): Promise<boolean> {
emailAddress = emailAddress.toLowerCase().trim();
const parts = emailAddress.split('@');
const parts = emailAddress.split("@");
if (!parts[0]) {
throw new WebError(400, 'Invalid email address');
throw new WebError(400, "Invalid email address");
}
const domain = parts[1] ? parts[1] : parts[0];
if (this.disposableEmailDomains) {
if (this.disposableEmailDomains.includes(domain)) {
this.log.alert('blacklisted email domain blocked', { emailAddress, domain });
throw new WebError(400, 'Invalid email address');
this.log.alert("blacklisted email domain blocked", {
emailAddress,
domain,
});
throw new WebError(400, "Invalid email address");
}
}
this.log.debug('checking email domain for blacklist', { domain });
const blacklistRecord = await EmailBlacklist.findOne({ email: emailAddress });
this.log.debug("checking email domain for blacklist", { domain });
const blacklistRecord = await EmailBlacklist.findOne({
email: emailAddress,
});
if (blacklistRecord) {
throw new WebError(401, 'Email address has requested to not receive emails');
throw new WebError(
401,
"Email address has requested to not receive emails"
);
}
return false;
@ -298,7 +324,7 @@ export class EmailService extends DtpService {
* be created.
* @returns new email verification token (for use in creating a link to it)
*/
async createVerificationToken (user: IUser) : Promise<IEmailVerify> {
async createVerificationToken(user: IUser): Promise<IEmailVerify> {
const NOW = new Date();
const verify = new EmailVerify();
@ -308,7 +334,7 @@ export class EmailService extends DtpService {
await verify.save();
this.log.info('created email verification token for user', {
this.log.info("created email verification token for user", {
user: {
_id: user._id,
},
@ -317,9 +343,8 @@ export class EmailService extends DtpService {
return verify.toObject();
}
async getVerificationToken (token: string) : Promise<IEmailVerify | null> {
const emailVerify = await EmailVerify
.findOne({ token })
async getVerificationToken(token: string): Promise<IEmailVerify | null> {
const emailVerify = await EmailVerify.findOne({ token })
.populate(this.populateEmailVerify)
.lean();
return emailVerify;
@ -331,32 +356,37 @@ export class EmailService extends DtpService {
* @param token string The token received from the email verification
* request.
*/
async verifyToken (token: string) : Promise<void> {
async verifyToken(token: string): Promise<void> {
const NOW = new Date();
const userService = this.platform.components.getService<UserService>('user');
const userService =
this.platform.components.getService<UserService>("user");
// fetch the token from the db
const emailVerify = await EmailVerify
.findOne({ token: token })
const emailVerify = await EmailVerify.findOne({ token: token })
.populate(this.populateEmailVerify)
.lean();
// verify that the token is at least valid (it exists)
if (!emailVerify) {
this.log.error('email verify token not found', { token });
throw new WebError(403, 'Email verification token is invalid');
this.log.error("email verify token not found", { token });
throw new WebError(403, "Email verification token is invalid");
}
// verify that it hasn't already been verified (user clicked link more than once)
if (emailVerify.verified) {
this.log.error('email verify token already claimed', { token });
throw new WebError(403, 'Email verification token is invalid');
this.log.error("email verify token already claimed", { token });
throw new WebError(403, "Email verification token is invalid");
}
this.log.info('marking user email verified', { userId: emailVerify.user._id });
this.log.info("marking user email verified", {
userId: emailVerify.user._id,
});
await userService.setEmailVerification(emailVerify.user, true);
await EmailVerify.updateOne({ _id: emailVerify._id }, { $set: { verified: NOW } });
await EmailVerify.updateOne(
{ _id: emailVerify._id },
{ $set: { verified: NOW } }
);
}
/**
@ -367,12 +397,15 @@ export class EmailService extends DtpService {
* @param user IUser the User for whom all pending verification tokens are
* to be removed.
*/
async removeVerificationTokensForUser (user: IUser) : Promise<void> {
this.log.info('removing all pending email address verification tokens for user', { user: user._id });
async removeVerificationTokensForUser(user: IUser): Promise<void> {
this.log.info(
"removing all pending email address verification tokens for user",
{ user: user._id }
);
await EmailVerify.deleteMany({ user: user._id });
}
createMessageModel (viewModel: WebViewModel) : WebViewModel {
createMessageModel(viewModel: WebViewModel): WebViewModel {
const messageModel: WebViewModel = {
config: env,
site: env.site,

78
src/app/services/feed.ts

@ -2,14 +2,14 @@
// Copyright (C) 2025 DTP Technologies, LLC
// All Rights Reserved
import { PopulateOptions, Types } from 'mongoose';
import { PopulateOptions, Types } from "mongoose";
import UserAgent from "user-agents";
import FeedItem, { IFeedItem } from '../models/feed-item.js';
import Feed, { IFeed } from '../models/feed.js';
import FeedItem, { IFeedItem } from "../models/feed-item.js";
import Feed, { IFeed } from "../models/feed.js";
import TextService from './text.js';
import TextService from "./text.js";
import { extractFromXml, FeedData } from "@extractus/feed-extractor";
@ -19,7 +19,7 @@ import {
WebPaginationParameters,
DtpServiceUpdate,
WebError,
} from '../../lib/dtplib.js';
} from "../../lib/dtplib.js";
/**
* Interface to be used when creating and updating RSS feed records.
@ -40,25 +40,28 @@ export interface FeedItemLibrary {
}
export class FeedService extends DtpService {
static get name ( ) { return 'FeedService'; }
static get slug ( ) { return 'feed'; }
static get name() {
return "FeedService";
}
static get slug() {
return "feed";
}
userAgent: UserAgent = new UserAgent();
populateFeed: Array<PopulateOptions>;
populateFeedItem: Array<PopulateOptions>;
constructor (platform: DtpPlatform) {
constructor(platform: DtpPlatform) {
super(platform, FeedService);
this.populateFeed = [
{
path: "latestItem",
}
},
];
this.populateFeedItem = [
{
path: 'feed',
path: "feed",
},
];
}
@ -69,7 +72,7 @@ export class FeedService extends DtpService {
* creating the feed.
* @returns An IFeed interface to the newly-created feed.
*/
async create (definition: FeedDefinition) : Promise<IFeed> {
async create(definition: FeedDefinition): Promise<IFeed> {
const textService = this.getService<TextService>("text");
const feed = new Feed();
@ -84,22 +87,25 @@ export class FeedService extends DtpService {
return feed.toObject();
}
async update (
async update(
feed: IFeed | Types.ObjectId,
definition: FeedDefinition,
) : Promise<IFeed> {
definition: FeedDefinition
): Promise<IFeed> {
const textService = this.getService<TextService>("text");
const update: DtpServiceUpdate = { };
update.$set = { };
update.$unset = { };
const update: DtpServiceUpdate = {};
update.$set = {};
update.$unset = {};
update.$set.title = textService.filter(definition.title);
update.$set.description = textService.filter(definition.description);
update.$set.url = textService.filter(definition.url);
update.$set.web = textService.filter(definition.web);
this.log.info("updating RSS feed", { _id: feed._id, title: definition.title });
this.log.info("updating RSS feed", {
_id: feed._id,
title: definition.title,
});
const newFeed = await Feed.findByIdAndUpdate(feed._id, update, {
new: true,
populate: this.populateFeed,
@ -111,21 +117,17 @@ export class FeedService extends DtpService {
return newFeed;
}
async getById (feedId: Types.ObjectId) : Promise<IFeed | null> {
const feed = await Feed
.findById(feedId)
.populate(this.populateFeed)
.lean();
async getById(feedId: Types.ObjectId): Promise<IFeed | null> {
const feed = await Feed.findById(feedId).populate(this.populateFeed).lean();
return feed;
}
async getItemsForFeed (
async getItemsForFeed(
feed: IFeed | Types.ObjectId,
pagination: WebPaginationParameters,
) : Promise<FeedItemLibrary> {
pagination: WebPaginationParameters
): Promise<FeedItemLibrary> {
const search = { feed: feed._id };
const items = await FeedItem
.find(search)
const items = await FeedItem.find(search)
.sort({ created: -1 })
.skip(pagination.skip)
.limit(pagination.cpp)
@ -135,11 +137,10 @@ export class FeedService extends DtpService {
return { items, totalItemCount };
}
async getUnifiedFeed (
pagination: WebPaginationParameters,
) : Promise<FeedItemLibrary> {
const items = await FeedItem
.find()
async getUnifiedFeed(
pagination: WebPaginationParameters
): Promise<FeedItemLibrary> {
const items = await FeedItem.find()
.sort({ created: -1 })
.skip(pagination.skip)
.limit(pagination.cpp)
@ -149,12 +150,11 @@ export class FeedService extends DtpService {
return { items, totalItemCount };
}
async fetchRssFeed (feed: IFeed) : Promise<FeedData> {
async fetchRssFeed(feed: IFeed): Promise<FeedData> {
const userAgent = this.userAgent.toString();
const headers = {
"User-Agent":
userAgent ||
`DtpNewsroom/1.0 (https://digitaltelepresence.com/)`,
userAgent || `DtpNewsroom/1.0 (https://digitaltelepresence.com/)`,
Accept:
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"Accept-Encoding": "gzip, deflate, br",
@ -166,7 +166,9 @@ export class FeedService extends DtpService {
const response = await fetch(feed.url, { method: "GET", headers });
if (!response.ok) {
throw new Error(`Failed to fetch RSS feed: ${response.statusText} (${response.status})`);
throw new Error(
`Failed to fetch RSS feed: ${response.statusText} (${response.status})`
);
}
const xml = await response.text();

28
src/app/services/image.ts

@ -13,10 +13,7 @@ const WebImage = mongoose.model<IImage>("Image");
import { IUser } from "../models/user.js";
import { IImageMetadata } from "../models/lib/image-metadata.js";
import {
DtpPlatform,
DtpService,
} from "../../lib/dtplib.js";
import { DtpPlatform, DtpService } from "../../lib/dtplib.js";
import { MinioObjectMetadata, MinioService } from "./minio.js";
import { UserService } from "./user.js";
@ -33,11 +30,14 @@ export type ImageServiceStatsReport = {
};
export class ImageService extends DtpService {
static get name() {
return "ImageService";
}
static get slug() {
return "image";
}
static get name ( ) { return "ImageService"; }
static get slug ( ) { return "image"; }
populateImage: Array<mongoose.PopulateOptions> = [ ];
populateImage: Array<mongoose.PopulateOptions> = [];
constructor(platform: DtpPlatform) {
super(platform, ImageService);
@ -46,7 +46,8 @@ export class ImageService extends DtpService {
async start() {
await super.start();
const userService = this.platform.components.getService<UserService>("user");
const userService =
this.platform.components.getService<UserService>("user");
this.populateImage.push({
path: "owner",
select: userService.USER_SELECT,
@ -55,7 +56,8 @@ export class ImageService extends DtpService {
async create(owner: IUser, file: Express.Multer.File): Promise<IImage> {
const NOW = new Date();
const minioService = this.platform.components.getService<MinioService>("minio");
const minioService =
this.platform.components.getService<MinioService>("minio");
this.log.debug("processing uploaded image", { file });
const sharpImage: Sharp = sharp(file.path);
@ -107,7 +109,8 @@ export class ImageService extends DtpService {
bufferMetadata?: MinioObjectMetadata
): Promise<IImage> {
const NOW = new Date();
const minioService = this.platform.components.getService<MinioService>("minio");
const minioService =
this.platform.components.getService<MinioService>("minio");
const image = new WebImage();
image.created = NOW;
@ -173,7 +176,8 @@ export class ImageService extends DtpService {
}
async remove(image: IImage): Promise<void> {
const minioService = this.platform.components.getService<MinioService>("minio");
const minioService =
this.platform.components.getService<MinioService>("minio");
try {
if (image.file && image.file.bucket && image.file.key) {
// this.log.debug('removing image from storage', { file: image.file });

32
src/app/services/job-queue.ts

@ -2,25 +2,28 @@
// Copyright (C) 2025 DTP Technologies, LLC
// All Rights Reserved
import env from '../../config/env.js';
import assert from 'node:assert';
import env from "../../config/env.js";
import assert from "node:assert";
import Bull from 'bull';
import Bull from "bull";
import { DtpPlatform, DtpService } from '../../lib/dtplib.js';
import { DtpPlatform, DtpService } from "../../lib/dtplib.js";
export class JobQueueService extends DtpService {
static get slug() {
return "jobQueue";
}
static get name() {
return "JobQueueService";
}
static get slug () { return 'jobQueue'; }
static get name ( ) { return 'JobQueueService'; }
queues: Record<string, Bull.Queue> = { };
queues: Record<string, Bull.Queue> = {};
constructor (platform: DtpPlatform) {
constructor(platform: DtpPlatform) {
super(platform, JobQueueService);
}
getJobQueue (name: string, defaultJobOptions: Bull.JobOptions) : Bull.Queue {
getJobQueue(name: string, defaultJobOptions: Bull.JobOptions): Bull.Queue {
let queue = this.queues[name];
if (queue) {
return queue;
@ -42,13 +45,10 @@ export class JobQueueService extends DtpService {
return queue;
}
async discoverJobQueues (pattern: string) : Promise<Array<string> > {
assert(this.platform.redis, 'Redis connection required');
async discoverJobQueues(pattern: string): Promise<Array<string>> {
assert(this.platform.redis, "Redis connection required");
const bullQueues: Array<string> = await this.platform.redis.keys(pattern);
return bullQueues
.map((queue) => queue.split(':')[1] || '---')
.sort()
;
return bullQueues.map((queue) => queue.split(":")[1] || "---").sort();
}
}

20
src/app/services/minio.ts

@ -46,8 +46,12 @@ export interface MinioBuffer {
}
export class MinioService extends DtpService {
static get name ( ) { return "MinioService"; }
static get slug ( ) { return "minio"; }
static get name() {
return "MinioService";
}
static get slug() {
return "minio";
}
minio?: MinioClient;
@ -55,7 +59,7 @@ export class MinioService extends DtpService {
super(platform, MinioService);
}
async start ( ) : Promise<void> {
async start(): Promise<void> {
await super.start();
this.minio = new MinioClient({
@ -67,7 +71,7 @@ export class MinioService extends DtpService {
});
}
async uploadBuffer (buffer: MinioBuffer) : Promise<void> {
async uploadBuffer(buffer: MinioBuffer): Promise<void> {
if (!this.minio) {
throw new WebError(500, "Not connected to storage");
}
@ -81,7 +85,7 @@ export class MinioService extends DtpService {
this.log.debug("uploaded buffer to storage", { response });
}
async uploadFile (fileInfo: MinioFile) : Promise<UploadedObjectInfo> {
async uploadFile(fileInfo: MinioFile): Promise<UploadedObjectInfo> {
if (!this.minio) {
throw new WebError(500, "Not connected to storage");
}
@ -96,11 +100,11 @@ export class MinioService extends DtpService {
);
}
async statFile (file: MinioFile) : Promise<MinioBucketItemStat> {
async statFile(file: MinioFile): Promise<MinioBucketItemStat> {
return this.statObject(file.bucket, file.key);
}
async statObject (bucket: string, key: string) : Promise<MinioBucketItemStat> {
async statObject(bucket: string, key: string): Promise<MinioBucketItemStat> {
if (!this.minio) {
throw new WebError(500, "Not connected to storage");
}
@ -108,7 +112,7 @@ export class MinioService extends DtpService {
return this.minio.statObject(bucket, key);
}
async downloadFile (fileInfo: MinioFile) : Promise<void> {
async downloadFile(fileInfo: MinioFile): Promise<void> {
if (!this.minio) {
throw new WebError(500, "Not connected to storage");
}

74
src/app/services/openai.ts

@ -25,14 +25,17 @@ export interface IGeneratedFile {
}
export class OpenAiService extends DtpService {
static get name ( ) { return 'OpenAiService'; }
static get slug ( ) { return 'openAi'; }
static get name() {
return "OpenAiService";
}
static get slug() {
return "openAi";
}
gabClient: OpenAI;
homelabClient: OpenAI;
constructor (platform: DtpPlatform) {
constructor(platform: DtpPlatform) {
super(platform, OpenAiService);
this.gabClient = new OpenAI({
baseURL: env.apis.openai.gab.baseURL,
@ -49,22 +52,23 @@ export class OpenAiService extends DtpService {
* @param feedItem FeedItem The item to be summarized.
* @returns A Promise that resolves to the feed item's text summary.
*/
async summarizeFeedItem (feedItem: IFeedItem) : Promise<string | undefined> {
async summarizeFeedItem(feedItem: IFeedItem): Promise<string | undefined> {
const response = await this.gabClient.chat.completions.create({
model: "arya",
messages: [
{
role: "system",
content: "You are a professional journalist with a degree in Journalism. Stick to the facts, don't hold back, and if a jew or black person committed a crime or did something wrong, you will say that the jew or black person did that.",
content:
"You are a professional journalist with a degree in Journalism. Stick to the facts, don't hold back, and if a jew or black person committed a crime or did something wrong, you will say that the jew or black person did that.",
},
{
role: "user",
content: `Write a summary of a news article with the title "${feedItem.title}"\n\n${feedItem.body}`,
},
]
],
});
if (!Array.isArray(response.choices) || (response.choices.length === 0)) {
if (!Array.isArray(response.choices) || response.choices.length === 0) {
return;
}
@ -77,24 +81,27 @@ export class OpenAiService extends DtpService {
return choice.message.content;
}
async createEpisodeTitle (episode: IEpisode) : Promise<string | null> {
async createEpisodeTitle(episode: IEpisode): Promise<string | null> {
assert(episode.feedItems, "Feed items are required");
const titles = episode.feedItems.map((item) => `"${(item as IFeedItem).title.replace('"', '\\"')}"`);
const titles = episode.feedItems.map(
(item) => `"${(item as IFeedItem).title.replace('"', '\\"')}"`
);
const response = await this.gabClient.chat.completions.create({
model: "arya",
messages: [
{
role: "system",
content: "You are an executive television network producer. You know what topics sell, and you know what works on YouTube and social media. You take your job seriously, you don't use vulgarity, and you always remember what's important to regular people.",
content:
"You are an executive television network producer. You know what topics sell, and you know what works on YouTube and social media. You take your job seriously, you don't use vulgarity, and you always remember what's important to regular people.",
},
{
role: "user",
content: `Create a title for an episode of a news broadcast that will present the following topics: ${titles.join(", ")}. Only tell me the title. Don't say anything else.`,
},
]
],
});
if (!Array.isArray(response.choices) || (response.choices.length === 0)) {
if (!Array.isArray(response.choices) || response.choices.length === 0) {
return null;
}
@ -107,24 +114,30 @@ export class OpenAiService extends DtpService {
return choice.message.content;
}
async createEpisodeDescription (episode: IEpisode) : Promise<string | null> {
assert(Array.isArray(episode.feedItems) && (episode.feedItems.length > 0), "Feed items are required");
const titles = episode.feedItems.map((item) => `"${(item as IFeedItem).title.replace('"', '\\"')}"`);
async createEpisodeDescription(episode: IEpisode): Promise<string | null> {
assert(
Array.isArray(episode.feedItems) && episode.feedItems.length > 0,
"Feed items are required"
);
const titles = episode.feedItems.map(
(item) => `"${(item as IFeedItem).title.replace('"', '\\"')}"`
);
const response = await this.gabClient.chat.completions.create({
model: "arya",
messages: [
{
role: "system",
content: "You are an executive television network producer. You know what topics sell, and you know what works on YouTube and social media. You take your job seriously, you don't use vulgarity, and you always remember what's important to regular people.",
content:
"You are an executive television network producer. You know what topics sell, and you know what works on YouTube and social media. You take your job seriously, you don't use vulgarity, and you always remember what's important to regular people.",
},
{
role: "user",
content: `Think up a title for an episode of a news broadcast that will present the following topics: ${titles.join(", ")}. Only say the description of the episode. Don't say anything else at all.`,
},
]
],
});
if (!Array.isArray(response.choices) || (response.choices.length === 0)) {
if (!Array.isArray(response.choices) || response.choices.length === 0) {
return null;
}
@ -137,7 +150,11 @@ export class OpenAiService extends DtpService {
return choice.message.content;
}
async generateSpeech (input: string, model: string, voice: SpeechVoice) : Promise<IGeneratedFile> {
async generateSpeech(
input: string,
model: string,
voice: SpeechVoice
): Promise<IGeneratedFile> {
const audioId = new Types.ObjectId();
const audioFile = path.join(env.root, `${audioId.toString()}.wav`);
@ -149,7 +166,10 @@ export class OpenAiService extends DtpService {
speed: 1.0,
});
if (!response.ok) {
throw new WebError(response.status, `failed to generate speech audio: ${response.statusText}`);
throw new WebError(
response.status,
`failed to generate speech audio: ${response.statusText}`
);
}
assert(response.body, "A response body is required");
@ -162,15 +182,19 @@ export class OpenAiService extends DtpService {
return { _id: audioId, file: audioFile };
}
async streamResponseToFile (stream: NodeJS.ReadableStream, path: string) : Promise<void> {
async streamResponseToFile(
stream: NodeJS.ReadableStream,
path: string
): Promise<void> {
return new Promise((resolve, reject) => {
const writeStream = fs.createWriteStream(path);
stream.pipe(writeStream)
.on('error', (error) => {
stream
.pipe(writeStream)
.on("error", (error) => {
writeStream.close();
reject(error);
})
.on('finish', resolve);
.on("finish", resolve);
});
}
}

131
src/app/services/process.ts

@ -3,44 +3,49 @@
// All Rights Reserved
import { Types as MongooseTypes } from "mongoose";
import dayjs from 'dayjs';
import dayjs from "dayjs";
import { IDtpProcessEndpoint } from '../models/lib/process-endpoint.js';
import { DtpProcessStatus } from '../models/lib/process-status.js';
import { IDtpProcessEndpoint } from "../models/lib/process-endpoint.js";
import { DtpProcessStatus } from "../models/lib/process-status.js";
import {
IDtpProcessStats,
IDtpWebProcessStats,
IDtpWorkerProcessStats,
} from 'app/models/lib/process-stats.js';
} from "app/models/lib/process-stats.js";
import {
DtpProcessType,
IDtpProcess,
DtpProcess,
IDtpWebProcess,
DtpWebProcess,
IDtpWorkerProcess,
DtpWorkerProcess,
} from "../models/process.ts";
import { DtpService, DtpPlatform } from '../../lib/dtplib.js';
import { WebPaginationParameters } from '../../lib/web/pagination-parameters.js';
import { DtpWebProcessStatsHistory, DtpWorkerProcessStatsHistory } from "../models/process-stats.ts";
import { DtpService, DtpPlatform } from "../../lib/dtplib.js";
import { WebPaginationParameters } from "../../lib/web/pagination-parameters.js";
import {
DtpWebProcessStatsHistory,
DtpWorkerProcessStatsHistory,
} from "../models/process-stats.ts";
export class ProcessService extends DtpService {
static get name() {
return "ProcessService";
}
static get slug() {
return "process";
}
static get name ( ) { return 'ProcessService'; }
static get slug ( ) { return 'process'; }
constructor (platform: DtpPlatform) {
constructor(platform: DtpPlatform) {
super(platform, ProcessService);
}
async createWebProcess (endpoint: IDtpProcessEndpoint) : Promise<IDtpWebProcess> {
async createWebProcess(
endpoint: IDtpProcessEndpoint
): Promise<IDtpWebProcess> {
const NOW = new Date();
const dtpProcess = new DtpWebProcess();
dtpProcess.created = NOW;
@ -63,7 +68,9 @@ export class ProcessService extends DtpService {
return dtpProcess.toObject();
}
async createWorkerProcess (endpoint: IDtpProcessEndpoint) : Promise<IDtpWorkerProcess> {
async createWorkerProcess(
endpoint: IDtpProcessEndpoint
): Promise<IDtpWorkerProcess> {
const NOW = new Date();
const dtpProcess = new DtpWorkerProcess();
dtpProcess.created = NOW;
@ -85,12 +92,12 @@ export class ProcessService extends DtpService {
return dtpProcess.toObject();
}
async setStatus (
async setStatus(
dtpProcess: IDtpProcess,
status: DtpProcessStatus,
) : Promise<void> {
status: DtpProcessStatus
): Promise<void> {
const NOW = new Date();
this.log.debug('setting process status', {
this.log.debug("setting process status", {
endpoint: dtpProcess.endpoint,
status,
});
@ -101,18 +108,18 @@ export class ProcessService extends DtpService {
updated: NOW,
status,
},
},
}
);
}
async reportWebProcessStats (
async reportWebProcessStats(
webProcess: IDtpWebProcess,
webStats: IDtpWebProcessStats,
processStats: IDtpProcessStats,
) : Promise<void> {
processStats: IDtpProcessStats
): Promise<void> {
const NOW = new Date();
this.log.debug('reporting web process stats', {
this.log.debug("reporting web process stats", {
hostname: webProcess.endpoint.hostname,
stats: webStats,
});
@ -125,7 +132,7 @@ export class ProcessService extends DtpService {
processStats,
webStats,
},
},
}
);
await DtpWebProcessStatsHistory.create({
created: NOW,
@ -135,14 +142,14 @@ export class ProcessService extends DtpService {
});
}
async reportWorkerProcessStats (
async reportWorkerProcessStats(
workerProcess: IDtpWorkerProcess,
workerStats: IDtpWorkerProcessStats,
processStats: IDtpProcessStats,
) : Promise<void> {
processStats: IDtpProcessStats
): Promise<void> {
const NOW = new Date();
this.log.debug('reporting worker process stats', {
this.log.debug("reporting worker process stats", {
hostname: workerProcess.endpoint.hostname,
workerStats,
processStats,
@ -156,7 +163,7 @@ export class ProcessService extends DtpService {
processStats,
workerStats,
},
},
}
);
await DtpWorkerProcessStatsHistory.create({
created: NOW,
@ -166,24 +173,36 @@ export class ProcessService extends DtpService {
});
}
async getWebProcessById (webProcessId: MongooseTypes.ObjectId): Promise<IDtpWebProcess | null> {
const dtpProcess = await DtpWebProcess.findOne({ _id: webProcessId }).lean();
async getWebProcessById(
webProcessId: MongooseTypes.ObjectId
): Promise<IDtpWebProcess | null> {
const dtpProcess = await DtpWebProcess.findOne({
_id: webProcessId,
}).lean();
return dtpProcess;
}
async getWorkerProcessById (workerProcessId: MongooseTypes.ObjectId): Promise<IDtpWorkerProcess | null> {
const dtpProcess = await DtpWorkerProcess.findOne({ _id: workerProcessId }).lean();
async getWorkerProcessById(
workerProcessId: MongooseTypes.ObjectId
): Promise<IDtpWorkerProcess | null> {
const dtpProcess = await DtpWorkerProcess.findOne({
_id: workerProcessId,
}).lean();
return dtpProcess;
}
async getProcessById (dtpProcessId: MongooseTypes.ObjectId) : Promise<IDtpProcess | null> {
async getProcessById(
dtpProcessId: MongooseTypes.ObjectId
): Promise<IDtpProcess | null> {
const dtpProcess = await DtpProcess.findOne({ _id: dtpProcessId }).lean();
return dtpProcess;
}
async getProcessesByStatus (status: [string], pagination: WebPaginationParameters) : Promise<Array<IDtpProcess>> {
const dtpProcesses = await DtpProcess
.find({ status })
async getProcessesByStatus(
status: [string],
pagination: WebPaginationParameters
): Promise<Array<IDtpProcess>> {
const dtpProcesses = await DtpProcess.find({ status })
.sort({ created: -1 })
.skip(pagination.skip)
.limit(pagination.cpp)
@ -191,9 +210,10 @@ export class ProcessService extends DtpService {
return dtpProcesses;
}
async getAllProcesses (pagination: WebPaginationParameters) : Promise<Array<IDtpProcess>> {
const dtpProcesses = await DtpProcess
.find()
async getAllProcesses(
pagination: WebPaginationParameters
): Promise<Array<IDtpProcess>> {
const dtpProcesses = await DtpProcess.find()
.sort({ created: -1 })
.skip(pagination.skip)
.limit(pagination.cpp)
@ -201,32 +221,33 @@ export class ProcessService extends DtpService {
return dtpProcesses;
}
async expireProcesses ( ) : Promise<void> {
async expireProcesses(): Promise<void> {
const NOW = new Date();
const OLDEST_DATE = dayjs(NOW).subtract(3, 'minute').toDate();
const OLDEST_DATE = dayjs(NOW).subtract(3, "minute").toDate();
this.log.info("expiring processes", { oldest: OLDEST_DATE });
await DtpProcess
.find({
$and: [
{ status: { $ne: DtpProcessStatus.Shutdown } },
{ updated: { $lt: OLDEST_DATE } },
]
})
await DtpProcess.find({
$and: [
{ status: { $ne: DtpProcessStatus.Shutdown } },
{ updated: { $lt: OLDEST_DATE } },
],
})
.cursor()
.eachAsync(async (dtpProcess) => {
this.log.info('expiring process', {
this.log.info("expiring process", {
processId: dtpProcess._id,
hostname: dtpProcess.endpoint.hostname,
});
await DtpProcess.updateOne(
{ _id: dtpProcess._id },
{ $set: { status: DtpProcessStatus.Shutdown } },
{ $set: { status: DtpProcessStatus.Shutdown } }
);
});
}
async remove (dtpProcess: IDtpProcess | MongooseTypes.ObjectId) : Promise<void> {
this.log.info('removing process', { processId: dtpProcess._id });
async remove(
dtpProcess: IDtpProcess | MongooseTypes.ObjectId
): Promise<void> {
this.log.info("removing process", { processId: dtpProcess._id });
await DtpProcess.deleteOne({ _id: dtpProcess._id });
}
}

168
src/app/services/session.ts

@ -2,7 +2,7 @@
// Copyright (C) 2025 DTP Technologies, LLC
// All Rights Reserved
'use strict';
"use strict";
import env from "../../config/env.js";
import assert from "node:assert";
@ -27,11 +27,7 @@ import {
Profile as GoogleProfile,
} from "passport-google-oauth20";
import {
DtpPlatform,
DtpService,
WebError,
} from "../../lib/dtplib.js";
import { DtpPlatform, DtpService, WebError } from "../../lib/dtplib.js";
import { IUser } from "../models/user.js";
import { UserService } from "./user.js";
@ -56,15 +52,18 @@ export type SessionAuthCheckOptions = {
};
export class SessionService extends DtpService {
static get name() {
return "SessionService";
}
static get slug() {
return "session";
}
static get name ( ) { return "SessionService"; }
static get slug ( ) { return "session"; }
constructor (platform: DtpPlatform) {
constructor(platform: DtpPlatform) {
super(platform, SessionService);
}
async start ( ) {
async start() {
await super.start();
passport.serializeUser(this.serializeUser.bind(this));
@ -74,16 +73,23 @@ export class SessionService extends DtpService {
usernameField: "email",
passwordField: "password",
};
passport.use("mt-local", new PassportLocal(strategyOptions, this.processLocalLogin.bind(this)));
passport.use("mt-admin", new PassportLocal(strategyOptions, this.processAdminLogin.bind(this)));
passport.use(
"mt-local",
new PassportLocal(strategyOptions, this.processLocalLogin.bind(this))
);
passport.use(
"mt-admin",
new PassportLocal(strategyOptions, this.processAdminLogin.bind(this))
);
if (env.apis.google.enabled) {
assert(env.apis.google.clientId, "MT_GOOGLE_CLIENT_ID is required");
assert(env.apis.google.secret, "MT_GOOGLE_SECRET is required");
const callbackURL = (env.NODE_ENV === "production")
? `https://${env.site.host}/auth/google/callback`
: "https://localhost:3000/auth/google/callback";
const callbackURL =
env.NODE_ENV === "production"
? `https://${env.site.host}/auth/google/callback`
: "https://localhost:3000/auth/google/callback";
const googleOptions: GoogleStrategyOptions = {
clientID: env.apis.google.clientId,
@ -93,11 +99,13 @@ export class SessionService extends DtpService {
// state: true,
// sessionKey: "goauth2",
};
passport.use(new GoogleStrategy(googleOptions, this.processGoogleLogin.bind(this)));
passport.use(
new GoogleStrategy(googleOptions, this.processGoogleLogin.bind(this))
);
}
}
async stop ( ) {
async stop() {
this.log.info(`stopping ${module.exports.name} service`);
await super.stop();
}
@ -106,8 +114,12 @@ export class SessionService extends DtpService {
* Generates a middleware function to process session data per request.
* @returns A middleware function to process per-request session data.
*/
middleware ( ) : RequestHandler {
return async (req: Request, res: Response, next: NextFunction) : Promise<void> => {
middleware(): RequestHandler {
return async (
req: Request,
res: Response,
next: NextFunction
): Promise<void> => {
res.locals.util = util;
res.locals.user = req.user;
@ -129,30 +141,43 @@ export class SessionService extends DtpService {
* @returns A middleware function that can protect a route with authentication
* and permission requirements.
*/
authCheckMiddleware (options: SessionAuthCheckOptions) : RequestHandler {
options = Object.assign({
useRedirect: true,
loginUri: '/welcome/login',
}, options);
return async (req: Request, res: Response, next: NextFunction) : Promise<void> => {
authCheckMiddleware(options: SessionAuthCheckOptions): RequestHandler {
options = Object.assign(
{
useRedirect: true,
loginUri: "/welcome/login",
},
options
);
return async (
req: Request,
res: Response,
next: NextFunction
): Promise<void> => {
if (options.permissions.requireCanLogin && !req.user) {
if (options.useRedirect) {
req.session.loginReturnTo = req.url;
await this.saveSession(req);
this.log.info('redirecting to login', { returnTo: req.url });
this.log.info("redirecting to login", { returnTo: req.url });
return res.redirect(options.loginUri);
}
return next(new WebError(403, 'Must sign in to continue'));
return next(new WebError(403, "Must sign in to continue"));
}
/*
* Flags checks
*/
if (options.flags.requireAdmin && (!req.user || !req.user.flags.isAdmin)) {
return next(new WebError(403, 'Administrator privileges are required'));
if (
options.flags.requireAdmin &&
(!req.user || !req.user.flags.isAdmin)
) {
return next(new WebError(403, "Administrator privileges are required"));
}
if (options.flags.requireEmailVerified && (!req.user || !req.user.flags.isEmailVerified)) {
if (
options.flags.requireEmailVerified &&
(!req.user || !req.user.flags.isEmailVerified)
) {
return next(new WebError(403, "Email verification is required"));
}
@ -160,14 +185,27 @@ export class SessionService extends DtpService {
* Permission checks
*/
if (options.permissions.requireCanComment && (!req.user || !req.user.permissions.canComment)) {
return next(new WebError(403, 'Commenting privileges are required'));
if (
options.permissions.requireCanComment &&
(!req.user || !req.user.permissions.canComment)
) {
return next(new WebError(403, "Commenting privileges are required"));
}
if (options.permissions.requireCanManageFeeds && (!req.user || !req.user.permissions.canManageFeeds)) {
return next(new WebError(403, 'Feed management privileges are required'));
if (
options.permissions.requireCanManageFeeds &&
(!req.user || !req.user.permissions.canManageFeeds)
) {
return next(
new WebError(403, "Feed management privileges are required")
);
}
if (options.permissions.requireCanManageContent && (!req.user || !req.user.permissions.canManageContent)) {
return next(new WebError(403, 'Content management privileges are required'));
if (
options.permissions.requireCanManageContent &&
(!req.user || !req.user.permissions.canManageContent)
) {
return next(
new WebError(403, "Content management privileges are required")
);
}
return next();
@ -183,9 +221,9 @@ export class SessionService extends DtpService {
* @returns A promise that resolves when the User is serialized to the
* session.
*/
async serializeUser (
async serializeUser(
user: IUser,
done: (error: Error | null, id?: string | undefined) => void,
done: (error: Error | null, id?: string | undefined) => void
) {
return done(null, user._id.toHexString());
}
@ -196,19 +234,22 @@ export class SessionService extends DtpService {
* @param done function PassportJS callback for async completion.
* @returns A promise that resolves when the user is fetched.
*/
async deserializeUser (
async deserializeUser(
userId: string,
done: (err: Error | null, user?: Express.User | false | null) => void,
) : Promise<void> {
const userService = this.platform.components.getService<UserService>("user");
done: (err: Error | null, user?: Express.User | false | null) => void
): Promise<void> {
const userService =
this.platform.components.getService<UserService>("user");
try {
const user = await userService.getAccountById(Types.ObjectId.createFromHexString(userId));
const user = await userService.getAccountById(
Types.ObjectId.createFromHexString(userId)
);
if (user && user.permissions && !user.permissions.canLogin) {
return done(null, null); // destroy the session
}
return done(null, user);
} catch (error) {
this.log.error('failed to deserialize user from session', { error });
this.log.error("failed to deserialize user from session", { error });
return done(error as Error, null);
}
}
@ -219,7 +260,7 @@ export class SessionService extends DtpService {
* @param req Express.Request The request being processed.
* @returns A promise that resolves when the session is saved.
*/
async saveSession (req: Request) : Promise<void> {
async saveSession(req: Request): Promise<void> {
return new Promise((resolve, reject) => {
req.session.save((err) => {
if (err) {
@ -236,13 +277,18 @@ export class SessionService extends DtpService {
* @param password string The password for the authenticating user.
* @param done function PassportJS callback for async completion.
*/
async processLocalLogin (
async processLocalLogin(
username: string,
password: string,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
done: (error: any, user?: Express.User | false, options?: IVerifyOptions) => void,
) : Promise<void> {
const userService = this.platform.components.getService<UserService>("user");
done: (
error: any,
user?: Express.User | false,
options?: IVerifyOptions
) => void
): Promise<void> {
const userService =
this.platform.components.getService<UserService>("user");
try {
const user: IUser = await userService.login(username, password);
this.log.info("user login", { user: { _id: user._id } });
@ -260,12 +306,16 @@ export class SessionService extends DtpService {
* @param password string The password for the authenticating user.
* @param done function PassportJS callback for async completion.
*/
async processAdminLogin (
async processAdminLogin(
username: string,
password: string,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
done: (error: any, user?: Express.User | false, options?: IVerifyOptions) => void,
) : Promise<void> {
done: (
error: any,
user?: Express.User | false,
options?: IVerifyOptions
) => void
): Promise<void> {
const userService = this.getService<UserService>("user");
try {
this.log.info("processing admin login", { username });
@ -288,17 +338,21 @@ export class SessionService extends DtpService {
accessToken: string,
refreshToken: string,
profile: GoogleProfile,
done: GoogleVerifyCallback,
) : Promise<void> {
done: GoogleVerifyCallback
): Promise<void> {
const userService = this.getService<UserService>("user");
try {
const user = await userService.grantGoogle(profile, accessToken, refreshToken);
const user = await userService.grantGoogle(
profile,
accessToken,
refreshToken
);
if (!user) {
return done(new Error("Failed to process User account"), false);
}
return done(null, user);
} catch (error) {
this.log.error('failed to process Google login', { profile, error });
this.log.error("failed to process Google login", { profile, error });
return done(error, false);
}
}

21
src/app/services/sidebar.ts

@ -4,21 +4,26 @@
import { Request, Response, NextFunction, RequestHandler } from "express";
import {
DtpPlatform,
DtpService,
} from "../../lib/dtplib.js";
import { DtpPlatform, DtpService } from "../../lib/dtplib.js";
export class SidebarService extends DtpService {
static get name ( ) { return "SidebarService"; }
static get slug ( ) { return "sidebar"; }
static get name() {
return "SidebarService";
}
static get slug() {
return "sidebar";
}
constructor(platform: DtpPlatform) {
super(platform, SidebarService);
}
middleware ( ) : RequestHandler {
return async (_req: Request, _res: Response, next: NextFunction) : Promise<void> => {
middleware(): RequestHandler {
return async (
_req: Request,
_res: Response,
next: NextFunction
): Promise<void> => {
try {
/*
* Populate sidebar content here

15
src/app/services/text.ts

@ -24,9 +24,12 @@ import { DtpPlatform, DtpService, DtpUnzalgo } from "../../lib/dtplib.js";
export type ReplacerFunction = (mention: string) => string;
export class TextService extends DtpService {
static get name ( ) { return "TextService"; }
static get slug ( ) { return "text"; }
static get name() {
return "TextService";
}
static get slug() {
return "text";
}
markedRenderer?: MarkedRenderer;
marked?: Marked;
@ -35,11 +38,11 @@ export class TextService extends DtpService {
super(platform, TextService);
}
async start ( ) : Promise<void> {
async start(): Promise<void> {
await super.start();
this.markedRenderer = new MarkedRenderer();
this.markedRenderer.link = (link: MarkedTokens.Link) : string => {
this.markedRenderer.link = (link: MarkedTokens.Link): string => {
if (link.title) {
return `<a href="${link.href}" title="${link.title}">${link.text}</a>`;
}
@ -99,7 +102,7 @@ export class TextService extends DtpService {
return this.clean(text);
}
async renderMarkdown (markdown: string) : Promise<string> {
async renderMarkdown(markdown: string): Promise<string> {
assert(this.marked, "Marked instance is required");
return this.marked.parse(markdown);
}

100
src/app/services/user.ts

@ -48,13 +48,19 @@ export interface UserPasswordChangeParams {
}
export class UserService extends DtpService {
static get name() {
return "UserService";
}
static get slug() {
return "user";
}
static get name ( ) { return "UserService"; }
static get slug ( ) { return "user"; }
USER_SELECT: string = "_id created email displayName bio image flags permissions";
SESSION_SELECT: string = "_id created email displayName bio image theme flags permissions";
ADMIN_SELECT: string = "_id created email displayName bio image flags permissions";
USER_SELECT: string =
"_id created email displayName bio image flags permissions";
SESSION_SELECT: string =
"_id created email displayName bio image theme flags permissions";
ADMIN_SELECT: string =
"_id created email displayName bio image flags permissions";
populateUser: Array<mongoose.PopulateOptions> = [];
@ -62,8 +68,12 @@ export class UserService extends DtpService {
super(platform, UserService);
}
requireAccountOwner (field: string = "userProfile") : RequestHandler {
return async (req: Request, res: Response, next?: NextFunction) : Promise<void> => {
requireAccountOwner(field: string = "userProfile"): RequestHandler {
return async (
req: Request,
res: Response,
next?: NextFunction
): Promise<void> => {
assert(next, "NextFunction is required");
if (!req.user) {
throw new WebError(403, "Login required");
@ -77,9 +87,12 @@ export class UserService extends DtpService {
}
async create(params: UserCreateParams): Promise<IUser> {
const cryptoService = this.platform.components.getService<CryptoService>("crypto");
const emailService = this.platform.components.getService<EmailService>("email");
const textService = this.platform.components.getService<TextService>("text");
const cryptoService =
this.platform.components.getService<CryptoService>("crypto");
const emailService =
this.platform.components.getService<EmailService>("email");
const textService =
this.platform.components.getService<TextService>("text");
const NOW = new Date();
@ -114,17 +127,22 @@ export class UserService extends DtpService {
return user.toObject();
}
async grantGoogle (
async grantGoogle(
profile: GoogleProfile,
accessToken: string,
refreshToken: string,
) : Promise<IUser> {
assert(Array.isArray(profile.emails) && profile.emails.length > 0, "An email address is required");
refreshToken: string
): Promise<IUser> {
assert(
Array.isArray(profile.emails) && profile.emails.length > 0,
"An email address is required"
);
const email = profile.emails.find((email) => email.verified);
assert(email, "A verified email address is required");
let dbUser: IUser | null = await this.getByEmailAddress(email.value.trim().toLowerCase());
let dbUser: IUser | null = await this.getByEmailAddress(
email.value.trim().toLowerCase()
);
if (dbUser) {
await User.updateOne(
{ _id: dbUser._id },
@ -132,13 +150,13 @@ export class UserService extends DtpService {
$set: {
"apis.google": { accessToken, refreshToken },
},
},
}
);
return dbUser;
}
const randomPassword = Randomstring.generate({ length: 12 });
this.log.info('creating new User account for Google authorization', {
this.log.info("creating new User account for Google authorization", {
profile: {
id: profile.id,
email: email.value,
@ -163,10 +181,11 @@ export class UserService extends DtpService {
}
async update(user: IUser, params: UserUpdateParams): Promise<IUser> {
const textService = this.platform.components.getService<TextService>("text");
const textService =
this.platform.components.getService<TextService>("text");
const NOW = new Date();
const update: DtpServiceUpdate = { };
const update: DtpServiceUpdate = {};
update.$set = { updated: NOW };
update.$unset = {};
@ -186,7 +205,9 @@ export class UserService extends DtpService {
update.$set.theme = params.theme;
}
const newUser = await User.findOneAndUpdate({ _id: user._id }, update, { new: true });
const newUser = await User.findOneAndUpdate({ _id: user._id }, update, {
new: true,
});
assert(newUser, "failed to update User record");
return newUser.toObject();
@ -196,7 +217,8 @@ export class UserService extends DtpService {
user: IUser,
params: UserPasswordChangeParams
): Promise<IUser> {
const cryptoService = this.platform.components.getService<CryptoService>("crypto");
const cryptoService =
this.platform.components.getService<CryptoService>("crypto");
if (!params.currentPassword) {
throw new WebError(400, "Must provide current password");
@ -249,11 +271,14 @@ export class UserService extends DtpService {
return User.populate(newUser, this.populateUser);
}
async setProfilePicture (user: IUser, file: Express.Multer.File) : Promise<IUser> {
async setProfilePicture(
user: IUser,
file: Express.Multer.File
): Promise<IUser> {
const imageService = this.getService<ImageService>("image");
const NOW = new Date();
const update: DtpServiceUpdate = { };
const update: DtpServiceUpdate = {};
update.$set = { updated: NOW };
if (user.picture) {
@ -279,7 +304,7 @@ export class UserService extends DtpService {
return User.populate(newUser, this.populateUser);
}
async removeProfilePicture (user: IUser) : Promise<IUser> {
async removeProfilePicture(user: IUser): Promise<IUser> {
const imageService = this.getService<ImageService>("image");
if (!user.picture) {
@ -295,11 +320,11 @@ export class UserService extends DtpService {
);
return User.populate(newUser, this.populateUser);
}
async login(email: string, password: string) : Promise<IUser> {
const cryptoService = this.platform.components.getService<CryptoService>("crypto");
async login(email: string, password: string): Promise<IUser> {
const cryptoService =
this.platform.components.getService<CryptoService>("crypto");
const user = await this.getAccountByEmailAddress(email);
if (!user) {
@ -310,7 +335,10 @@ export class UserService extends DtpService {
throw new WebError(403, "Account disabled");
}
const maskedPassword = cryptoService.maskPassword(user.passwordSalt, password);
const maskedPassword = cryptoService.maskPassword(
user.passwordSalt,
password
);
if (maskedPassword !== user.password) {
throw new WebError(400, "Credentials don't match user records");
}
@ -318,7 +346,7 @@ export class UserService extends DtpService {
return user;
}
async getAccountById (userId: mongoose.Types.ObjectId) : Promise<IUser | null> {
async getAccountById(userId: mongoose.Types.ObjectId): Promise<IUser | null> {
const user = await User.findOne({ _id: userId })
.select("+passwordSalt +password +flags +permissions")
.populate(this.populateUser)
@ -326,7 +354,7 @@ export class UserService extends DtpService {
return user;
}
async getAccountByEmailAddress (email: string) : Promise<IUser | null> {
async getAccountByEmailAddress(email: string): Promise<IUser | null> {
email = email.trim().toLowerCase();
const user = await User.findOne({ email_lc: email })
.select("+passwordSalt +password +flags +permissions")
@ -335,18 +363,18 @@ export class UserService extends DtpService {
return user;
}
async getById (userId: mongoose.Types.ObjectId) : Promise<IUser | null> {
async getById(userId: mongoose.Types.ObjectId): Promise<IUser | null> {
const user = await User.findOne({ _id: userId })
.select('+flags +permissions')
.select("+flags +permissions")
.populate(this.populateUser)
.lean();
return user;
}
async getByEmailAddress (email: string): Promise<IUser | null> {
async getByEmailAddress(email: string): Promise<IUser | null> {
email = email.trim().toLowerCase();
const user = await User.findOne({ email_lc: email })
.select('+flags +permissions')
.select("+flags +permissions")
.populate(this.populateUser)
.lean();
return user;
@ -355,7 +383,7 @@ export class UserService extends DtpService {
async getRandom(
exclude: IUser | mongoose.Types.ObjectId | null | undefined,
count: number
) : Promise<Array<IUser>> {
): Promise<Array<IUser>> {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const search: FilterQuery<any> = {};
search.$and = [];

79
src/app/services/video.ts

@ -4,10 +4,15 @@
import env from "../../config/env.js";
import { Types } from 'mongoose';
import { Types } from "mongoose";
import Video, { IVideo, VideoStatus } from '../models/video.js';
import TextService from './text.js';
import Bull from "bull";
import Video, { IVideo, VideoStatus } from "../models/video.js";
import TextService from "./text.js";
import JobQueueService from "./job-queue.js";
import MinioService from "./minio.js";
import {
DtpService,
@ -15,10 +20,7 @@ import {
WebPaginationParameters,
DtpServiceUpdate,
WebError,
} from '../../lib/dtplib.js';
import Bull from 'bull';
import JobQueueService from './job-queue.js';
import MinioService from "./minio.js";
} from "../../lib/dtplib.js";
/**
* Interface to be used when creating and updating videos.
@ -37,22 +39,28 @@ export interface VideoLibrary {
}
export class VideoService extends DtpService {
static get name ( ) { return 'VideoService'; }
static get slug ( ) { return 'video'; }
static get name() {
return "VideoService";
}
static get slug() {
return "video";
}
jobQueue?: Bull.Queue;
constructor (platform: DtpPlatform) {
constructor(platform: DtpPlatform) {
super(platform, VideoService);
}
async start ( ) : Promise<void> {
async start(): Promise<void> {
const jobQueueService = this.getService<JobQueueService>("jobQueue");
this.jobQueue = jobQueueService.getJobQueue("video", env.jobQueues.video);
}
async create (definition: VideoDefinition, file: Express.Multer.File) : Promise<IVideo> {
async create(
definition: VideoDefinition,
file: Express.Multer.File
): Promise<IVideo> {
const textService = this.getService<TextService>("text");
const minioService = this.getService<MinioService>("minio");
const NOW = new Date();
@ -87,51 +95,57 @@ export class VideoService extends DtpService {
key: video.file.key,
filePath: file.path,
metadata: {
'Content-Type': file.mimetype,
'Content-Length': file.size.toString(),
"Content-Type": file.mimetype,
"Content-Length": file.size.toString(),
},
});
this.log.info("video file uploaded to storage", { etag: minioFile.etag });
this.log.info("creating video record", { videoId: video._id, title: video.title });
this.log.info("creating video record", {
videoId: video._id,
title: video.title,
});
await video.save();
return video.toObject();
}
async update (video: IVideo | Types.ObjectId, definition: VideoDefinition) : Promise<IVideo> {
async update(
video: IVideo | Types.ObjectId,
definition: VideoDefinition
): Promise<IVideo> {
const textService = this.getService<TextService>("text");
const update: DtpServiceUpdate = { };
update.$set = { };
const update: DtpServiceUpdate = {};
update.$set = {};
update.$set.title = textService.filter(definition.title);
update.$set.description = textService.filter(definition.description);
this.log.info("updating video", { videoId: video._id });
const newVideo = await Video
.findByIdAndUpdate(video._id, update, {
new: true,
})
.lean();
const newVideo = await Video.findByIdAndUpdate(video._id, update, {
new: true,
}).lean();
if (!newVideo) {
throw new WebError(500, "Failed to update video");
}
return newVideo;
}
async setStatus (video: IVideo | Types.ObjectId, status: VideoStatus) : Promise<void> {
async setStatus(
video: IVideo | Types.ObjectId,
status: VideoStatus
): Promise<void> {
this.log.info("setting video status", { videoId: video._id, status });
await Video.findByIdAndUpdate(video._id, { $set: { status } });
}
async getById (videoId: Types.ObjectId) : Promise<IVideo | null> {
async getById(videoId: Types.ObjectId): Promise<IVideo | null> {
const video = await Video.findById(videoId).lean();
return video;
}
async getAll (pagination: WebPaginationParameters) : Promise<VideoLibrary> {
const videos = await Video
.find({ })
async getAll(pagination: WebPaginationParameters): Promise<VideoLibrary> {
const videos = await Video.find({})
.sort({ created: -1 })
.skip(pagination.skip)
.limit(pagination.cpp)
@ -140,7 +154,7 @@ export class VideoService extends DtpService {
return { videos, totalVideoCount };
}
async remove (video: IVideo) : Promise<void> {
async remove(video: IVideo): Promise<void> {
if (video.file) {
const minioService = this.getService<MinioService>("minio");
this.log.info("removing video file", {
@ -152,7 +166,10 @@ export class VideoService extends DtpService {
await minioService.removeObject(video.file?.bucket, video.file?.key);
}
this.log.info("removing video record", { videoId: video._id, title: video.title });
this.log.info("removing video record", {
videoId: video._id,
title: video.title,
});
await Video.deleteOne({ _id: video._id });
}
}

2
src/browsersync.ts

@ -5,7 +5,7 @@
import env from "./config/env.js";
import path from "node:path";
import { Options as BrowserSyncOptions} from "browser-sync";
import { Options as BrowserSyncOptions } from "browser-sync";
const options: BrowserSyncOptions = {
proxy: {

5
src/client/js/lib/app.ts

@ -5,14 +5,13 @@
import { WebLog } from "./log";
export class WebApp {
log: WebLog;
constructor ( ) {
constructor() {
this.log = new WebLog("WebApp");
}
async start ( ) {
async start() {
this.log.info("start", "The application is alive!");
}
}

25
src/client/js/lib/log.ts

@ -41,7 +41,6 @@ export interface WebLogData {
}
export class WebLog {
componentName: string;
options: WebLogOptions;
@ -67,7 +66,7 @@ export class WebLog {
ui.toggleAttribute("hidden", true);
}
constructor (componentName: string, options?: WebLogOptions) {
constructor(componentName: string, options?: WebLogOptions) {
this.componentName = componentName;
this.uiEntries = [];
@ -153,36 +152,42 @@ export class WebLog {
}
}
enable (enabled: boolean = true) : void {
enable(enabled: boolean = true): void {
this.enabled = enabled;
}
debug (event: string, msg: string, data?: WebLogData) : void {
debug(event: string, msg: string, data?: WebLogData): void {
this.write("debug", this.styles.debug, event, msg, data);
}
log (event: string, msg: string, data?: WebLogData) : void {
log(event: string, msg: string, data?: WebLogData): void {
this.info(event, msg, data);
}
info (event: string, msg: string, data?: WebLogData) : void {
info(event: string, msg: string, data?: WebLogData): void {
this.write("log", this.styles.info, event, msg, data);
}
warn (event: string, msg: string, data?: WebLogData) : void {
warn(event: string, msg: string, data?: WebLogData): void {
// alias for warning
this.warning(event, msg, data);
}
warning (event: string, msg: string, data?: WebLogData) : void {
warning(event: string, msg: string, data?: WebLogData): void {
this.write("warn", this.styles.warn, event, msg, data);
}
error (event: string, msg: string, data?: WebLogData) : void {
error(event: string, msg: string, data?: WebLogData): void {
this.write("error", this.styles.error, event, msg, data);
}
write(method: string, css: WebLogCss, event: string, msg: string, data?: WebLogData) {
write(
method: string,
css: WebLogCss,
event: string,
msg: string,
data?: WebLogData
) {
if (!this.enabled) {
return;
}

7
src/client/js/newsroom-app.ts

@ -9,20 +9,17 @@ export interface WebAppGlobals {
}
declare global {
interface Window {
dtp: WebAppGlobals;
}
}
(async ( ) => {
(async () => {
try {
window.dtp = window.dtp || { };
window.dtp = window.dtp || {};
window.dtp.app = new WebApp();
await window.dtp.app.start();
} catch (error) {
console.error("failed to start DTP Newsroom application", { error });
}
})();

40
src/config/env.ts

@ -18,31 +18,39 @@ async function readJsonFile<T>(path: string): Promise<T> {
return JSON.parse(file.toString("utf-8")) as T;
}
/* eslint-disable no-process-env */
export default {
NODE_ENV: process.env.NODE_ENV,
timezone: process.env.DTP_TIMEZONE || "America/New_York",
root: ROOT_DIR,
src: SRC_DIR,
pkg: await readJsonFile<typeof PackageJson>(path.join(ROOT_DIR, "package.json")),
pkg: await readJsonFile<typeof PackageJson>(
path.join(ROOT_DIR, "package.json")
),
site: {
company: process.env.DTP_SITE_COMPANY || "DTP Technologies, LLC",
companyShort: process.env.DTP_SITE_COMPANY_SHORT || "DTP",
name: process.env.DTP_SITE_NAME || "DTP Newsroom",
shortName: process.env.DTP_SITE_NAME || "Newsroom",
description: process.env.DTP_SITE_DESCRIPTION || "A virtual newsroom powered by RSS and AI.",
domain: process.env.DTP_SITE_DOMAIN || "dev.newsroom.digitaltelepresence.com",
domainKey: process.env.DTP_SITE_DOMAIN_KEY || "newsroom.digitaltelepresence.com",
host: process.env.DTP_SITE_HOST || "dev.newsroom.digitaltelepresence.com:3443",
description:
process.env.DTP_SITE_DESCRIPTION ||
"A virtual newsroom powered by RSS and AI.",
domain:
process.env.DTP_SITE_DOMAIN || "dev.newsroom.digitaltelepresence.com",
domainKey:
process.env.DTP_SITE_DOMAIN_KEY || "newsroom.digitaltelepresence.com",
host:
process.env.DTP_SITE_HOST || "dev.newsroom.digitaltelepresence.com:3443",
},
session: {
secret: process.env.DTP_SESSION_SECRET,
trustProxy: (process.env.NODE_ENV === "production") || (process.env.DTP_SESSION_TRUST_PROXY === "enabled"),
trustProxy:
process.env.NODE_ENV === "production" ||
process.env.DTP_SESSION_TRUST_PROXY === "enabled",
cookie: {
secure: process.env.DTP_SESSION_COOKIE_SECURE === "enabled",
sameSite: process.env.DTP_SESSION_COOKIE_SAMESITE || false,
}
},
},
mongodb: {
host: process.env.DTP_MONGODB_HOST || "localhost",
@ -106,13 +114,21 @@ export default {
host: process.env.DTP_EMAIL_SMTP_HOST || "localhost",
port: parseInt(process.env.DTP_EMAIL_SMTP_PORT || "465", 10),
secure: process.env.DTP_EMAIL_SMTP_SECURE === "enabled",
from: process.env.DTP_EMAIL_SMTP_FROM || "Support <[email protected]>",
from:
process.env.DTP_EMAIL_SMTP_FROM ||
"Support <[email protected]>",
user: process.env.DTP_EMAIL_SMTP_USER,
password: process.env.DTP_EMAIL_SMTP_PASS,
pool: {
enabled: process.env.DTP_EMAIL_SMTP_POOL_ENABLED === "enabled",
maxConnections: parseInt(process.env.DTP_EMAIL_SMTP_POOL_MAX_CONN || "5", 10),
maxMessages: parseInt(process.env.DTP_EMAIL_SMTP_POOL_MAX_MSGS || "100", 10),
maxConnections: parseInt(
process.env.DTP_EMAIL_SMTP_POOL_MAX_CONN || "5",
10
),
maxMessages: parseInt(
process.env.DTP_EMAIL_SMTP_POOL_MAX_MSGS || "100",
10
),
},
},
},
@ -157,7 +173,7 @@ export default {
debug: process.env.DTP_LOG_DEBUG === "enabled",
info: process.env.DTP_LOG_INFO === "enabled",
warn: process.env.DTP_LOG_WARN === "enabled",
}
},
},
};
/* eslint-enable no-process-env */

46
src/lib/core/base.ts

@ -2,25 +2,27 @@
// Copyright (C) 2025 DTP Technologies, LLC
// All Rights Reserved
import env from '../../config/env.js';
import path from 'node:path';
import pug from 'pug';
import env from "../../config/env.js";
import path from "node:path";
import pug from "pug";
import { EventEmitter } from 'node:events';
import { EventEmitter } from "node:events";
import { DtpPlatform } from './platform.js';
import { DtpLog } from './log.js';
import { DtpComponent } from './component.js';
import { DtpPlatform } from "./platform.js";
import { DtpLog } from "./log.js";
import { DtpComponent } from "./component.js";
import { DisplayEngineService } from '../../app/services/display-engine.js';
import { DisplayEngineService } from "../../app/services/display-engine.js";
/**
* Base class for most components in the system.
*/
export class DtpBase extends EventEmitter {
platform: DtpPlatform;
component: DtpComponent;
log: DtpLog;
constructor (platform: DtpPlatform, component: DtpComponent) {
constructor(platform: DtpPlatform, component: DtpComponent) {
super();
this.platform = platform;
@ -29,21 +31,31 @@ export class DtpBase extends EventEmitter {
this.log = new DtpLog(this.component, platform.logFile);
}
get name ( ) { return this.component.name; }
get slug ( ) { return this.component.slug; }
get name() {
return this.component.name;
}
get slug() {
return this.component.slug;
}
getService<T> (slug: string) : T {
getService<T>(slug: string): T {
return this.platform.components.getService<T>(slug);
}
loadAppTemplate (type: string, name: string) : pug.compileTemplate {
this.log.debug('loading application template', { type, name, root: env.src });
loadAppTemplate(type: string, name: string): pug.compileTemplate {
this.log.debug("loading application template", {
type,
name,
root: env.src,
});
const templateRoot = path.join(env.src, "app", "templates");
return pug.compileFile(path.join(templateRoot, type, name));
}
createDisplayList (name: string) {
const displayEngineService = this.getService<DisplayEngineService>('displayEngine');
createDisplayList(name: string) {
const displayEngineService =
this.getService<DisplayEngineService>("displayEngine");
return displayEngineService.createDisplayList(name);
}
}

33
src/lib/core/component-registry.ts

@ -23,19 +23,23 @@ type ServiceRegistry = Record<string, DtpService>;
type ControllerRegistry = Record<string, WebController>;
export class DtpComponentRegistry extends DtpBase {
models: ModelRegistry = {};
services: ServiceRegistry = {};
controllers: ControllerRegistry = {};
static get name ( ) { return "DtpComponentRegistry"; }
static get slug ( ) { return "DtpComponentRegistry"; }
static get name() {
return "DtpComponentRegistry";
}
models: ModelRegistry = { };
services: ServiceRegistry = { };
controllers: ControllerRegistry = { };
static get slug() {
return "DtpComponentRegistry";
}
constructor (platform: DtpPlatform) {
constructor(platform: DtpPlatform) {
super(platform, DtpComponentRegistry);
}
async loadModels ( ) {
async loadModels() {
const basePath = path.join(env.src, "app", "models");
this.log.info("loading models", { basePath });
@ -61,7 +65,7 @@ export class DtpComponentRegistry extends DtpBase {
}
}
async loadServices ( ) : Promise<void> {
async loadServices(): Promise<void> {
const basePath = path.join(env.src, "app", "services");
this.log.debug("loading services", { basePath });
@ -77,7 +81,9 @@ export class DtpComponentRegistry extends DtpBase {
}
try {
const ServiceClass = (await import(path.join(basePath, entry.name))).default;
const ServiceClass = (await import(path.join(basePath, entry.name)))
.default;
this.log.info("loading service", {
script: entry.name,
name: ServiceClass.name,
@ -99,7 +105,7 @@ export class DtpComponentRegistry extends DtpBase {
}
}
getService<T> (slug: string) : T {
getService<T>(slug: string): T {
const service = this.services[slug];
if (!service) {
throw new Error(`Service ${slug} is not loaded`);
@ -107,7 +113,7 @@ export class DtpComponentRegistry extends DtpBase {
return service as T;
}
async loadControllers (server: WebServer) : Promise<void> {
async loadControllers(server: WebServer): Promise<void> {
assert(server.app, "ExpressJS instance required");
const basePath = path.join(env.src, "app", "controllers");
@ -125,7 +131,8 @@ export class DtpComponentRegistry extends DtpBase {
}
try {
const ControllerClass = (await import(path.join(basePath, entry.name))).default;
const ControllerClass = (await import(path.join(basePath, entry.name)))
.default;
if (!ControllerClass) {
this.log.error("failed to receive a default export from controller", {
script: entry.name,
@ -160,7 +167,7 @@ export class DtpComponentRegistry extends DtpBase {
}
}
getController<T> (slug: string) : T {
getController<T>(slug: string): T {
const controller = this.controllers[slug];
if (!controller) {
throw new Error(`Controller ${slug} is not loaded`);

4
src/lib/core/component.ts

@ -3,6 +3,6 @@
// All Rights Reserved
export interface DtpComponent {
get name ( ) : string;
get slug ( ) : string;
get name(): string;
get slug(): string;
}

23
src/lib/core/log-file.ts

@ -2,12 +2,12 @@
// Copyright (C) 2025 DTP Technologies, LLC
// All Rights Reserved
import fs from 'node:fs';
import path from 'node:path';
import fs from "node:fs";
import path from "node:path";
import numeral from 'numeral';
import numeral from "numeral";
import { Writable, WritableOptions } from 'stream';
import { Writable, WritableOptions } from "stream";
type StreamCallback = (error?: Error | null) => void;
@ -19,29 +19,32 @@ export interface DtpLogFileOptions extends WritableOptions {
}
export class DtpLogFile extends Writable {
options: DtpLogFileOptions;
file?: fs.WriteStream;
fileIdx: number = 0;
writeCount: number = 0;
constructor (options: DtpLogFileOptions) {
constructor(options: DtpLogFileOptions) {
super(options);
this.options = options;
}
open ( ) : void {
open(): void {
fs.mkdirSync(this.options.basePath, { recursive: true });
const filename = path.join(
this.options.basePath,
`${this.options.name}.${numeral(this.fileIdx).format('000')}.log`,
`${this.options.name}.${numeral(this.fileIdx).format("000")}.log`
);
this.file = fs.createWriteStream(filename, { encoding: 'utf-8' });
this.file = fs.createWriteStream(filename, { encoding: "utf-8" });
this.writeCount = 0;
}
_write(chunk: unknown, encoding: BufferEncoding, callback: StreamCallback): boolean {
_write(
chunk: unknown,
encoding: BufferEncoding,
callback: StreamCallback
): boolean {
if (!this.file) {
return false;
}

10
src/lib/core/log-transport-console.ts

@ -7,11 +7,7 @@ import * as util from "node:util";
import dayjs from "dayjs";
import color from "ansicolor";
import {
DtpLogLevel,
DtpLogTransport,
DtpComponent,
} from "../dtplib.js";
import { DtpLogLevel, DtpLogTransport, DtpComponent } from "../dtplib.js";
export class DtpLogTransportConsole implements DtpLogTransport {
async writeLog(
@ -19,8 +15,8 @@ export class DtpLogTransportConsole implements DtpLogTransport {
component: DtpComponent,
level: DtpLogLevel,
message: string,
metadata?: unknown,
) : Promise<void> {
metadata?: unknown
): Promise<void> {
let clevel = level.padEnd(5);
switch (level) {
case "debug":

25
src/lib/core/log-transport-file.ts

@ -2,17 +2,16 @@
// Copyright (C) 2025 DTP Technologies, LLC
// All Rights Reserved
import { Writable } from 'node:stream';
import { Writable } from "node:stream";
import { DtpLogLevel } from './log.js';
import { DtpLogTransport } from './log-transport.js';
import { DtpLogLevel } from "./log.js";
import { DtpLogTransport } from "./log-transport.js";
import { DtpComponent } from "./component.js";
export class DtpLogTransportFile implements DtpLogTransport {
file: Writable;
constructor (file: Writable) {
constructor(file: Writable) {
this.file = file;
}
@ -21,12 +20,18 @@ export class DtpLogTransportFile implements DtpLogTransport {
component: DtpComponent,
level: DtpLogLevel,
message: string,
metadata?: unknown,
) : Promise<void> {
metadata?: unknown
): Promise<void> {
return new Promise<void>((resolve, reject) => {
const logMessage = JSON.stringify({ timestamp, component, level, message, metadata });
const chunk = Buffer.from(logMessage + '\r\n');
this.file.write(chunk, (error: Error | null | undefined) : void => {
const logMessage = JSON.stringify({
timestamp,
component,
level,
message,
metadata,
});
const chunk = Buffer.from(logMessage + "\r\n");
this.file.write(chunk, (error: Error | null | undefined): void => {
if (error) {
return reject(error);
}

8
src/lib/core/log-transport.ts

@ -2,8 +2,8 @@
// Copyright (C) 2025 DTP Technologies, LLC
// All Rights Reserved
import { DtpComponent } from './component.js';
import { DtpLogLevel } from './log.js';
import { DtpComponent } from "./component.js";
import { DtpLogLevel } from "./log.js";
export abstract class DtpLogTransport {
abstract writeLog(
@ -11,6 +11,6 @@ export abstract class DtpLogTransport {
component: DtpComponent,
level: DtpLogLevel,
message: string,
metadata?: unknown,
) : Promise<void>;
metadata?: unknown
): Promise<void>;
}

6
src/lib/core/log.ts

@ -7,8 +7,8 @@ import env from "../../config/env.js";
import { DtpComponent } from "./component.js";
import { DtpLogTransportConsole } from "./log-transport-console.js";
import { DtpLogTransportFile } from "./log-transport-file.js";
import { DtpLogTransport } from './log-transport.js';
import { DtpLogFile } from './log-file.js';
import { DtpLogTransport } from "./log-transport.js";
import { DtpLogFile } from "./log-file.js";
export enum DtpLogLevel {
debug = "debug",
@ -22,7 +22,7 @@ export enum DtpLogLevel {
export class DtpLog {
component: DtpComponent;
transports: Array<DtpLogTransport> = [ ];
transports: Array<DtpLogTransport> = [];
constructor(component: DtpComponent, file?: DtpLogFile) {
this.component = component;

27
src/lib/core/platform.ts

@ -1,7 +1,7 @@
// lib/core/platform.ts
// Copyright (C) 2025 DTP Technologies, LLC
// All Rights Reserved
console.log('importing DtpPlatform');
console.log("importing DtpPlatform");
import env from "../../config/env.js";
@ -18,8 +18,9 @@ interface ProcessWarning extends Error {
}
export class DtpPlatform {
static get REPORT_INTERVAL ( ) { return 1000 * 60; }
static get REPORT_INTERVAL() {
return 1000 * 60;
}
log: DtpLog;
logFile: DtpLogFile;
@ -30,8 +31,8 @@ export class DtpPlatform {
component: DtpComponent;
components: DtpComponentRegistry;
constructor (component: DtpComponent) {
console.log('DtpPlatform constructor');
constructor(component: DtpComponent) {
console.log("DtpPlatform constructor");
this.component = component;
const logOptions: DtpLogFileOptions = {
@ -48,7 +49,7 @@ export class DtpPlatform {
this.components = new DtpComponentRegistry(this);
}
async start ( ) {
async start() {
try {
this.hookProcessSignals();
@ -58,11 +59,11 @@ export class DtpPlatform {
await this.components.loadModels();
await this.components.loadServices();
} catch (error) {
throw new Error('failed to start DTP core', { cause: error });
throw new Error("failed to start DTP core", { cause: error });
}
}
async stop ( ): Promise<number> {
async stop(): Promise<number> {
if (this.redis) {
this.log.info("closing Redis connection(s)");
this.redis.quit();
@ -79,7 +80,7 @@ export class DtpPlatform {
return 0;
}
hookProcessSignals ( ) : void {
hookProcessSignals(): void {
process.title = this.component.name;
process.on("unhandledRejection", async (error: Error) => {
this.log.error("Unhandled rejection", {
@ -109,15 +110,15 @@ export class DtpPlatform {
});
}
async connectMongoDB ( ) {
async connectMongoDB() {
const mongoConnectUri = `mongodb://${env.mongodb.host}/${env.mongodb.database}`;
this.log.info('connecting to MongoDB', { uri: mongoConnectUri });
this.log.info("connecting to MongoDB", { uri: mongoConnectUri });
await mongoose.connect(mongoConnectUri, {
socketTimeoutMS: 0,
dbName: env.mongodb.database,
});
this.db = mongoose.connection;
this.log.debug('connected to MongoDB', { id: this.db.id });
this.log.debug("connected to MongoDB", { id: this.db.id });
}
async resetIndexes(args: Array<string>) {
@ -175,7 +176,7 @@ export class DtpPlatform {
}
}
getService<T> (slug: string) : T {
getService<T>(slug: string): T {
return this.components.getService<T>(slug);
}
}

51
src/lib/core/process.ts

@ -29,7 +29,6 @@ interface CpuTimes {
}
export abstract class DtpProcess extends DtpPlatform {
maxCpuTime: number = 0;
processStats: IDtpProcessStats = {
cpu: 0,
@ -41,13 +40,13 @@ export abstract class DtpProcess extends DtpPlatform {
reportJob?: CronJob;
abstract reportStats ( ) : Promise<void>;
abstract reportStats(): Promise<void>;
constructor (component: DtpComponent) {
constructor(component: DtpComponent) {
super(component);
}
async start ( ) : Promise<void> {
async start(): Promise<void> {
await super.start();
this.reportJob = new CronJob(
@ -55,11 +54,11 @@ export abstract class DtpProcess extends DtpPlatform {
this.reportStats.bind(this),
null,
true,
env.timezone,
env.timezone
);
}
async stop ( ) : Promise<number> {
async stop(): Promise<number> {
if (this.reportJob) {
this.log.info("stopping host report job");
this.reportJob.stop();
@ -68,20 +67,20 @@ export abstract class DtpProcess extends DtpPlatform {
return super.stop();
}
async updateStats ( ) : Promise<void> {
async updateStats(): Promise<void> {
await this.getCpuUsage();
await this.getMemoryUsage();
await this.getDiskUsage(env.https.uploadPath);
await this.getNetworkUsage();
}
async getMemoryUsage ( ) {
async getMemoryUsage() {
const freeMem = os.freemem();
const totalMem = os.totalmem();
this.processStats.memory = freeMem / totalMem;
}
async getCpuUsage ( ) {
async getCpuUsage() {
this.processStats.cpu = 0;
const cpus = os.cpus();
@ -108,31 +107,39 @@ export abstract class DtpProcess extends DtpPlatform {
cpuTimes.sys /= cpus.length;
cpuTimes.user /= cpus.length;
const cpuTime = cpuTimes.idle + cpuTimes.irq + cpuTimes.nice + cpuTimes.sys + cpuTimes.user;
const cpuTime =
cpuTimes.idle +
cpuTimes.irq +
cpuTimes.nice +
cpuTimes.sys +
cpuTimes.user;
this.processStats.cpu = cpuTime / this.maxCpuTime;
}
}
async getDiskUsage (pathname: string) : Promise<void> {
async getDiskUsage(pathname: string): Promise<void> {
return new Promise<void>((resolve, reject) => {
diskusage.check(pathname, (error?: Error, usage?: diskusage.DiskUsage) => {
if (error) {
return reject(error);
}
if (usage) {
this.processStats.storage = usage.available / usage.total;
return resolve();
diskusage.check(
pathname,
(error?: Error, usage?: diskusage.DiskUsage) => {
if (error) {
return reject(error);
}
if (usage) {
this.processStats.storage = usage.available / usage.total;
return resolve();
}
return 0;
}
return 0;
});
);
});
}
async getNetworkUsage ( ) : Promise<void> {
async getNetworkUsage(): Promise<void> {
this.processStats.netBytesRx = 0;
this.processStats.netBytesTx = 0;
const interfaces = await SystemInformation.networkStats('*');
const interfaces = await SystemInformation.networkStats("*");
for (const iface of interfaces) {
this.processStats.netBytesTx += iface.tx_bytes;
this.processStats.netBytesRx += iface.rx_bytes;

23
src/lib/core/service.ts

@ -2,31 +2,26 @@
// Copyright (C) 2025 DTP Technologies, LLC
// All Rights Reserved
import {
DtpPlatform,
DtpBase,
DtpComponent,
} from "../dtplib.js";
import { DtpPlatform, DtpBase, DtpComponent } from "../dtplib.js";
export type DtpServiceUpdate = {
$set?: Record<string, unknown>,
$unset?: Record<string, unknown>,
$push?: Record<string, unknown>,
$pull?: Record<string, unknown>,
$inc?: Record<string, number>,
$set?: Record<string, unknown>;
$unset?: Record<string, unknown>;
$push?: Record<string, unknown>;
$pull?: Record<string, unknown>;
$inc?: Record<string, number>;
};
export abstract class DtpService extends DtpBase {
constructor (platform: DtpPlatform, component: DtpComponent) {
constructor(platform: DtpPlatform, component: DtpComponent) {
super(platform, component);
}
async start ( ) {
async start() {
this.log.info(`starting ${this.name} service`);
}
async stop ( ) {
async stop() {
this.log.info(`stopping ${this.name} service`);
}
}

1
src/lib/core/unzalgo.ts

@ -17,7 +17,6 @@ const computeZalgoDensity = (text: string) =>
const clamp = (x: number) => Math.max(Math.min(x, 1), 0);
export class DtpUnzalgo {
/**
* Computes a score [0, 1] for every word in the input string. Each score
* represents the ratio of combining characters to total characters in a word.

17
src/lib/core/worker.ts

@ -18,7 +18,6 @@ import JobQueueService from "app/services/job-queue.js";
* Base class for all Worker processes.
*/
export class DtpWorker extends DtpProcess {
workerProcess?: IDtpWorkerProcess;
workerStats: IDtpWorkerProcessStats = {
jobRunCount: 0,
@ -26,15 +25,15 @@ export class DtpWorker extends DtpProcess {
jobQueue?: Bull.Queue;
constructor (component: DtpComponent) {
console.log('DtpWorker constructor');
constructor(component: DtpComponent) {
console.log("DtpWorker constructor");
super(component);
}
async startWorker (
async startWorker(
queueName: string,
queueOptions: Bull.JobOptions,
) : Promise<void> {
queueOptions: Bull.JobOptions
): Promise<void> {
await super.start();
const jobQueueService = this.getService<JobQueueService>("jobQueue");
@ -43,17 +42,17 @@ export class DtpWorker extends DtpProcess {
await this.cleanJobQueue();
}
async reportStats ( ) : Promise<void> {
async reportStats(): Promise<void> {
assert(this.workerProcess, "Worker process required");
const processService = this.getService<ProcessService>("process");
await processService.reportWorkerProcessStats(
this.workerProcess,
this.workerStats,
this.processStats,
this.processStats
);
}
async cleanJobQueue ( ) : Promise<void> {
async cleanJobQueue(): Promise<void> {
assert(this.jobQueue, "Job queue required");
const CHUNK_SIZE = 50;

45
src/lib/web/controller.ts

@ -10,7 +10,10 @@ import { Router, Request, Response, NextFunction } from "express";
import multer from "multer";
import { ICsrfToken } from "../../app/models/csrf-token.js";
import { CsrfTokenService, CsrfTokenOptions } from "../../app/services/csrf-token.js";
import {
CsrfTokenService,
CsrfTokenOptions,
} from "../../app/services/csrf-token.js";
import {
DtpBase,
@ -29,7 +32,6 @@ import {
* or JSON object responses.
*/
export abstract class WebController extends DtpBase {
server: WebServer;
router: Router;
@ -41,9 +43,9 @@ export abstract class WebController extends DtpBase {
this.router.use(this.middleware.bind(this));
}
abstract get route ( ) : string;
abstract get route(): string;
abstract start ( ) : Promise<void>;
abstract start(): Promise<void>;
/**
* Middleware common to all controllers that populates the view model with
@ -52,7 +54,7 @@ export abstract class WebController extends DtpBase {
* @param res Response The response being generated.
* @param next NextFunction The next function to call when done.
*/
middleware (req: Request, res: Response, next: NextFunction) {
middleware(req: Request, res: Response, next: NextFunction) {
res.locals.request = req;
res.locals.currentView = this.component.slug;
res.locals.signupEnabled = env.user.signupEnabled;
@ -83,7 +85,10 @@ export abstract class WebController extends DtpBase {
name: ControllerClass.name,
slug: ControllerClass.slug,
});
const controller: WebController = new ControllerClass(this.server, ControllerClass);
const controller: WebController = new ControllerClass(
this.server,
ControllerClass
);
this.log.info("starting child controller", {
name: ControllerClass.name,
@ -115,8 +120,12 @@ export abstract class WebController extends DtpBase {
pageParamName: string = "p",
cppParamName: string = "cpp"
): WebPaginationParameters {
const pageParam: string = req.query[pageParamName] ? req.query[pageParamName] as string : "1";
const cppParam: string = req.query[cppParamName] ? req.query[cppParamName] as string : maxPerPage.toString();
const pageParam: string = req.query[pageParamName]
? (req.query[pageParamName] as string)
: "1";
const cppParam: string = req.query[cppParamName]
? (req.query[cppParamName] as string)
: maxPerPage.toString();
const pagination: WebPaginationParameters = {
p: parseInt(pageParam, 10),
skip: 0,
@ -139,17 +148,20 @@ export abstract class WebController extends DtpBase {
* @param options multer.Options Options for the form processor.
* @returns An ExpressJS middleware that enables a route to receive files.
*/
createMulter (slug: string, options: multer.Options) : multer.Multer {
if (!!slug && (typeof slug === 'object')) {
createMulter(slug: string, options: multer.Options): multer.Multer {
if (!!slug && typeof slug === "object") {
options = slug;
slug = this.component.slug;
} else {
slug = slug || this.component.slug;
}
options = Object.assign({
dest: path.join(env.https.uploadPath, slug),
}, options || { });
options = Object.assign(
{
dest: path.join(env.https.uploadPath, slug),
},
options || {}
);
return multer(options);
}
@ -177,7 +189,10 @@ export abstract class WebController extends DtpBase {
* @param error WebError|Error|unknown The error being reported.
* @returns
*/
async renderErrorJSON (res: Response, error: WebError | Error | unknown) : Promise<void> {
async renderErrorJSON(
res: Response,
error: WebError | Error | unknown
): Promise<void> {
if (error instanceof WebError) {
res.status(error.status).json({
success: false,
@ -207,7 +222,7 @@ export abstract class WebController extends DtpBase {
* @param req Express.Request The request being processed.
* @returns A promise that resolves when the session is saved.
*/
async saveSession (req: Request) : Promise<void> {
async saveSession(req: Request): Promise<void> {
return new Promise((resolve, reject) => {
req.session.save((err) => {
if (err) {

100
src/lib/web/display-list.ts

@ -11,133 +11,149 @@ export interface WebDisplayListCommand {
}
export class WebDisplayList {
id: string = uuidv4();
name: string = 'default';
commands: Array<WebDisplayListCommand> = [ ];
name: string = "default";
commands: Array<WebDisplayListCommand> = [];
constructor (name?: string) {
constructor(name?: string) {
if (name) {
this.name = name;
}
}
showNotification (message: string, status: string, pos: string, timeout: number) {
showNotification(
message: string,
status: string,
pos: string,
timeout: number
) {
this.commands.push({
action: 'showNotification',
action: "showNotification",
params: { message, status, pos, timeout },
});
}
showModal (html: string) {
showModal(html: string) {
this.commands.push({
action: 'showModal',
action: "showModal",
params: { html },
});
}
addElement (selector: string, where: string, html: string) {
addElement(selector: string, where: string, html: string) {
this.commands.push({
selector, action: 'addElement',
selector,
action: "addElement",
params: { where, html },
});
}
removeChildren (selector: string) {
removeChildren(selector: string) {
this.commands.push({
selector, action: 'removeChildren',
params: { },
selector,
action: "removeChildren",
params: {},
});
}
setTextContent (selector: string, text: string) {
setTextContent(selector: string, text: string) {
this.commands.push({
selector, action: 'setTextContent',
selector,
action: "setTextContent",
params: { text },
});
}
replaceElement (selector: string, html: string) {
replaceElement(selector: string, html: string) {
this.commands.push({
selector, action: 'replaceElement',
selector,
action: "replaceElement",
params: { html },
});
}
removeElement (selector: string) {
removeElement(selector: string) {
this.commands.push({
selector, action: 'removeElement',
params: { },
selector,
action: "removeElement",
params: {},
});
}
setAttribute (selector: string, name: string, value: string) {
setAttribute(selector: string, name: string, value: string) {
this.commands.push({
selector, action: 'setAttribute',
selector,
action: "setAttribute",
params: { name, value },
});
}
removeAttribute (selector: string, name: string) {
removeAttribute(selector: string, name: string) {
this.commands.push({
selector, action: 'removeAttribute',
selector,
action: "removeAttribute",
params: { name },
});
}
toggleAttribute (selector: string, name: string, force: boolean) {
toggleAttribute(selector: string, name: string, force: boolean) {
this.commands.push({
selector, action: 'toggleAttribute',
selector,
action: "toggleAttribute",
params: { name, force },
});
}
addClass (selector: string, add: string) {
addClass(selector: string, add: string) {
this.commands.push({
selector, action: 'addClass',
selector,
action: "addClass",
params: { add },
});
}
removeClass (selector: string, remove: string) {
removeClass(selector: string, remove: string) {
this.commands.push({
selector, action: 'removeClass',
selector,
action: "removeClass",
params: { remove },
});
}
replaceClass (selector: string, remove: string, add: string) {
replaceClass(selector: string, remove: string, add: string) {
this.commands.push({
selector, action: 'replaceClass',
selector,
action: "replaceClass",
params: { remove, add },
});
}
setValue (selector: string, value: string) {
setValue(selector: string, value: string) {
this.commands.push({
selector, action: 'setValue',
selector,
action: "setValue",
params: { value },
});
}
navigateTo (href: string) {
navigateTo(href: string) {
this.commands.push({
action: 'navigateTo',
action: "navigateTo",
params: { href },
});
}
navigateBack ( ) {
navigateBack() {
this.commands.push({
action: 'navigateBack',
params: { },
action: "navigateBack",
params: {},
});
}
reloadView ( ) {
reloadView() {
this.commands.push({
action: 'reloadView',
params: { },
action: "reloadView",
params: {},
});
}
}

3
src/lib/web/error.ts

@ -3,10 +3,9 @@
// All Rights Reserved
export class WebError extends Error {
status: number;
constructor (status: number, message: string, options?: ErrorOptions) {
constructor(status: number, message: string, options?: ErrorOptions) {
super(message, options);
this.status = status;
}

6
src/lib/web/file.ts

@ -2,7 +2,7 @@
// Copyright (C) 2025 DTP Technologies, LLC
// All Rights Reserved
import { v4 as uuidv4 } from 'uuid';
import { v4 as uuidv4 } from "uuid";
export class WebFile {
uuid: string;
@ -11,11 +11,11 @@ export class WebFile {
mimetype: string;
size: number;
constructor (
constructor(
filePath: string,
filename: string,
mimetype: string,
size: number,
size: number
) {
this.uuid = uuidv4();
this.filePath = filePath;

188
src/lib/web/server.ts

@ -17,7 +17,7 @@ import svgCaptcha from "svg-captcha";
import * as rfs from "rotating-file-stream";
import express, { Request, Response, NextFunction } from 'express';
import express, { Request, Response, NextFunction } from "express";
import morgan from "morgan";
import cookieParser from "cookie-parser";
@ -31,12 +31,7 @@ import { RedisStore } from "connect-redis";
import SessionService from "../../app/services/session.js";
import TextService from "../../app/services/text.js";
import {
DtpTools,
DtpPlatform,
DtpBase,
WebError,
} from "../dtplib.js";
import { DtpTools, DtpPlatform, DtpBase, WebError } from "../dtplib.js";
type SameSiteOption = boolean | "lax" | "strict" | "none" | undefined;
@ -46,29 +41,41 @@ export interface WebServerStats {
}
export class WebServer extends DtpBase {
static get name ( ) { return "WebServer"; }
static get slug ( ) { return "webServer"; }
static get name() {
return "WebServer";
}
static get slug() {
return "webServer";
}
app?: express.Application;
https?: https.Server;
stats: WebServerStats = { requestCount: 0, errorCount: 0 };
constructor (platform: DtpPlatform) {
constructor(platform: DtpPlatform) {
super(platform, WebServer);
}
get name ( ) { return "WebServer"; }
get slug ( ) { return "webServer"; }
get name() {
return "WebServer";
}
get slug() {
return "webServer";
}
async createExpressApp ( ) {
async createExpressApp() {
const textService = this.getService<TextService>("text");
/*
* Load the CAPTCHA font
*/
const captchaFontPath = path.resolve(env.src, "client", "fonts", "green-nature.ttf");
const captchaFontPath = path.resolve(
env.src,
"client",
"fonts",
"green-nature.ttf"
);
svgCaptcha.loadFont(captchaFontPath);
/*
@ -84,7 +91,9 @@ export class WebServer extends DtpBase {
this.app.locals.MT_SCRIPT_DEBUG = env.NODE_ENV === "development";
this.app.locals.env = env;
this.app.locals.pkg = await DtpTools.readJsonFile<typeof PackageJson>(path.join(env.root, "package.json"));
this.app.locals.pkg = await DtpTools.readJsonFile<typeof PackageJson>(
path.join(env.root, "package.json")
);
this.app.locals.platform = this;
this.app.locals.site = env.site;
@ -94,7 +103,7 @@ export class WebServer extends DtpBase {
this.app.locals.marked = textService.marked;
if (env.log.http.enabled) {
this.log.info('creating HTTP access log', {
this.log.info("creating HTTP access log", {
name: env.log.http.name,
path: env.log.http.path,
});
@ -109,13 +118,13 @@ export class WebServer extends DtpBase {
this.app.use(morgan(env.log.http.format, { stream: httpLogStream }));
}
this.app.use((_req: Request, _res: Response, next: NextFunction) : void => {
this.app.use((_req: Request, _res: Response, next: NextFunction): void => {
++this.stats.requestCount;
return next();
});
}
registerStaticPaths ( ) {
registerStaticPaths() {
assert(this.app, "ExpressJS app instance is required");
function cacheOneDay(_req: Request, res: Response, next: NextFunction) {
@ -123,26 +132,74 @@ export class WebServer extends DtpBase {
return next();
}
function serviceWorkerAllowed(_req: Request, res: Response, next: NextFunction) {
function serviceWorkerAllowed(
_req: Request,
res: Response,
next: NextFunction
) {
res.set("Service-Worker-Allowed", "/");
return next();
}
this.app.use(["/dist", "/"], serviceWorkerAllowed, express.static(path.join(env.root, "dist", "client")));
this.app.use("/socket.io", cacheOneDay, express.static(path.join(env.root, "node_modules", "socket.io", "client-dist")));
this.app.use("/uikit/images", cacheOneDay, express.static(path.join(env.root, "node_modules", "uikit", "src", "images")));
this.app.use("/uikit", cacheOneDay, express.static(path.join(env.root, "node_modules", "uikit", "dist")));
this.app.use("/fontawesome", cacheOneDay, express.static(path.join(env.root, "node_modules", "@fortawesome", "fontawesome-free")));
this.app.use("/pretty-checkbox", cacheOneDay, express.static(path.join(env.root, "node_modules", "pretty-checkbox", "dist")));
this.app.use("/cropperjs", cacheOneDay, express.static(path.join(env.root, "node_modules", "cropperjs", "dist")));
this.app.use("/dayjs", cacheOneDay, express.static(path.join(env.root, "node_modules", "dayjs")));
this.app.use("/numeral", cacheOneDay, express.static(path.join(env.root, "node_modules", "numeral", "min")));
this.app.use(
["/dist", "/"],
serviceWorkerAllowed,
express.static(path.join(env.root, "dist", "client"))
);
this.app.use(
"/socket.io",
cacheOneDay,
express.static(
path.join(env.root, "node_modules", "socket.io", "client-dist")
)
);
this.app.use(
"/uikit/images",
cacheOneDay,
express.static(
path.join(env.root, "node_modules", "uikit", "src", "images")
)
);
this.app.use(
"/uikit",
cacheOneDay,
express.static(path.join(env.root, "node_modules", "uikit", "dist"))
);
this.app.use(
"/fontawesome",
cacheOneDay,
express.static(
path.join(env.root, "node_modules", "@fortawesome", "fontawesome-free")
)
);
this.app.use(
"/pretty-checkbox",
cacheOneDay,
express.static(
path.join(env.root, "node_modules", "pretty-checkbox", "dist")
)
);
this.app.use(
"/cropperjs",
cacheOneDay,
express.static(path.join(env.root, "node_modules", "cropperjs", "dist"))
);
this.app.use(
"/dayjs",
cacheOneDay,
express.static(path.join(env.root, "node_modules", "dayjs"))
);
this.app.use(
"/numeral",
cacheOneDay,
express.static(path.join(env.root, "node_modules", "numeral", "min"))
);
}
registerPreSessionHandlers ( ) {
registerPreSessionHandlers() {
assert(this.app, "ExpressJS app instance is required");
this.app.use(express.json());
this.app.use(express.urlencoded({ extended: true }));
@ -151,11 +208,12 @@ export class WebServer extends DtpBase {
this.app.use(methodOverride());
}
registerSessionHandlers ( ) {
registerSessionHandlers() {
assert(env.session.secret, "Must define an HTTP session secret");
assert(this.app, "ExpressJS app instance is required");
const sessionService = this.platform.components.getService<SessionService>("session");
const sessionService =
this.platform.components.getService<SessionService>("session");
const ONE_WEEK = 1000 * 60 * 60 * 24 * 7;
const SESSION_DURATION = ONE_WEEK;
@ -185,7 +243,7 @@ export class WebServer extends DtpBase {
},
store,
};
this.log.debug('ExpressJS session configuration', sessionConfig);
this.log.debug("ExpressJS session configuration", sessionConfig);
this.app.use(session(sessionConfig));
/*
@ -200,31 +258,45 @@ export class WebServer extends DtpBase {
this.app.use(sessionService.middleware());
}
registerDefaultHandlers ( ) {
registerDefaultHandlers() {
assert(this.app, "ExpressJS app instance is required");
this.app.use(async (err: WebError, _req: Request, res: Response, next: NextFunction) => {
const errorCode = err.status || 500;
++this.stats.errorCount;
this.log.error("HTTP error", { error: err });
res.status(errorCode).render("error", {
message: err.message,
error: err,
title: "error",
});
return next();
});
this.app.use(
async (
err: WebError,
_req: Request,
res: Response,
next: NextFunction
) => {
const errorCode = err.status || 500;
++this.stats.errorCount;
this.log.error("HTTP error", { error: err });
res.status(errorCode).render("error", {
message: err.message,
error: err,
title: "error",
});
return next();
}
);
}
async startHttpsServer ( ) {
async startHttpsServer() {
assert(env.https.keyFile, "SSL private key file is required");
assert(env.https.crtFile, "SSL certificate file is required");
const serverOptions = Object.assign({
key: await fs.promises.readFile(env.https.keyFile, { encoding: "utf-8" }),
cert: await fs.promises.readFile(env.https.crtFile, { encoding: "utf-8" }),
}, env.https);
const serverOptions = Object.assign(
{
key: await fs.promises.readFile(env.https.keyFile, {
encoding: "utf-8",
}),
cert: await fs.promises.readFile(env.https.crtFile, {
encoding: "utf-8",
}),
},
env.https
);
return new Promise<void>((resolve) => {
assert(this.app, "ExpressJS app instance is required");
@ -235,7 +307,7 @@ export class WebServer extends DtpBase {
serverOptions.port,
serverOptions.address,
serverOptions.backlog || 64,
( ) => {
() => {
this.log.debug("HTTPS server started", env.https);
resolve();
}
@ -243,7 +315,7 @@ export class WebServer extends DtpBase {
});
}
async stopHttpsServer ( ) : Promise<void> {
async stopHttpsServer(): Promise<void> {
return new Promise<void>((resolve, reject) => {
if (!this.https) {
return;
@ -268,7 +340,7 @@ export class WebServer extends DtpBase {
});
}
async close ( ) : Promise<void> {
async close(): Promise<void> {
this.log.info("closing web server");
await this.stopHttpsServer();
delete this.app;

58
src/newsroom-web.ts

@ -23,14 +23,16 @@ import { IDtpWebProcess } from "./app/models/process.js";
import ProcessService from "./app/services/process.js";
import { DtpProcess, WebServer } from "./lib/dtplib.js";
/**
* The DTP Newsroom Web application.
*/
class DtpNewsroomWebProcess extends DtpProcess {
static get name ( ) { return "DtpNewsroomWebProcess"; }
static get slug ( ) { return "newsroom-web"; }
static get name() {
return "DtpNewsroomWebProcess";
}
static get slug() {
return "newsroom-web";
}
webServer?: WebServer;
bs?: BrowserSyncInstance;
@ -41,11 +43,11 @@ class DtpNewsroomWebProcess extends DtpProcess {
webErrorCount: 0,
};
constructor ( ) {
constructor() {
super(DtpNewsroomWebProcess);
}
async start ( ) : Promise<void> {
async start(): Promise<void> {
await super.start();
const processService = this.getService<ProcessService>("process");
@ -75,7 +77,7 @@ class DtpNewsroomWebProcess extends DtpProcess {
/**
* Starts the HTTPS server that will drive request processing for ExpressJS.
*/
async startWebServer ( ) {
async startWebServer() {
this.log.info("starting web application server");
this.webServer = new WebServer(this);
await this.webServer.createExpressApp();
@ -91,12 +93,14 @@ class DtpNewsroomWebProcess extends DtpProcess {
await this.webServer.startHttpsServer();
}
async startBrowserSync ( ) {
async startBrowserSync() {
if (env.NODE_ENV !== "development") {
throw new Error("BrowserSync is only available in development environments");
throw new Error(
"BrowserSync is only available in development environments"
);
}
this.log.debug('BrowserSync create', { options: bsOptions });
this.log.debug("BrowserSync create", { options: bsOptions });
this.bs = browserSync.create("mt-control");
this.bs.init(bsOptions);
@ -115,7 +119,7 @@ class DtpNewsroomWebProcess extends DtpProcess {
this.bs.reload();
}
} catch (error) {
this.log.error('Less build failed', { error });
this.log.error("Less build failed", { error });
}
});
@ -144,7 +148,7 @@ class DtpNewsroomWebProcess extends DtpProcess {
});
}
async stop ( ) : Promise<number> {
async stop(): Promise<number> {
this.log.info("stopping Newsroom web app");
if (this.webServer) {
@ -158,20 +162,20 @@ class DtpNewsroomWebProcess extends DtpProcess {
return super.stop();
}
async runLessBuild ( ) : Promise<void> {
async runLessBuild(): Promise<void> {
await this.lessCompile(
path.join(env.src, "client", "less", "newsroom-dark.less"),
path.join(env.root, "dist", "client", "css", "newsroom-dark.css"),
path.join(env.root, "dist", "client", "css", "newsroom-dark.map"),
path.join(env.root, "dist", "client", "css", "newsroom-dark.map")
);
await this.lessCompile(
path.join(env.src, "client", "less", "newsroom-light.less"),
path.join(env.root, "dist", "client", "css", "newsroom-light.css"),
path.join(env.root, "dist", "client", "css", "newsroom-light.map"),
path.join(env.root, "dist", "client", "css", "newsroom-light.map")
);
}
async lessCompile (inFile: string, outFile: string, mapFile?: string) {
async lessCompile(inFile: string, outFile: string, mapFile?: string) {
const options: Less.Options = {
filename: inFile,
sourceMap: { sourceMapFileInline: false },
@ -185,11 +189,9 @@ class DtpNewsroomWebProcess extends DtpProcess {
}
}
async runEsBuild ( ) : Promise<void> {
async runEsBuild(): Promise<void> {
const options: esbuild.BuildOptions = {
entryPoints: [
path.join(env.src, "client", "js", "newsroom-app.ts"),
],
entryPoints: [path.join(env.src, "client", "js", "newsroom-app.ts")],
outdir: path.join(env.root, "dist", "client", "js"),
bundle: true,
minify: false,
@ -197,24 +199,26 @@ class DtpNewsroomWebProcess extends DtpProcess {
format: "esm",
sourcemap: "linked",
loader: { ".ts": "ts" },
plugins: [ ],
plugins: [],
};
const output = await esbuild.build(options);
this.log.info('esbuild finished', { output });
this.log.info("esbuild finished", { output });
}
async reportStats ( ) : Promise<void> {
async reportStats(): Promise<void> {
assert(this.webProcess, "registered Web process required");
const processService = this.getService<ProcessService>("process");
await this.updateStats();
await processService.reportWebProcessStats(this.webProcess, this.webStats, this.processStats);
await processService.reportWebProcessStats(
this.webProcess,
this.webStats,
this.processStats
);
}
}
(async ( ) => {
(async () => {
const webApp = new DtpNewsroomWebProcess();
await webApp.start();
})();

41
src/release.ts

@ -12,7 +12,10 @@ import semver from "semver";
import SimpleGit from "simple-git";
const release = process.argv[2] as semver.ReleaseType;
assert (release && semver.RELEASE_TYPES.includes(release), "Must specify the release type: major, minor, patch, build");
assert(
release && semver.RELEASE_TYPES.includes(release),
"Must specify the release type: major, minor, patch, build"
);
const pkgVersion = new semver.SemVer(env.pkg.version);
assert(pkgVersion, "A valid version is required in package.json");
@ -22,11 +25,16 @@ newVersion.inc(release);
env.pkg.version = newVersion.format();
const releaseTag = `v${newVersion.format()}`;
console.log('tagging release:', pkgVersion.format(), release, newVersion.format(), releaseTag);
(async ( ) => {
console.log(
"tagging release:",
pkgVersion.format(),
release,
newVersion.format(),
releaseTag
);
(async () => {
try {
const simpleGit = SimpleGit(env.root);
/*
@ -47,14 +55,24 @@ console.log('tagging release:', pkgVersion.format(), release, newVersion.format(
*/
const status = await simpleGit.status();
assert(!status.detached, "git repo is detached, please fix.");
assert(status.not_added.length === 0, "You have untracked files, please fix.");
assert(status.modified.length === 0, "You have modified files, please fix.");
assert(
status.not_added.length === 0,
"You have untracked files, please fix."
);
assert(
status.modified.length === 0,
"You have modified files, please fix."
);
/*
* Write package.json to disk
*/
const pkgFilename = path.join(env.root, "package.json");
await fs.promises.writeFile(pkgFilename, JSON.stringify(env.pkg, null, 2), "utf-8");
await fs.promises.writeFile(
pkgFilename,
JSON.stringify(env.pkg, null, 2),
"utf-8"
);
/*
* Commit package.json and push to develop on origin.
@ -74,11 +92,12 @@ console.log('tagging release:', pkgVersion.format(), release, newVersion.format(
await simpleGit.checkout("master");
await simpleGit.pull("origin", "master");
await simpleGit.pull(".", "develop");
await simpleGit.push("origin","master");
await simpleGit.push("origin", "master");
simpleGit.checkout("develop");
} catch (error) {
console.error(`Failed to process release: ${(error as Error).message}`, { error });
console.error(`Failed to process release: ${(error as Error).message}`, {
error,
});
}
})();

23
src/speechgen.ts

@ -11,15 +11,22 @@ import { SpeechVoice } from "./app/models/lib/speech-personality.js";
import OpenAiService, { IGeneratedFile } from "app/services/openai.js";
class SpeechGenerator extends DtpProcess {
static get name() {
return "SpeechGenerator";
}
static get slug() {
return "speechgen";
}
static get name ( ) { return "SpeechGenerator"; }
static get slug ( ) { return "speechgen"; }
constructor ( ) {
constructor() {
super(SpeechGenerator);
}
async generate (model: string, voice: SpeechVoice, input: string) : Promise<IGeneratedFile> {
async generate(
model: string,
voice: SpeechVoice,
input: string
): Promise<IGeneratedFile> {
try {
console.log("requesting audio resource");
const openAiService = this.getService<OpenAiService>("openAi");
@ -29,13 +36,12 @@ class SpeechGenerator extends DtpProcess {
}
}
async reportStats ( ) : Promise<void> {
async reportStats(): Promise<void> {
this.log.info("this process does not report statistics");
}
}
(async ( ) => {
(async () => {
try {
console.log("Speech Generator: A command line tool to generate audio");
const generator = new SpeechGenerator();
@ -57,7 +63,6 @@ class SpeechGenerator extends DtpProcess {
} catch (error) {
console.error("audiogen has failed", error);
}
})();
/*

30
src/workers/newsroom.ts

@ -16,18 +16,22 @@ import { DtpProcessStatus } from "../app/models/lib/process-status.js";
import ProcessService from "../app/services/process.js";
class NewsroomWorker extends DtpWorker {
static get name ( ) { return "NewsroomWorker"; }
static get slug ( ) { return "newsroom-worker"; }
fetchNews?: FetchNewsJob;
fetchNewsJob?: CronJob;
constructor ( ) {
static get name() {
return "NewsroomWorker";
}
static get slug() {
return "newsroom-worker";
}
constructor() {
super(NewsroomWorker);
}
async start ( ) : Promise<void> {
async start(): Promise<void> {
await super.startWorker("newsroom", env.jobQueues.newsroom);
assert(this.jobQueue, "Job queue required");
@ -50,7 +54,11 @@ class NewsroomWorker extends DtpWorker {
this.fetchNews = new FetchNewsJob(this, this.jobQueue);
this.log.info("registering ingest-entry job processor");
this.jobQueue.process("ingest-entry", 1, this.fetchNews.ingestEntry.bind(this.fetchNews));
this.jobQueue.process(
"ingest-entry",
1,
this.fetchNews.ingestEntry.bind(this.fetchNews)
);
/*
* Create cron job schedules
@ -63,7 +71,7 @@ class NewsroomWorker extends DtpWorker {
this.fetchNews.run.bind(this.fetchNews),
null,
true,
env.timezone,
env.timezone
);
/*
@ -73,7 +81,7 @@ class NewsroomWorker extends DtpWorker {
this.log.info(`${this.component.name}:${this.component.slug} online`);
}
async stop ( ) : Promise<number> {
async stop(): Promise<number> {
this.log.info("stopping worker jobs");
if (this.fetchNewsJob) {
this.fetchNewsJob.stop();
@ -85,8 +93,7 @@ class NewsroomWorker extends DtpWorker {
}
}
(async ( ) => {
(async () => {
process.on("unhandledRejection", async (error: Error, p) => {
console.error("Unhandled rejection", {
error: error,
@ -103,5 +110,4 @@ class NewsroomWorker extends DtpWorker {
console.log("worker error", (error as Error).message);
process.exit(-1);
}
})();

71
src/workers/newsroom/fetch-news.ts

@ -24,43 +24,44 @@ import { HumanGender } from "app/models/lib/human-gender.js";
import { SpeechVoice } from "app/models/lib/speech-personality.js";
export class FetchNewsJob extends DtpBase {
static get name ( ) { return "FetchNewsJob"; }
static get slug ( ) { return "job:fetch-news"; }
worker: DtpWorker;
jobQueue: Bull.Queue;
userAgent: UserAgent = new UserAgent();
aiService: OpenAiService;
constructor (worker: DtpWorker, jobQueue: Bull.Queue) {
static get name() {
return "FetchNewsJob";
}
static get slug() {
return "job:fetch-news";
}
constructor(worker: DtpWorker, jobQueue: Bull.Queue) {
super(worker, FetchNewsJob);
this.worker = worker;
this.jobQueue = jobQueue;
this.aiService = this.getService<OpenAiService>("openAi");
}
async run ( ) : Promise<void> {
async run(): Promise<void> {
const processor = this.ingestFeed.bind(this);
await Feed
.find()
.cursor()
.eachAsync(processor);
await Feed.find().cursor().eachAsync(processor);
}
async ingestFeed (feed: IFeed) : Promise<void> {
async ingestFeed(feed: IFeed): Promise<void> {
const feedService = this.getService<FeedService>("feed");
this.log.info("running news fetch job", { title: feed.title });
const rss: FeedData = await feedService.fetchRssFeed(feed);
this.log.debug('feed loaded', {
this.log.debug("feed loaded", {
feed: {
title: rss.title,
},
});
if (!Array.isArray(rss.entries) || (rss.entries.length === 0)) {
if (!Array.isArray(rss.entries) || rss.entries.length === 0) {
this.log.warn("the RSS feed is empty", { title: rss.title });
return;
}
@ -83,10 +84,13 @@ export class FetchNewsJob extends DtpBase {
* output media.
* @param job
*/
async ingestEntry (job: Bull.Job) : Promise<void> {
async ingestEntry(job: Bull.Job): Promise<void> {
const feed: IFeed = job.data.feed as IFeed;
const entry: FeedEntry = job.data.entry as FeedEntry;
this.log.info("ingesting news article", { jobId: job.id, title: entry.title });
this.log.info("ingesting news article", {
jobId: job.id,
title: entry.title,
});
const feedItem = await FeedItem.findOneAndUpdate(
{ link: entry.link },
@ -101,7 +105,7 @@ export class FetchNewsJob extends DtpBase {
link: entry.link,
},
},
{ upsert: true, new: true },
{ upsert: true, new: true }
);
if (!feedItem.isNew) {
@ -119,22 +123,24 @@ export class FetchNewsJob extends DtpBase {
model: "tts-1-hd",
voice: SpeechVoice.Shimmer,
role: "You are a female anchor of a television news broadcast.",
}
},
});
}
async fetchArticleBody (rssEntry: IFeedItem) : Promise<void> {
async fetchArticleBody(rssEntry: IFeedItem): Promise<void> {
const url = new URL(rssEntry.link);
const response = await fetch(rssEntry.link, {
method: "GET",
headers: {
"Host": url.host,
Host: url.host,
"User-Agent": this.userAgent.toString(),
}
},
});
if (!response.ok) {
throw new Error(`Failed to fetch news article: ${response.statusText} (${response.status})`);
throw new Error(
`Failed to fetch news article: ${response.statusText} (${response.status})`
);
}
const html = await response.text();
@ -143,7 +149,7 @@ export class FetchNewsJob extends DtpBase {
const paragraphs = document.body.querySelectorAll("p");
let body: string = '';
let body: string = "";
for (const p of paragraphs) {
body += `[${p.classList.toString()}] ${p.textContent}\n\n`;
}
@ -160,22 +166,29 @@ export class FetchNewsJob extends DtpBase {
);
}
async generateItemSummary (feedItem: IFeedItem) : Promise<void> {
async generateItemSummary(feedItem: IFeedItem): Promise<void> {
const openAiService = this.getService<OpenAiService>("openAi");
feedItem.summary = await openAiService.summarizeFeedItem(feedItem);
await FeedItem.updateOne(
{ _id: feedItem._id },
{ $set: { summary: feedItem.summary } },
{ $set: { summary: feedItem.summary } }
);
this.log.debug('article summarized', { title: feedItem.title, summary: feedItem.summary });
this.log.debug("article summarized", {
title: feedItem.title,
summary: feedItem.summary,
});
}
async generateItemNarration (
async generateItemNarration(
feedItem: IFeedItem,
host: IBroadcastShowHost,
) : Promise<IGeneratedFile> {
host: IBroadcastShowHost
): Promise<IGeneratedFile> {
const openAiService = this.getService<OpenAiService>("openAi");
assert(feedItem.summary, "Feed item summary is required");
return openAiService.generateSpeech(feedItem.summary, host.speech.model, host.speech.voice);
return openAiService.generateSpeech(
feedItem.summary,
host.speech.model,
host.speech.voice
);
}
}

Loading…
Cancel
Save