dtp-base/dtp-chat.js

// dtp-chat.js
// Copyright (C) 2024 DTP Technologies, LLC
// All Rights Reserved

'use strict';

import 'dotenv/config';

import path, { dirname } from 'path';
import fs from 'fs';
import { fileURLToPath } from 'url';
const __dirname = dirname(fileURLToPath(import.meta.url)); // jshint ignore:line

import { createRequire } from 'module';
const require = createRequire(import.meta.url); // jshint ignore:line

import * as rfs from 'rotating-file-stream';

import webpack from 'webpack';
import webpackDevMiddleware from 'webpack-dev-middleware';

import WEBPACK_CONFIG from './webpack.config.js';

import numeral from 'numeral';
import dayjs from 'dayjs';
import * as Marked from 'marked';
import hljs from 'highlight.js';

import mongoose from 'mongoose';
import { Redis } from 'ioredis';

import { SiteLog } from './lib/site-lib.js';
import { SiteTripwire } from './lib/site-tripwire.js';

import http from 'node:http';

import express from 'express';
import morgan from 'morgan';
import session from 'express-session';
import RedisStore from 'connect-redis';
import passport from 'passport';

import { Emitter } from '@socket.io/redis-emitter';

const APP_CONFIG = {
  pkg: require('./package.json'),
};

class Harness {

  static get name ( ) { return 'Harness'; }
  static get slug ( ) { return 'harness'; }

  constructor ( ) {
    this.config = { root: __dirname };
    this.log = new SiteLog(this, Harness);
    this.models = [ ];
  }

  async start ( ) {
    await this.loadConfig();

    this.log.info('loading SiteTripwire (known-malicious request protection)');
    this.tripwire = new SiteTripwire(this);
    await this.tripwire.start();

    await this.connectMongoDB();
    await this.connectRedis();

    await this.loadModels();
    await this.loadServices();
    await this.startExpressJS();
  }

  async loadConfig ( ) {
    this.config.site = (await import('./config/site.js')).default;
    this.config.limiter = (await import('./config/limiter.js')).default;
    this.config.jobQueues = (await import('./config/job-queues.js')).default;
  }

  async connectMongoDB ( ) {
    try {
      this.log.info('starting MongoDB');

      /*
       * For some time, strictQuery=true was the Mongoose default. It's being
       * changed. The option has to be set manually now.
       */
      mongoose.set('strictQuery', true);
  
      this.log.info('connecting to MongoDB database', {
        pid: process.pid,
        host: process.env.MONGODB_HOST,
        database: process.env.MONGODB_DATABASE,
      });
      const mongoConnectUri = `mongodb://${process.env.DTP_MONGODB_HOST}/${process.env.DTP_MONGODB_DATABASE}`;
      await mongoose.connect(mongoConnectUri, {
        socketTimeoutMS: 0,
        dbName: process.env.MONGODB_DATABASE,
      });
      this.db = mongoose.connection;
      this.log.info('connected to MongoDB');
    } catch (error) {
      this.log.error('failed to connect to database', { error });
      throw error;
    }
  }

  async loadModels ( ) {
    const basePath = path.join(this.config.root, 'app', 'models');
    const entries = await fs.promises.readdir(basePath, { withFileTypes: true });

    this.log.info('loading models', { count: entries.length });
    for (const entry of entries) {
      if (!entry.isFile()) {
        continue;
      }
      const filename = path.join(basePath, entry.name);
      const model = (await import(filename)).default;
      if (this.models[model.modelName]) {
        this.log.error('model name collision', { name: model.modelName });
        process.exit(-1);
      }
      this.models.push(model);
      this.log.info('model loaded', { name: model.modelName});
    }
  }
  
  async resetIndexes (target) {
    if (target === 'all') {
      for (const model of this.models) {
        await this.resetIndex(model);
      }
      return;
    }
  
    const model = this.models.find((model) => model.modelName === target);
    if (!model) {
      throw new Error(`requested Mongoose model does not exist: ${target}`);
    }

    return this.resetIndex(model);
  }
  
  async resetIndex (model) {
    return new Promise(async (resolve, reject) => {
      this.log.info('dropping model indexes', { model: model.modelName });
      model.collection.dropIndexes((err) => {
        if (err) {
          return reject(err);
        }
        this.log.info('creating model indexes', { model: model.modelName });
        model.ensureIndexes((err) => {
          if (err) {
            return reject(err);
          }
          return resolve(model);
        });
      });
    });
  }

  async connectRedis ( ) {
    try {
      const options = {
        host: process.env.DTP_REDIS_HOST,
        port: parseInt(process.env.DTP_REDIS_PORT || '6379', 10),
        password: process.env.DTP_REDIS_PASSWORD,
        keyPrefix: process.env.DTP_REDIS_KEY_PREFIX,
        lazyConnect: false,
      };
      this.log.info('connecting to Redis', {
        host: options.host,
        port: options.port,
        prefix: options.keyPrefix || 'dtp',
      });
  
      this.redis = new Redis(options);
      this.redis.setMaxListeners(64); // prevents warnings/errors with Bull Queue
  
      this.log.info('creating Socket.io Emitter');
      this.emitter = new Emitter(this.redis);
  
      this.log.info('Redis connected');
    } catch (error) {
      this.log.error('failed to connect to Redis', error);
      throw new Error('failed to connect to Redis', { cause: error });
    }
  }
  
  async getRedisKeys (pattern) {
    return new Promise((resolve, reject) => {
      return this.redis.keys(pattern, (err, response) => {
        if (err) {
          return reject(err);
        }
        return resolve(response);
      });
    });
  }

  async startExpressJS ( ) {
    const { session: sessionService } = this.services;

    this.app = express();

    this.app.locals.config = APP_CONFIG;
    this.app.locals.pkg = APP_CONFIG.pkg; // convenience
    this.app.locals.site = (await import(path.join(this.config.root, 'config', 'site.js'))).default;
  
    this.app.set('view engine', 'pug');
    this.app.set('views', path.join(__dirname, 'app', 'views'));
    this.app.set('x-powered-by', false);
    
    await this.populateViewModel(this.app.locals);

    this.app.use(this.tripwire.guard.bind(this.tripwire));

    this.app.use(express.json());
    this.app.use(express.urlencoded({ extended: true }));

    const sessionStore = new RedisStore({ client: this.redis });
    const cookieDomain = (process.env.NODE_ENV === 'production') ? process.env.DTP_SITE_DOMAIN_KEY : `localhost`;
    const SESSION_DURATION = 1000 * 60 * 60 * 24 * 30; // 30 days

    this.sessionOptions = {
      name: `dtp.${process.env.DTP_SITE_DOMAIN_KEY}.${process.env.NODE_ENV}`,
      secret: process.env.DTP_HTTP_SESSION_SECRET,
      resave: false,
      saveUninitialized: true,
      proxy: (process.env.NODE_ENV === 'production') || (process.env.HTTP_SESSION_TRUST_PROXY === 'enabled'),
      cookie: {
        domain: cookieDomain,
        path: '/',
        httpOnly: true,
        secure: process.env.HTTP_COOKIE_SECURE === 'enabled',
        sameSite: process.env.HTTP_COOKIE_SAMESITE || false,
        maxAge: SESSION_DURATION,
      },
      store: sessionStore,
    };
    this.app.use(session(this.sessionOptions));

    /*
     * PassportJS setup
     */
    this.log.info('initializting PassportJS');
    this.app.use(passport.initialize());
    this.app.use(passport.session());

    /*
     * DTP Session Service integration
     */
    this.app.use(this.services.session.middleware());

    /*
     * HTTP request logging
     */
    if (process.env.DTP_LOG_FILE === 'enabled') {
      const httpLogStream = rfs.createStream(process.env.DTP_LOG_FILE_NAME_HTTP || 'dtp-sites-access.log', {
        interval: '1d',
        path: process.env.DTP_LOG_FILE_PATH || '/tmp',
        compress: 'gzip',
      });
      this.app.use(morgan(process.env.DTP_LOG_HTTP_FORMAT || 'combined', { stream: httpLogStream }));
    }

    function cacheOneDay (req, res, next) {
      res.set('Cache-Control', 'public, maxage=86400, s-maxage=86400, immutable');
      return next();
    }
    function serviceWorkerAllowed (req, res, next) {
      res.set('Service-Worker-Allowed', '/');
      return next();
    }

    /*
     * Static content and file services
     */
    const staticOptions = {
      cacheControl: true,
      immutable: true,
      maxAge: '4h',
      dotfiles: 'ignore',
    };

    this.app.use('/fontawesome', cacheOneDay, express.static(path.join(__dirname, 'node_modules', '@fortawesome', 'fontawesome-free')));
    this.app.use('/uikit', cacheOneDay, express.static(path.join(__dirname, 'node_modules', 'uikit')));
    this.app.use('/pretty-checkbox', cacheOneDay, express.static(path.join(__dirname, 'node_modules', 'pretty-checkbox', 'dist')));
    this.app.use('/cropperjs', cacheOneDay, express.static(path.join(this.config.root, 'node_modules', 'cropperjs', 'dist')));

    this.app.use('/dist', cacheOneDay, serviceWorkerAllowed, express.static(path.join(__dirname, 'dist')));
    this.app.use('/js', cacheOneDay, serviceWorkerAllowed, express.static(path.join(__dirname, 'client', 'js')));
    this.app.use('/lib', cacheOneDay, serviceWorkerAllowed, express.static(path.join(__dirname, 'lib', 'client', 'js')));
    this.app.use('/static', cacheOneDay, serviceWorkerAllowed, express.static(path.join(__dirname, 'client', 'static'), staticOptions));
    this.app.use('/img', cacheOneDay, serviceWorkerAllowed, express.static(path.join(__dirname, 'client', 'img'), staticOptions));

    this.app.use('/dayjs', cacheOneDay, express.static(path.join(this.config.root, 'node_modules', 'dayjs')));
    this.app.use('/numeral', cacheOneDay, express.static(path.join(this.config.root, 'node_modules', 'numeral', 'min')));
  
    /*
     * Webpack integration
     */
    if (process.env.NODE_ENV !== 'production') {
      this.compiler = webpack(WEBPACK_CONFIG);
      this.webpackDevMiddleware = webpackDevMiddleware(this.compiler, {
        publicPath: WEBPACK_CONFIG.output.publicPath,
        writeToDisk: true,
      });
      this.app.use(this.webpackDevMiddleware);
    }

    this.app.use(
      (req, res, next) => {
        res.locals.dtp = {
          request: req,
        };
        return next();
      },
      sessionService.middleware(),
    );

    await this.loadControllers();

    /*
     * Default error handler
     */
    this.log.info('registering ExpressJS error handler');
    this.app.use((error, req, res, next) => { // jshint ignore:line
      res.locals.errorCode = error.statusCode || error.status || 500;
      this.log.error('ExpressJS error', { url: req.url, error });
      res.status(res.locals.errorCode).render('error', {
        message: error.message,
        error,
        errorCode: res.locals.errorCode,
      });
    });

    /*
     * Create the HTTP server, do not start it.
     */

    this.httpServer = http.createServer(this.app);

    /*
     * Create Socket.io server bound to HTTP server
     */

    const { SiteIoServer } = await import('./lib/site-ioserver.js');
    this.log.info('creating Socket.io server');
    this.io = new SiteIoServer(this);
    await this.io.start(this.httpServer);

    /*
     * Start the HTTP server
     */

    return new Promise((resolve, reject) => {
      const host = process.env.DTP_HTTP_HOST || '127.0.0.1';
      const port = parseInt(process.env.DTP_HTTP_PORT || '3000', 10);

      this.log.info('starting HTTP server', { host, port });
      this.httpServer.listen(port, host, (err) => {
        if (err) {
          this.log.error('failed to start application services', { err });
          return reject(new Error('failed to start application services', { cause: err }));
        }
        this.log.info(`${APP_CONFIG.pkg.name} application services online.`);
        return resolve();
      });
    });
  }

  async loadServices ( ) {
    const basePath = path.join(this.config.root, 'app', 'services');
    const entries = await fs.promises.readdir(basePath, { withFileTypes: true });
    const inits = [ ];
  
    this.services = { };
  
    for await (const entry of entries) {
      if (!entry.isFile()) {
        continue;
      }
      try {
        const ServiceClass = (await import(path.join(basePath, entry.name))).default;
        this.log.info('loading service', {
          script: entry.name,
          name: ServiceClass.name,
          slug: ServiceClass.slug,
        });

        const service = new ServiceClass(this);

        this.services[ServiceClass.slug] = service;
        inits.push(service);
      } catch (error) {
        this.log.error('failed to load service', { error });
        throw new Error('failed to load service', { cause: error });
      }
    }

    for await (const service of inits) {
      await service.start();
    }
  }

  async loadControllers ( ) {
    const basePath = path.join(this.config.root, 'app', 'controllers');
    const entries = await fs.promises.readdir(basePath, { withFileTypes: true });
    const inits = [ ];
  
    this.controllers = { };
  
    for await (const entry of entries) {
      if (!entry.isFile()) {
        continue;
      }
      try {
        const ControllerClass = (await import(path.join(basePath, entry.name))).default;
        if (!ControllerClass) {
          this.log.error('failed to receive a default export class from controller', { script: entry.name });
          throw new Error('controller failed to provide a default export');
        }
        this.log.info('loading controller', {
          script: entry.name,
          name: ControllerClass.name,
          slug: ControllerClass.slug,
        });
        const controller = new ControllerClass(this);

        this.controllers[ControllerClass.slug] = controller;
        inits.push(controller);
      } catch (error) {
        this.log.error('failed to load controller', { error });
        throw new Error('failed to load controller', { cause: error });
      }
    }

    for await (const controller of inits) {
      if (controller.name === 'HomeController') {
        continue;
      }
      await controller.start();
    }
  
    /*
     * Start the Home controller
     */
    await this.controllers.home.start();
  }

  async populateViewModel (viewModel) {
    viewModel.DTP_SCRIPT_DEBUG = (process.env.NODE_ENV !== 'production');
    viewModel.dtp = this;

    const pkg = await import(path.join(this.config.root, 'package.json'), { assert: { type: 'json' } }); // jshint ignore:line
    viewModel.pkg = pkg.default; // jshint ignore:line
    viewModel.dayjs = dayjs;
    viewModel.numeral = numeral;
    // viewModel.phoneNumberJS = require('libphonenumber-js');
    // viewModel.anchorme = require('anchorme').default;
    // viewModel.hljs = hljs;
    // viewModel.Color = require('color');
    // viewModel.numberToWords = require('number-to-words');
    viewModel.uuidv4 = (await import('uuid')).v4;
  
    /*
     * Set up the protected markdown renderer that will refuse to process links and images
     * for security reasons.
     */
  
    function safeImageRenderer (href, title, text) { return text; }
    function safeLinkRenderer (href, title, text) { return text; }
  
    function confirmedLinkRenderer (href, title, text) {
      return `<a href="${href}" uk-tooltip="${title || 'Visit link...'}" onclick="return dtp.app.confirmNavigation(event);">${text}</a>`;
    }
  
    viewModel.fullMarkedRenderer = new Marked.Renderer();
    viewModel.fullMarkedRenderer.image = safeImageRenderer;
    viewModel.fullMarkedRenderer.link = confirmedLinkRenderer;
  
    viewModel.safeMarkedRenderer = new Marked.Renderer();
    viewModel.safeMarkedRenderer.link = safeLinkRenderer;
    viewModel.safeMarkedRenderer.image = safeImageRenderer;
  
    viewModel.markedConfigChat = {
      renderer: this.safeMarkedRenderer,
      highlight: function(code, lang) {
        const language = hljs.getLanguage(lang) ? lang : 'plaintext';
        return hljs.highlight(code, { language }).value;
      },
      langPrefix: 'hljs language-',
      pedantic: false,
      gfm: true,
      breaks: true,
      sanitize: false,
      smartLists: true,
      smartypants: false,
      xhtml: false,
    };
    Marked.setOptions(viewModel.markedConfigChat);
    viewModel.marked = Marked;
  }
}

(async ( ) => {

  try {
    const harness = new Harness();
    await harness.start();
  } catch (error) {
    console.error('failed to start application harness', error);
  }

})();