host management and new reeeper to clean up crashed hosts

app/controllers/admin/host.js

@@ -30,6 +30,8 @@ class HostController extends SiteController {
 
     router.param('hostId', this.populateHostId.bind(this));
 
+    router.post('/:hostId/deactivate', this.postDeactiveHost.bind(this));
+
     router.get('/:hostId', this.getHostView.bind(this));
 
     router.get('/', this.getHomeView.bind(this));
@@ -46,6 +48,39 @@ class HostController extends SiteController {
     }
   }
 
+  async postDeactiveHost (req, res) {
+    const { displayEngine: displayEngineService } = this.dtp.services;
+    try {
+      const displayList = displayEngineService.createDisplayList('deactivate-host');
+
+      await NetHost.updateOne(
+        { _id: res.locals.host._id },
+        {
+          $set: { status: 'inactive' },
+        },
+      );
+
+      displayList.removeElement(`tr[data-host-id="${res.locals.host._id}"]`);
+      displayList.showNotification(
+        `Host "${res.locals.host.hostname}" deactivated`,
+        'success',
+        'bottom-center',
+        3000,
+      );
+      res.status(200).json({ success: true, displayList });
+
+    } catch (error) {
+      this.log.error('failed to deactivate host', {
+        hostId: res.local.host._id,
+        error,
+      });
+      res.status(error.statusCode || 500).json({
+        success: false,
+        message: error.message,
+      });
+    }
+  }
+
   async getHostView (req, res, next) {
     try {
       res.locals.stats = await NetHostStats
@@ -62,7 +97,20 @@ class HostController extends SiteController {
 
   async getHomeView (req, res, next) {
     try {
-      res.locals.hosts = await NetHost.find({ status: { $ne: 'inactive' } });
+      const HOST_SELECT = '_id created updated hostname status platform arch totalmem freemem';
+
+      res.locals.activeHosts = await NetHost
+        .find({ status: 'active' })
+        .select(HOST_SELECT)
+        .sort({ updated: 1 })
+        .lean();
+
+      res.locals.crashedHosts = await NetHost
+        .find({ status: 'crashed' })
+        .select(HOST_SELECT)
+        .sort({ updated: 1 })
+        .lean();
+
       res.render('admin/host/index');
     } catch (error) {
       return next(error);

app/views/admin/host/index.pug

@@ -1,23 +1,35 @@
 extends ../layouts/main
 block content
 
-  table.uk-table.uk-table-small.uk-table-divider
-    thead
-      th Host
-      th Status
-      th Memory
-      th Platform
-      th Arch
-      th Created
-      th Updated
-    tbody
-      each host in hosts
-        tr
-          td
-            a(href=`/admin/host/${host._id}`)= host.hostname
-          td= host.status
-          td= numeral((host.totalmem - host.freemem) / host.totalmem).format('0.00%')
-          td= host.platform
-          td= host.arch
-          td= moment(host.created).fromNow()
-          td= host.updated ? moment(host.updated).fromNow() : 'N/A'
+  mixin renderHostList (hosts)
+    if Array.isArray(hosts) && (hosts.length > 0)
+      table.uk-table.uk-table-small.uk-table-divider
+        thead
+          th Host
+          th Status
+          th Memory
+          th Platform
+          th Arch
+          th Created
+          th Updated
+        tbody
+          each host in hosts
+            tr(data-host-id= host._id)
+              td
+                a(href=`/admin/host/${host._id}`)= host.hostname
+              td= host.status
+              td= numeral((host.totalmem - host.freemem) / host.totalmem).format('0.00%')
+              td= host.platform
+              td= host.arch
+              td= moment(host.created).fromNow()
+              td= host.updated ? moment(host.updated).fromNow() : 'N/A'
+    else
+      div The host list is empty
+  
+  if Array.isArray(activeHosts) && (activeHosts.length > 0)
+    h2 Active hosts
+    +renderHostList(activeHosts)
+
+  if Array.isArray(crashedHosts) && (crashedHosts.length > 0)
+    h2 Crashed hosts
+    +renderHostList(crashedHosts)

app/workers/reeeper.js

@@ -0,0 +1,80 @@
+// host-services.js
+// Copyright (C) 2021 Digital Telepresence, LLC
+// License: Apache-2.0
+
+'use strict';
+
+const path = require('path');
+
+require('dotenv').config({ path: path.resolve(__dirname, '..', '..', '.env') });
+
+const mongoose = require('mongoose');
+
+const {
+  SitePlatform,
+  SiteAsync,
+  SiteLog,
+  SiteError,
+} = require(path.join(__dirname, '..', '..', 'lib', 'site-lib'));
+
+const { CronJob } = require('cron');
+
+const CRON_TIMEZONE = 'America/New_York';
+
+module.pkg = require(path.resolve(__dirname, '..', '..', 'package.json'));
+module.config = {
+  componentName: 'reeeper',
+  root: path.resolve(__dirname, '..', '..'),
+};
+
+module.log = new SiteLog(module, module.config.componentName);
+
+module.expireCrashedHosts = async ( ) => {
+  const NetHost = mongoose.model('NetHost');
+  try {
+    await NetHost
+      .find({ status: 'crashed' })
+      .select('_id hostname')
+      .lean()
+      .cursor()
+      .eachAsync(async (host) => {
+        module.log.info('deactivating crashed host', { hostname: host.hostname });
+        await NetHost.updateOne({ _id: host._id }, { $set: { status: 'inactive' } });
+      });
+  } catch (error) {
+    module.log.error('failed to expire crashed hosts', { error });
+  }
+};
+
+(async ( ) => {
+  try {
+    process.once('SIGINT', async ( ) => {
+      module.log.info('SIGINT received');
+      module.log.info('requesting shutdown...');
+
+      const exitCode = await SitePlatform.shutdown();
+      process.nextTick(( ) => {
+        process.exit(exitCode);
+      });
+    });
+
+    /*
+     * Site Platform startup
+     */
+    await SitePlatform.startPlatform(module);
+
+    await module.expireCrashedHosts(); // first-run the expirations
+
+    module.expireJob = new CronJob(
+      '*/5 * * * * *',
+      module.expireCrashedHosts,
+      null, true, CRON_TIMEZONE,
+    );
+
+    module.log.info(`${module.pkg.name} v${module.pkg.version} ${module.config.componentName} started`);
+  } catch (error) {
+    module.log.error('failed to start Host Cache worker', { error });
+    process.exit(-1);
+  }
+
+})();

client/less/site/dashboard.less

@@ -1,7 +1,86 @@
-.links-dashboard {
+.dtp-dashboard-cluster {
 
   canvas.visit-graph {
     width: 960px;
     height: 360px;
   }
+
+  fieldset {
+    border-color: #9e9e9e;
+    color: #c8c8c8;
+
+    legend {
+      font-family: 'Courier New', Courier, monospace;
+      font-size: 11px;
+      color: #9e9e9e;
+    }
+  }
+
+  .dtp-cpu-graph {
+    background: none;
+
+    &.cpu-overload {
+      background: #4e0000;
+    }
+  }
+
+  .dtp-stat-cell {
+    line-height: 1;
+  }
+
+  .dtp-stats-bar {
+    width: 100%;
+    height: 16px;
+    margin: 0;
+    padding: 0;
+  
+    .dtp-cpu-stat-bar {
+      display: inline-block;
+      height: 16px;
+      text-align: center;
+      font-size: 10px;
+      overflow: hidden;
+  
+      &.dtp-cpu-user {
+        background-color: #008000;
+      }
+      &.dtp-cpu-nice {
+        background-color: #808080;
+      }
+      &.dtp-cpu-sys {
+        background-color: #808000;
+      }
+      &.dtp-cpu-idle {
+        background-color: #484848;
+      }
+      &.dtp-cpu-irq {
+        background-color: #800000;
+      }
+    }
+  
+    .dtp-mem-stat-bar {
+      display: inline-block;
+      height: 16px;
+      text-align: center;
+      font-size: 10px;
+      overflow: hidden;
+  
+      &.dtp-mem-used {
+        background-color: #008000;
+      }
+      &.dtp-mem-available {
+        background-color: #404040;
+      }
+  
+      &.dtp-mem-cached {
+        background-color: #008000;
+      }
+      &.dtp-mem-buffers {
+        background-color: #004000;
+      }
+      &.dtp-mem-slab {
+        background-color: #808000;
+      }
+    }
+  }
 }

dtp-libertylinks-cli.js

@@ -140,7 +140,7 @@ module.deleteOtpAccount = async (target) => {
         case 'delete-otp':
           await module.deleteOtpAccount(target);
           break;
-          
+
         default:
           throw new Error(`invalid action: ${module.app.options.action}`);
       }