dtp-base-v1/app/services/feed.js

// announcement.js
// Copyright (C) 2022 DTP Technologies, LLC
// License: Apache-2.0

'use strict';

const mongoose = require('mongoose');
const Feed = mongoose.model('Feed');
const FeedEntry = mongoose.model('FeedEntry');

const { SiteService, SiteError, SiteAsync } = require('../../lib/site-lib');

const FeedExtractor = require('@extractus/feed-extractor');
const UserAgent = require('user-agents');

const { getLinkPreview } = require('link-preview-js');

class FeedService extends SiteService {

  constructor (dtp) {
    super(dtp, module.exports);

    this.populateFeedEntry = [
      {
        path: 'feed',
      },
    ];
  }

  async start ( ) {
    this.userAgent = new UserAgent();
    this.jobQueue = await this.getJobQueue('newsroom', this.dtp.config.jobQueues.newsroom);
  }

  middleware (options) {
    options = Object.assign({ maxEntryCount: 5 }, options);
    return async (req, res, next) => {
      if (this.isSystemRoute(req.path)) {
        return next(); // don't load newsfeeds for non-content routes
      }
      res.locals.newsfeed = await this.getNewsfeed({ skip: 0, cpp: options.maxEntryCount });
      return next();
    };
  }

  async create (feedDefinition) {
    feedDefinition.url = feedDefinition.url.trim();
    const feedContent = await this.load(feedDefinition.url);
    if (!feedContent) {
      throw new SiteError(404, 'Feed failed to load');
    }

    const feed = new Feed();
    feed.url = feedDefinition.url;
    feed.title = feedDefinition.title || feedContent.title || 'New Feed';
    feed.link = feedDefinition.link || feedContent.link;
    feed.description = feedDefinition.description || feedContent.description;
    feed.language = feedContent.language;
    feed.generator = feedContent.generator;
    feed.published = feedContent.published;
    await feed.save();

    this.jobQueue.add('update-feed', { feedId: feed._id });

    return feed.toObject();
  }

  async update (feed, feedDefinition) {
    feedDefinition.url = feedDefinition.url.trim();
    const feedContent = await this.load(feedDefinition.url);
    if (!feedContent) {
      throw new SiteError(404, 'Feed failed to load');
    }

    const updateOp = { $set: { }, $unset: { } };

    updateOp.$set.url = feedDefinition.url;
    updateOp.$set.title = feedDefinition.title || feedContent.title || 'New Feed';
    updateOp.$set.link = feedDefinition.link || feedContent.link;
    updateOp.$set.description = feedDefinition.description || feedContent.description;

    updateOp.$set.language = feedDefinition.language || feedContent.language;
    updateOp.$set.generator = feedDefinition.generator || feedContent.generator;

    await Feed.updateOne({ _id: feed._id }, updateOp);

    this.jobQueue.add('update-feed', { feedId: feed._id });
  }

  async updateFavicon (feed) {
    const linkPreview = await getLinkPreview(feed.link || feed.url, {
      headers: {
        'User-Agent': this.userAgent.toString(),
        'Accept-Language': 'en-US',
      },
      followRedirects: true,
      resolveDNSHost: module.resolveHost,
      timeout: 15000,
    });
    await Feed.updateOne(
      { _id: feed._id },
      {
        $set: {
          favicons: linkPreview.favicons,
        },
      },
    );
  }

  async getFeeds (pagination, options) {
    options = Object.assign({ withEntries: false, entryCount: 3 }, options);
    pagination = Object.assign({ skip: 0, cpp: 10 }, pagination);
    const feeds = await Feed
      .find()
      .sort({ title: 1 })
      .skip(pagination.skip)
      .limit(pagination.cpp)
      .lean();

    if (options.withEntries) {
      await SiteAsync.each(feeds, async (feed) => {
        try {
          feed.recent = await this.getFeedEntries(feed, { skip: 0, cpp: options.entryCount });
        } catch (error) {
          this.log.error('failed to populate recent entries for feed', { feedId: feed._id, error });
          // fall through
        }
      }, 2);
    }

    const totalFeedCount = await Feed.countDocuments();

    return { feeds, totalFeedCount };
  }

  async getById (feedId) {
    const feed = await Feed.findOne({ _id: feedId }).lean();
    return feed;
  }

  async getFeedEntries (feed, pagination) {
    pagination = Object.assign({ skip: 0, cpp: 10 }, pagination);
    const entries = await FeedEntry
      .find({ feed: feed._id })
      .sort({ published: -1 })
      .skip(pagination.skip)
      .limit(pagination.cpp)
      .populate(this.populateFeedEntry)
      .lean();
    const totalFeedEntryCount = await FeedEntry.countDocuments({ feed: feed._id });
    return { entries, totalFeedEntryCount };
  }

  async getNewsfeed (pagination) {
    pagination = Object.assign({ skip: 0, cpp: 5 }, pagination);
    const entries = await FeedEntry
      .find()
      .sort({ published: -1 })
      .skip(pagination.skip)
      .limit(pagination.cpp)
      .populate(this.populateFeedEntry)
      .lean();
    const totalFeedEntryCount = await FeedEntry.estimatedDocumentCount();
    return { entries, totalFeedEntryCount };
  }

  async remove (feed) {
    this.log.info('removing all feed entries', { feedId: feed._id, title: feed.title });
    await FeedEntry.deleteMany({ feed: feed._id });

    this.log.info('removing feed', { feedId: feed._id, title: feed.title });
    await Feed.deleteOne({ _id: feed._id });
  }

  async load (url) {
    const response = await fetch(url, {
      method: 'GET',
      headers: {
        'User-Agent': `DtpNewsroom/${this.dtp.pkg.version} (https://sites.digitaltelepresence.com/newsroom)`,
      },
    });
    if (!response.ok) {
      throw new SiteError(response.status, response.statusText);
    }
    const xml = await response.text();
    return FeedExtractor.extractFromXml(xml);
  }

  async createEntry (feed, entryDefinition) {
    const NOW = new Date();
    const updateOp = { $setOnInsert: { }, $set: { }, $unset: { } };

    updateOp.$setOnInsert.feed = feed._id;
    updateOp.$setOnInsert.link = entryDefinition.link.trim();
    updateOp.$setOnInsert.published = new Date(entryDefinition.published || NOW);

    updateOp.$set.title = entryDefinition.title.trim();
    updateOp.$set.description = entryDefinition.description.trim();

    await FeedEntry.updateOne(
      {
        feed: feed._id,
        link: updateOp.$setOnInsert.link,
      },
      updateOp,
      { upsert: true },
    );
  }
}

module.exports = {
  logId: 'svc:feed',
  index: 'feed',
  className: 'FeedService',
  create: (dtp) => { return new FeedService(dtp); },
};