dtp-base/app/services/link.js

// link.js
// Copyright (C) 2024 DTP Technologies, LLC
// All Rights Reserved

'use strict';

import mongoose from 'mongoose';
const Link = mongoose.model('Link');

import { JSDOM } from 'jsdom';

import { SiteService, SiteError } from '../../lib/site-lib.js';

export default class LinkService extends SiteService {

  static get name ( ) { return 'LinkService'; }
  static get slug ( ) { return 'link'; }

  static get DOMAIN_BLACKLIST_KEY ( ) { return 'dblklist'; }

  constructor (dtp) {
    super(dtp, LinkService);
  }

  async start ( ) {
    await super.start();

    const { jobQueue: jobQueueService, user: userService } = this.dtp.services;
    this.linksQueue = jobQueueService.getJobQueue('links', this.dtp.config.jobQueues.links);

    this.userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36";

    this.templates = {
      linkPreview: this.loadViewTemplate('link/components/preview-standalone.pug'),
    };

    this.populateLink = [
      {
        path: 'submittedBy',
        select: userService.USER_SELECT,
      },
    ];
  }

  async getRecent (pagination) {
    const search = { };
    const links = await Link
      .find(search)
      .sort({ created: -1 })
      .skip(pagination.skip)
      .limit(pagination.cpp)
      .populate(this.populateLink)
      .lean();
    const totalLinkCount = await Link.estimatedDocumentCount();
    return { links, totalLinkCount };
  }

  async getById (linkId) {
    return Link
      .findOne({ _id: linkId })
      .populate(this.populateLink)
      .lean();
  }

  async recordVisit (link) {
    await Link.updateOne(
      { _id: link._id },
      {
        $inc: { 'stats.visitCount': 1 },
      },
    );
  }

  async isDomainBlocked (domain) {
    const isBlocked = await this.dtp.redis.sismember(LinkService.DOMAIN_BLACKLIST_KEY, domain);
    return (isBlocked !== 0);
  }

  async ingest (author, url) {
    const NOW = new Date();
    const domain = new URL(url).hostname.toLowerCase();
    if (domain.endsWith('.cn')) {
      throw new SiteError(403, 'Linking to Chinese websites is prohibited.');
    }
    if (domain.endsWith('.il')) {
      throw new SiteError(403, 'Linking to websites in Israel is prohibited.');
    }
    if (domain.includes('tiktok.com')) {
      throw new SiteError(403, 'Linking to TikTok is prohibited.');
    }

    if (await this.isDomainBlocked(domain)) {
      this.log.alert('detected blocked domain in shared link', {
        author: { _id: author._id, username: author.username },
        domain, url,
      });
      throw new SiteError(403, `All links/URLs pointing to ${domain} are prohibited.`);
    }

    /*
     * An upsert is used to create a document if one doesn't exist. The domain
     * and url are set on insert, and lastShared is always set so it will be
     * current.
     * 
     * submittedBy is an array that holds the User._id of each member that
     * submitted the link. This enables their Link History view, which becomes
     * it's own feed.
     */
    const link = await Link.findOneAndUpdate(
      { domain, url },
      {
        $setOnInsert: {
          created: NOW,
          domain, url,
        },
        $addToSet: { submittedBy: author._id },
        $set: { lastShared: NOW },
      },
      { upsert: true, new: true },
    );

    this.linksQueue.add('link-ingest', {
      submitterId: author._id,
      linkId: link._id,
    });

    return link;
  }

  async generatePagePreview (url, options) {
    const NOW = new Date();
    const linkUrlObj = new URL(url);
    this.log.debug('generating page preview', { url, linkUrlObj });

    const { /*window,*/ document } = await this.loadUrlAsDOM(url, options);
    const preview = {
      fetched: NOW,
      domain: linkUrlObj.hostname,
      tags: [ ],
      images: [ ],
      videos: [ ],
      audios: [ ],
      favicons: [ ],
    };

    function getMetaContent (selector) {
      const element = document.querySelector(selector);
      if (!element) {
        return;
      }
      return element.getAttribute('content');
    }

    function getElementContent (selector) {
      const element = document.querySelector(selector);
      if (!element) {
        return;
      }
      return element.textContent;
    }

    function getLinkHref (selector) {
      const element = document.querySelector(selector);
      if (!element) {
        return;
      }
      return element.getAttribute('href');
    }

    preview.mediaType = getMetaContent('head meta[property="og:type"]');
    preview.title =
      getMetaContent('head meta[property="og:title"]') ||
      getElementContent(`head title`);
    preview.siteName =
      getMetaContent('head meta[property="og:site_name') ||
      getElementContent(`head title`);

    preview.description =
      getMetaContent('head meta[property="og:description"]') ||
      getMetaContent('head meta[name="description"]');

    let href =
      getMetaContent('head meta[property="og:image:secure_url') ||
      getMetaContent('head meta[property="og:image') ||
      getMetaContent('head meta[name="twitter:image:src"]');
    if (href) {
      preview.images.push(href);
    }

    href = getLinkHref('head link[rel="shortcut icon"]');
    if (href) {
      preview.favicons.push(href);
    }

    const keywords = getMetaContent('head meta[name="keywords"]');
    if (keywords) {
      preview.tags = keywords.split(',').map((keyword) => keyword.trim());
    }

    const videoTags = document.querySelectorAll('head meta[property="og:video:tag"]');
    if (videoTags) {
      videoTags.forEach((tag) => {
        tag = tag.getAttribute('content');
        if (!tag) {
          return;
        }
        tag = tag.trim().toLowerCase();
        if (!tag.length) {
          return;
        }
        preview.tags.push(tag);
      });
    }

    const icons = document.querySelectorAll('head link[rel="icon"]');
    if (icons) {
      icons.forEach((icon) => {
        preview.favicons.push(icon.getAttribute('href'));
      });
    }

    //TODO: oEmbed spec allows for JSON and XML. May need to implement an XML
    // reader for `head link[rel="alternate"][type="text/xml+oembed"]`

    preview.oembed = { };
    preview.oembed.href = getLinkHref('head link[type="application/json+oembed"]');
    if (preview.oembed.href) {
      this.log.info('fetching oEmbed data for url', { url, href: preview.oembed.href });
      const json = await this.fetchOembedJson(preview.oembed.href);

      preview.oembed.version = json.version;
      preview.oembed.type = json.type;
      if (json.cache_age) {
        preview.oembed.cache_age = json.cache_age;
      }
      preview.oembed.title = json.title;
      preview.oembed.provider_name = json.provider_name;
      preview.oembed.provider_url = json.provider_url;
      preview.oembed.author_name = json.author_name;
      preview.oembed.author_url = json.author_url;
      preview.oembed.thumbnail_url = json.thumbnail_url;
      preview.oembed.thumbnail_width = json.thumbnail_width;
      preview.oembed.thumbnail_height = json.thumbnail_height;

      switch (json.type) {
        case 'video':
          preview.oembed.html = json.html;
          preview.oembed.width = json.width;
          preview.oembed.height = json.height;
          break;

        case 'photo':
          preview.oembed.url = json.url;
          preview.oembed.width = json.width;
          preview.oembed.height = json.height;
          break;
      }
    }

    return preview;
  }

  async fetchOembedJson (url) {
    const response = await fetch(url);
    const json = await response.json();
    return json;
  }

  async loadUrlAsDOM (url, options) {
    options = Object.assign({
      userAgent: this.userAgent,
      acceptLanguage: 'en-US',
    }, options);

    const response = await fetch(url, {
      method: "GET",
      headers: {
        "User-Agent": options.userAgent,
        "Accept-Language": options.acceptLanguage,
      },
    });

    const html = await response.text();
    const { window } = new JSDOM(html);
    return { window, document: window.document };
  }

  async renderPreview (viewModel) {
    return this.renderTemplate(this.templates.linkPreview, viewModel);
  }

  async removeForUser (user) {
    this.log.info('removing all links for user', {
      user: {
        _id: user._id,
        username: user.username,
      },
    });
    await Link
      .find({ submittedBy: user._id })
      .populate(this.populateLink)
      .cursor()
      .eachAsync(async (link) => {
        if (link.submittedBy.length > 1) {
          return Link.updateOne({ _id: link._id }, { $pull: { submittedBy: user._id } });
        }
        await Link.deleteOne({ _id: link._id });
      });
  }
}