// link.js // Copyright (C) 2024 DTP Technologies, LLC // All Rights Reserved 'use strict'; import mongoose from 'mongoose'; const Link = mongoose.model('Link'); import { JSDOM } from 'jsdom'; import { SiteService, SiteError } from '../../lib/site-lib.js'; export default class LinkService extends SiteService { static get name ( ) { return 'LinkService'; } static get slug ( ) { return 'link'; } static get DOMAIN_BLACKLIST_KEY ( ) { return 'dblklist'; } constructor (dtp) { super(dtp, LinkService); } async start ( ) { await super.start(); const { jobQueue: jobQueueService, user: userService } = this.dtp.services; this.linksQueue = jobQueueService.getJobQueue('links', this.dtp.config.jobQueues.links); this.userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"; this.templates = { linkPreview: this.loadViewTemplate('link/components/preview-standalone.pug'), }; this.populateLink = [ { path: 'submittedBy', select: userService.USER_SELECT, }, ]; } async getRecent (pagination) { const search = { }; const links = await Link .find(search) .sort({ created: -1 }) .skip(pagination.skip) .limit(pagination.cpp) .populate(this.populateLink) .lean(); const totalLinkCount = await Link.estimatedDocumentCount(); return { links, totalLinkCount }; } async getById (linkId) { return Link .findOne({ _id: linkId }) .populate(this.populateLink) .lean(); } async recordVisit (link) { await Link.updateOne( { _id: link._id }, { $inc: { 'stats.visitCount': 1 }, }, ); } async isDomainBlocked (domain) { const isBlocked = await this.dtp.redis.sismember(LinkService.DOMAIN_BLACKLIST_KEY, domain); return (isBlocked !== 0); } async ingest (author, url) { const NOW = new Date(); const domain = new URL(url).hostname.toLowerCase(); if (domain.endsWith('.cn')) { throw new SiteError(403, 'Linking to Chinese websites is prohibited.'); } if (domain.endsWith('.il')) { throw new SiteError(403, 'Linking to websites in Israel is prohibited.'); } if (domain.includes('tiktok.com')) { throw new SiteError(403, 'Linking to TikTok is prohibited.'); } if (await this.isDomainBlocked(domain)) { this.log.alert('detected blocked domain in shared link', { author: { _id: author._id, username: author.username }, domain, url, }); throw new SiteError(403, `All links/URLs pointing to ${domain} are prohibited.`); } /* * An upsert is used to create a document if one doesn't exist. The domain * and url are set on insert, and lastShared is always set so it will be * current. * * submittedBy is an array that holds the User._id of each member that * submitted the link. This enables their Link History view, which becomes * it's own feed. */ const link = await Link.findOneAndUpdate( { domain, url }, { $setOnInsert: { created: NOW, domain, url, }, $addToSet: { submittedBy: author._id }, $set: { lastShared: NOW }, }, { upsert: true, new: true }, ); this.linksQueue.add('link-ingest', { submitterId: author._id, linkId: link._id, }); return link; } async generatePagePreview (url, options) { const NOW = new Date(); const linkUrlObj = new URL(url); this.log.debug('generating page preview', { url, linkUrlObj }); const { /*window,*/ document } = await this.loadUrlAsDOM(url, options); const preview = { fetched: NOW, domain: linkUrlObj.hostname, tags: [ ], images: [ ], videos: [ ], audios: [ ], favicons: [ ], }; function getMetaContent (selector) { const element = document.querySelector(selector); if (!element) { return; } return element.getAttribute('content'); } function getElementContent (selector) { const element = document.querySelector(selector); if (!element) { return; } return element.textContent; } function getLinkHref (selector) { const element = document.querySelector(selector); if (!element) { return; } return element.getAttribute('href'); } preview.mediaType = getMetaContent('head meta[property="og:type"]'); preview.title = getMetaContent('head meta[property="og:title"]') || getElementContent(`head title`); preview.siteName = getMetaContent('head meta[property="og:site_name') || getElementContent(`head title`); preview.description = getMetaContent('head meta[property="og:description"]') || getMetaContent('head meta[name="description"]'); let href = getMetaContent('head meta[property="og:image:secure_url') || getMetaContent('head meta[property="og:image') || getMetaContent('head meta[name="twitter:image:src"]'); if (href) { preview.images.push(href); } href = getLinkHref('head link[rel="shortcut icon"]'); if (href) { preview.favicons.push(href); } const keywords = getMetaContent('head meta[name="keywords"]'); if (keywords) { preview.tags = keywords.split(',').map((keyword) => keyword.trim()); } const videoTags = document.querySelectorAll('head meta[property="og:video:tag"]'); if (videoTags) { videoTags.forEach((tag) => { tag = tag.getAttribute('content'); if (!tag) { return; } tag = tag.trim().toLowerCase(); if (!tag.length) { return; } preview.tags.push(tag); }); } const icons = document.querySelectorAll('head link[rel="icon"]'); if (icons) { icons.forEach((icon) => { preview.favicons.push(icon.getAttribute('href')); }); } //TODO: oEmbed spec allows for JSON and XML. May need to implement an XML // reader for `head link[rel="alternate"][type="text/xml+oembed"]` preview.oembed = { }; preview.oembed.href = getLinkHref('head link[type="application/json+oembed"]'); if (preview.oembed.href) { this.log.info('fetching oEmbed data for url', { url, href: preview.oembed.href }); const json = await this.fetchOembedJson(preview.oembed.href); preview.oembed.version = json.version; preview.oembed.type = json.type; if (json.cache_age) { preview.oembed.cache_age = json.cache_age; } preview.oembed.title = json.title; preview.oembed.provider_name = json.provider_name; preview.oembed.provider_url = json.provider_url; preview.oembed.author_name = json.author_name; preview.oembed.author_url = json.author_url; preview.oembed.thumbnail_url = json.thumbnail_url; preview.oembed.thumbnail_width = json.thumbnail_width; preview.oembed.thumbnail_height = json.thumbnail_height; switch (json.type) { case 'video': preview.oembed.html = json.html; preview.oembed.width = json.width; preview.oembed.height = json.height; break; case 'photo': preview.oembed.url = json.url; preview.oembed.width = json.width; preview.oembed.height = json.height; break; } } return preview; } async fetchOembedJson (url) { const response = await fetch(url); const json = await response.json(); return json; } async loadUrlAsDOM (url, options) { options = Object.assign({ userAgent: this.userAgent, acceptLanguage: 'en-US', }, options); const response = await fetch(url, { method: "GET", headers: { "User-Agent": options.userAgent, "Accept-Language": options.acceptLanguage, }, }); const html = await response.text(); const { window } = new JSDOM(html); return { window, document: window.document }; } async renderPreview (viewModel) { return this.renderTemplate(this.templates.linkPreview, viewModel); } }