rudeshark.net/src/services/fetch-instance-metadata.ts

266 lines
7.2 KiB
TypeScript
Raw Normal View History

2020-10-27 08:16:59 +01:00
import { DOMWindow, JSDOM } from 'jsdom';
import fetch from 'node-fetch';
import { getJson, getHtml, getAgentByUrl } from '../misc/fetch';
import { Instance } from '../models/entities/instance';
import { Instances } from '../models';
import { getFetchInstanceMetadataLock } from '../misc/app-lock';
import Logger from './logger';
import { URL } from 'url';
const logger = new Logger('metadata', 'cyan');
2020-10-27 08:44:54 +01:00
export async function fetchInstanceMetadata(instance: Instance, force = false): Promise<void> {
const unlock = await getFetchInstanceMetadataLock(instance.host);
2020-10-27 08:44:54 +01:00
if (!force) {
const _instance = await Instances.findOne({ host: instance.host });
const now = Date.now();
if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24)) {
unlock();
return;
}
}
logger.info(`Fetching metadata of ${instance.host} ...`);
try {
2020-10-27 08:16:59 +01:00
const [info, dom, manifest] = await Promise.all([
fetchNodeinfo(instance).catch(() => null),
2020-10-27 08:16:59 +01:00
fetchDom(instance).catch(() => null),
fetchManifest(instance).catch(() => null),
]);
const [favicon, icon, themeColor, name, description] = await Promise.all([
2020-10-27 08:44:54 +01:00
fetchFaviconUrl(instance, dom).catch(() => null),
2020-10-27 08:16:59 +01:00
fetchIconUrl(instance, dom, manifest).catch(() => null),
getThemeColor(dom, manifest).catch(() => null),
getSiteName(info, dom, manifest).catch(() => null),
getDescription(info, dom, manifest).catch(() => null),
]);
logger.succ(`Successfuly fetched metadata of ${instance.host}`);
const updates = {
infoUpdatedAt: new Date(),
} as Record<string, any>;
if (info) {
2020-10-27 08:16:59 +01:00
updates.softwareName = info.software?.name.toLowerCase();
updates.softwareVersion = info.software?.version;
updates.openRegistrations = info.openRegistrations;
updates.maintainerName = info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.name || null) : null : null;
updates.maintainerEmail = info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.email || null) : null : null;
}
2020-10-27 08:16:59 +01:00
if (name) updates.name = name;
if (description) updates.description = description;
if (icon || favicon) updates.iconUrl = icon || favicon;
if (favicon) updates.faviconUrl = favicon;
if (themeColor) updates.themeColor = themeColor;
await Instances.update(instance.id, updates);
logger.succ(`Successfuly updated metadata of ${instance.host}`);
} catch (e) {
logger.error(`Failed to update metadata of ${instance.host}: ${e}`);
} finally {
unlock();
}
}
2020-10-27 08:16:59 +01:00
type NodeInfo = {
openRegistrations?: any;
software?: {
name?: any;
version?: any;
};
metadata?: {
name?: any;
nodeName?: any;
nodeDescription?: any;
description?: any;
maintainer?: {
name?: any;
email?: any;
};
};
};
async function fetchNodeinfo(instance: Instance): Promise<NodeInfo> {
logger.info(`Fetching nodeinfo of ${instance.host} ...`);
try {
const wellknown = await getJson('https://' + instance.host + '/.well-known/nodeinfo')
.catch(e => {
if (e.statusCode === 404) {
throw 'No nodeinfo provided';
} else {
throw e.statusCode || e.message;
}
});
if (wellknown.links == null || !Array.isArray(wellknown.links)) {
throw 'No wellknown links';
}
const links = wellknown.links as any[];
const lnik1_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/1.0');
const lnik2_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.0');
const lnik2_1 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.1');
const link = lnik2_1 || lnik2_0 || lnik1_0;
if (link == null) {
throw 'No nodeinfo link provided';
}
const info = await getJson(link.href)
.catch(e => {
throw e.statusCode || e.message;
});
logger.succ(`Successfuly fetched nodeinfo of ${instance.host}`);
return info;
} catch (e) {
logger.error(`Failed to fetch nodeinfo of ${instance.host}: ${e}`);
throw e;
}
}
2020-10-27 08:16:59 +01:00
async function fetchDom(instance: Instance): Promise<DOMWindow['document']> {
logger.info(`Fetching HTML of ${instance.host} ...`);
const url = 'https://' + instance.host;
const html = await getHtml(url);
const { window } = new JSDOM(html);
const doc = window.document;
2020-10-27 08:16:59 +01:00
return doc;
}
2020-10-27 08:16:59 +01:00
async function fetchManifest(instance: Instance): Promise<Record<string, any> | null> {
const url = 'https://' + instance.host;
2020-10-27 08:16:59 +01:00
const manifestUrl = url + '/manifest.json';
const manifest = await getJson(manifestUrl);
2020-10-27 08:16:59 +01:00
return manifest;
}
2020-10-27 08:44:54 +01:00
async function fetchFaviconUrl(instance: Instance, doc: DOMWindow['document'] | null): Promise<string | null> {
2020-10-27 08:16:59 +01:00
const url = 'https://' + instance.host;
2020-10-27 08:44:54 +01:00
if (doc) {
const href = doc.querySelector('link[rel="icon"]')?.getAttribute('href');
if (href) {
return (new URL(href, url)).href;
}
}
const faviconUrl = url + '/favicon.ico';
const favicon = await fetch(faviconUrl, {
timeout: 10000,
agent: getAgentByUrl,
});
if (favicon.ok) {
return faviconUrl;
}
return null;
}
2020-10-27 08:16:59 +01:00
async function fetchIconUrl(instance: Instance, doc: DOMWindow['document'] | null, manifest: Record<string, any> | null): Promise<string | null> {
2020-10-27 08:44:54 +01:00
if (manifest && manifest.icons && manifest.icons.length > 0 && manifest.icons[0].src) {
const url = 'https://' + instance.host;
return (new URL(manifest.icons[0].src, url)).href;
}
2020-10-27 08:16:59 +01:00
if (doc) {
const url = 'https://' + instance.host;
const hrefAppleTouchIconPrecomposed = doc.querySelector('link[rel="apple-touch-icon-precomposed"]')?.getAttribute('href');
const hrefAppleTouchIcon = doc.querySelector('link[rel="apple-touch-icon"]')?.getAttribute('href');
const hrefIcon = doc.querySelector('link[rel="icon"]')?.getAttribute('href');
const href = hrefAppleTouchIconPrecomposed || hrefAppleTouchIcon || hrefIcon;
if (href) {
return (new URL(href, url)).href;
}
}
return null;
}
async function getThemeColor(doc: DOMWindow['document'] | null, manifest: Record<string, any> | null): Promise<string | null> {
if (doc) {
const themeColor = doc.querySelector('meta[name="theme-color"]')?.getAttribute('content');
if (themeColor) {
return themeColor;
}
}
if (manifest) {
return manifest.theme_color;
}
return null;
}
async function getSiteName(info: NodeInfo | null, doc: DOMWindow['document'] | null, manifest: Record<string, any> | null): Promise<string | null> {
if (info && info.metadata) {
if (info.metadata.nodeName || info.metadata.name) {
return info.metadata.nodeName || info.metadata.name;
}
}
if (doc) {
const og = doc.querySelector('meta[property="og:title"]')?.getAttribute('content');
if (og) {
return og;
}
}
if (manifest) {
return manifest?.name || manifest?.short_name;
}
return null;
}
async function getDescription(info: NodeInfo | null, doc: DOMWindow['document'] | null, manifest: Record<string, any> | null): Promise<string | null> {
if (info && info.metadata) {
if (info.metadata.nodeDescription || info.metadata.description) {
return info.metadata.nodeDescription || info.metadata.description;
}
}
if (doc) {
const meta = doc.querySelector('meta[name="description"]')?.getAttribute('content');
if (meta) {
return meta;
}
const og = doc.querySelector('meta[property="og:description"]')?.getAttribute('content');
if (og) {
return og;
}
}
if (manifest) {
return manifest?.name || manifest?.short_name;
}
return null;
}