275 lines
8.7 KiB
TypeScript
275 lines
8.7 KiB
TypeScript
import { lookup } from 'node:dns/promises';
|
|
import { isIP } from 'node:net';
|
|
import http from 'node:http';
|
|
import https from 'node:https';
|
|
import metascraper from 'metascraper';
|
|
import description from 'metascraper-description';
|
|
import image from 'metascraper-image';
|
|
import publisher from 'metascraper-publisher';
|
|
import title from 'metascraper-title';
|
|
|
|
export type PresentationMode = 'text' | 'link' | 'video';
|
|
export type VideoProvider = 'youtube' | 'tiktok';
|
|
|
|
export type PresentationMetadata = {
|
|
title: string;
|
|
description: string;
|
|
imageUrl: string;
|
|
siteName: string;
|
|
provider: VideoProvider | '';
|
|
providerId: string;
|
|
};
|
|
|
|
export type PresentationSource = {
|
|
label: string;
|
|
url: string;
|
|
timecodeSeconds: number | null;
|
|
};
|
|
|
|
export type PresentationContent = {
|
|
mode: PresentationMode;
|
|
title: string;
|
|
html: string;
|
|
source: PresentationSource | null;
|
|
previewTitle: string;
|
|
description: string;
|
|
imageUrl: string;
|
|
siteName: string;
|
|
provider: VideoProvider | '';
|
|
providerId: string;
|
|
embedUrl: string;
|
|
};
|
|
|
|
const scrape = metascraper([title(), description({ truncateLength: 500 }), image(), publisher()]);
|
|
const MAX_HTML_BYTES = 1024 * 1024;
|
|
const MAX_IMAGE_BYTES = 5 * 1024 * 1024;
|
|
const REQUEST_TIMEOUT_MS = 5_000;
|
|
const MAX_REDIRECTS = 3;
|
|
|
|
function isPrivateIpv4(address: string) {
|
|
const parts = address.split('.').map(Number);
|
|
return (
|
|
parts[0] === 0 ||
|
|
parts[0] === 10 ||
|
|
parts[0] === 127 ||
|
|
(parts[0] === 169 && parts[1] === 254) ||
|
|
(parts[0] === 172 && parts[1] >= 16 && parts[1] <= 31) ||
|
|
(parts[0] === 192 && parts[1] === 168) ||
|
|
(parts[0] === 100 && parts[1] >= 64 && parts[1] <= 127) ||
|
|
parts[0] >= 224
|
|
);
|
|
}
|
|
|
|
function isPrivateAddress(address: string) {
|
|
if (isIP(address) === 4) return isPrivateIpv4(address);
|
|
const normalized = address.toLowerCase().split('%')[0];
|
|
if (normalized === '::' || normalized === '::1') return true;
|
|
if (normalized.startsWith('fc') || normalized.startsWith('fd') || normalized.startsWith('fe8')) {
|
|
return true;
|
|
}
|
|
if (
|
|
normalized.startsWith('fe9') ||
|
|
normalized.startsWith('fea') ||
|
|
normalized.startsWith('feb')
|
|
) {
|
|
return true;
|
|
}
|
|
const mapped = normalized.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/)?.[1];
|
|
return mapped ? isPrivateIpv4(mapped) : false;
|
|
}
|
|
|
|
async function publicAddress(url: URL) {
|
|
if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password) {
|
|
throw new Error('FETCH_BLOCKED');
|
|
}
|
|
if (url.port && url.port !== (url.protocol === 'https:' ? '443' : '80')) {
|
|
throw new Error('FETCH_BLOCKED');
|
|
}
|
|
const hostname = url.hostname.startsWith('[') ? url.hostname.slice(1, -1) : url.hostname;
|
|
if (hostname === 'localhost' || hostname.endsWith('.localhost')) {
|
|
throw new Error('FETCH_BLOCKED');
|
|
}
|
|
const addresses = isIP(hostname)
|
|
? [{ address: hostname, family: isIP(hostname) }]
|
|
: await lookup(hostname, { all: true, verbatim: true }).catch(() => {
|
|
throw new Error('FETCH_UNAVAILABLE');
|
|
});
|
|
if (!addresses.length || addresses.some((entry) => isPrivateAddress(entry.address))) {
|
|
throw new Error('FETCH_BLOCKED');
|
|
}
|
|
return addresses[0];
|
|
}
|
|
|
|
async function requestPublic(
|
|
input: string,
|
|
accept: string,
|
|
maximumBytes: number,
|
|
redirects = 0
|
|
): Promise<{ url: string; contentType: string; data: Buffer }> {
|
|
let url: URL;
|
|
try {
|
|
url = new URL(input);
|
|
} catch {
|
|
throw new Error('FETCH_INVALID_URL');
|
|
}
|
|
const target = await publicAddress(url);
|
|
const client = url.protocol === 'https:' ? https : http;
|
|
return new Promise((resolve, reject) => {
|
|
const request = client.get(
|
|
url,
|
|
{
|
|
headers: {
|
|
Accept: accept,
|
|
'User-Agent': 'Sendewerk/0.5.1 (+https://git.hnz.io/hnzio/sendewerk)'
|
|
},
|
|
lookup: (_hostname, _options, callback) =>
|
|
callback(null, target.address, target.family as 4 | 6)
|
|
},
|
|
(response) => {
|
|
const status = response.statusCode ?? 0;
|
|
if (status >= 300 && status < 400 && response.headers.location) {
|
|
response.resume();
|
|
if (redirects >= MAX_REDIRECTS) return reject(new Error('FETCH_REDIRECTS'));
|
|
const next = new URL(response.headers.location, url).href;
|
|
requestPublic(next, accept, maximumBytes, redirects + 1).then(resolve, reject);
|
|
return;
|
|
}
|
|
if (status < 200 || status >= 300) {
|
|
response.resume();
|
|
return reject(new Error('FETCH_UNAVAILABLE'));
|
|
}
|
|
const chunks: Buffer[] = [];
|
|
let size = 0;
|
|
response.on('data', (chunk: Buffer) => {
|
|
size += chunk.length;
|
|
if (size > maximumBytes) {
|
|
request.destroy(new Error('FETCH_TOO_LARGE'));
|
|
return;
|
|
}
|
|
chunks.push(chunk);
|
|
});
|
|
response.on('end', () =>
|
|
resolve({
|
|
url: url.href,
|
|
contentType: String(response.headers['content-type'] ?? ''),
|
|
data: Buffer.concat(chunks)
|
|
})
|
|
);
|
|
}
|
|
);
|
|
request.setTimeout(REQUEST_TIMEOUT_MS, () => request.destroy(new Error('FETCH_TIMEOUT')));
|
|
request.on('error', reject);
|
|
});
|
|
}
|
|
|
|
export function detectVideoProvider(input: string): {
|
|
provider: VideoProvider;
|
|
id: string;
|
|
} | null {
|
|
let url: URL;
|
|
try {
|
|
url = new URL(input);
|
|
} catch {
|
|
return null;
|
|
}
|
|
if (url.protocol !== 'https:' || url.username || url.password || url.port) return null;
|
|
const host = url.hostname.toLowerCase();
|
|
let youtubeId = '';
|
|
if (host === 'youtu.be') youtubeId = url.pathname.split('/').filter(Boolean)[0] ?? '';
|
|
if (host === 'youtube.com' || host === 'www.youtube.com' || host === 'm.youtube.com') {
|
|
if (url.pathname === '/watch') youtubeId = url.searchParams.get('v') ?? '';
|
|
else if (/^\/(shorts|embed|live)\//.test(url.pathname)) {
|
|
youtubeId = url.pathname.split('/').filter(Boolean)[1] ?? '';
|
|
}
|
|
}
|
|
if (/^[A-Za-z0-9_-]{11}$/.test(youtubeId)) {
|
|
return { provider: 'youtube', id: youtubeId };
|
|
}
|
|
if (host === 'tiktok.com' || host === 'www.tiktok.com' || host === 'm.tiktok.com') {
|
|
const id = url.pathname.match(/\/video\/(\d+)/)?.[1] ?? '';
|
|
if (/^\d+$/.test(id)) return { provider: 'tiktok', id };
|
|
}
|
|
return null;
|
|
}
|
|
|
|
export function buildEmbedUrl(
|
|
provider: VideoProvider | '',
|
|
providerId: string,
|
|
timecodeSeconds: number | null
|
|
) {
|
|
if (provider === 'youtube' && /^[A-Za-z0-9_-]{11}$/.test(providerId)) {
|
|
const url = new URL(`https://www.youtube-nocookie.com/embed/${providerId}`);
|
|
if (timecodeSeconds && timecodeSeconds > 0)
|
|
url.searchParams.set('start', String(timecodeSeconds));
|
|
return url.href;
|
|
}
|
|
if (provider === 'tiktok' && /^\d+$/.test(providerId)) {
|
|
const url = new URL(`https://www.tiktok.com/player/v1/${providerId}`);
|
|
url.searchParams.set('autoplay', '0');
|
|
url.searchParams.set('controls', '1');
|
|
return url.href;
|
|
}
|
|
return '';
|
|
}
|
|
|
|
export async function extractMetadataFromHtml(
|
|
html: string,
|
|
url: string
|
|
): Promise<PresentationMetadata> {
|
|
const metadata = await scrape({ html, url });
|
|
return {
|
|
title: String(metadata.title ?? '').slice(0, 300),
|
|
description: String(metadata.description ?? '').slice(0, 1_000),
|
|
imageUrl: String(metadata.image ?? '').slice(0, 2_000),
|
|
siteName: String(metadata.publisher ?? new URL(url).hostname).slice(0, 200),
|
|
provider: '',
|
|
providerId: ''
|
|
};
|
|
}
|
|
|
|
async function providerMetadata(url: string, provider: VideoProvider) {
|
|
const endpoint =
|
|
provider === 'youtube'
|
|
? `https://www.youtube.com/oembed?format=json&url=${encodeURIComponent(url)}`
|
|
: `https://www.tiktok.com/oembed?url=${encodeURIComponent(url)}`;
|
|
const response = await requestPublic(endpoint, 'application/json', 256 * 1024);
|
|
if (!response.contentType.toLowerCase().includes('json')) throw new Error('FETCH_INVALID_TYPE');
|
|
const value = JSON.parse(response.data.toString('utf8')) as Record<string, unknown>;
|
|
return {
|
|
title: String(value.title ?? '').slice(0, 300),
|
|
description: '',
|
|
imageUrl: String(value.thumbnail_url ?? '').slice(0, 2_000),
|
|
siteName: String(value.provider_name ?? (provider === 'youtube' ? 'YouTube' : 'TikTok')).slice(
|
|
0,
|
|
200
|
|
)
|
|
};
|
|
}
|
|
|
|
export async function fetchPresentationMetadata(url: string): Promise<PresentationMetadata> {
|
|
const video = detectVideoProvider(url);
|
|
if (video) {
|
|
const metadata = await providerMetadata(url, video.provider);
|
|
return { ...metadata, provider: video.provider, providerId: video.id };
|
|
}
|
|
const response = await requestPublic(url, 'text/html,application/xhtml+xml', MAX_HTML_BYTES);
|
|
const contentType = response.contentType.toLowerCase();
|
|
if (!contentType.includes('text/html') && !contentType.includes('application/xhtml+xml')) {
|
|
throw new Error('FETCH_INVALID_TYPE');
|
|
}
|
|
return extractMetadataFromHtml(response.data.toString('utf8'), response.url);
|
|
}
|
|
|
|
export async function fetchPresentationImage(url: string) {
|
|
const response = await requestPublic(
|
|
url,
|
|
'image/jpeg,image/png,image/webp,image/gif',
|
|
MAX_IMAGE_BYTES
|
|
);
|
|
const mimeType = response.contentType.split(';')[0].trim().toLowerCase();
|
|
if (!['image/jpeg', 'image/png', 'image/webp', 'image/gif'].includes(mimeType)) {
|
|
throw new Error('FETCH_INVALID_TYPE');
|
|
}
|
|
return { data: response.data, mimeType };
|
|
}
|