Release 0.5.1
This commit is contained in:
hnzio committed
1 parent
5a9d7735d4
commit
fb24d171ae
36 files changed
+5099
-263
No files matched your search
@@ -0,0 +1,274 @@
|
||||
import { lookup } from 'node:dns/promises';
|
||||
import { isIP } from 'node:net';
|
||||
import http from 'node:http';
|
||||
import https from 'node:https';
|
||||
import metascraper from 'metascraper';
|
||||
import description from 'metascraper-description';
|
||||
import image from 'metascraper-image';
|
||||
import publisher from 'metascraper-publisher';
|
||||
import title from 'metascraper-title';
|
||||
|
||||
export type PresentationMode = 'text' | 'link' | 'video';
|
||||
export type VideoProvider = 'youtube' | 'tiktok';
|
||||
|
||||
export type PresentationMetadata = {
|
||||
title: string;
|
||||
description: string;
|
||||
imageUrl: string;
|
||||
siteName: string;
|
||||
provider: VideoProvider | '';
|
||||
providerId: string;
|
||||
};
|
||||
|
||||
export type PresentationSource = {
|
||||
label: string;
|
||||
url: string;
|
||||
timecodeSeconds: number | null;
|
||||
};
|
||||
|
||||
export type PresentationContent = {
|
||||
mode: PresentationMode;
|
||||
title: string;
|
||||
html: string;
|
||||
source: PresentationSource | null;
|
||||
previewTitle: string;
|
||||
description: string;
|
||||
imageUrl: string;
|
||||
siteName: string;
|
||||
provider: VideoProvider | '';
|
||||
providerId: string;
|
||||
embedUrl: string;
|
||||
};
|
||||
|
||||
const scrape = metascraper([title(), description({ truncateLength: 500 }), image(), publisher()]);
|
||||
const MAX_HTML_BYTES = 1024 * 1024;
|
||||
const MAX_IMAGE_BYTES = 5 * 1024 * 1024;
|
||||
const REQUEST_TIMEOUT_MS = 5_000;
|
||||
const MAX_REDIRECTS = 3;
|
||||
|
||||
function isPrivateIpv4(address: string) {
|
||||
const parts = address.split('.').map(Number);
|
||||
return (
|
||||
parts[0] === 0 ||
|
||||
parts[0] === 10 ||
|
||||
parts[0] === 127 ||
|
||||
(parts[0] === 169 && parts[1] === 254) ||
|
||||
(parts[0] === 172 && parts[1] >= 16 && parts[1] <= 31) ||
|
||||
(parts[0] === 192 && parts[1] === 168) ||
|
||||
(parts[0] === 100 && parts[1] >= 64 && parts[1] <= 127) ||
|
||||
parts[0] >= 224
|
||||
);
|
||||
}
|
||||
|
||||
function isPrivateAddress(address: string) {
|
||||
if (isIP(address) === 4) return isPrivateIpv4(address);
|
||||
const normalized = address.toLowerCase().split('%')[0];
|
||||
if (normalized === '::' || normalized === '::1') return true;
|
||||
if (normalized.startsWith('fc') || normalized.startsWith('fd') || normalized.startsWith('fe8')) {
|
||||
return true;
|
||||
}
|
||||
if (
|
||||
normalized.startsWith('fe9') ||
|
||||
normalized.startsWith('fea') ||
|
||||
normalized.startsWith('feb')
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
const mapped = normalized.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/)?.[1];
|
||||
return mapped ? isPrivateIpv4(mapped) : false;
|
||||
}
|
||||
|
||||
async function publicAddress(url: URL) {
|
||||
if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password) {
|
||||
throw new Error('FETCH_BLOCKED');
|
||||
}
|
||||
if (url.port && url.port !== (url.protocol === 'https:' ? '443' : '80')) {
|
||||
throw new Error('FETCH_BLOCKED');
|
||||
}
|
||||
const hostname = url.hostname.startsWith('[') ? url.hostname.slice(1, -1) : url.hostname;
|
||||
if (hostname === 'localhost' || hostname.endsWith('.localhost')) {
|
||||
throw new Error('FETCH_BLOCKED');
|
||||
}
|
||||
const addresses = isIP(hostname)
|
||||
? [{ address: hostname, family: isIP(hostname) }]
|
||||
: await lookup(hostname, { all: true, verbatim: true }).catch(() => {
|
||||
throw new Error('FETCH_UNAVAILABLE');
|
||||
});
|
||||
if (!addresses.length || addresses.some((entry) => isPrivateAddress(entry.address))) {
|
||||
throw new Error('FETCH_BLOCKED');
|
||||
}
|
||||
return addresses[0];
|
||||
}
|
||||
|
||||
async function requestPublic(
|
||||
input: string,
|
||||
accept: string,
|
||||
maximumBytes: number,
|
||||
redirects = 0
|
||||
): Promise<{ url: string; contentType: string; data: Buffer }> {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(input);
|
||||
} catch {
|
||||
throw new Error('FETCH_INVALID_URL');
|
||||
}
|
||||
const target = await publicAddress(url);
|
||||
const client = url.protocol === 'https:' ? https : http;
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = client.get(
|
||||
url,
|
||||
{
|
||||
headers: {
|
||||
Accept: accept,
|
||||
'User-Agent': 'Sendewerk/0.5.1 (+https://git.hnz.io/hnzio/sendewerk)'
|
||||
},
|
||||
lookup: (_hostname, _options, callback) =>
|
||||
callback(null, target.address, target.family as 4 | 6)
|
||||
},
|
||||
(response) => {
|
||||
const status = response.statusCode ?? 0;
|
||||
if (status >= 300 && status < 400 && response.headers.location) {
|
||||
response.resume();
|
||||
if (redirects >= MAX_REDIRECTS) return reject(new Error('FETCH_REDIRECTS'));
|
||||
const next = new URL(response.headers.location, url).href;
|
||||
requestPublic(next, accept, maximumBytes, redirects + 1).then(resolve, reject);
|
||||
return;
|
||||
}
|
||||
if (status < 200 || status >= 300) {
|
||||
response.resume();
|
||||
return reject(new Error('FETCH_UNAVAILABLE'));
|
||||
}
|
||||
const chunks: Buffer[] = [];
|
||||
let size = 0;
|
||||
response.on('data', (chunk: Buffer) => {
|
||||
size += chunk.length;
|
||||
if (size > maximumBytes) {
|
||||
request.destroy(new Error('FETCH_TOO_LARGE'));
|
||||
return;
|
||||
}
|
||||
chunks.push(chunk);
|
||||
});
|
||||
response.on('end', () =>
|
||||
resolve({
|
||||
url: url.href,
|
||||
contentType: String(response.headers['content-type'] ?? ''),
|
||||
data: Buffer.concat(chunks)
|
||||
})
|
||||
);
|
||||
}
|
||||
);
|
||||
request.setTimeout(REQUEST_TIMEOUT_MS, () => request.destroy(new Error('FETCH_TIMEOUT')));
|
||||
request.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
export function detectVideoProvider(input: string): {
|
||||
provider: VideoProvider;
|
||||
id: string;
|
||||
} | null {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(input);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (url.protocol !== 'https:' || url.username || url.password || url.port) return null;
|
||||
const host = url.hostname.toLowerCase();
|
||||
let youtubeId = '';
|
||||
if (host === 'youtu.be') youtubeId = url.pathname.split('/').filter(Boolean)[0] ?? '';
|
||||
if (host === 'youtube.com' || host === 'www.youtube.com' || host === 'm.youtube.com') {
|
||||
if (url.pathname === '/watch') youtubeId = url.searchParams.get('v') ?? '';
|
||||
else if (/^\/(shorts|embed|live)\//.test(url.pathname)) {
|
||||
youtubeId = url.pathname.split('/').filter(Boolean)[1] ?? '';
|
||||
}
|
||||
}
|
||||
if (/^[A-Za-z0-9_-]{11}$/.test(youtubeId)) {
|
||||
return { provider: 'youtube', id: youtubeId };
|
||||
}
|
||||
if (host === 'tiktok.com' || host === 'www.tiktok.com' || host === 'm.tiktok.com') {
|
||||
const id = url.pathname.match(/\/video\/(\d+)/)?.[1] ?? '';
|
||||
if (/^\d+$/.test(id)) return { provider: 'tiktok', id };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function buildEmbedUrl(
|
||||
provider: VideoProvider | '',
|
||||
providerId: string,
|
||||
timecodeSeconds: number | null
|
||||
) {
|
||||
if (provider === 'youtube' && /^[A-Za-z0-9_-]{11}$/.test(providerId)) {
|
||||
const url = new URL(`https://www.youtube-nocookie.com/embed/${providerId}`);
|
||||
if (timecodeSeconds && timecodeSeconds > 0)
|
||||
url.searchParams.set('start', String(timecodeSeconds));
|
||||
return url.href;
|
||||
}
|
||||
if (provider === 'tiktok' && /^\d+$/.test(providerId)) {
|
||||
const url = new URL(`https://www.tiktok.com/player/v1/${providerId}`);
|
||||
url.searchParams.set('autoplay', '0');
|
||||
url.searchParams.set('controls', '1');
|
||||
return url.href;
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
export async function extractMetadataFromHtml(
|
||||
html: string,
|
||||
url: string
|
||||
): Promise<PresentationMetadata> {
|
||||
const metadata = await scrape({ html, url });
|
||||
return {
|
||||
title: String(metadata.title ?? '').slice(0, 300),
|
||||
description: String(metadata.description ?? '').slice(0, 1_000),
|
||||
imageUrl: String(metadata.image ?? '').slice(0, 2_000),
|
||||
siteName: String(metadata.publisher ?? new URL(url).hostname).slice(0, 200),
|
||||
provider: '',
|
||||
providerId: ''
|
||||
};
|
||||
}
|
||||
|
||||
async function providerMetadata(url: string, provider: VideoProvider) {
|
||||
const endpoint =
|
||||
provider === 'youtube'
|
||||
? `https://www.youtube.com/oembed?format=json&url=${encodeURIComponent(url)}`
|
||||
: `https://www.tiktok.com/oembed?url=${encodeURIComponent(url)}`;
|
||||
const response = await requestPublic(endpoint, 'application/json', 256 * 1024);
|
||||
if (!response.contentType.toLowerCase().includes('json')) throw new Error('FETCH_INVALID_TYPE');
|
||||
const value = JSON.parse(response.data.toString('utf8')) as Record<string, unknown>;
|
||||
return {
|
||||
title: String(value.title ?? '').slice(0, 300),
|
||||
description: '',
|
||||
imageUrl: String(value.thumbnail_url ?? '').slice(0, 2_000),
|
||||
siteName: String(value.provider_name ?? (provider === 'youtube' ? 'YouTube' : 'TikTok')).slice(
|
||||
0,
|
||||
200
|
||||
)
|
||||
};
|
||||
}
|
||||
|
||||
export async function fetchPresentationMetadata(url: string): Promise<PresentationMetadata> {
|
||||
const video = detectVideoProvider(url);
|
||||
if (video) {
|
||||
const metadata = await providerMetadata(url, video.provider);
|
||||
return { ...metadata, provider: video.provider, providerId: video.id };
|
||||
}
|
||||
const response = await requestPublic(url, 'text/html,application/xhtml+xml', MAX_HTML_BYTES);
|
||||
const contentType = response.contentType.toLowerCase();
|
||||
if (!contentType.includes('text/html') && !contentType.includes('application/xhtml+xml')) {
|
||||
throw new Error('FETCH_INVALID_TYPE');
|
||||
}
|
||||
return extractMetadataFromHtml(response.data.toString('utf8'), response.url);
|
||||
}
|
||||
|
||||
export async function fetchPresentationImage(url: string) {
|
||||
const response = await requestPublic(
|
||||
url,
|
||||
'image/jpeg,image/png,image/webp,image/gif',
|
||||
MAX_IMAGE_BYTES
|
||||
);
|
||||
const mimeType = response.contentType.split(';')[0].trim().toLowerCase();
|
||||
if (!['image/jpeg', 'image/png', 'image/webp', 'image/gif'].includes(mimeType)) {
|
||||
throw new Error('FETCH_INVALID_TYPE');
|
||||
}
|
||||
return { data: response.data, mimeType };
|
||||
}
|
||||
Reference in new issue
Block a user