Release 0.7.0
This commit is contained in:
hnzio committed
1 parent
d586fbd8ce
commit
6d96900d81
27 files changed
+2060
-25
No files matched your search
@@ -0,0 +1,47 @@
|
||||
import { json } from '@sveltejs/kit';
|
||||
|
||||
export async function apiBody(request: Request) {
|
||||
const type = request.headers.get('content-type') ?? '';
|
||||
if (!type.includes('application/json')) throw new Error('API_JSON_REQUIRED');
|
||||
try {
|
||||
return (await request.json()) as Record<string, unknown>;
|
||||
} catch {
|
||||
throw new Error('API_INVALID_JSON');
|
||||
}
|
||||
}
|
||||
|
||||
export function apiFailure(cause: unknown) {
|
||||
const code = cause instanceof Error ? cause.message : 'API_ERROR';
|
||||
const status =
|
||||
code === 'API_UNAUTHORIZED'
|
||||
? 401
|
||||
: code === 'API_FORBIDDEN'
|
||||
? 403
|
||||
: code === 'NOT_FOUND'
|
||||
? 404
|
||||
: code === 'CONFLICT' || code === 'API_IDEMPOTENCY_CONFLICT'
|
||||
? 409
|
||||
: 400;
|
||||
return json(
|
||||
{
|
||||
error: {
|
||||
code,
|
||||
message:
|
||||
status === 401
|
||||
? 'Authentifizierung erforderlich.'
|
||||
: status === 403
|
||||
? 'Diese Berechtigung fehlt.'
|
||||
: status === 404
|
||||
? 'Nicht gefunden.'
|
||||
: 'Anfrage konnte nicht verarbeitet werden.'
|
||||
}
|
||||
},
|
||||
{ status }
|
||||
);
|
||||
}
|
||||
|
||||
export function pageInput(url: URL) {
|
||||
const limit = Math.min(100, Math.max(1, Number(url.searchParams.get('limit') ?? 50) || 50));
|
||||
const offset = Math.max(0, Number(url.searchParams.get('offset') ?? 0) || 0);
|
||||
return { limit, offset };
|
||||
}
|
||||
@@ -62,7 +62,7 @@ describe('database setup', () => {
|
||||
database.close();
|
||||
|
||||
expect(counts).toEqual({
|
||||
migrations: 17,
|
||||
migrations: 18,
|
||||
pieces: 6,
|
||||
shows: 2,
|
||||
usages: 5,
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { mkdtempSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { closeDatabase, getDatabase } from './database.js';
|
||||
import {
|
||||
discardFeedEntries,
|
||||
fetchDueFeeds,
|
||||
fetchFeed,
|
||||
importFeedEntries,
|
||||
listFeedEntries,
|
||||
saveFeedSource
|
||||
} from './feeds.js';
|
||||
import { getPiece } from './content.js';
|
||||
|
||||
vi.mock('node:dns/promises', () => ({
|
||||
lookup: vi.fn(async () => [{ address: '93.184.216.34', family: 4 }])
|
||||
}));
|
||||
|
||||
const validFeed = `<?xml version="1.0"?><rss version="2.0"><channel><title>Test</title><item>
|
||||
<guid>entry-1</guid><title>Ein Beitrag</title><link>https://example.org/post</link>
|
||||
<description><![CDATA[<p>Kurz <strong>und</strong> gut.</p><script>evil()</script>]]></description>
|
||||
<pubDate>Sat, 24 Oct 2026 12:00:00 GMT</pubDate></item></channel></rss>`;
|
||||
|
||||
let directory: string;
|
||||
|
||||
beforeEach(() => {
|
||||
directory = mkdtempSync(path.join(tmpdir(), 'sendewerk-feed-test-'));
|
||||
process.env.SENDEWERK_DB_PATH = path.join(directory, 'test.sqlite');
|
||||
process.env.SENDEWERK_UPLOAD_DIR = path.join(directory, 'uploads');
|
||||
execFileSync(process.execPath, ['scripts/migrate.mjs'], { cwd: process.cwd(), env: process.env });
|
||||
execFileSync(process.execPath, ['scripts/seed.mjs'], { cwd: process.cwd(), env: process.env });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
closeDatabase();
|
||||
delete process.env.SENDEWERK_DB_PATH;
|
||||
delete process.env.SENDEWERK_UPLOAD_DIR;
|
||||
rmSync(directory, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('RSS import inbox', () => {
|
||||
it('keeps one entry across fetches and preserves a discard decision', async () => {
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(
|
||||
async () => new Response(validFeed, { headers: { 'content-type': 'application/rss+xml' } })
|
||||
)
|
||||
);
|
||||
const feedId = saveFeedSource(null, {
|
||||
name: 'Feed',
|
||||
url: 'https://example.org/feed.xml',
|
||||
isEnabled: true,
|
||||
refreshIntervalMinutes: 60,
|
||||
defaultCategoryId: null
|
||||
});
|
||||
await fetchFeed(feedId);
|
||||
const [entry] = listFeedEntries();
|
||||
expect(entry.title).toBe('Ein Beitrag');
|
||||
expect(discardFeedEntries([entry.id])).toBe(1);
|
||||
await fetchFeed(feedId);
|
||||
expect(listFeedEntries()).toHaveLength(0);
|
||||
expect(listFeedEntries('discarded')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('sanitizes foreign HTML and retains the original source on explicit import', async () => {
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(async () => new Response(validFeed))
|
||||
);
|
||||
const feedId = saveFeedSource(null, {
|
||||
name: 'Feed',
|
||||
url: 'https://example.org/feed.xml',
|
||||
isEnabled: true,
|
||||
refreshIntervalMinutes: 60,
|
||||
defaultCategoryId: 'cat-jc'
|
||||
});
|
||||
await fetchFeed(feedId);
|
||||
const [entry] = listFeedEntries();
|
||||
const [pieceId] = importFeedEntries([entry.id], []);
|
||||
const piece = getPiece(pieceId)!;
|
||||
expect(piece.summary).toContain('Kurz und gut.');
|
||||
expect(piece.summary).not.toContain('<script>');
|
||||
expect(piece.sources).toEqual([
|
||||
{ label: 'RSS: Originalbeitrag', url: 'https://example.org/post', timecodeSeconds: null }
|
||||
]);
|
||||
expect(piece.categoryIds).toEqual(['cat-jc']);
|
||||
});
|
||||
|
||||
it('isolates a broken feed while fetching other due feeds', async () => {
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(
|
||||
async (url: URL) => new Response(url.href.includes('broken') ? '<not-feed />' : validFeed)
|
||||
)
|
||||
);
|
||||
saveFeedSource(null, {
|
||||
name: 'A broken',
|
||||
url: 'https://example.org/broken.xml',
|
||||
isEnabled: true,
|
||||
refreshIntervalMinutes: 60,
|
||||
defaultCategoryId: null
|
||||
});
|
||||
saveFeedSource(null, {
|
||||
name: 'B valid',
|
||||
url: 'https://example.org/valid.xml',
|
||||
isEnabled: true,
|
||||
refreshIntervalMinutes: 60,
|
||||
defaultCategoryId: null
|
||||
});
|
||||
await fetchDueFeeds();
|
||||
expect(listFeedEntries()).toHaveLength(1);
|
||||
const errors = getDatabase()
|
||||
.prepare("SELECT count(*) AS count FROM feed_sources WHERE last_error <> ''")
|
||||
.get() as { count: number };
|
||||
expect(errors.count).toBe(1);
|
||||
});
|
||||
|
||||
it('blocks loopback targets before sending a request', async () => {
|
||||
const request = vi.fn();
|
||||
vi.stubGlobal('fetch', request);
|
||||
const feedId = saveFeedSource(null, {
|
||||
name: 'Local',
|
||||
url: 'http://127.0.0.1/feed.xml',
|
||||
isEnabled: false,
|
||||
refreshIntervalMinutes: 60,
|
||||
defaultCategoryId: null
|
||||
});
|
||||
await expect(fetchFeed(feedId)).rejects.toThrow('FEED_BLOCKED');
|
||||
expect(request).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,457 @@
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { lookup } from 'node:dns/promises';
|
||||
import { isIP } from 'node:net';
|
||||
import sanitizeHtml from 'sanitize-html';
|
||||
import { XMLParser } from 'fast-xml-parser';
|
||||
import { createPiece, getPiece, savePiece } from './content.js';
|
||||
import { getDatabase } from './database.js';
|
||||
|
||||
const MAX_FEED_BYTES = 2 * 1024 * 1024;
|
||||
const FETCH_TIMEOUT_MS = 8_000;
|
||||
const MAX_REDIRECTS = 3;
|
||||
|
||||
export type FeedSource = {
|
||||
id: string;
|
||||
name: string;
|
||||
url: string;
|
||||
isEnabled: boolean;
|
||||
refreshIntervalMinutes: number;
|
||||
defaultCategoryId: string | null;
|
||||
lastFetchedAtUtc: string | null;
|
||||
nextFetchAtUtc: string | null;
|
||||
lastError: string;
|
||||
};
|
||||
|
||||
export type FeedEntry = {
|
||||
id: string;
|
||||
feedId: string;
|
||||
feedName: string;
|
||||
title: string;
|
||||
url: string;
|
||||
summary: string;
|
||||
publishedAtUtc: string | null;
|
||||
status: 'new' | 'imported' | 'discarded';
|
||||
pieceId: string | null;
|
||||
defaultCategoryId: string | null;
|
||||
};
|
||||
|
||||
function clean(value: unknown, maximum: number) {
|
||||
return String(value ?? '')
|
||||
.trim()
|
||||
.slice(0, maximum);
|
||||
}
|
||||
|
||||
function validHttpUrl(value: string) {
|
||||
try {
|
||||
const url = new URL(value);
|
||||
return ['http:', 'https:'].includes(url.protocol) && !url.username && !url.password;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function privateIpv4(address: string) {
|
||||
const p = address.split('.').map(Number);
|
||||
return (
|
||||
p[0] === 0 ||
|
||||
p[0] === 10 ||
|
||||
p[0] === 127 ||
|
||||
(p[0] === 169 && p[1] === 254) ||
|
||||
(p[0] === 172 && p[1] >= 16 && p[1] <= 31) ||
|
||||
(p[0] === 192 && p[1] === 168) ||
|
||||
p[0] >= 224
|
||||
);
|
||||
}
|
||||
|
||||
function privateAddress(address: string) {
|
||||
if (isIP(address) === 4) return privateIpv4(address);
|
||||
const value = address.toLowerCase().split('%')[0];
|
||||
if (value === '::' || value === '::1' || value.startsWith('fc') || value.startsWith('fd'))
|
||||
return true;
|
||||
if (/^fe[89ab]/.test(value)) return true;
|
||||
const mapped = value.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/)?.[1];
|
||||
return mapped ? privateIpv4(mapped) : false;
|
||||
}
|
||||
|
||||
async function assertPublic(url: URL) {
|
||||
if (!validHttpUrl(url.href) || (url.port && !['80', '443'].includes(url.port)))
|
||||
throw new Error('FEED_BLOCKED');
|
||||
const hostname = url.hostname.replace(/^\[|\]$/g, '');
|
||||
if (hostname === 'localhost' || hostname.endsWith('.localhost')) throw new Error('FEED_BLOCKED');
|
||||
const addresses = isIP(hostname)
|
||||
? [{ address: hostname }]
|
||||
: await lookup(hostname, { all: true, verbatim: true });
|
||||
if (!addresses.length || addresses.some((entry) => privateAddress(entry.address)))
|
||||
throw new Error('FEED_BLOCKED');
|
||||
}
|
||||
|
||||
async function fetchXml(
|
||||
input: string,
|
||||
headers: Record<string, string>,
|
||||
redirects = 0
|
||||
): Promise<{ xml: string; etag: string; lastModified: string; unchanged: boolean }> {
|
||||
const url = new URL(input);
|
||||
await assertPublic(url);
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
Accept: 'application/rss+xml, application/atom+xml, application/xml, text/xml',
|
||||
...headers
|
||||
},
|
||||
redirect: 'manual',
|
||||
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS)
|
||||
});
|
||||
if ([301, 302, 303, 307, 308].includes(response.status)) {
|
||||
if (redirects >= MAX_REDIRECTS) throw new Error('FEED_REDIRECT');
|
||||
const location = response.headers.get('location');
|
||||
if (!location) throw new Error('FEED_FETCH');
|
||||
return fetchXml(new URL(location, url).href, headers, redirects + 1);
|
||||
}
|
||||
if (response.status === 304)
|
||||
return {
|
||||
xml: '',
|
||||
etag: headers['If-None-Match'] ?? '',
|
||||
lastModified: headers['If-Modified-Since'] ?? '',
|
||||
unchanged: true
|
||||
};
|
||||
if (!response.ok) throw new Error(`FEED_HTTP_${response.status}`);
|
||||
const declared = Number(response.headers.get('content-length') ?? 0);
|
||||
if (declared > MAX_FEED_BYTES) throw new Error('FEED_TOO_LARGE');
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
if (buffer.length > MAX_FEED_BYTES) throw new Error('FEED_TOO_LARGE');
|
||||
return {
|
||||
xml: buffer.toString('utf8'),
|
||||
etag: response.headers.get('etag') ?? '',
|
||||
lastModified: response.headers.get('last-modified') ?? '',
|
||||
unchanged: false
|
||||
};
|
||||
}
|
||||
|
||||
function values<T>(value: T | T[] | undefined): T[] {
|
||||
return value === undefined ? [] : Array.isArray(value) ? value : [value];
|
||||
}
|
||||
|
||||
function textFrom(value: unknown): string {
|
||||
if (typeof value === 'string' || typeof value === 'number') return String(value);
|
||||
if (value && typeof value === 'object') {
|
||||
const record = value as Record<string, unknown>;
|
||||
return textFrom(record['#text'] ?? record.__cdata ?? '');
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
function entryLink(value: unknown) {
|
||||
for (const link of values(
|
||||
value as Record<string, unknown> | Record<string, unknown>[] | string | undefined
|
||||
)) {
|
||||
if (typeof link === 'string') return link;
|
||||
if (link && typeof link === 'object') {
|
||||
const href = clean((link as Record<string, unknown>)['@_href'], 2_000);
|
||||
const rel = clean((link as Record<string, unknown>)['@_rel'], 30);
|
||||
if (href && (!rel || rel === 'alternate')) return href;
|
||||
}
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
function parseDate(value: unknown) {
|
||||
const raw = textFrom(value);
|
||||
if (!raw) return null;
|
||||
const date = new Date(raw);
|
||||
return Number.isNaN(date.valueOf()) ? null : date.toISOString();
|
||||
}
|
||||
|
||||
function parseFeed(xml: string) {
|
||||
let document: Record<string, unknown>;
|
||||
try {
|
||||
document = new XMLParser({
|
||||
ignoreAttributes: false,
|
||||
processEntities: false,
|
||||
trimValues: true
|
||||
}).parse(xml) as Record<string, unknown>;
|
||||
} catch {
|
||||
throw new Error('INVALID_FEED');
|
||||
}
|
||||
const rssChannel = (document.rss as Record<string, unknown> | undefined)?.channel as
|
||||
Record<string, unknown> | undefined;
|
||||
const atom = document.feed as Record<string, unknown> | undefined;
|
||||
if (!rssChannel && !atom) throw new Error('INVALID_FEED');
|
||||
const rawEntries = values(
|
||||
(rssChannel?.item ?? atom?.entry) as
|
||||
Record<string, unknown> | Record<string, unknown>[] | undefined
|
||||
);
|
||||
return rawEntries.slice(0, 200).flatMap((entry) => {
|
||||
const title = clean(textFrom(entry.title), 300);
|
||||
const url = clean(entryLink(entry.link) || textFrom(entry.link), 2_000);
|
||||
if (!title || !validHttpUrl(url)) return [];
|
||||
const guid = clean(textFrom(entry.guid ?? entry.id), 2_000);
|
||||
const externalId = guid || createHash('sha256').update(url).digest('hex');
|
||||
const rawSummary = textFrom(entry.description ?? entry.summary ?? entry.content);
|
||||
const summary = clean(
|
||||
sanitizeHtml(rawSummary, { allowedTags: [], allowedAttributes: {} }),
|
||||
5_000
|
||||
);
|
||||
return [
|
||||
{
|
||||
externalId,
|
||||
title,
|
||||
url,
|
||||
summary,
|
||||
publishedAtUtc: parseDate(entry.pubDate ?? entry.published ?? entry.updated)
|
||||
}
|
||||
];
|
||||
});
|
||||
}
|
||||
|
||||
export function listFeedSources(): FeedSource[] {
|
||||
return (
|
||||
getDatabase()
|
||||
.prepare(
|
||||
`SELECT id, name, url, is_enabled, refresh_interval_minutes,
|
||||
default_category_id, last_fetched_at_utc, next_fetch_at_utc, last_error FROM feed_sources ORDER BY name`
|
||||
)
|
||||
.all() as Array<Record<string, unknown>>
|
||||
).map((row) => ({
|
||||
id: String(row.id),
|
||||
name: String(row.name),
|
||||
url: String(row.url),
|
||||
isEnabled: row.is_enabled === 1,
|
||||
refreshIntervalMinutes: Number(row.refresh_interval_minutes),
|
||||
defaultCategoryId: row.default_category_id ? String(row.default_category_id) : null,
|
||||
lastFetchedAtUtc: row.last_fetched_at_utc ? String(row.last_fetched_at_utc) : null,
|
||||
nextFetchAtUtc: row.next_fetch_at_utc ? String(row.next_fetch_at_utc) : null,
|
||||
lastError: String(row.last_error)
|
||||
}));
|
||||
}
|
||||
|
||||
export function saveFeedSource(
|
||||
id: string | null,
|
||||
input: {
|
||||
name: string;
|
||||
url: string;
|
||||
isEnabled: boolean;
|
||||
refreshIntervalMinutes: number;
|
||||
defaultCategoryId: string | null;
|
||||
}
|
||||
) {
|
||||
const name = clean(input.name, 120);
|
||||
const url = clean(input.url, 2_000);
|
||||
if (
|
||||
!name ||
|
||||
!validHttpUrl(url) ||
|
||||
!Number.isInteger(input.refreshIntervalMinutes) ||
|
||||
input.refreshIntervalMinutes < 15 ||
|
||||
input.refreshIntervalMinutes > 10080
|
||||
)
|
||||
throw new Error('INVALID_FEED');
|
||||
const database = getDatabase();
|
||||
if (
|
||||
input.defaultCategoryId &&
|
||||
!database.prepare('SELECT 1 FROM section_categories WHERE id = ?').get(input.defaultCategoryId)
|
||||
)
|
||||
throw new Error('INVALID_TAXONOMY');
|
||||
const now = new Date().toISOString();
|
||||
if (!id) {
|
||||
id = randomUUID();
|
||||
database
|
||||
.prepare(
|
||||
`INSERT INTO feed_sources (id, name, url, is_enabled, refresh_interval_minutes,
|
||||
default_category_id, next_fetch_at_utc, created_at_utc, updated_at_utc) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
)
|
||||
.run(
|
||||
id,
|
||||
name,
|
||||
url,
|
||||
input.isEnabled ? 1 : 0,
|
||||
input.refreshIntervalMinutes,
|
||||
input.defaultCategoryId,
|
||||
input.isEnabled ? now : null,
|
||||
now,
|
||||
now
|
||||
);
|
||||
} else {
|
||||
const result = database
|
||||
.prepare(
|
||||
`UPDATE feed_sources SET name = ?, url = ?, is_enabled = ?, refresh_interval_minutes = ?,
|
||||
default_category_id = ?, next_fetch_at_utc = CASE WHEN ? = 1 THEN COALESCE(next_fetch_at_utc, ?) ELSE NULL END,
|
||||
updated_at_utc = ? WHERE id = ?`
|
||||
)
|
||||
.run(
|
||||
name,
|
||||
url,
|
||||
input.isEnabled ? 1 : 0,
|
||||
input.refreshIntervalMinutes,
|
||||
input.defaultCategoryId,
|
||||
input.isEnabled ? 1 : 0,
|
||||
now,
|
||||
now,
|
||||
id
|
||||
);
|
||||
if (!result.changes) throw new Error('NOT_FOUND');
|
||||
}
|
||||
return id;
|
||||
}
|
||||
|
||||
export function deleteFeedSource(id: string) {
|
||||
if (!getDatabase().prepare('DELETE FROM feed_sources WHERE id = ?').run(id).changes)
|
||||
throw new Error('NOT_FOUND');
|
||||
}
|
||||
|
||||
export function listFeedEntries(status: 'new' | 'imported' | 'discarded' = 'new'): FeedEntry[] {
|
||||
return (
|
||||
getDatabase()
|
||||
.prepare(
|
||||
`SELECT e.id, e.feed_id, f.name AS feed_name, e.title, e.url, e.summary,
|
||||
e.published_at_utc, e.status, e.piece_id, f.default_category_id FROM feed_entries e
|
||||
JOIN feed_sources f ON f.id = e.feed_id WHERE e.status = ? ORDER BY COALESCE(e.published_at_utc, e.first_seen_at_utc) DESC`
|
||||
)
|
||||
.all(status) as Array<Record<string, unknown>>
|
||||
).map((row) => ({
|
||||
id: String(row.id),
|
||||
feedId: String(row.feed_id),
|
||||
feedName: String(row.feed_name),
|
||||
title: String(row.title),
|
||||
url: String(row.url),
|
||||
summary: String(row.summary),
|
||||
publishedAtUtc: row.published_at_utc ? String(row.published_at_utc) : null,
|
||||
status: row.status as FeedEntry['status'],
|
||||
pieceId: row.piece_id ? String(row.piece_id) : null,
|
||||
defaultCategoryId: row.default_category_id ? String(row.default_category_id) : null
|
||||
}));
|
||||
}
|
||||
|
||||
export async function fetchFeed(feedId: string) {
|
||||
const database = getDatabase();
|
||||
const feed = database.prepare('SELECT * FROM feed_sources WHERE id = ?').get(feedId) as
|
||||
Record<string, unknown> | undefined;
|
||||
if (!feed) throw new Error('NOT_FOUND');
|
||||
const now = new Date();
|
||||
try {
|
||||
const response = await fetchXml(String(feed.url), {
|
||||
...(feed.etag ? { 'If-None-Match': String(feed.etag) } : {}),
|
||||
...(feed.last_modified ? { 'If-Modified-Since': String(feed.last_modified) } : {})
|
||||
});
|
||||
let inserted = 0;
|
||||
if (!response.unchanged) {
|
||||
const add =
|
||||
database.prepare(`INSERT INTO feed_entries (id, feed_id, external_id, title, url, summary,
|
||||
published_at_utc, first_seen_at_utc, updated_at_utc) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(feed_id, external_id) DO UPDATE SET title = excluded.title, url = excluded.url,
|
||||
summary = excluded.summary, published_at_utc = excluded.published_at_utc, updated_at_utc = excluded.updated_at_utc`);
|
||||
for (const entry of parseFeed(response.xml)) {
|
||||
const existed = database
|
||||
.prepare('SELECT 1 FROM feed_entries WHERE feed_id = ? AND external_id = ?')
|
||||
.get(feedId, entry.externalId);
|
||||
add.run(
|
||||
randomUUID(),
|
||||
feedId,
|
||||
entry.externalId,
|
||||
entry.title,
|
||||
entry.url,
|
||||
entry.summary,
|
||||
entry.publishedAtUtc,
|
||||
now.toISOString(),
|
||||
now.toISOString()
|
||||
);
|
||||
if (!existed) inserted += 1;
|
||||
}
|
||||
}
|
||||
const next = new Date(
|
||||
now.getTime() + Number(feed.refresh_interval_minutes) * 60_000
|
||||
).toISOString();
|
||||
database
|
||||
.prepare(
|
||||
`UPDATE feed_sources SET etag = ?, last_modified = ?, last_fetched_at_utc = ?,
|
||||
next_fetch_at_utc = ?, last_error = '', updated_at_utc = ? WHERE id = ?`
|
||||
)
|
||||
.run(
|
||||
response.etag,
|
||||
response.lastModified,
|
||||
now.toISOString(),
|
||||
next,
|
||||
now.toISOString(),
|
||||
feedId
|
||||
);
|
||||
return { inserted, unchanged: response.unchanged };
|
||||
} catch (cause) {
|
||||
const message = cause instanceof Error ? cause.message.slice(0, 500) : 'FEED_FETCH';
|
||||
const next = new Date(
|
||||
now.getTime() + Number(feed.refresh_interval_minutes) * 60_000
|
||||
).toISOString();
|
||||
database
|
||||
.prepare(
|
||||
'UPDATE feed_sources SET last_fetched_at_utc = ?, next_fetch_at_utc = ?, last_error = ?, updated_at_utc = ? WHERE id = ?'
|
||||
)
|
||||
.run(now.toISOString(), next, message, now.toISOString(), feedId);
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchDueFeeds() {
|
||||
const rows = getDatabase()
|
||||
.prepare(
|
||||
`SELECT id FROM feed_sources WHERE is_enabled = 1
|
||||
AND (next_fetch_at_utc IS NULL OR next_fetch_at_utc <= ?) ORDER BY next_fetch_at_utc LIMIT 10`
|
||||
)
|
||||
.all(new Date().toISOString()) as { id: string }[];
|
||||
for (const row of rows) await fetchFeed(row.id).catch(() => undefined);
|
||||
}
|
||||
|
||||
export function importFeedEntries(entryIds: string[], categoryIds: string[]) {
|
||||
const database = getDatabase();
|
||||
const categories = [...new Set(categoryIds.filter(Boolean))];
|
||||
if (categories.length > 20) throw new Error('INVALID_TAXONOMY');
|
||||
for (const category of categories)
|
||||
if (!database.prepare('SELECT 1 FROM section_categories WHERE id = ?').get(category))
|
||||
throw new Error('INVALID_TAXONOMY');
|
||||
return database.transaction(() =>
|
||||
entryIds.map((entryId) => {
|
||||
const entry = database
|
||||
.prepare(
|
||||
`SELECT e.*, f.default_category_id FROM feed_entries e JOIN feed_sources f ON f.id = e.feed_id
|
||||
WHERE e.id = ? AND e.status = 'new'`
|
||||
)
|
||||
.get(entryId) as Record<string, unknown> | undefined;
|
||||
if (!entry) throw new Error('NOT_FOUND');
|
||||
const pieceId = createPiece('blank');
|
||||
const piece = getPiece(pieceId)!;
|
||||
const assigned = categories.length
|
||||
? categories
|
||||
: entry.default_category_id
|
||||
? [String(entry.default_category_id)]
|
||||
: [];
|
||||
savePiece(pieceId, piece.revision, {
|
||||
...piece,
|
||||
title: String(entry.title),
|
||||
summary: String(entry.summary),
|
||||
status: 'inbox',
|
||||
categoryIds: assigned,
|
||||
sources: [{ label: 'RSS: Originalbeitrag', url: String(entry.url), timecodeSeconds: null }]
|
||||
});
|
||||
database
|
||||
.prepare(
|
||||
"UPDATE feed_entries SET status = 'imported', piece_id = ?, updated_at_utc = ? WHERE id = ?"
|
||||
)
|
||||
.run(pieceId, new Date().toISOString(), entryId);
|
||||
return pieceId;
|
||||
})
|
||||
)();
|
||||
}
|
||||
|
||||
export function discardFeedEntries(entryIds: string[]) {
|
||||
const database = getDatabase();
|
||||
const update = database.prepare(
|
||||
"UPDATE feed_entries SET status = 'discarded', updated_at_utc = ? WHERE id = ? AND status = 'new'"
|
||||
);
|
||||
const now = new Date().toISOString();
|
||||
return database.transaction(() =>
|
||||
entryIds.reduce((count, id) => count + update.run(now, id).changes, 0)
|
||||
)();
|
||||
}
|
||||
|
||||
let schedulerStarted = false;
|
||||
export function startFeedScheduler() {
|
||||
if (schedulerStarted || process.env.NODE_ENV === 'test') return;
|
||||
schedulerStarted = true;
|
||||
void fetchDueFeeds();
|
||||
setInterval(() => void fetchDueFeeds(), 60_000).unref();
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { mkdtempSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import type { AuthUser } from './auth.js';
|
||||
import { closeDatabase, getDatabase } from './database.js';
|
||||
import {
|
||||
authenticateTechnical,
|
||||
createTechnicalAccess,
|
||||
idempotentResponse,
|
||||
revokeTechnicalAccess
|
||||
} from './technical-access.js';
|
||||
|
||||
let directory: string;
|
||||
let actor: AuthUser;
|
||||
|
||||
beforeEach(() => {
|
||||
directory = mkdtempSync(path.join(tmpdir(), 'sendewerk-api-test-'));
|
||||
process.env.SENDEWERK_DB_PATH = path.join(directory, 'test.sqlite');
|
||||
process.env.SENDEWERK_UPLOAD_DIR = path.join(directory, 'uploads');
|
||||
execFileSync(process.execPath, ['scripts/migrate.mjs'], { cwd: process.cwd(), env: process.env });
|
||||
execFileSync(process.execPath, ['scripts/seed.mjs'], { cwd: process.cwd(), env: process.env });
|
||||
const now = new Date().toISOString();
|
||||
getDatabase()
|
||||
.prepare(
|
||||
`INSERT INTO users (id, email, email_normalized, display_name, password_hash, role,
|
||||
password_changed_at_utc, created_at_utc, updated_at_utc) VALUES ('test-user', 'test@example.org',
|
||||
'test@example.org', 'Test', 'not-used', 'barkeeper', ?, ?, ?)`
|
||||
)
|
||||
.run(now, now, now);
|
||||
actor = {
|
||||
id: 'test-user',
|
||||
email: 'test@example.org',
|
||||
displayName: 'Test',
|
||||
role: 'barkeeper',
|
||||
lockedAtUtc: null
|
||||
};
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
closeDatabase();
|
||||
delete process.env.SENDEWERK_DB_PATH;
|
||||
delete process.env.SENDEWERK_UPLOAD_DIR;
|
||||
rmSync(directory, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function request(token: string, key?: string) {
|
||||
return new Request('https://sendewerk.test/api/v1/pieces', {
|
||||
headers: { Authorization: `Bearer ${token}`, ...(key ? { 'Idempotency-Key': key } : {}) }
|
||||
});
|
||||
}
|
||||
|
||||
describe('technical API access', () => {
|
||||
it('enforces permissions and revocation without storing the raw token', () => {
|
||||
const created = createTechnicalAccess(actor, 'Leser', 'read');
|
||||
expect(authenticateTechnical(request(created.token))).toMatchObject({
|
||||
name: 'Leser',
|
||||
permissions: 'read'
|
||||
});
|
||||
expect(() => authenticateTechnical(request(created.token), true)).toThrow('API_FORBIDDEN');
|
||||
expect(
|
||||
getDatabase()
|
||||
.prepare('SELECT token_hash FROM technical_accesses WHERE id = ?')
|
||||
.get(created.id)
|
||||
).not.toMatchObject({ token_hash: created.token });
|
||||
revokeTechnicalAccess(actor, created.id);
|
||||
expect(() => authenticateTechnical(request(created.token))).toThrow('API_UNAUTHORIZED');
|
||||
});
|
||||
|
||||
it('replays an identical idempotent write and rejects a changed payload', () => {
|
||||
const created = createTechnicalAccess(actor, 'Schreiber', 'read_write');
|
||||
const technical = authenticateTechnical(request(created.token), true);
|
||||
let calls = 0;
|
||||
const first = idempotentResponse(
|
||||
technical,
|
||||
request(created.token, 'command-1'),
|
||||
{ title: 'A' },
|
||||
() => ({ status: 201, body: { calls: ++calls } })
|
||||
);
|
||||
const replay = idempotentResponse(
|
||||
technical,
|
||||
request(created.token, 'command-1'),
|
||||
{ title: 'A' },
|
||||
() => ({ status: 201, body: { calls: ++calls } })
|
||||
);
|
||||
expect(first.duplicate).toBe(false);
|
||||
expect(replay).toMatchObject({ duplicate: true, body: { calls: 1 } });
|
||||
expect(calls).toBe(1);
|
||||
expect(() =>
|
||||
idempotentResponse(technical, request(created.token, 'command-1'), { title: 'B' }, () => ({
|
||||
status: 201,
|
||||
body: {}
|
||||
}))
|
||||
).toThrow('API_IDEMPOTENCY_CONFLICT');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,151 @@
|
||||
import { createHash, randomBytes, randomUUID } from 'node:crypto';
|
||||
import type { AuthUser } from './auth.js';
|
||||
import { getDatabase } from './database.js';
|
||||
|
||||
export type TechnicalPermission = 'read' | 'read_write';
|
||||
export type TechnicalActor = { id: string; name: string; permissions: TechnicalPermission };
|
||||
|
||||
function tokenHash(token: string) {
|
||||
return createHash('sha256').update(token).digest('hex');
|
||||
}
|
||||
|
||||
export function listTechnicalAccesses() {
|
||||
return getDatabase()
|
||||
.prepare(
|
||||
`SELECT id, name, permissions, created_at_utc AS createdAtUtc,
|
||||
last_used_at_utc AS lastUsedAtUtc, revoked_at_utc AS revokedAtUtc FROM technical_accesses ORDER BY name`
|
||||
)
|
||||
.all() as {
|
||||
id: string;
|
||||
name: string;
|
||||
permissions: TechnicalPermission;
|
||||
createdAtUtc: string;
|
||||
lastUsedAtUtc: string | null;
|
||||
revokedAtUtc: string | null;
|
||||
}[];
|
||||
}
|
||||
|
||||
export function createTechnicalAccess(actor: AuthUser, nameValue: string, permissions: string) {
|
||||
const name = nameValue.trim();
|
||||
if (!name || name.length > 120 || !['read', 'read_write'].includes(permissions))
|
||||
throw new Error('INVALID_ACCESS');
|
||||
const token = randomBytes(32).toString('base64url');
|
||||
const id = randomUUID();
|
||||
const now = new Date().toISOString();
|
||||
getDatabase()
|
||||
.prepare(
|
||||
`INSERT INTO technical_accesses
|
||||
(id, name, token_hash, permissions, created_by_user_id, created_at_utc) VALUES (?, ?, ?, ?, ?, ?)`
|
||||
)
|
||||
.run(id, name, tokenHash(token), permissions, actor.id, now);
|
||||
auditTechnical(
|
||||
{ id: actor.id, name: actor.displayName, permissions: 'read_write' },
|
||||
'technical_access.create',
|
||||
'technical_access',
|
||||
id,
|
||||
'user'
|
||||
);
|
||||
return { id, token };
|
||||
}
|
||||
|
||||
export function revokeTechnicalAccess(actor: AuthUser, id: string) {
|
||||
const result = getDatabase()
|
||||
.prepare(
|
||||
'UPDATE technical_accesses SET revoked_at_utc = ? WHERE id = ? AND revoked_at_utc IS NULL'
|
||||
)
|
||||
.run(new Date().toISOString(), id);
|
||||
if (!result.changes) throw new Error('NOT_FOUND');
|
||||
auditTechnical(
|
||||
{ id: actor.id, name: actor.displayName, permissions: 'read_write' },
|
||||
'technical_access.revoke',
|
||||
'technical_access',
|
||||
id,
|
||||
'user'
|
||||
);
|
||||
}
|
||||
|
||||
export function authenticateTechnical(request: Request, write = false): TechnicalActor {
|
||||
const authorization = request.headers.get('authorization') ?? '';
|
||||
const token = authorization.startsWith('Bearer ') ? authorization.slice(7).trim() : '';
|
||||
if (!token) throw new Error('API_UNAUTHORIZED');
|
||||
const row = getDatabase()
|
||||
.prepare(
|
||||
`SELECT id, name, permissions FROM technical_accesses
|
||||
WHERE token_hash = ? AND revoked_at_utc IS NULL`
|
||||
)
|
||||
.get(tokenHash(token)) as TechnicalActor | undefined;
|
||||
if (!row) throw new Error('API_UNAUTHORIZED');
|
||||
if (write && row.permissions !== 'read_write') throw new Error('API_FORBIDDEN');
|
||||
getDatabase()
|
||||
.prepare('UPDATE technical_accesses SET last_used_at_utc = ? WHERE id = ?')
|
||||
.run(new Date().toISOString(), row.id);
|
||||
return row;
|
||||
}
|
||||
|
||||
export function auditTechnical(
|
||||
actor: TechnicalActor,
|
||||
action: string,
|
||||
targetType: string,
|
||||
targetId: string,
|
||||
actorType: 'technical' | 'user' = 'technical'
|
||||
) {
|
||||
getDatabase()
|
||||
.prepare(
|
||||
`INSERT INTO audit_events (id, actor_type, actor_id, action, target_type, target_id, created_at_utc)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)`
|
||||
)
|
||||
.run(
|
||||
randomUUID(),
|
||||
actorType,
|
||||
actor.id,
|
||||
action.slice(0, 120),
|
||||
targetType.slice(0, 80),
|
||||
targetId.slice(0, 200),
|
||||
new Date().toISOString()
|
||||
);
|
||||
}
|
||||
|
||||
export function idempotentResponse(
|
||||
actor: TechnicalActor,
|
||||
request: Request,
|
||||
body: unknown,
|
||||
operation: () => { status: number; body: unknown }
|
||||
) {
|
||||
const key = request.headers.get('idempotency-key')?.trim() ?? '';
|
||||
if (!key || key.length > 200) throw new Error('API_IDEMPOTENCY_REQUIRED');
|
||||
const hash = createHash('sha256')
|
||||
.update(`${request.method} ${new URL(request.url).pathname}\n${JSON.stringify(body)}`)
|
||||
.digest('hex');
|
||||
const database = getDatabase();
|
||||
return database.transaction(() => {
|
||||
const existing = database
|
||||
.prepare(
|
||||
'SELECT request_hash, status_code, response_json FROM api_idempotency WHERE access_id = ? AND request_key = ?'
|
||||
)
|
||||
.get(actor.id, key) as
|
||||
{ request_hash: string; status_code: number; response_json: string } | undefined;
|
||||
if (existing) {
|
||||
if (existing.request_hash !== hash) throw new Error('API_IDEMPOTENCY_CONFLICT');
|
||||
return {
|
||||
status: existing.status_code,
|
||||
body: JSON.parse(existing.response_json),
|
||||
duplicate: true
|
||||
};
|
||||
}
|
||||
const result = operation();
|
||||
database
|
||||
.prepare(
|
||||
`INSERT INTO api_idempotency
|
||||
(access_id, request_key, request_hash, status_code, response_json, created_at_utc) VALUES (?, ?, ?, ?, ?, ?)`
|
||||
)
|
||||
.run(
|
||||
actor.id,
|
||||
key,
|
||||
hash,
|
||||
result.status,
|
||||
JSON.stringify(result.body),
|
||||
new Date().toISOString()
|
||||
);
|
||||
return { ...result, duplicate: false };
|
||||
})();
|
||||
}
|
||||
Reference in new issue
Block a user