devsyncwrld/backspace
1
0
Fork 0
Code Issues Pull Requests Actions 26 Packages Projects Releases Wiki Activity

refactor: extract metadata fetcher into reusable utility

Browse Source
This commit is contained in:
Jannis Braun
2026-03-20 23:30:01 +01:00
parent ebc1868dd1
commit de1c0b4ed8
2 changed files with 116 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/server/src/routes/utils.ts
+3 -79
View File
@@ -1,22 +1,6 @@
import type { FastifyInstance } from 'fastify';
import dns from 'dns';
import { authenticate } from '../utils/auth.js';
import * as cheerio from 'cheerio';
function isPrivateIp(ip: string): boolean {
// IPv4
if (ip.startsWith('127.') || ip.startsWith('0.') || ip === '0.0.0.0') return true;
if (ip.startsWith('10.')) return true;
if (ip.startsWith('192.168.')) return true;
if (ip.startsWith('169.254.')) return true;
if (ip.startsWith('172.')) {
const second = parseInt(ip.split('.')[1] ?? '', 10);
if (second >= 16 && second <= 31) return true;
}
// IPv6
if (ip === '::1' || ip.startsWith('fc') || ip.startsWith('fd') || ip.startsWith('fe80')) return true;
return false;
}
import { fetchUrlMetadata } from '../utils/metadataFetcher.js';
export async function utilRoutes(app: FastifyInstance): Promise<void> {
app.get<{ Querystring: { url: string } }>('/api/utils/metadata', {
@@ -28,71 +12,11 @@ export async function utilRoutes(app: FastifyInstance): Promise<void> {
return reply.code(400).send({ error: 'URL is required' });
}
// Validate URL scheme
let parsed: URL;
try {
parsed = new URL(url);
} catch {
return reply.code(400).send({ error: 'Invalid URL' });
}
if (!['http:', 'https:'].includes(parsed.protocol)) {
return reply.code(400).send({ error: 'Only HTTP(S) URLs are supported' });
}
// Resolve hostname and block private/internal IPs
let address: string;
try {
const result = await dns.promises.lookup(parsed.hostname);
address = result.address;
} catch {
const metadata = await fetchUrlMetadata(url);
if (!metadata) {
return reply.code(200).send({});
}
if (isPrivateIp(address)) {
return reply.code(400).send({ error: 'URLs pointing to private/internal addresses are not allowed' });
}
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 5000);
try {
const response = await fetch(url, {
headers: {
'User-Agent': 'BackspaceBot/1.0',
},
signal: controller.signal,
redirect: 'follow',
});
clearTimeout(timeout);
if (!response.ok) {
return reply.code(200).send({});
}
// Reject oversized responses
const contentLength = parseInt(response.headers.get('content-length') ?? '0', 10);
if (contentLength > 512_000) {
return reply.code(200).send({});
}
const html = await response.text();
const safeHtml = html.length > 512_000 ? html.slice(0, 512_000) : html;
const $ = cheerio.load(safeHtml);
const metadata = {
title: $('meta[property="og:title"]').attr('content') || $('title').text(),
description: $('meta[property="og:description"]').attr('content') || $('meta[name="description"]').attr('content'),
image: $('meta[property="og:image"]').attr('content'),
siteName: $('meta[property="og:site_name"]').attr('content'),
url: url,
};
return reply.code(200).send(metadata);
} catch (err) {
return reply.code(200).send({}); // Fail silently with empty object
} finally {
clearTimeout(timeout);
}
});
}
packages/server/src/utils/metadataFetcher.ts
+112
View File
@@ -0,0 +1,112 @@
import dns from 'dns';
import * as cheerio from 'cheerio';
export function isPrivateIp(ip: string): boolean {
// IPv4
if (ip.startsWith('127.') || ip.startsWith('0.') || ip === '0.0.0.0') return true;
if (ip.startsWith('10.')) return true;
if (ip.startsWith('192.168.')) return true;
if (ip.startsWith('169.254.')) return true;
if (ip.startsWith('172.')) {
const second = parseInt(ip.split('.')[1] ?? '', 10);
if (second >= 16 && second <= 31) return true;
}
// IPv6
if (ip === '::1' || ip.startsWith('fc') || ip.startsWith('fd') || ip.startsWith('fe80')) return true;
return false;
}
export interface UrlMetadata {
title: string | null;
description: string | null;
image: string | null;
siteName: string | null;
url: string;
}
export async function fetchUrlMetadata(url: string): Promise<UrlMetadata | null> {
// Validate URL scheme
let parsed: URL;
try {
parsed = new URL(url);
} catch {
return null;
}
if (!['http:', 'https:'].includes(parsed.protocol)) {
return null;
}
// Resolve hostname and block private/internal IPs
let address: string;
try {
const result = await dns.promises.lookup(parsed.hostname);
address = result.address;
} catch {
return null;
}
if (isPrivateIp(address)) {
return null;
}
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 5000);
try {
const response = await fetch(url, {
headers: {
'User-Agent': 'BackspaceBot/1.0',
},
signal: controller.signal,
redirect: 'follow',
});
clearTimeout(timeout);
if (!response.ok) {
return null;
}
// Early exit if Content-Length > 512KB
const contentLength = parseInt(response.headers.get('content-length') ?? '0', 10);
if (contentLength > 512_000) {
return null;
}
// Stream-read the body with a hard 512KB byte limit to prevent OOM from chunked responses
if (!response.body) return null;
const reader = response.body.getReader();
const decoder = new TextDecoder();
let html = '';
let bytesRead = 0;
while (true) {
const { done, value } = await reader.read();
if (done) break;
bytesRead += value.length;
if (bytesRead > 512_000) {
reader.cancel();
break;
}
html += decoder.decode(value, { stream: true });
}
html += decoder.decode(); // flush remaining
const $ = cheerio.load(html);
const metadata: UrlMetadata = {
title: $('meta[property="og:title"]').attr('content') ?? $('title').text() ?? null,
description:
$('meta[property="og:description"]').attr('content') ??
$('meta[name="description"]').attr('content') ??
null,
image: $('meta[property="og:image"]').attr('content') ?? null,
siteName: $('meta[property="og:site_name"]').attr('content') ?? null,
url,
};
return metadata;
} catch {
return null;
} finally {
clearTimeout(timeout);
}
}