diff --git a/README.md b/README.md index 770e335..946f829 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ data/ The app serves crawler and answer-engine metadata without adding visible page copy: - `/robots.txt` -- `/sitemap.xml` with approved meme image entries +- `/sitemap.xml` with the home page and approved meme URLs - `/feed.json` - `/llms.txt` - `/site.webmanifest` diff --git a/server.js b/server.js index fae0492..ca6eebf 100644 --- a/server.js +++ b/server.js @@ -19,6 +19,13 @@ const PAGE_SIZE_MAX = 48; const UPLOAD_MAX_BYTES = 5 * 1024 * 1024; const REQUEST_MAX_BYTES = 6 * 1024 * 1024; const events = new Set(); +const DISCOVERY_ROUTES = new Set([ + '/robots.txt', + '/llms.txt', + '/site.webmanifest', + '/feed.json', + '/sitemap.xml' +]); const ADMIN_TOKEN = process.env.ADMIN_TOKEN || crypto.randomBytes(24).toString('hex'); const indexTemplate = await fs.readFile('./public/index.html', 'utf8'); @@ -32,10 +39,9 @@ if (!process.env.ADMIN_TOKEN) { } const server = http.createServer(async (req, res) => { - withSecurityHeaders(res); - try { const url = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`); + withSecurityHeaders(res, { contentSecurityPolicy: !isDiscoveryRoute(req, url) }); const baseUrl = publicBaseUrl(req); if (req.method === 'GET' && url.pathname === '/') { @@ -253,6 +259,11 @@ function positiveInt(value, fallback) { return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; } +function isDiscoveryRoute(req, url) { + if (req.method !== 'GET') return false; + return DISCOVERY_ROUTES.has(url.pathname); +} + function downloadName(meme) { return `meme-protocol-${meme.id.slice(0, 12)}.${meme.ext}`; } diff --git a/src/http.js b/src/http.js index 3a929f5..dc3cb0f 100644 --- a/src/http.js +++ b/src/http.js @@ -19,6 +19,8 @@ export function withSecurityHeaders(res, options = {}) { res.setHeader('X-Content-Type-Options', 'nosniff'); res.setHeader('X-Frame-Options', 'DENY'); res.setHeader('Permissions-Policy', 'camera=(), microphone=(), geolocation=()'); + if (options.contentSecurityPolicy === false) return; + const scriptSrc = options.scriptNonce ? `script-src 'self' 'nonce-${options.scriptNonce}'; ` : ''; res.setHeader( 'Content-Security-Policy', diff --git a/src/seo.js b/src/seo.js index e6f183b..ff51d07 100644 --- a/src/seo.js +++ b/src/seo.js @@ -87,7 +87,7 @@ export function llmsTxt(baseUrl) { 'Crawler guidance:', '- Public approved meme images are available under /media/.', '- Admin, review, upload, and API mutation routes are not public knowledge sources.', - '- The site is intentionally visual and sparse; use metadata, sitemap image entries, and JSON feed for machine summaries.', + '- The site is intentionally visual and sparse; use metadata, sitemap URLs, and JSON feed for machine summaries.', '' ].join('\n'); } @@ -132,22 +132,26 @@ export function feedJson(baseUrl, memes) { } export function sitemapXml(baseUrl, memes) { - const images = memes.slice(0, 1000).map((meme) => [ - ' ', - ` ${xmlEscape(`${baseUrl}/media/${meme.id}`)}`, - ` ${xmlEscape(`Meme ${shortId(meme.id)} with MEME_CONSENSUS_SCORE ${meme.moderationScore}/100`)}`, - ' ' - ].join('\n')).join('\n'); + const latest = memes[0]?.createdAt || new Date().toISOString(); + const urls = [ + [ + ' ', + ` ${xmlEscape(`${baseUrl}/`)}`, + ` ${xmlEscape(latest)}`, + ' ' + ].join('\n'), + ...memes.slice(0, 1000).map((meme) => [ + ' ', + ` ${xmlEscape(`${baseUrl}/media/${meme.id}`)}`, + ` ${xmlEscape(meme.createdAt)}`, + ' ' + ].join('\n')) + ].join('\n'); return [ '', - '', - ' ', - ` ${xmlEscape(`${baseUrl}/`)}`, - ' hourly', - ' 1.0', - images, - ' ', + '', + urls, '', '' ].join('\n');