From 082f715c392d092f268fa309355f73b95427e909 Mon Sep 17 00:00:00 2001 From: Beda Schmid Date: Wed, 30 Sep 2026 16:24:21 -0300 Subject: [PATCH] feat: persist image captions and managed tags with fuzzy search --- src/metadata.js | 39 +++++++ src/search.js | 61 ++++++++++ src/store.js | 279 ++++++++++++++++++++++++-------------------- tests/store.test.js | 94 +++++++++++++++ 4 files changed, 348 insertions(+), 125 deletions(-) create mode 100644 src/metadata.js create mode 100644 src/search.js diff --git a/src/metadata.js b/src/metadata.js new file mode 100644 index 0000000..b6af503 --- /dev/null +++ b/src/metadata.js @@ -0,0 +1,39 @@ +import { HttpError } from './errors.js'; + +export const CAPTION_MAX_LENGTH = 2000; +export const TAG_MAX_LENGTH = 60; +export const TAG_LIMIT = 200; + +export function normalizeTag(value) { + if (typeof value !== 'string') throw new HttpError(400, 'Tag must be a string.'); + const name = value.normalize('NFC').trim().replace(/\s+/gu, ' ').toLowerCase(); + if (!name || name.length > TAG_MAX_LENGTH || !/^[\p{L}\p{N}][\p{L}\p{N} _-]*$/u.test(name)) { + throw new HttpError(400, 'Tags must be 1–60 characters: letters, numbers, spaces, underscores, or hyphens.'); + } + return name; +} + +export function validateMetadata(input, availableTags) { + if (!input || typeof input !== 'object' || Array.isArray(input)) throw new HttpError(400, 'Expected a metadata object.'); + const result = {}; + if ('caption' in input) { + if (typeof input.caption !== 'string' || input.caption.length > CAPTION_MAX_LENGTH) { + throw new HttpError(400, `Caption must be a string of at most ${CAPTION_MAX_LENGTH} characters.`); + } + result.caption = input.caption.trim(); + } + if ('tags' in input) { + if (!Array.isArray(input.tags) || input.tags.length > TAG_LIMIT) throw new HttpError(400, 'Tags must be an array of at most 200 managed tag names.'); + result.tags = [...new Set(input.tags.map(normalizeTag))]; + if (result.tags.some((tag) => !availableTags.includes(tag))) throw new HttpError(400, 'Choose tags from the managed tag catalog.'); + } + return result; +} + +export function normalizeAiMetadata(raw, availableTags) { + return { + caption: typeof raw?.caption === 'string' ? raw.caption.trim().slice(0, CAPTION_MAX_LENGTH) : '', + tags: Array.isArray(raw?.tags) ? [...new Set(raw.tags.filter((tag) => typeof tag === 'string') + .map((tag) => tag.trim().toLowerCase()).filter((tag) => availableTags.includes(tag)))] : [] + }; +} diff --git a/src/search.js b/src/search.js new file mode 100644 index 0000000..152c659 --- /dev/null +++ b/src/search.js @@ -0,0 +1,61 @@ +import { HttpError } from './errors.js'; +import { normalizeTag } from './metadata.js'; + +export function searchOptions(params) { + const q = (params.get('q') || '').trim(); + if (q.length > 200) throw new HttpError(400, 'Search query must be at most 200 characters.'); + const tags = params.getAll('tag'); + if (tags.length > 20) throw new HttpError(400, 'Use at most 20 tag filters.'); + return { q, tags: [...new Set(tags.map(normalizeTag))] }; +} + +export function rankMemes(entries, { q = '', tags = [], missingCaption = false } = {}) { + const query = fold(q); + const terms = [...new Set(words(query))]; + return entries.filter((entry) => tags.every((tag) => entry.tags.includes(tag)) && (!missingCaption || !entry.caption)) + .map((entry) => ({ entry, score: relevance(entry, query, terms) })) + .filter(({ score }) => score > 0) + .sort((a, b) => b.score - a.score || b.entry.createdAt.localeCompare(a.entry.createdAt) || a.entry.id.localeCompare(b.entry.id)) + .map(({ entry }) => entry); +} + +function relevance(entry, query, terms) { + if (!query) return 1; + if (!terms.length) return 0; + const text = fold([entry.caption, ...entry.tags].join(' ')); + const candidates = [...new Set(words(text))]; + let score = text.includes(query) ? 20 : 0; + for (const term of terms) { + let best = 0; + for (const candidate of candidates) { + if (candidate === term) best = Math.max(best, 10); + else if (candidate.startsWith(term)) best = Math.max(best, 6); + else if (term.length >= 4 && near(term, candidate, term.length >= 7 ? 2 : 1)) best = Math.max(best, 3); + } + if (!best) return 0; // Every query term must match; results remain useful for multiword searches. + score += best; + } + return score; +} + +function fold(value) { + return value.normalize('NFKD').replace(/\p{M}/gu, '').toLowerCase(); +} + +function words(value) { + return value.match(/[\p{L}\p{N}]+/gu) || []; +} + +function near(a, b, limit) { + if (Math.abs(a.length - b.length) > limit) return false; + let previous = Array.from({ length: b.length + 1 }, (_, i) => i); + for (let i = 1; i <= a.length; i += 1) { + const row = [i]; + for (let j = 1; j <= b.length; j += 1) { + row[j] = Math.min(row[j - 1] + 1, previous[j] + 1, previous[j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1)); + } + if (Math.min(...row) > limit) return false; + previous = row; + } + return previous[b.length] <= limit; +} diff --git a/src/store.js b/src/store.js index a3e0b19..59f3ce4 100644 --- a/src/store.js +++ b/src/store.js @@ -1,15 +1,32 @@ import crypto from 'node:crypto'; import fs from 'node:fs/promises'; import path from 'node:path'; +import { HttpError } from './errors.js'; +import { normalizeTag, validateMetadata, TAG_LIMIT } from './metadata.js'; +import { rankMemes } from './search.js'; export async function createStore({ dataDir }) { const root = path.resolve(dataDir); const indexDir = path.join(root, 'index'); const indexFile = path.join(indexDir, 'memes.jsonl'); + const tagsFile = path.join(indexDir, 'tags.json'); await fs.mkdir(indexDir, { recursive: true }); const entries = await loadIndex(indexFile, root); + let tags = []; + try { + tags = [...new Set(JSON.parse(await fs.readFile(tagsFile, 'utf8')).map(normalizeTag))].sort(); + } catch (error) { + if (error.code !== 'ENOENT') throw error; + } + for (const entry of entries) entry.tags = entry.tags.filter((tag) => tags.includes(tag)); const byId = new Map(entries.map((entry) => [entry.id, entry])); let writeChain = Promise.resolve(); + function mutate(operation) { + const result = writeChain.then(operation); + writeChain = result.catch(() => {}); + return result; + } + const persist = () => persistAll(root, indexFile, entries); return { count(status) { @@ -21,123 +38,142 @@ export async function createStore({ dataDir }) { absolutePath(storageKey) { return path.join(root, storageKey); }, - list({ page, pageSize }) { - const visible = entries.filter((entry) => entry.status === 'approved'); - const total = visible.length; - const totalPages = Math.max(1, Math.ceil(total / pageSize)); - const safePage = Math.min(page, totalPages); - const start = (safePage - 1) * pageSize; - return { - page: safePage, - pageSize, - total, - totalPages, - memes: visible.slice(start, start + pageSize).map(toPublicMeme) - }; + listTags() { + return [...tags]; + }, + addTag(name) { + return mutate(async () => { + const tag = normalizeTag(name); + if (tags.includes(tag)) throw new HttpError(409, 'Tag already exists.'); + if (tags.length >= TAG_LIMIT) throw new HttpError(400, 'The tag catalog is limited to 200 tags.'); + tags = [...tags, tag].sort(); + await atomicWrite(tagsFile, `${JSON.stringify(tags, null, 2)}\n`); + return tag; + }); + }, + deleteTag(name) { + return mutate(async () => { + const tag = normalizeTag(name); + if (!tags.includes(tag)) throw new HttpError(404, 'Tag not found.'); + tags = tags.filter((value) => value !== tag); + for (const entry of entries) { + if (entry.tags.includes(tag)) { + entry.tags = entry.tags.filter((value) => value !== tag); + entry.metadataVersion += 1; + } + } + await persist(); + await atomicWrite(tagsFile, `${JSON.stringify(tags, null, 2)}\n`); + return tag; + }); + }, + list({ page, pageSize, q = '', tags: filters = [] }) { + return paginate(rankMemes(entries.filter((entry) => entry.status === 'approved'), { q, tags: filters }), page, pageSize); + }, + listForAdmin({ page = 1, pageSize = 12, status = 'all', ...search } = {}) { + if (!['all', 'approved', 'pending'].includes(status)) throw new HttpError(400, 'Status must be all, approved, or pending.'); + return paginate(rankMemes(entries.filter((entry) => status === 'all' || entry.status === status), search), page, pageSize); }, listForReview({ status = 'pending' } = {}) { - return { - total: entries.filter((entry) => entry.status === status).length, - memes: entries.filter((entry) => entry.status === status).map(toPublicMeme) - }; + const memes = entries.filter((entry) => entry.status === status); + return { total: memes.length, memes: memes.map(toPublicMeme) }; }, - async approve(ids) { - const approved = []; - for (const id of ids) { + updateMetadata(id, input, { expectedVersion } = {}) { + return mutate(async () => { const record = byId.get(id); - if (!record || record.status !== 'pending') continue; - record.status = 'approved'; - record.reviewedAt = new Date().toISOString(); - record.moderationReason = `${record.moderationReason || 'Queued for review.'} Admin approved.`; - approved.push(toPublicMeme(record)); - } - if (approved.length > 0) { - writeChain = writeChain.then(() => persistAll(root, indexFile, entries)); - await writeChain; - } - return approved; + if (!record) throw new HttpError(404, 'Meme not found.'); + if (expectedVersion !== undefined && record.metadataVersion !== expectedVersion) { + throw new HttpError(409, 'Metadata changed during AI generation. Refresh and try again.'); + } + const metadata = validateMetadata(input, tags); + if (!Object.keys(metadata).length) throw new HttpError(400, 'Supply caption and/or tags.'); + Object.assign(record, metadata, { metadataVersion: record.metadataVersion + 1 }); + await persist(); + return toPublicMeme(record); + }); }, - async delete(ids) { - const deleted = []; - for (const id of ids) { - const index = entries.findIndex((entry) => entry.id === id); - if (index === -1) continue; - const [record] = entries.splice(index, 1); - byId.delete(id); - deleted.push(id); - await fs.rm(path.join(root, record.storageKey), { force: true }); - await fs.rm(path.join(root, record.metaKey), { force: true }); - } - if (deleted.length > 0) { - writeChain = writeChain.then(() => persistAll(root, indexFile, entries)); - await writeChain; - } - return deleted; + approve(ids) { + return mutate(async () => { + const approved = []; + for (const id of ids) { + const record = byId.get(id); + if (!record || record.status !== 'pending') continue; + record.status = 'approved'; + record.reviewedAt = new Date().toISOString(); + record.moderationReason = `${record.moderationReason || 'Queued for review.'} Admin approved.`; + approved.push(toPublicMeme(record)); + } + if (approved.length) await persist(); + return approved; + }); }, - async incrementMetric(id, metric) { - const record = byId.get(id); - if (!record) return null; - if (metric !== 'viewCount' && metric !== 'downloadCount') return toPublicMeme(record); - - record[metric] = Number.isFinite(record[metric]) ? record[metric] + 1 : 1; - writeChain = writeChain.then(() => persistRecord(root, indexFile, entries, record)); - await writeChain; - return toPublicMeme(record); + delete(ids) { + return mutate(async () => { + const deleted = []; + for (const id of ids) { + const index = entries.findIndex((entry) => entry.id === id); + if (index === -1) continue; + const [record] = entries.splice(index, 1); + byId.delete(id); + deleted.push(id); + await fs.rm(path.join(root, record.storageKey), { force: true }); + await fs.rm(path.join(root, record.metaKey), { force: true }); + } + if (deleted.length) await persist(); + return deleted; + }); }, - async save({ - buffer, - image, - originalName, - createdAt = new Date().toISOString(), - demo = false, - status = 'approved', - moderationScore = demo ? 90 : 0, - moderationReason = '', - originalMime = image.mime - }) { - const id = crypto.createHash('sha256').update(buffer).digest('hex'); - const existing = byId.get(id); - if (existing) return toPublicMeme(existing); - - const date = createdAt.slice(0, 10).replaceAll('-', '/'); - const shard = `${id.slice(0, 2)}/${id.slice(2, 4)}`; - const storageKey = `memes/${date}/${shard}/${id}.${image.ext}`; - const metaKey = `meta/${date}/${shard}/${id}.json`; - const record = { - id, - createdAt, - originalName, - byteSize: image.byteSize, - width: image.width, - height: image.height, - mime: image.mime, - originalMime, - ext: image.ext, - status, - moderationScore, - moderationReason, - viewCount: 0, - downloadCount: 0, - storageKey, - metaKey, - demo - }; - - await fs.mkdir(path.dirname(path.join(root, storageKey)), { recursive: true }); - await fs.mkdir(path.dirname(path.join(root, metaKey)), { recursive: true }); - await fs.writeFile(path.join(root, storageKey), buffer, { flag: 'wx' }).catch((error) => { - if (error.code !== 'EEXIST') throw error; + incrementMetric(id, metric) { + return mutate(async () => { + const record = byId.get(id); + if (!record) return null; + if (metric !== 'viewCount' && metric !== 'downloadCount') return toPublicMeme(record); + record[metric] = Number.isFinite(record[metric]) ? record[metric] + 1 : 1; + await persist(); + return toPublicMeme(record); + }); + }, + save({ buffer, image, originalName, createdAt = new Date().toISOString(), demo = false, + status = 'approved', moderationScore = demo ? 90 : 0, moderationReason = '', originalMime = image.mime, + caption = '', tags: assignedTags = [] }) { + return mutate(async () => { + const id = crypto.createHash('sha256').update(buffer).digest('hex'); + const existing = byId.get(id); + if (existing) return toPublicMeme(existing); + const date = createdAt.slice(0, 10).replaceAll('-', '/'); + const shard = `${id.slice(0, 2)}/${id.slice(2, 4)}`; + const storageKey = `memes/${date}/${shard}/${id}.${image.ext}`; + const metaKey = `meta/${date}/${shard}/${id}.json`; + // Tags can be removed while an upload's AI request is running. + const metadata = validateMetadata({ caption, tags: assignedTags.filter((tag) => tags.includes(tag)) }, tags); + const record = { + id, createdAt, originalName, byteSize: image.byteSize, width: image.width, height: image.height, + mime: image.mime, originalMime, ext: image.ext, status, moderationScore, moderationReason, + ...metadata, metadataVersion: 0, viewCount: 0, downloadCount: 0, storageKey, metaKey, demo + }; + await fs.mkdir(path.dirname(path.join(root, storageKey)), { recursive: true }); + await fs.mkdir(path.dirname(path.join(root, metaKey)), { recursive: true }); + await fs.writeFile(path.join(root, storageKey), buffer, { flag: 'wx' }).catch((error) => { + if (error.code !== 'EEXIST') throw error; + }); + entries.unshift(record); + entries.sort((a, b) => b.createdAt.localeCompare(a.createdAt)); + byId.set(id, record); + await persist(); + return toPublicMeme(record); }); - await persistRecord(root, indexFile, [...entries, record], record, { writeFileFlag: 'wx' }); - - entries.unshift(record); - entries.sort((a, b) => b.createdAt.localeCompare(a.createdAt)); - byId.set(id, record); - return toPublicMeme(record); } }; } +function paginate(entries, page, pageSize) { + const total = entries.length; + const totalPages = Math.max(1, Math.ceil(total / pageSize)); + const safePage = Math.min(page, totalPages); + const start = (safePage - 1) * pageSize; + return { page: safePage, pageSize, total, totalPages, memes: entries.slice(start, start + pageSize).map(toPublicMeme) }; +} + async function loadIndex(indexFile, root) { let content = ''; try { @@ -145,7 +181,6 @@ async function loadIndex(indexFile, root) { } catch (error) { if (error.code !== 'ENOENT') throw error; } - const recordsById = new Map(); for (const line of content.split('\n')) { if (!line.trim()) continue; @@ -157,6 +192,9 @@ async function loadIndex(indexFile, root) { record.originalMime = record.originalMime || record.mime; record.viewCount = Number.isFinite(record.viewCount) ? record.viewCount : 0; record.downloadCount = Number.isFinite(record.downloadCount) ? record.downloadCount : 0; + record.caption = typeof record.caption === 'string' ? record.caption : ''; + record.tags = Array.isArray(record.tags) ? record.tags : []; + record.metadataVersion = Number.isInteger(record.metadataVersion) ? record.metadataVersion : 0; const mediaPath = path.join(root, record.storageKey); const relative = path.relative(root, mediaPath); if (relative.startsWith('..') || path.isAbsolute(relative)) continue; @@ -166,32 +204,21 @@ async function loadIndex(indexFile, root) { // Ignore corrupt index lines; individual metadata files remain on disk for recovery. } } - const records = [...recordsById.values()]; - records.sort((a, b) => b.createdAt.localeCompare(a.createdAt)); - return records; + return [...recordsById.values()].sort((a, b) => b.createdAt.localeCompare(a.createdAt)); } -async function persistRecord(root, indexFile, entries, record, options = {}) { - await fs.writeFile(path.join(root, record.metaKey), `${JSON.stringify(record, null, 2)}\n`, { - flag: options.writeFileFlag || 'w' - }).catch((error) => { - if (options.writeFileFlag === 'wx' && error.code === 'EEXIST') return; - throw error; - }); - await persistAll(root, indexFile, entries); +async function atomicWrite(file, content) { + const temporary = `${file}.tmp`; + await fs.writeFile(temporary, content); + await fs.rename(temporary, file); } async function persistAll(root, indexFile, entries) { - await Promise.all(entries.map((entry) => fs.writeFile( - path.join(root, entry.metaKey), - `${JSON.stringify(entry, null, 2)}\n` - ).catch((error) => { - if (error.code !== 'ENOENT') throw error; - }))); - await fs.writeFile(indexFile, `${entries.map((entry) => JSON.stringify(entry)).join('\n')}\n`); + await Promise.all(entries.map((entry) => atomicWrite(path.join(root, entry.metaKey), `${JSON.stringify(entry, null, 2)}\n`))); + await atomicWrite(indexFile, `${entries.map((entry) => JSON.stringify(entry)).join('\n')}\n`); } -function toPublicMeme(record) { +export function toPublicMeme(record) { return { id: record.id, createdAt: record.createdAt, @@ -203,6 +230,8 @@ function toPublicMeme(record) { status: record.status, moderationScore: record.moderationScore, moderationReason: record.moderationReason, + caption: record.caption || '', + tags: [...(record.tags || [])], viewCount: record.viewCount, downloadCount: record.downloadCount, originalName: record.originalName, diff --git a/tests/store.test.js b/tests/store.test.js index 73aa41e..c49d621 100644 --- a/tests/store.test.js +++ b/tests/store.test.js @@ -55,3 +55,97 @@ test('increments view and download counters', async () => { assert.equal(listed.viewCount, 1); assert.equal(listed.downloadCount, 1); }); + +async function metadataStore(t) { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'meme-metadata-')); + t.after(() => fs.rm(dir, { recursive: true, force: true })); + const store = await createStore({ dataDir: dir }); + let seed = 0; + async function save(options = {}) { + const color = ++seed; + const buffer = encodePng(8, 8, () => [color, 200, 50, 255]); + const image = validateImage(buffer, { maxBytes: 100_000, maxWidth: 6000, maxHeight: 6000, maxPixels: 20_000_000 }); + return store.save({ buffer, image, originalName: 'test.png', ...options }); + } + return { dir, store, save }; +} + +test('persists editable metadata and managed tags, removing assignments on tag deletion', async (t) => { + const { dir, store, save } = await metadataStore(t); + await store.addTag(' Cats '); + await store.addTag('work'); + await assert.rejects(store.addTag('CATS'), { statusCode: 409 }); + await assert.rejects(store.addTag('