init

src/moderation.js

@@ -0,0 +1,112 @@
+const DEFAULT_MODEL = process.env.OPENAI_MODERATION_MODEL || 'gpt-4o-mini';
+
+export async function moderateImage({ buffer, mime }) {
+  if (!process.env.OPENAI_API_KEY) {
+    return {
+      status: 'pending',
+      score: 50,
+      reason: 'AI moderation is not configured; queued for review.'
+    };
+  }
+
+  try {
+    const response = await fetch('https://api.openai.com/v1/responses', {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,
+        'Content-Type': 'application/json'
+      },
+      body: JSON.stringify({
+        model: DEFAULT_MODEL,
+        temperature: 0,
+        max_output_tokens: 220,
+        input: [
+          {
+            role: 'user',
+            content: [
+              {
+                type: 'input_text',
+                text: moderationPrompt()
+              },
+              {
+                type: 'input_image',
+                image_url: `data:${mime};base64,${buffer.toString('base64')}`
+              }
+            ]
+          }
+        ]
+      })
+    });
+
+    if (!response.ok) {
+      return {
+        status: 'pending',
+        score: 50,
+        reason: `AI moderation unavailable (${response.status}); queued for review.`
+      };
+    }
+
+    const payload = await response.json();
+    return normalizeDecision(parseJsonOutput(payload.output_text || extractOutputText(payload)));
+  } catch {
+    return {
+      status: 'pending',
+      score: 50,
+      reason: 'AI moderation failed; queued for review.'
+    };
+  }
+}
+
+function moderationPrompt() {
+  return [
+    'You are moderating image uploads for THE_MEME_PROTOCOL, a meme gallery.',
+    'This is a meme site. Images may be sarcastic, roasting, edgy, political, profane, absurd, or not PG-13.',
+    'Do not reject merely because a meme is rude, critical, weird, darkly humorous, or adult in tone.',
+    'Reject only if the image appears illegal or likely illegal to host or distribute, including child sexual content, sexual content involving minors, explicit non-consensual sexual content, bestiality, credible illegal activity instructions, terrorist or extremist recruitment, doxxing/private identity documents, or explicit threats that appear actionable.',
+    'If legality or age is ambiguous, choose pending.',
+    'Assign MEME_CONSENSUS_SCORE from 0-100: higher means it is clearly a meme/reaction/roast/remix and suitable for this site; lower means random photo, spam, ad, QR scam, screenshot dump, or unclear.',
+    'Return only compact JSON with keys: decision, score, reason.',
+    'decision must be one of: approved, pending, rejected.',
+    'Use approved only when it appears legal and score is at least 65.',
+    'Use pending for ambiguity, uncertainty, low meme relevance, or anything that needs human review.',
+    'Use rejected only for likely illegal content.'
+  ].join('\n');
+}
+
+function parseJsonOutput(text) {
+  const trimmed = String(text || '').trim();
+  const match = trimmed.match(/\{[\s\S]*\}/);
+  if (!match) return null;
+  try {
+    return JSON.parse(match[0]);
+  } catch {
+    return null;
+  }
+}
+
+function normalizeDecision(raw) {
+  if (!raw || typeof raw !== 'object') {
+    return { status: 'pending', score: 50, reason: 'AI response was ambiguous; queued for review.' };
+  }
+
+  const decision = ['approved', 'pending', 'rejected'].includes(raw.decision) ? raw.decision : 'pending';
+  const score = Math.max(0, Math.min(100, Number.parseInt(raw.score, 10) || 50));
+  const reason = typeof raw.reason === 'string' && raw.reason.trim()
+    ? raw.reason.trim().slice(0, 300)
+    : 'No moderation reason supplied.';
+
+  if (decision === 'approved' && score < 65) {
+    return { status: 'pending', score, reason: `${reason} Low MEME_CONSENSUS_SCORE queued for review.` };
+  }
+  return { status: decision, score, reason };
+}
+
+function extractOutputText(payload) {
+  const parts = [];
+  for (const item of payload.output || []) {
+    for (const content of item.content || []) {
+      if (content.type === 'output_text' && content.text) parts.push(content.text);
+    }
+  }
+  return parts.join('\n');
+}