bitsforfree/src/moderation.js

const DEFAULT_MODEL = process.env.OPENAI_MODERATION_MODEL || 'gpt-4o-mini';
const DEFAULT_AUTO_APPROVE_MIN_SCORE = 80;

export async function moderateImage({ buffer, mime }) {
  if (!process.env.OPENAI_API_KEY) {
    return {
      status: 'pending',
      score: 50,
      reason: 'AI moderation is not configured; queued for review.'
    };
  }

  try {
    const response = await fetch('https://api.openai.com/v1/responses', {
      method: 'POST',
      headers: {
        Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,
        'Content-Type': 'application/json'
      },
      body: JSON.stringify({
        model: DEFAULT_MODEL,
        temperature: 0,
        max_output_tokens: 220,
        input: [
          {
            role: 'user',
            content: [
              {
                type: 'input_text',
                text: moderationPrompt()
              },
              {
                type: 'input_image',
                image_url: `data:${mime};base64,${buffer.toString('base64')}`
              }
            ]
          }
        ]
      })
    });

    if (!response.ok) {
      return {
        status: 'pending',
        score: 50,
        reason: `AI moderation unavailable (${response.status}); queued for review.`
      };
    }

    const payload = await response.json();
    return normalizeDecision(parseJsonOutput(payload.output_text || extractOutputText(payload)));
  } catch {
    return {
      status: 'pending',
      score: 50,
      reason: 'AI moderation failed; queued for review.'
    };
  }
}

function moderationPrompt() {
  const autoApproveMinScore = autoApproveMinScoreValue();
  return [
    'You are moderating image uploads for THE_MEME_PROTOCOL, a meme gallery.',
    'This is a meme site. Images may be sarcastic, roasting, edgy, political, profane, absurd, or not PG-13.',
    'Do not reject merely because a meme is rude, critical, weird, darkly humorous, or adult in tone.',
    'Reject only if the image appears illegal or likely illegal to host or distribute, including child sexual content, sexual content involving minors, explicit non-consensual sexual content, bestiality, credible illegal activity instructions, terrorist or extremist recruitment, doxxing/private identity documents, or explicit threats that appear actionable.',
    'If legality or age is ambiguous, choose pending.',
    'Quality matters. Do not auto-approve low-quality, generic, or AI-slop memes; send them to pending for admin review.',
    'Low-quality memes usually have generic stock meme templates with large caption text, captions that could fit dozens of unrelated images, jokes that rely almost entirely on text, images that add little or nothing, literal descriptions instead of punchlines, excessive text that obscures the image, no clear comedic payoff, or a humorless generated feel.',
    'High-quality memes usually have strong visual composition, a joke contained in or strengthened by the image, visual storytelling, an unexpected punchline, original artwork or a meaningful remix, and a caption/image pairing where the joke would lose meaning if the image changed.',
    'Before deciding, ask: is there an actual punchline; could the exact same caption work on 50 unrelated images; does the image meaningfully contribute; is the text overwhelming the artwork; does it feel like a specific joke someone wanted to make; would users share it because it is clever rather than merely because it exists?',
    'Assign MEME_CONSENSUS_SCORE from 0-100: higher means it is legal, clearly a meme/reaction/roast/remix, visually meaningful, has a real punchline, and is suitable for this site; lower means random photo, spam, ad, QR scam, screenshot dump, unclear, generic template caption, text-only joke, weak payoff, or AI-slop.',
    'Return only compact JSON with keys: decision, score, reason.',
    'decision must be one of: approved, pending, rejected.',
    `Use approved only when it appears legal and score is at least ${autoApproveMinScore}.`,
    'Use pending for ambiguity, uncertainty, low meme relevance, weak quality, generic/slop memes, or anything that needs human review.',
    'Use rejected only for likely illegal content.'
  ].join('\n');
}

function parseJsonOutput(text) {
  const trimmed = String(text || '').trim();
  const match = trimmed.match(/\{[\s\S]*\}/);
  if (!match) return null;
  try {
    return JSON.parse(match[0]);
  } catch {
    return null;
  }
}

function normalizeDecision(raw) {
  if (!raw || typeof raw !== 'object') {
    return { status: 'pending', score: 50, reason: 'AI response was ambiguous; queued for review.' };
  }

  const decision = ['approved', 'pending', 'rejected'].includes(raw.decision) ? raw.decision : 'pending';
  const score = Math.max(0, Math.min(100, Number.parseInt(raw.score, 10) || 50));
  const reason = typeof raw.reason === 'string' && raw.reason.trim()
    ? raw.reason.trim().slice(0, 300)
    : 'No moderation reason supplied.';

  const autoApproveMinScore = autoApproveMinScoreValue();
  if (decision === 'approved' && score < autoApproveMinScore) {
    return { status: 'pending', score, reason: `${reason} Low MEME_CONSENSUS_SCORE queued for review.` };
  }
  return { status: decision, score, reason };
}

function autoApproveMinScoreValue() {
  const value = Number.parseInt(process.env.AUTO_APPROVE_MIN_SCORE || '', 10);
  if (!Number.isFinite(value)) return DEFAULT_AUTO_APPROVE_MIN_SCORE;
  return Math.max(0, Math.min(100, value));
}

function extractOutputText(payload) {
  const parts = [];
  for (const item of payload.output || []) {
    for (const content of item.content || []) {
      if (content.type === 'output_text' && content.text) parts.push(content.text);
    }
  }
  return parts.join('\n');
}