Content Moderation

from openai import OpenAI

client = OpenAI()

def moderate_output(text: str) -> dict:
    """Check LLM output against OpenAI moderation."""
    response = client.moderations.create(input=text)
    result = response.results[0]

    if result.flagged:
        # Identify which categories were flagged
        flagged_categories = [
            cat for cat, flagged
            in result.categories.model_dump().items()
            if flagged
        ]
        return {
            "safe": False,
            "categories": flagged_categories,
            "scores": result.category_scores.model_dump()
        }
    return {"safe": True}

from transformers import pipeline

class CustomModerator:
    def __init__(self):
        self.toxicity = pipeline(
            "text-classification",
            model="unitary/toxic-bert"
        )
        self.threshold = 0.7

    def check(self, text: str) -> dict:
        # Split long text into chunks for classification
        chunks = [text[i:i+512] for i in range(0, len(text), 512)]
        max_score = 0

        for chunk in chunks:
            result = self.toxicity(chunk)[0]
            if result["label"] == "toxic":
                max_score = max(max_score, result["score"])

        return {
            "toxic": max_score > self.threshold,
            "score": max_score
        }