Verdict examples

import os
import requests

def verdicts(state, questions, model="qwen3-8b", images=None):
    body = {"model": model, "state": state, "questions": questions}
    if images:
        body["images"] = images
    r = requests.post(
        "http://localhost:8088/v1/verdicts",
        headers={"Authorization": f"Bearer {os.environ['OBLETH_KEY']}"},
        json=body,
        timeout=120,
    )
    r.raise_for_status()
    return r.json()["verdicts"]

import base64
from pathlib import Path

def data_url(path):
    return "data:image/jpeg;base64," + base64.b64encode(Path(path).read_bytes()).decode()

DAMAGE = {
    "visible_damage": {
        "type": "boolean",
        "instructions": "Is the phone's screen or body visibly damaged, such as cracked glass "
                        "or a dent? Ignore smudges, reflections and the case.",
    },
    "severity": {
        "type": "score",
        "instructions": "How badly is the phone damaged?",
        "criteria": [
            "No visible damage",
            "Scuffs or scratches only",
            "Cracked, but the screen is readable",
            "Shattered or unusable",
        ],
    },
}

v = verdicts("A customer's photo of the phone they are returning.", DAMAGE,
             model="auto", images=[data_url("returned-phone.jpg")])
if v["visible_damage"]["probabilities"]["true"] >= 0.9:
    print("damaged, severity", v["severity"]["value"])

from concurrent.futures import ThreadPoolExecutor

RELEVANCE = {
    "relevance": {
        "type": "score",
        "instructions": "How well does the passage answer the query?",
        "criteria": [
            "Unrelated to the query",
            "On the topic, but does not help answer it",
            "Partly answers the query",
            "Directly answers the query",
        ],
    }
}

def rerank(query, passages):
    def score(passage):
        state = {"query": query, "passage": passage}
        return verdicts(state, RELEVANCE)["relevance"]["expected_value"]

    with ThreadPoolExecutor(max_workers=8) as pool:
        scores = list(pool.map(score, passages))
    return sorted(zip(scores, passages), key=lambda pair: pair[0], reverse=True)

query = "Why does my sourdough starter smell like nail polish remover?"
passages = [
    "Acetone is a common solvent found in nail polish remover and some paint thinners.",
    "Sourdough starters are flour and water colonized by wild yeast and lactic acid bacteria. Bakers have kept them for thousands of years.",
    "A starter that smells of acetone is hungry: the yeast and bacteria have run out of food and start making ethyl acetate. Feed it more often, or with more flour and water at each feeding.",
]
for score, passage in rerank(query, passages):
    print(f"{score:.2f}  {passage[:60]}")

4.00  A starter that smells of acetone is hungry: the yeast and ba
2.00  Acetone is a common solvent found in nail polish remover and
2.00  Sourdough starters are flour and water colonized by wild yea

curl -s http://localhost:8088/v1/verdicts \
  -H "Authorization: Bearer $OBLETH_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "qwen3-8b",
    "state": {
      "question": "When does the library close on Saturdays?",
      "source": "Hours: Monday to Thursday 8 am to 10 pm. Friday 8 am to 6 pm. Saturday 10 am to 6 pm. Sunday noon to 10 pm. Holiday hours are posted at the front desk.",
      "answer": "The library closes at 6 pm on Saturdays. It is closed on public holidays."
    },
    "questions": {
      "supported": {
        "type": "boolean",
        "instructions": "Is every statement in the answer supported by the source?",
        "criteria": {
          "true": "Each statement can be checked against the source",
          "false": "At least one statement is missing from the source or contradicts it"
        }
      },
      "complete": {
        "type": "score",
        "instructions": "How fully does the answer respond to the question?",
        "criteria": ["Does not answer it", "Answers part of it", "Answers all of it"]
      }
    }
  }'

"verdicts": {
  "complete": {"type": "score", "value": 3,
    "probabilities": [0.0, 0.0, 1.0], "confidence": 1.0, "expected_value": 3.0},
  "supported": {"type": "boolean", "value": false,
    "probabilities": {"false": 0.9998, "true": 0.0002}, "confidence": 0.998}
}

SAME_PERSON = {
    "same_person": {
        "type": "boolean",
        "instructions": "Do records a and b describe the same person?",
        "criteria": {
            "true": "Names, affiliation and contact details are consistent with one person",
            "false": "A detail rules out one person, or the records are too sparse to tell",
        },
    }
}

def match(a, b):
    v = verdicts({"a": a, "b": b}, SAME_PERSON)["same_person"]
    p = v["probabilities"]["true"]
    if p >= 0.95 and v["confidence"] >= 0.5:
        return "merge"
    if p >= 0.5:
        return "review"
    return "distinct"

a = {"name": "J. Rivera-Lopez", "department": "Dept. of Chemistry",
     "institution": "Univ. of Example", "email": "jrivera@example.edu"}

match(a, {"name": "Julia Rivera López", "department": "Chemistry",
          "institution": "University of Example", "orcid": "0000-0002-1825-0097"})
# "merge": p(true) = 1.0

match(a, {"name": "Jorge Rivera", "department": "History",
          "institution": "Example State College", "email": "jorge.rivera@esc.example"})
# "distinct": p(true) = 0.0

const QUESTIONS = {
  in_scope: {
    type: "boolean",
    instructions: "Does the tool call do only what the user asked for, and nothing more?",
  },
  outside_project: {
    type: "boolean",
    instructions: "Does the command delete or change anything outside the current project directory?",
  },
};

async function checkToolCall(userRequest, toolCall) {
  const r = await fetch("http://localhost:8088/v1/verdicts", {
    method: "POST",
    headers: {
      Authorization: `Bearer ${process.env.OBLETH_KEY}`,
      "Content-Type": "application/json",
    },
    body: JSON.stringify({
      model: "qwen3-8b",
      state: { user_request: userRequest, tool_call: toolCall },
      questions: QUESTIONS,
    }),
  });
  if (!r.ok) throw new Error(`verdicts ${r.status}: ${await r.text()}`);
  const { verdicts } = await r.json();
  const inScope = verdicts.in_scope.probabilities.true;
  const outside = verdicts.outside_project.probabilities.true;
  return inScope >= 0.9 && outside <= 0.1 ? "run" : "ask_user";
}

await checkToolCall("Clean up the build output so I can do a fresh build.", {
  name: "shell",
  arguments: { command: "rm -rf ./build ./dist ~/.cache" },
});

IS_BUG = {
    "is_bug": {
        "type": "boolean",
        "instructions": "Does this describe something in the product not working as intended?",
    }
}
SEVERITY = {
    "severity": {
        "type": "score",
        "instructions": "How severe is the problem for the person who reported it?",
        "criteria": [
            "Cosmetic: looks wrong, works fine",
            "A task fails, but another way works",
            "A task fails with no way around it",
            "Data is lost or the product is unusable",
        ],
    }
}

def triage(report):
    if verdicts(report, IS_BUG)["is_bug"]["probabilities"]["true"] < 0.5:
        return {"queue": "requests"}
    severity = verdicts(report, SEVERITY)["severity"]
    return {"queue": "bugs", "severity": severity["value"]}

triage("Since the update, exporting a report to PDF crashes the app every time. CSV export still works.")
# {"queue": "bugs", "severity": 2}

triage("Could you add a dark mode? The white background is hard on the eyes at night.")
# {"queue": "requests"}

labeled = [
    ("Payouts have failed for three days and payroll runs tomorrow.", True),
    ("Could you add an export to Excel?", False),
    # ... a few hundred more
]
URGENT = {"is_urgent": {"type": "boolean", "instructions": "Does this need action today?"}}

results = [
    (verdicts(text, URGENT)["is_urgent"]["probabilities"]["true"], label)
    for text, label in labeled
]
positives = sum(label for _, label in results)

for t in (0.5, 0.7, 0.8, 0.9, 0.95, 0.99):
    flagged = [label for p, label in results if p >= t]
    precision = sum(flagged) / len(flagged) if flagged else 1.0
    recall = sum(flagged) / positives
    print(f"{t:.2f}  precision {precision:.2f}  recall {recall:.2f}  flagged {len(flagged)}")