Sync Offline Evaluations

import uuid
from honeyhive import HoneyHive
from honeyhive.models import PostExperimentRunRequest, PutExperimentRunRequest

client = HoneyHive()  # reads HH_API_KEY (and HH_API_URL) from your environment

# Results you computed offline, with any framework (or none).
results = [
    {
        "inputs": {"question": "What is the capital of France?"},
        "outputs": {"answer": "Paris"},
        "metrics": {"exact_match": 1.0, "relevance": 0.95},
    },
    {
        "inputs": {"question": "Who wrote Hamlet?"},
        "outputs": {"answer": "Shakespeare"},
        "metrics": {"exact_match": 1.0, "relevance": 0.88},
    },
]

# 1. Create a pending experiment run (the project is scoped by your API key)
run = client.experiments.create_run(
    PostExperimentRunRequest(name="offline-eval-v1", event_ids=[], status="pending")
)
run_id = run.run_id

# 2. Log each result as a session, attaching your precomputed scores as metrics
session_ids = []
for item in results:
    response = client.sessions.start(
        {
            "session": {
                "session_name": "offline-eval-v1",
                "source": "evaluation",
                "session_id": str(uuid.uuid4()),
                "inputs": item["inputs"],
                "outputs": item["outputs"],
                "metrics": item["metrics"],
                "metadata": {"run_id": run_id, "datapoint_id": str(uuid.uuid4())},
            }
        }
    )
    session_ids.append(response.session_id)

# 3. Complete the run, linking every session
client.experiments.update_run(
    run_id, PutExperimentRunRequest(event_ids=session_ids, status="completed")
)

import os
import uuid
import requests

HH_API_KEY = os.environ["HH_API_KEY"]
# Standard cloud uses https://api.dp1.us.honeyhive.ai. Self-hosted/dedicated
# deployments set their own host (the value of HH_API_URL).
BASE_URL = os.getenv("HH_API_URL", "https://api.dp1.us.honeyhive.ai")
headers = {"Authorization": f"Bearer {HH_API_KEY}", "Content-Type": "application/json"}

# Results you computed offline, with any framework (or none).
results = [
    {
        "inputs": {"question": "What is the capital of France?"},
        "outputs": {"answer": "Paris"},
        "metrics": {"exact_match": 1.0, "relevance": 0.95},
    },
    {
        "inputs": {"question": "Who wrote Hamlet?"},
        "outputs": {"answer": "Shakespeare"},
        "metrics": {"exact_match": 1.0, "relevance": 0.88},
    },
]

# 1. Create a pending experiment run (the project is scoped by your API key)
run = requests.post(
    f"{BASE_URL}/v1/runs",
    headers=headers,
    json={"name": "offline-eval-v1", "event_ids": [], "status": "pending"},
).json()
run_id = run["run_id"]

# 2. Log each result as a session, attaching your precomputed scores as metrics
session_ids = []
for item in results:
    response = requests.post(
        f"{BASE_URL}/session/start",
        headers=headers,
        json={
            "session": {
                "session_name": "offline-eval-v1",
                "source": "evaluation",
                "session_id": str(uuid.uuid4()),
                "inputs": item["inputs"],
                "outputs": item["outputs"],
                "metrics": item["metrics"],
                "metadata": {
                    "run_id": run_id,
                    "datapoint_id": str(uuid.uuid4()),
                },
            }
        },
    )
    session_ids.append(response.json()["session_id"])

# 3. Complete the run, linking every session
requests.put(
    f"{BASE_URL}/v1/runs/{run_id}",
    headers=headers,
    json={"event_ids": session_ids, "status": "completed"},
)