Evaluation

use std::sync::Arc;
use cognis::eval::{EvalCase, EvalRunner, ExactMatch};
use cognis::prelude::*;

let cases = vec![
    EvalCase::new("greeting".to_string(), "hello".to_string()).with_name("greet"),
    EvalCase::new("math".to_string(), "4".to_string()).with_name("two-plus-two"),
];

let runner = EvalRunner::new(
    Arc::new(my_runnable),       // anything Runnable<String, String>
    Arc::new(ExactMatch),        // built-in evaluator
    cases,
)
.with_concurrency(8);

let report = runner.run().await?;
println!("mean score: {:.2}", report.mean());
println!("pass rate (>= 0.8): {:.2}", report.pass_rate(0.8));

use async_trait::async_trait;
use cognis::eval::Evaluator;
use cognis::Result;

struct LengthSimilar;

#[async_trait]
impl Evaluator<String> for LengthSimilar {
    async fn score(&self, actual: &String, expected: &String) -> Result<f32> {
        let diff = (actual.len() as f32 - expected.len() as f32).abs();
        let scale = expected.len().max(1) as f32;
        Ok((1.0 - diff / scale).max(0.0))
    }
}

use cognis_trace::{ScoreRecord, ScoreValue};
use cognis_trace::exporters::langfuse::{LangfuseConfig, LangfuseScorer};

let scorer = LangfuseScorer::new(LangfuseConfig::from_env()?)?;
for (row, run_id) in report.rows.iter().zip(case_run_ids) {
    scorer.submit(ScoreRecord {
        run_id,
        // Set at least one of `trace_id` or `session_id` — `LangfuseScorer`
        // skips submission when both are None (a score with nothing to link
        // to is dropped).
        trace_id: Some(run_id),       // attach to the case's trace
        session_id: None,
        name: "exact_match".into(),
        value: ScoreValue::Numeric(row.score as f64),
        comment: row.name.clone(),
    }).await?;
}