Shared Contracts

from dataclasses import dataclass

from agentcompass.benchmarks.config import RuntimeBenchmarkConfig
from agentcompass.runtime.config import config_field, parse_bool


@dataclass(slots=True)
class ExampleExactMatchConfig(RuntimeBenchmarkConfig):
    case_sensitive: bool = config_field(
        default=False,
        description="Compare answers with case sensitivity.",
    )

    def __post_init__(self) -> None:
        RuntimeBenchmarkConfig.__post_init__(self)
        self.case_sensitive = parse_bool(self.case_sensitive, "case_sensitive")

def load_tasks(self, req: RunRequest) -> list[TaskSpec]:
    _ = req
    return [
        TaskSpec(
            task_id="capital-france",
            question="What is the capital of France? Answer with only the city name.",
            category="geography",
            ground_truth="Paris",
            metadata={"dataset_revision": "tutorial-v1"},
        )
    ]

agentcompass run deepswe claude_code "$MODEL_NAME" \
  --env docker \
  --model-base-url "$MODEL_BASE_URL" \
  --model-api-key "$MODEL_API_KEY" \
  --execution-params '{
    "save_artifacts": true,
    "artifacts": [
      {"source": "/app/submission", "destination": "submission", "exclude": ["*.tmp"]},
      {"source": "/app/result.json", "destination": "result.json"}
    ],
    "artifact_collect": [
      {"command": "mkdir -p /app/submission && printf example > /app/submission/note.txt", "timeout_seconds": 60}
    ]
  }'

artifacts = ["/logs/artifacts/result.txt"]

[[verifier.collect]]
command = "mkdir -p /logs/artifacts && cp /app/result.txt /logs/artifacts/result.txt"
timeout_sec = 300.0

from dataclasses import dataclass

from agentcompass.runtime import BenchmarkPlan, EnvironmentSpec, RunRequest, TaskSpec


@dataclass(slots=True)
class ExampleExactMatchPlan(BenchmarkPlan):
    expected: str = ""
    case_sensitive: bool = False


def build_plan(
    self,
    task: TaskSpec,
    req: RunRequest,
    environment: EnvironmentSpec,
) -> ExampleExactMatchPlan:
    _ = environment
    config = self.build_config(req)
    if not isinstance(config, ExampleExactMatchConfig):
        raise TypeError("example_exact_match requires ExampleExactMatchConfig")
    return ExampleExactMatchPlan(
        expected=str(task.ground_truth),
        case_sensitive=config.case_sensitive,
    )

async def prepare_task(
    self,
    task: TaskSpec,
    env: EnvironmentSession,
    req: RunRequest,
    plan: BenchmarkPlan,
) -> PreparedTask:
    _ = env, req
    self._require_plan(plan)
    return PreparedTask(
        task_id=task.task_id,
        category=task.category,
        ground_truth=None,
        input=TaskInput(prompt=task.question),
        output=TaskOutput(answer="Return only the city name."),
        metadata={"dataset_revision": task.metadata["dataset_revision"]},
    )

from .example_exact_match import ExampleExactMatchBenchmark

uv run agentcompass list benchmark
uv run agentcompass config docs benchmark example_exact_match