Eval Builder V2: spec-driven evals with generated data, real traces, and calibrated judges

app/desktop/desktop_server.py

@@ -33,10 +33,12 @@
 from app.desktop.studio_server.data_gen_api import connect_data_gen_api
 from app.desktop.studio_server.dev_tools import connect_dev_tools
 from app.desktop.studio_server.eval_api import connect_evals_api
+from app.desktop.studio_server.eval_builder_api import connect_eval_builder_api
 from app.desktop.studio_server.finetune_api import connect_fine_tune_api
 from app.desktop.studio_server.import_api import connect_import_api
 from app.desktop.studio_server.jobs.api import connect_jobs_api
 from app.desktop.studio_server.jobs.registry import job_registry
+from app.desktop.studio_server.multiturn_sdg_api import connect_multiturn_sdg_api
 from app.desktop.studio_server.prompt_api import connect_prompt_api
 from app.desktop.studio_server.prompt_optimization_job_api import (
     connect_prompt_optimization_job_api,
@@ -153,6 +155,8 @@ def make_app(tk_root: tk.Tk | None = None):
     connect_skill_api(app)
     connect_prompt_optimization_job_api(app)
     connect_copilot_api(app)
+    connect_eval_builder_api(app)
+    connect_multiturn_sdg_api(app)
     connect_batch_plan_api(app)
     connect_git_sync_api(app)
     connect_agent_api(app)

app/desktop/studio_server/api_client/kiln_ai_server_client/api/copilot/build_claim_evidence_v1_copilot_build_claim_evidence_post.py

@@ -0,0 +1,180 @@
+from http import HTTPStatus
+from typing import Any
+
+import httpx
+
+from ... import errors
+from ...client import AuthenticatedClient, Client
+from ...models.build_claim_evidence_input import BuildClaimEvidenceInput
+from ...models.build_claim_evidence_output import BuildClaimEvidenceOutput
+from ...models.http_validation_error import HTTPValidationError
+from ...models.unauthorized_response import UnauthorizedResponse
+from ...types import Response
+
+
+def _get_kwargs(
+    *,
+    body: BuildClaimEvidenceInput,
+) -> dict[str, Any]:
+    headers: dict[str, Any] = {}
+
+    _kwargs: dict[str, Any] = {
+        "method": "post",
+        "url": "/v1/copilot/build_claim_evidence",
+    }
+
+    _kwargs["json"] = body.to_dict()
+
+    headers["Content-Type"] = "application/json"
+
+    _kwargs["headers"] = headers
+    return _kwargs
+
+
+def _parse_response(
+    *, client: AuthenticatedClient | Client, response: httpx.Response
+) -> BuildClaimEvidenceOutput | HTTPValidationError | UnauthorizedResponse | None:
+    if response.status_code == 200:
+        response_200 = BuildClaimEvidenceOutput.from_dict(response.json())
+
+        return response_200
+
+    if response.status_code == 401:
+        response_401 = UnauthorizedResponse.from_dict(response.json())
+
+        return response_401
+
+    if response.status_code == 422:
+        response_422 = HTTPValidationError.from_dict(response.json())
+
+        return response_422
+
+    if client.raise_on_unexpected_status:
+        raise errors.UnexpectedStatus(response.status_code, response.content)
+    else:
+        return None
+
+
+def _build_response(
+    *, client: AuthenticatedClient | Client, response: httpx.Response
+) -> Response[BuildClaimEvidenceOutput | HTTPValidationError | UnauthorizedResponse]:
+    return Response(
+        status_code=HTTPStatus(response.status_code),
+        content=response.content,
+        headers=response.headers,
+        parsed=_parse_response(client=client, response=response),
+    )
+
+
+def sync_detailed(
+    *,
+    client: AuthenticatedClient,
+    body: BuildClaimEvidenceInput,
+) -> Response[BuildClaimEvidenceOutput | HTTPValidationError | UnauthorizedResponse]:
+    """Build Claim Evidence
+
+     Build a review card (overview plus claims) for one eval trace + judge decision.
+
+    Args:
+        body (BuildClaimEvidenceInput):
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        Response[BuildClaimEvidenceOutput | HTTPValidationError | UnauthorizedResponse]
+    """
+
+    kwargs = _get_kwargs(
+        body=body,
+    )
+
+    response = client.get_httpx_client().request(
+        **kwargs,
+    )
+
+    return _build_response(client=client, response=response)
+
+
+def sync(
+    *,
+    client: AuthenticatedClient,
+    body: BuildClaimEvidenceInput,
+) -> BuildClaimEvidenceOutput | HTTPValidationError | UnauthorizedResponse | None:
+    """Build Claim Evidence
+
+     Build a review card (overview plus claims) for one eval trace + judge decision.
+
+    Args:
+        body (BuildClaimEvidenceInput):
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        BuildClaimEvidenceOutput | HTTPValidationError | UnauthorizedResponse
+    """
+
+    return sync_detailed(
+        client=client,
+        body=body,
+    ).parsed
+
+
+async def asyncio_detailed(
+    *,
+    client: AuthenticatedClient,
+    body: BuildClaimEvidenceInput,
+) -> Response[BuildClaimEvidenceOutput | HTTPValidationError | UnauthorizedResponse]:
+    """Build Claim Evidence
+
+     Build a review card (overview plus claims) for one eval trace + judge decision.
+
+    Args:
+        body (BuildClaimEvidenceInput):
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        Response[BuildClaimEvidenceOutput | HTTPValidationError | UnauthorizedResponse]
+    """
+
+    kwargs = _get_kwargs(
+        body=body,
+    )
+
+    response = await client.get_async_httpx_client().request(**kwargs)
+
+    return _build_response(client=client, response=response)
+
+
+async def asyncio(
+    *,
+    client: AuthenticatedClient,
+    body: BuildClaimEvidenceInput,
+) -> BuildClaimEvidenceOutput | HTTPValidationError | UnauthorizedResponse | None:
+    """Build Claim Evidence
+
+     Build a review card (overview plus claims) for one eval trace + judge decision.
+
+    Args:
+        body (BuildClaimEvidenceInput):
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        BuildClaimEvidenceOutput | HTTPValidationError | UnauthorizedResponse
+    """
+
+    return (
+        await asyncio_detailed(
+            client=client,
+            body=body,
+        )
+    ).parsed