Get the JSON report, with the execution trace

GET

https://api.moveworks.ai/rest/v1alpha1/eval_runs/:eval_run_id/report

GET

/rest/v1alpha1/eval_runs/:eval_run_id/report

curl https://api.moveworks.ai/rest/v1alpha1/eval_runs/eval_run_id/report \
     -H "Authorization: Bearer <token>"

{
  "eval_run_id": "b3d1a9e2-6f7c-4d2e-9a1b-2c3d4e5f6a7b",
  "title": "Calendar Eval",
  "generated_at": "2026-10-01T20:03:10Z",
  "summary": {
    "tasks": {
      "total": 40,
      "completed": 39,
      "failed": 1
    },
    "metrics": [
      {
        "name": "Entity Resolution",
        "kind": "METRIC_KIND_JUDGE_CATEGORY",
        "value": 0.87,
        "sample_count": 39
      },
      {
        "name": "database_state_assertions",
        "kind": "METRIC_KIND_DETERMINISTIC",
        "value": 0.92,
        "sample_count": 39
      },
      {
        "name": "end_to_end_latency_seconds",
        "kind": "METRIC_KIND_LATENCY",
        "value": 14.2,
        "sample_count": 39
      },
      {
        "name": "expected_tool_call_present",
        "kind": "METRIC_KIND_TOOL_SEQUENCE",
        "value": 0.95,
        "sample_count": 39
      },
      {
        "name": "user_turns",
        "kind": "METRIC_KIND_TURNS",
        "value": 1.4,
        "sample_count": 39
      }
    ],
    "usage": {
      "input_tokens": 1834211,
      "output_tokens": 96410,
      "total_tokens": 1930621
    }
  },
  "tasks": [
    {
      "task_id": "cal_create_001",
      "file": "calendar_suite.yaml",
      "status": "TASK_STATUS_COMPLETED",
      "index": 0,
      "conversation": {
        "conversation_id": "conv_01J...",
        "turns": [
          {
            "index": 0,
            "user": {
              "content": "Book a 30 min sync with Priya tomorrow at 10am",
              "created_at": "2026-10-01T18:01:02Z"
            },
            "assistant": {
              "status": "ASSISTANT_TURN_STATUS_COMPLETED",
              "response_id": "resp_01J...",
              "messages": [
                {
                  "content": "Done. I scheduled \"Sync\" with Priya Nair for Oct 2, 10:00–10:30 PT.",
                  "created_at": "2026-10-01T18:01:14Z"
                }
              ],
              "created_at": "2026-10-01T18:01:02Z",
              "completed_at": "2026-10-01T18:01:14Z"
            },
            "trajectory": {
              "tool_calls": [
                {
                  "call_id": "call_01",
                  "tool": {
                    "name": "search_people",
                    "namespace": "people"
                  },
                  "status": "TOOL_CALL_STATUS_COMPLETED",
                  "connectors": [
                    {
                      "name": "hr_directory",
                      "mode": "CONNECTOR_MODE_AS_IS"
                    }
                  ],
                  "arguments": {
                    "query": "Priya"
                  },
                  "output": {
                    "results": [
                      {
                        "email": "priya.nair@example.com"
                      }
                    ]
                  },
                  "requested_at": "2026-10-01T18:01:04Z",
                  "dispatched_at": "2026-10-01T18:01:04Z",
                  "completed_at": "2026-10-01T18:01:05Z",
                  "output_truncation": {
                    "truncated": true,
                    "original_bytes": 48213
                  }
                },
                {
                  "call_id": "call_02",
                  "tool": {
                    "name": "create_event",
                    "namespace": "calendar"
                  },
                  "status": "TOOL_CALL_STATUS_COMPLETED",
                  "connectors": [
                    {
                      "name": "events",
                      "mode": "CONNECTOR_MODE_MOCKED"
                    }
                  ],
                  "arguments": {
                    "attendees": [
                      "priya.nair@example.com"
                    ],
                    "end": "2026-10-02T10:30:00-07:00",
                    "start": "2026-10-02T10:00:00-07:00",
                    "subject": "Sync"
                  },
                  "output": {
                    "id": "evt_9c1"
                  },
                  "requested_at": "2026-10-01T18:01:08Z",
                  "dispatched_at": "2026-10-01T18:01:08Z",
                  "completed_at": "2026-10-01T18:01:09Z"
                }
              ],
              "available_tools": {
                "recorded": true,
                "tools": [
                  {
                    "name": "search_people",
                    "namespace": "people"
                  },
                  {
                    "name": "create_event",
                    "namespace": "calendar"
                  },
                  {
                    "name": "list_events",
                    "namespace": "calendar"
                  }
                ]
              },
              "usage": {
                "input_tokens": 41230,
                "output_tokens": 1830,
                "total_tokens": 43060,
                "cached_tokens": 12800
              }
            }
          }
        ]
      },
      "graders": [
        {
          "name": "event_created",
          "type": "GRADER_TYPE_DETERMINISTIC",
          "status": "GRADER_STATUS_PASSED",
          "score": 1,
          "deterministic": {
            "expected": 1,
            "actual": 1,
            "comparison_type": "equals"
          }
        },
        {
          "name": "resolution",
          "type": "GRADER_TYPE_LLM_JUDGE",
          "status": "GRADER_STATUS_FAILED",
          "score": 0.5,
          "judge": {
            "assertions": [
              {
                "text": "Resolves 'Priya' to exactly one employee without asking",
                "category": "Entity Resolution",
                "passed": true
              },
              {
                "text": "Confirms the resolved person's full name in the reply",
                "category": "Entity Resolution",
                "passed": false
              }
            ]
          }
        },
        {
          "name": "tool_order",
          "type": "GRADER_TYPE_TOOL_SEQUENCE",
          "status": "GRADER_STATUS_PASSED",
          "score": 1,
          "tool_sequence": {
            "expected": [
              "search_people",
              "create_event"
            ],
            "observed": [
              "search_people",
              "create_event"
            ]
          }
        }
      ],
      "metrics": {
        "latency_seconds": 12.1,
        "user_turns": 1,
        "total_turns": 2,
        "usage": {
          "input_tokens": 41230,
          "output_tokens": 1830,
          "total_tokens": 43060,
          "cached_tokens": 12800
        }
      }
    },
    {
      "task_id": "cal_reschedule_014",
      "file": "calendar_suite.yaml",
      "status": "TASK_STATUS_FAILED",
      "index": 13,
      "error": {
        "code": "TASK_ERROR_CODE_TURN_TIMEOUT",
        "phase": "TASK_ERROR_PHASE_SIMULATION",
        "attempts": 3,
        "message": "The assistant did not complete turn 2 within 180 s on any of 3 attempts."
      },
      "conversation": {
        "conversation_id": "conv_01K...",
        "turns": [
          {
            "index": 0,
            "user": {
              "content": "Move my 3pm with Diego to Friday"
            },
            "assistant": {
              "status": "ASSISTANT_TURN_STATUS_FAILED",
              "response_id": "resp_01K...",
              "error": {
                "code": "TIMEOUT",
                "message": "response not completed"
              }
            },
            "trajectory": {
              "tool_calls": [
                {
                  "call_id": "call_11",
                  "tool": {
                    "name": "list_events",
                    "namespace": "calendar"
                  },
                  "status": "TOOL_CALL_STATUS_INCOMPLETE",
                  "connectors": [
                    {
                      "name": "events",
                      "mode": "CONNECTOR_MODE_MOCKED"
                    }
                  ],
                  "arguments": {
                    "date": "today"
                  },
                  "status_reason": "TOOL_CALL_STATUS_REASON_PENDING_USER_INPUT"
                }
              ],
              "available_tools": {
                "recorded": true,
                "tools": [
                  {
                    "name": "list_events",
                    "namespace": "calendar"
                  },
                  {
                    "name": "update_event",
                    "namespace": "calendar"
                  }
                ]
              }
            }
          }
        ]
      },
      "graders": [
        {
          "name": "event_moved",
          "type": "GRADER_TYPE_DETERMINISTIC",
          "status": "GRADER_STATUS_ERROR"
        },
        {
          "name": "resolution",
          "type": "GRADER_TYPE_LLM_JUDGE",
          "status": "GRADER_STATUS_ERROR"
        }
      ]
    }
  ]
}

Available once lifecycle.report.status is REPORT_STATUS_AVAILABLE, which can precede cleanup. One entry per accepted task with its conversation, the per-turn trajectory (tool calls with arguments, outputs, status and timing; the tools the agent could see; token usage), and the result of every grader by name. Failed tasks are included with an error and no scores.

Responses are large for big batches; send Accept-Encoding: gzip. Visible to the user who created the run and to the eval admins Moveworks designates for the org; any other id answers 404 RESOURCE_NOT_FOUND.

Authentication

AuthorizationBearer

OAuth 2.0 client-credentials bearer from POST /oauth/v1/token (same as the Conversations API). The token is issued to one org and bound to one user; the run executes in that org and its conversations run as that user. If the eval config names a source_org, it must be the same org. An org that is not enabled for the Eval API sees 404 NOT_FOUND (“API not found”) on every route. An enabled org whose credential is not entitled to run evals receives 403 REQUEST_ACCESS_DENIED. submitted_by.email_addr is the email of the user the credential is bound to.

Path parameters

eval_run_idstringRequired

The run id returned by the submit call.

Response

The report.

eval_run_idstring

titlestring

generated_atdatetime

summaryobject

taskslist of objectsOptional

One entry per accepted task, in task-file order. Includes failed tasks.

Errors

400

Bad Request Error

401

Unauthorized Error

403

Forbidden Error

404

Not Found Error

409

Conflict Error

429

Too Many Requests Error

500

Internal Server Error