Eval runs

List a suite's per-run stage analytics

One materialized stage-analytics document per RUN, newest completion first: where trials fell out of the six-stage user-value chain (connection, discovery, selection, tool call, response, user value), overall and sliced marginally by intent, model and host.

Each item is one run’s COMPLETE document and stands alone. There is deliberately no cross-run aggregate — two funnels averaged together describe no run — so compare runs by rendering them side by side, never by summing these documents.

Counts are returned; RATES are not. A zero denominator is notMeasured, never 0, and every excluded observation is counted under a named class so a denominator can never shrink silently.

NOT backfilled: a run that completed before this shipped has no document at all, and that absence means UNMEASURED — it is never a funnel of zeros.

GET

projects

{projectId}

eval-suites

{suiteId}

stage-analytics

List a suite's per-run stage analytics

curl --request GET \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/stage-analytics \
  --header 'Authorization: Bearer <token>'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/stage-analytics"

headers = {"Authorization": "Bearer <token>"}

response = requests.get(url, headers=headers)

print(response.text)

const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/stage-analytics', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/stage-analytics",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/stage-analytics"

	req, _ := http.NewRequest("GET", url, nil)

	req.Header.Add("Authorization", "Bearer <token>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/stage-analytics")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'

response = http.request(request)
puts response.read_body

{
  "items": [
    {
      "schemaVersion": 1,
      "measurementUnit": "trial",
      "runId": "<string>",
      "suiteId": "<string>",
      "sourceIterationCount": 1,
      "stageAnalyzerVersion": 2,
      "measurementsSchemaVersion": 2,
      "materializationState": "provisional",
      "createdAt": 1,
      "updatedAt": 1,
      "includedTrials": 1,
      "excludedTrials": {
        "lifecycle": 1,
        "integrity": 1,
        "version": 1,
        "notApplicable": 1,
        "reachUnknown": 1,
        "notMeasured": 1
      },
      "totalTrials": 1,
      "excludedTrialDetail": {
        "notTerminal": 1,
        "skipped": 1,
        "cancelled": 1,
        "setupFailed": 1,
        "timedOut": 1,
        "executionFailed": 1,
        "evaluatorError": 1,
        "chainMissing": 1,
        "chainUnverified": 1,
        "chainVersionAhead": 1,
        "measurementsMissing": 1,
        "measurementsInvalid": 1,
        "measurementsVersionAhead": 1,
        "analyzerMismatch": 1
      },
      "slices": [
        {
          "slice": {
            "dimension": "overall"
          },
          "includedTrials": 1,
          "excludedTrials": {
            "lifecycle": 1,
            "integrity": 1,
            "version": 1,
            "notApplicable": 1,
            "reachUnknown": 1,
            "notMeasured": 1
          },
          "failureCategories": [
            {
              "category": "<string>",
              "count": 2
            }
          ],
          "stages": [
            {
              "stage": "connection",
              "applicable": 1,
              "reached": 1,
              "notReached": 1,
              "reachUnknown": 1,
              "measured": 1,
              "passed": 1,
              "failed": 1,
              "notMeasured": 1,
              "notApplicable": 1,
              "excluded": {
                "lifecycle": 1,
                "integrity": 1,
                "version": 1,
                "notApplicable": 1,
                "reachUnknown": 1,
                "notMeasured": 1
              },
              "reasons": [
                {
                  "reason": "<string>",
                  "count": 2
                }
              ],
              "latency": {
                "unit": "ms",
                "basis": "evidence_span_union",
                "sampleCount": 2,
                "totalMs": 1,
                "minMs": 1,
                "maxMs": 1
              }
            }
          ]
        }
      ],
      "setup": [
        {
          "phase": "connection",
          "uniqueAttempts": 1,
          "failedAttempts": 1,
          "serverAttributedFailures": 1,
          "impactedTrials": 1,
          "latency": {
            "unit": "ms",
            "basis": "setup_phase_wall",
            "sampleCount": 2,
            "totalMs": 1,
            "minMs": 1,
            "maxMs": 1
          }
        }
      ],
      "runGroupId": "<string>",
      "configRevision": "<string>",
      "caseSetFingerprint": "<string>",
      "organizationId": "<string>",
      "workspaceId": "<string>",
      "projectId": "<string>",
      "runCompletedAt": 1,
      "sourceMaxUpdatedAt": 1,
      "sourceStageAnalyzerVersions": [
        2
      ],
      "sourceMeasurementsSchemaVersions": [
        2
      ],
      "sliceTruncation": [
        {
          "dimension": "intent",
          "distinctValues": 2,
          "retained": 1
        }
      ]
    }
  ],
  "nextCursor": "<string>"
}

Authorizations

Authorization

string

header

required

MCPJam API key (sk_…). Create one at Settings → API keys. Guest sessions cannot use the API, and API keys cannot manage other API keys.

Headers

x-mcpjam-eval-vocabulary

enum<string>

Which vocabulary this request and its response speak. Absent means 1, which is byte-for-byte today's contract: the same request fields, the same refusals, the same response projection. 2 is the canonical vocabulary. Any other value is a 400 with code: "VALIDATION_ERROR".

Today it decides one thing: the spelling of an evaluator's policy role. Vocabulary 1 accepts and returns gating; vocabulary 2 accepts both spellings and returns the canonical required. Sending required without the header is a 400, deliberately — vocabulary 1 is not widened to meet vocabulary 2 half way, because a boundary that accepts a spelling it does not announce is one two implementations can disagree about.

A response that varies by vocabulary sends Vary: x-mcpjam-eval-vocabulary.

Available options:

1,

2

Path Parameters

projectId

string

required

ID of the hosted project that contains the server.

suiteId

string

required

Eval suite ID, as returned by POST /eval-runs.

Query Parameters

from

integer

Inclusive lower bound on the run's completion stamp, in epoch MILLISECONDS (not ISO). Runs that never completed carry no stamp and are excluded by any from bound. Must be less than or equal to to, or the request is a VALIDATION_ERROR.

Required range: x >= 0

integer

Inclusive upper bound on the run's completion stamp, in epoch MILLISECONDS (not ISO).

Required range: x >= 0

runGroupId

string

Narrow to one comparison group (a matrix leg, a schedule).

Minimum string length: 1

cursor

string

Opaque cursor from a previous page's nextCursor.

limit

integer

default:25

Maximum documents to return, 1–100. Defaults to 25 — these documents are large.

Required range: 1 <= x <= 100

Response

One page of per-run stage-analytics documents, newest run completion first. An empty items means this suite has no materialized documents in the requested window — which is not the same as a run whose stages all measured zero.

items

object[]

required

Show child attributes

nextCursor

string

Opaque cursor for the next page. Omitted on the last page.

⌘I

List a suite's per-run stage analytics

curl --request GET \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/stage-analytics \
  --header 'Authorization: Bearer <token>'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/stage-analytics"

headers = {"Authorization": "Bearer <token>"}

response = requests.get(url, headers=headers)

print(response.text)

const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/stage-analytics', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/stage-analytics",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/stage-analytics"

	req, _ := http.NewRequest("GET", url, nil)

	req.Header.Add("Authorization", "Bearer <token>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/stage-analytics")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'

response = http.request(request)
puts response.read_body

{
  "items": [
    {
      "schemaVersion": 1,
      "measurementUnit": "trial",
      "runId": "<string>",
      "suiteId": "<string>",
      "sourceIterationCount": 1,
      "stageAnalyzerVersion": 2,
      "measurementsSchemaVersion": 2,
      "materializationState": "provisional",
      "createdAt": 1,
      "updatedAt": 1,
      "includedTrials": 1,
      "excludedTrials": {
        "lifecycle": 1,
        "integrity": 1,
        "version": 1,
        "notApplicable": 1,
        "reachUnknown": 1,
        "notMeasured": 1
      },
      "totalTrials": 1,
      "excludedTrialDetail": {
        "notTerminal": 1,
        "skipped": 1,
        "cancelled": 1,
        "setupFailed": 1,
        "timedOut": 1,
        "executionFailed": 1,
        "evaluatorError": 1,
        "chainMissing": 1,
        "chainUnverified": 1,
        "chainVersionAhead": 1,
        "measurementsMissing": 1,
        "measurementsInvalid": 1,
        "measurementsVersionAhead": 1,
        "analyzerMismatch": 1
      },
      "slices": [
        {
          "slice": {
            "dimension": "overall"
          },
          "includedTrials": 1,
          "excludedTrials": {
            "lifecycle": 1,
            "integrity": 1,
            "version": 1,
            "notApplicable": 1,
            "reachUnknown": 1,
            "notMeasured": 1
          },
          "failureCategories": [
            {
              "category": "<string>",
              "count": 2
            }
          ],
          "stages": [
            {
              "stage": "connection",
              "applicable": 1,
              "reached": 1,
              "notReached": 1,
              "reachUnknown": 1,
              "measured": 1,
              "passed": 1,
              "failed": 1,
              "notMeasured": 1,
              "notApplicable": 1,
              "excluded": {
                "lifecycle": 1,
                "integrity": 1,
                "version": 1,
                "notApplicable": 1,
                "reachUnknown": 1,
                "notMeasured": 1
              },
              "reasons": [
                {
                  "reason": "<string>",
                  "count": 2
                }
              ],
              "latency": {
                "unit": "ms",
                "basis": "evidence_span_union",
                "sampleCount": 2,
                "totalMs": 1,
                "minMs": 1,
                "maxMs": 1
              }
            }
          ]
        }
      ],
      "setup": [
        {
          "phase": "connection",
          "uniqueAttempts": 1,
          "failedAttempts": 1,
          "serverAttributedFailures": 1,
          "impactedTrials": 1,
          "latency": {
            "unit": "ms",
            "basis": "setup_phase_wall",
            "sampleCount": 2,
            "totalMs": 1,
            "minMs": 1,
            "maxMs": 1
          }
        }
      ],
      "runGroupId": "<string>",
      "configRevision": "<string>",
      "caseSetFingerprint": "<string>",
      "organizationId": "<string>",
      "workspaceId": "<string>",
      "projectId": "<string>",
      "runCompletedAt": 1,
      "sourceMaxUpdatedAt": 1,
      "sourceStageAnalyzerVersions": [
        2
      ],
      "sourceMeasurementsSchemaVersions": [
        2
      ],
      "sliceTruncation": [
        {
          "dimension": "intent",
          "distinctValues": 2,
          "retained": 1
        }
      ]
    }
  ],
  "nextCursor": "<string>"
}

API Guides

Endpoints

List a suite's per-run stage analytics

Authorizations

Headers

Path Parameters

Query Parameters

Response