Eval runs

Compare a run against a baseline

Compare this run against a baseline run: per-case status (regressed, fixed, new_case, removed_case, changed), per-scorer pass-rate and mean deltas from the evaluation contract, and whether the evaluation config changed. Omit baseRunId to compare against the nearest earlier completed run in the same suite. Returns 404 with details.reason = BASELINE_NOT_FOUND when there is no comparable predecessor — that is an incomplete comparison, not a failing one. A scorer whose definitionChanged is true was graded by a different definition on each side, so its delta is not a regression.

GET

projects

{projectId}

eval-runs

{runId}

compare

Compare a run against a baseline

curl --request GET \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/compare \
  --header 'Authorization: Bearer <token>'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/compare"

headers = {"Authorization": "Bearer <token>"}

response = requests.get(url, headers=headers)

print(response.text)

const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/compare', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/compare",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/compare"

	req, _ := http.NewRequest("GET", url, nil)

	req.Header.Add("Authorization", "Bearer <token>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/compare")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'

response = http.request(request)
puts response.read_body

{
  "suite": {
    "id": "<string>",
    "name": "<string>"
  },
  "baseline": {
    "policy": "previous_completed",
    "baseRunId": "<string>",
    "baseCommitSha": "<string>",
    "matchCount": 123,
    "matchCountTruncated": true
  },
  "baseRun": {
    "id": "<string>",
    "runNumber": 123,
    "result": "<string>",
    "createdAt": 123,
    "completedAt": 123,
    "summary": {
      "total": 123,
      "passed": 123,
      "failed": 123,
      "passRate": 123
    },
    "environment": {
      "id": "<string>",
      "name": "<string>"
    },
    "effectiveModelId": "<string>",
    "client": {
      "name": "<string>"
    },
    "modelSource": "client_default"
  },
  "compareRun": {
    "id": "<string>",
    "runNumber": 123,
    "result": "<string>",
    "createdAt": 123,
    "completedAt": 123,
    "summary": {
      "total": 123,
      "passed": 123,
      "failed": 123,
      "passRate": 123
    },
    "environment": {
      "id": "<string>",
      "name": "<string>"
    },
    "effectiveModelId": "<string>",
    "client": {
      "name": "<string>"
    },
    "modelSource": "client_default"
  },
  "passSummary": {
    "passRatePercent": {
      "base": 123,
      "compare": 123,
      "delta": 123,
      "percentDelta": 123
    },
    "total": {
      "base": 123,
      "compare": 123,
      "delta": 123,
      "percentDelta": 123
    },
    "passed": {
      "base": 123,
      "compare": 123,
      "delta": 123,
      "percentDelta": 123
    },
    "failed": {
      "base": 123,
      "compare": 123,
      "delta": 123,
      "percentDelta": 123
    }
  },
  "metrics": {
    "wallDurationMs": {
      "base": 123,
      "compare": 123,
      "delta": 123,
      "percentDelta": 123
    },
    "totalTokens": {
      "base": 123,
      "compare": 123,
      "delta": 123,
      "percentDelta": 123
    },
    "estimatedCostUsd": {
      "base": 123,
      "compare": 123,
      "delta": 123,
      "percentDelta": 123
    },
    "costCoverage": {
      "base": {
        "costed": 123,
        "total": 123
      },
      "compare": {
        "costed": 123,
        "total": 123
      }
    }
  },
  "scoreContract": {
    "base": {
      "evaluationConfigHash": "<string>",
      "scoreIntegrity": "valid",
      "scoredIterations": 123,
      "quarantinedIterations": 123
    },
    "compare": {
      "evaluationConfigHash": "<string>",
      "scoreIntegrity": "valid",
      "scoredIterations": 123,
      "quarantinedIterations": 123
    },
    "evaluationConfigChanged": true,
    "scorers": [
      {
        "scorerId": "<string>",
        "gating": true,
        "deterministic": true,
        "definitionChanged": true,
        "passRate": {
          "base": 123,
          "compare": 123,
          "delta": 123,
          "percentDelta": 123
        },
        "meanValue": {
          "base": 123,
          "compare": 123,
          "delta": 123,
          "percentDelta": 123
        },
        "errorCount": {
          "base": 123,
          "compare": 123
        }
      }
    ]
  },
  "cases": [
    {
      "caseKey": "<string>",
      "title": "<string>",
      "status": "unchanged_passed",
      "configChanged": true,
      "evaluationConfigChanged": true,
      "scoreDeltas": [
        {
          "scorerId": "<string>",
          "gating": true,
          "deterministic": true,
          "definitionChanged": true,
          "base": {
            "status": "scored",
            "value": 123,
            "passed": true
          },
          "compare": {
            "status": "scored",
            "value": 123,
            "passed": true
          },
          "value": {
            "base": 123,
            "compare": 123,
            "delta": 123,
            "percentDelta": 123
          }
        }
      ],
      "base": {
        "outcome": "passed",
        "iterationIds": [
          "<string>"
        ],
        "representativeIterationId": "<string>",
        "error": "<string>"
      },
      "compare": {
        "outcome": "passed",
        "iterationIds": [
          "<string>"
        ],
        "representativeIterationId": "<string>",
        "error": "<string>"
      },
      "metrics": {
        "estimatedCostUsd": {
          "base": 123,
          "compare": 123,
          "delta": 123,
          "percentDelta": 123
        },
        "costCoverage": {
          "base": {
            "costed": 123,
            "total": 123
          },
          "compare": {
            "costed": 123,
            "total": 123
          }
        }
      }
    }
  ],
  "skills": {
    "base": {
      "excluded": true,
      "count": 123
    },
    "compare": {
      "excluded": true,
      "count": 123
    },
    "changes": [
      {
        "key": "<string>",
        "name": "<string>",
        "channels": [
          "host"
        ],
        "kind": "added",
        "modelRef": "<string>",
        "renamedFrom": "<string>",
        "base": {
          "contentHash": "<string>",
          "aggregateHash": "<string>",
          "versionNumber": 123,
          "serverSkillVersionNumber": 123
        },
        "compare": {
          "contentHash": "<string>",
          "aggregateHash": "<string>",
          "versionNumber": 123,
          "serverSkillVersionNumber": 123
        },
        "versionDelta": "<string>"
      }
    ],
    "unchangedCount": 123
  }
}

Authorizations

Authorization

string

header

required

MCPJam API key (sk_…). Create one at Settings → API keys. Guest sessions cannot use the API, and API keys cannot manage other API keys.

Headers

x-mcpjam-eval-vocabulary

enum<string>

Which vocabulary this request and its response speak. Absent means 1, which is byte-for-byte today's contract: the same request fields, the same refusals, the same response projection. 2 is the canonical vocabulary. Any other value is a 400 with code: "VALIDATION_ERROR".

Today it decides one thing: the spelling of an evaluator's policy role. Vocabulary 1 accepts and returns gating; vocabulary 2 accepts both spellings and returns the canonical required. Sending required without the header is a 400, deliberately — vocabulary 1 is not widened to meet vocabulary 2 half way, because a boundary that accepts a spelling it does not announce is one two implementations can disagree about.

A response that varies by vocabulary sends Vary: x-mcpjam-eval-vocabulary.

Available options:

1,

2

Path Parameters

projectId

string

required

ID of the hosted project that contains the server.

runId

string

required

Eval run ID, as returned by POST /eval-runs.

Query Parameters

baseRunId

string

Run ID to compare against. Omit to use the nearest earlier completed run in the same suite. Mutually exclusive with baseCommitSha.

baseCommitSha

string

Source commit SHA to compare against, resolved to the completed run in this suite recorded against it. Mutually exclusive with baseRunId; sending both is a 400. A SHA matching no completed run is the ordinary 404 with details.reason = BASELINE_NOT_FOUND - an incomplete comparison, not a regression.

Response

The comparison.

suite

object

required

Show child attributes

baseline

object

required

Show child attributes

baseRun

object

required

Show child attributes

compareRun

object

required

Show child attributes

passSummary

object

required

Run-summary counters. Named passSummary, not scores, so it cannot be confused with scoreContract — the two answer different questions.

Show child attributes

metrics

object

required

Show child attributes

scoreContract

object

required

Show child attributes

cases

object[]

required

Show child attributes

skills

object | null

Which skills changed between the two runs — the configuration attribution that usually explains the case-level differences. null when NEITHER run recorded pinned skills (an empty section would instead claim no skills were involved). Absent when the deployment predates skill attribution, so clients must tolerate all three states.

Show child attributes

⌘I

Compare a run against a baseline

curl --request GET \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/compare \
  --header 'Authorization: Bearer <token>'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/compare"

headers = {"Authorization": "Bearer <token>"}

response = requests.get(url, headers=headers)

print(response.text)

const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/compare', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/compare",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/compare"

	req, _ := http.NewRequest("GET", url, nil)

	req.Header.Add("Authorization", "Bearer <token>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-runs/{runId}/compare")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'

response = http.request(request)
puts response.read_body

{
  "suite": {
    "id": "<string>",
    "name": "<string>"
  },
  "baseline": {
    "policy": "previous_completed",
    "baseRunId": "<string>",
    "baseCommitSha": "<string>",
    "matchCount": 123,
    "matchCountTruncated": true
  },
  "baseRun": {
    "id": "<string>",
    "runNumber": 123,
    "result": "<string>",
    "createdAt": 123,
    "completedAt": 123,
    "summary": {
      "total": 123,
      "passed": 123,
      "failed": 123,
      "passRate": 123
    },
    "environment": {
      "id": "<string>",
      "name": "<string>"
    },
    "effectiveModelId": "<string>",
    "client": {
      "name": "<string>"
    },
    "modelSource": "client_default"
  },
  "compareRun": {
    "id": "<string>",
    "runNumber": 123,
    "result": "<string>",
    "createdAt": 123,
    "completedAt": 123,
    "summary": {
      "total": 123,
      "passed": 123,
      "failed": 123,
      "passRate": 123
    },
    "environment": {
      "id": "<string>",
      "name": "<string>"
    },
    "effectiveModelId": "<string>",
    "client": {
      "name": "<string>"
    },
    "modelSource": "client_default"
  },
  "passSummary": {
    "passRatePercent": {
      "base": 123,
      "compare": 123,
      "delta": 123,
      "percentDelta": 123
    },
    "total": {
      "base": 123,
      "compare": 123,
      "delta": 123,
      "percentDelta": 123
    },
    "passed": {
      "base": 123,
      "compare": 123,
      "delta": 123,
      "percentDelta": 123
    },
    "failed": {
      "base": 123,
      "compare": 123,
      "delta": 123,
      "percentDelta": 123
    }
  },
  "metrics": {
    "wallDurationMs": {
      "base": 123,
      "compare": 123,
      "delta": 123,
      "percentDelta": 123
    },
    "totalTokens": {
      "base": 123,
      "compare": 123,
      "delta": 123,
      "percentDelta": 123
    },
    "estimatedCostUsd": {
      "base": 123,
      "compare": 123,
      "delta": 123,
      "percentDelta": 123
    },
    "costCoverage": {
      "base": {
        "costed": 123,
        "total": 123
      },
      "compare": {
        "costed": 123,
        "total": 123
      }
    }
  },
  "scoreContract": {
    "base": {
      "evaluationConfigHash": "<string>",
      "scoreIntegrity": "valid",
      "scoredIterations": 123,
      "quarantinedIterations": 123
    },
    "compare": {
      "evaluationConfigHash": "<string>",
      "scoreIntegrity": "valid",
      "scoredIterations": 123,
      "quarantinedIterations": 123
    },
    "evaluationConfigChanged": true,
    "scorers": [
      {
        "scorerId": "<string>",
        "gating": true,
        "deterministic": true,
        "definitionChanged": true,
        "passRate": {
          "base": 123,
          "compare": 123,
          "delta": 123,
          "percentDelta": 123
        },
        "meanValue": {
          "base": 123,
          "compare": 123,
          "delta": 123,
          "percentDelta": 123
        },
        "errorCount": {
          "base": 123,
          "compare": 123
        }
      }
    ]
  },
  "cases": [
    {
      "caseKey": "<string>",
      "title": "<string>",
      "status": "unchanged_passed",
      "configChanged": true,
      "evaluationConfigChanged": true,
      "scoreDeltas": [
        {
          "scorerId": "<string>",
          "gating": true,
          "deterministic": true,
          "definitionChanged": true,
          "base": {
            "status": "scored",
            "value": 123,
            "passed": true
          },
          "compare": {
            "status": "scored",
            "value": 123,
            "passed": true
          },
          "value": {
            "base": 123,
            "compare": 123,
            "delta": 123,
            "percentDelta": 123
          }
        }
      ],
      "base": {
        "outcome": "passed",
        "iterationIds": [
          "<string>"
        ],
        "representativeIterationId": "<string>",
        "error": "<string>"
      },
      "compare": {
        "outcome": "passed",
        "iterationIds": [
          "<string>"
        ],
        "representativeIterationId": "<string>",
        "error": "<string>"
      },
      "metrics": {
        "estimatedCostUsd": {
          "base": 123,
          "compare": 123,
          "delta": 123,
          "percentDelta": 123
        },
        "costCoverage": {
          "base": {
            "costed": 123,
            "total": 123
          },
          "compare": {
            "costed": 123,
            "total": 123
          }
        }
      }
    }
  ],
  "skills": {
    "base": {
      "excluded": true,
      "count": 123
    },
    "compare": {
      "excluded": true,
      "count": 123
    },
    "changes": [
      {
        "key": "<string>",
        "name": "<string>",
        "channels": [
          "host"
        ],
        "kind": "added",
        "modelRef": "<string>",
        "renamedFrom": "<string>",
        "base": {
          "contentHash": "<string>",
          "aggregateHash": "<string>",
          "versionNumber": 123,
          "serverSkillVersionNumber": 123
        },
        "compare": {
          "contentHash": "<string>",
          "aggregateHash": "<string>",
          "versionNumber": 123,
          "serverSkillVersionNumber": 123
        },
        "versionDelta": "<string>"
      }
    ],
    "unchangedCount": 123
  }
}

API Guides

Endpoints

Compare a run against a baseline

Authorizations

Headers

Path Parameters

Query Parameters

Response