Eval runs

Generate eval cases from the suite's tools

Discovers the suite’s server tools over a live MCP connection, generates cases against them, and persists them — the only edit route that connects to a server. Synchronous: connect, generate, persist, disconnect, respond.

The authoring model is on MCPJam: no credits are consumed. The call still counts against the project’s daily generation quota, so it can be refused with 429 RATE_LIMITED once that is used up.

An environment-based suite generates against that environment’s closed server set, so the cases match the tools its runs will actually see.

Pass x-mcpjam-idempotency-key to make a retry safe: drafts are recorded backend-side BEFORE any case is persisted, so a replay reuses them instead of taking a second slice of the daily quota, and each case is persisted under a derived per-item key so the loop is resumable.

POST

projects

{projectId}

eval-suites

{suiteId}

cases

generate

Generate eval cases from the suite's tools

curl --request POST \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/generate \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "servers": [
    "<string>"
  ],
  "environmentId": "<string>",
  "caseModels": [
    {
      "model": "<string>",
      "provider": "<string>"
    }
  ],
  "caseMix": {
    "simple": 5,
    "multiTool": 5,
    "multiTurn": 5,
    "complex": 5,
    "negative": 5
  },
  "varyUserStyles": true,
  "idempotencyKey": "<string>"
}
'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/generate"

payload = {
    "servers": ["<string>"],
    "environmentId": "<string>",
    "caseModels": [
        {
            "model": "<string>",
            "provider": "<string>"
        }
    ],
    "caseMix": {
        "simple": 5,
        "multiTool": 5,
        "multiTurn": 5,
        "complex": 5,
        "negative": 5
    },
    "varyUserStyles": True,
    "idempotencyKey": "<string>"
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    servers: ['<string>'],
    environmentId: '<string>',
    caseModels: [{model: '<string>', provider: '<string>'}],
    caseMix: {simple: 5, multiTool: 5, multiTurn: 5, complex: 5, negative: 5},
    varyUserStyles: true,
    idempotencyKey: '<string>'
  })
};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/generate', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/generate",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'servers' => [
        '<string>'
    ],
    'environmentId' => '<string>',
    'caseModels' => [
        [
                'model' => '<string>',
                'provider' => '<string>'
        ]
    ],
    'caseMix' => [
        'simple' => 5,
        'multiTool' => 5,
        'multiTurn' => 5,
        'complex' => 5,
        'negative' => 5
    ],
    'varyUserStyles' => true,
    'idempotencyKey' => '<string>'
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/generate"

	payload := strings.NewReader("{\n  \"servers\": [\n    \"<string>\"\n  ],\n  \"environmentId\": \"<string>\",\n  \"caseModels\": [\n    {\n      \"model\": \"<string>\",\n      \"provider\": \"<string>\"\n    }\n  ],\n  \"caseMix\": {\n    \"simple\": 5,\n    \"multiTool\": 5,\n    \"multiTurn\": 5,\n    \"complex\": 5,\n    \"negative\": 5\n  },\n  \"varyUserStyles\": true,\n  \"idempotencyKey\": \"<string>\"\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/generate")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"servers\": [\n    \"<string>\"\n  ],\n  \"environmentId\": \"<string>\",\n  \"caseModels\": [\n    {\n      \"model\": \"<string>\",\n      \"provider\": \"<string>\"\n    }\n  ],\n  \"caseMix\": {\n    \"simple\": 5,\n    \"multiTool\": 5,\n    \"multiTurn\": 5,\n    \"complex\": 5,\n    \"negative\": 5\n  },\n  \"varyUserStyles\": true,\n  \"idempotencyKey\": \"<string>\"\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/generate")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"servers\": [\n    \"<string>\"\n  ],\n  \"environmentId\": \"<string>\",\n  \"caseModels\": [\n    {\n      \"model\": \"<string>\",\n      \"provider\": \"<string>\"\n    }\n  ],\n  \"caseMix\": {\n    \"simple\": 5,\n    \"multiTool\": 5,\n    \"multiTurn\": 5,\n    \"complex\": 5,\n    \"negative\": 5\n  },\n  \"varyUserStyles\": true,\n  \"idempotencyKey\": \"<string>\"\n}"

response = http.request(request)
puts response.read_body

{
  "generationModel": "<string>",
  "created": [
    {
      "id": "<string>",
      "title": "<string>",
      "steps": [
        {
          "id": "<string>",
          "kind": "prompt",
          "prompt": "<string>",
          "serverName": "<string>",
          "toolName": "<string>",
          "arguments": {},
          "action": {},
          "assertion": {}
        }
      ],
      "iterations": 5,
      "isNegative": true,
      "models": [
        {
          "model": "<string>",
          "provider": "<string>"
        }
      ],
      "judge": {
        "enabled": true
      },
      "declaredId": "<string>",
      "expectedOutput": "<string>",
      "repetitions": 2,
      "passThreshold": 0.5,
      "scenario": "<string>",
      "intent": "<string>",
      "kind": "capability",
      "matchOptions": {},
      "checks": {
        "mode": "inherit",
        "list": [
          {
            "type": "toolDescriptionsPresent",
            "minLength": 123,
            "role": "gating",
            "severity": "warn"
          }
        ]
      },
      "import": {
        "status": "exact",
        "sourceCaseKey": "<string>",
        "note": "<string>"
      },
      "source": {
        "format": "markdown",
        "method": "ai",
        "fileName": "<string>",
        "fileHash": "<string>",
        "excerpt": "<string>",
        "startLine": 2,
        "endLine": 2,
        "extractorVersion": "<string>"
      },
      "createdAt": 123,
      "updatedAt": 123,
      "suppressedSuiteStandardCheckIds": [
        "<string>"
      ]
    }
  ],
  "counts": {
    "normal": 123,
    "negative": 123
  },
  "skipped": [
    {}
  ]
}

Authorizations

Authorization

string

header

required

MCPJam API key (sk_…). Create one at Settings → API keys. Guest sessions cannot use the API, and API keys cannot manage other API keys.

Headers

x-mcpjam-eval-vocabulary

enum<string>

Which vocabulary this request and its response speak. Absent means 1, which is byte-for-byte today's contract: the same request fields, the same refusals, the same response projection. 2 is the canonical vocabulary. Any other value is a 400 with code: "VALIDATION_ERROR".

Today it decides one thing: the spelling of an evaluator's policy role. Vocabulary 1 accepts and returns gating; vocabulary 2 accepts both spellings and returns the canonical required. Sending required without the header is a 400, deliberately — vocabulary 1 is not widened to meet vocabulary 2 half way, because a boundary that accepts a spelling it does not announce is one two implementations can disagree about.

A response that varies by vocabulary sends Vary: x-mcpjam-eval-vocabulary.

Available options:

1,

2

x-mcpjam-idempotency-key

string

Makes a retry replay the recorded drafts instead of taking a second slice of the daily generation quota.

Path Parameters

projectId

string

required

ID of the hosted project that contains the server.

suiteId

string

required

Eval suite ID, as returned by POST /eval-runs.

Body

application/json

AI-generate cases from the suite's server tools and persist them. The authoring model is on MCPJam — no credits are consumed; it counts against the project's daily generation quota.

mode

enum<string>

Superseded by caseMix when that is present.

Available options:

normal,

negative

servers

string[]

Server ids or names to discover tools from. Ignored when the suite is environment-based.

Minimum string length: 1

environmentId

string

Discover tools from this attached environment's closed server set, so generated cases are written against the tools the suite's runs will actually see.

Minimum string length: 1

caseModels

object[]

Show child attributes

caseMix

object

Per-bucket case counts. Omitted buckets inherit the default mix; the backend bounds each bucket and the total.

Show child attributes

varyUserStyles

boolean

Condition generated cases on a range of user styles so the queries read like different users wrote them.

idempotencyKey

string

Write-idempotency key. A repeat call with the same key replays recorded drafts instead of spending credits again. The Idempotency-Key / x-mcpjam-idempotency-key header carries the same value and WINS over this field.

Required string length: 1 - 256

Response

The generated cases.

generationModel

string

required

created

object[]

required

Show child attributes

counts

object

required

Show child attributes

skipped

object[]

Drafts that were generated but failed to persist. Surfaced rather than silently dropped.

⌘I

Generate eval cases from the suite's tools

curl --request POST \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/generate \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "servers": [
    "<string>"
  ],
  "environmentId": "<string>",
  "caseModels": [
    {
      "model": "<string>",
      "provider": "<string>"
    }
  ],
  "caseMix": {
    "simple": 5,
    "multiTool": 5,
    "multiTurn": 5,
    "complex": 5,
    "negative": 5
  },
  "varyUserStyles": true,
  "idempotencyKey": "<string>"
}
'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/generate"

payload = {
    "servers": ["<string>"],
    "environmentId": "<string>",
    "caseModels": [
        {
            "model": "<string>",
            "provider": "<string>"
        }
    ],
    "caseMix": {
        "simple": 5,
        "multiTool": 5,
        "multiTurn": 5,
        "complex": 5,
        "negative": 5
    },
    "varyUserStyles": True,
    "idempotencyKey": "<string>"
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    servers: ['<string>'],
    environmentId: '<string>',
    caseModels: [{model: '<string>', provider: '<string>'}],
    caseMix: {simple: 5, multiTool: 5, multiTurn: 5, complex: 5, negative: 5},
    varyUserStyles: true,
    idempotencyKey: '<string>'
  })
};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/generate', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/generate",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'servers' => [
        '<string>'
    ],
    'environmentId' => '<string>',
    'caseModels' => [
        [
                'model' => '<string>',
                'provider' => '<string>'
        ]
    ],
    'caseMix' => [
        'simple' => 5,
        'multiTool' => 5,
        'multiTurn' => 5,
        'complex' => 5,
        'negative' => 5
    ],
    'varyUserStyles' => true,
    'idempotencyKey' => '<string>'
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/generate"

	payload := strings.NewReader("{\n  \"servers\": [\n    \"<string>\"\n  ],\n  \"environmentId\": \"<string>\",\n  \"caseModels\": [\n    {\n      \"model\": \"<string>\",\n      \"provider\": \"<string>\"\n    }\n  ],\n  \"caseMix\": {\n    \"simple\": 5,\n    \"multiTool\": 5,\n    \"multiTurn\": 5,\n    \"complex\": 5,\n    \"negative\": 5\n  },\n  \"varyUserStyles\": true,\n  \"idempotencyKey\": \"<string>\"\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/generate")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"servers\": [\n    \"<string>\"\n  ],\n  \"environmentId\": \"<string>\",\n  \"caseModels\": [\n    {\n      \"model\": \"<string>\",\n      \"provider\": \"<string>\"\n    }\n  ],\n  \"caseMix\": {\n    \"simple\": 5,\n    \"multiTool\": 5,\n    \"multiTurn\": 5,\n    \"complex\": 5,\n    \"negative\": 5\n  },\n  \"varyUserStyles\": true,\n  \"idempotencyKey\": \"<string>\"\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/generate")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"servers\": [\n    \"<string>\"\n  ],\n  \"environmentId\": \"<string>\",\n  \"caseModels\": [\n    {\n      \"model\": \"<string>\",\n      \"provider\": \"<string>\"\n    }\n  ],\n  \"caseMix\": {\n    \"simple\": 5,\n    \"multiTool\": 5,\n    \"multiTurn\": 5,\n    \"complex\": 5,\n    \"negative\": 5\n  },\n  \"varyUserStyles\": true,\n  \"idempotencyKey\": \"<string>\"\n}"

response = http.request(request)
puts response.read_body

{
  "generationModel": "<string>",
  "created": [
    {
      "id": "<string>",
      "title": "<string>",
      "steps": [
        {
          "id": "<string>",
          "kind": "prompt",
          "prompt": "<string>",
          "serverName": "<string>",
          "toolName": "<string>",
          "arguments": {},
          "action": {},
          "assertion": {}
        }
      ],
      "iterations": 5,
      "isNegative": true,
      "models": [
        {
          "model": "<string>",
          "provider": "<string>"
        }
      ],
      "judge": {
        "enabled": true
      },
      "declaredId": "<string>",
      "expectedOutput": "<string>",
      "repetitions": 2,
      "passThreshold": 0.5,
      "scenario": "<string>",
      "intent": "<string>",
      "kind": "capability",
      "matchOptions": {},
      "checks": {
        "mode": "inherit",
        "list": [
          {
            "type": "toolDescriptionsPresent",
            "minLength": 123,
            "role": "gating",
            "severity": "warn"
          }
        ]
      },
      "import": {
        "status": "exact",
        "sourceCaseKey": "<string>",
        "note": "<string>"
      },
      "source": {
        "format": "markdown",
        "method": "ai",
        "fileName": "<string>",
        "fileHash": "<string>",
        "excerpt": "<string>",
        "startLine": 2,
        "endLine": 2,
        "extractorVersion": "<string>"
      },
      "createdAt": 123,
      "updatedAt": 123,
      "suppressedSuiteStandardCheckIds": [
        "<string>"
      ]
    }
  ],
  "counts": {
    "normal": 123,
    "negative": 123
  },
  "skipped": [
    {}
  ]
}

API Guides

Endpoints

Generate eval cases from the suite's tools

Authorizations

Headers

Path Parameters

Body

Response